Added support for snapshot related metrics

README.md

@@ -152,6 +152,13 @@ To disable exporting cluster settings use:
 prometheus.cluster.settings: false
 ```
 
+#### Snapshot metrics
+
+By default, snapshot metrics are disabled. To enable exporting snapshot metrics use:
+```
+prometheus.snapshots: true
+```
+
 #### Nodes filter
 
 Metrics include statistics about individual OpenSearch nodes.

build.gradle

@@ -1,3 +1,4 @@
+import org.opensearch.gradle.PropertyNormalization
 import org.opensearch.gradle.test.RestIntegTestTask
 
 import java.util.concurrent.Callable
@@ -113,7 +114,7 @@ dependencies {
 
 restResources {
     restApi {
-        includeCore '_common', 'cat', 'cluster', 'nodes', 'indices', 'index'
+        includeCore '_common', 'cat', 'cluster', 'nodes', 'indices', 'index', 'snapshot'
     }
 }
 
@@ -139,8 +140,13 @@ tasks.named("check").configure { dependsOn(integTest) }
 // Temporary disable task :testingConventions
 testingConventions.enabled = false
 
+// Directory for snapshot repository
+File repositoryDir = new File(project.layout.buildDirectory.get().asFile, "shared-repository")
+
 testClusters.all {
     numberOfNodes = 2
+    // Configuring repo path for 'fs' type snapshot repository
+    setting 'path.repo', "${repositoryDir.absolutePath}", PropertyNormalization.IGNORE_VALUE
 
     // It seems cluster name can not be customized here. It gives an error:
     // Testclusters does not allow the following settings to be changed:[cluster.name] for node{::yamlRestTest-0}

src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java

@@ -18,6 +18,7 @@
 package org.compuscene.metrics.prometheus;
 
 import org.opensearch.action.ClusterStatsData;
+import org.opensearch.action.SnapshotsResponse;
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.cluster.node.stats.NodeStats;
 import org.opensearch.action.admin.indices.stats.CommonStats;
@@ -37,6 +38,8 @@
 import org.opensearch.monitor.os.OsStats;
 import org.opensearch.monitor.process.ProcessStats;
 import org.opensearch.script.ScriptStats;
+import org.opensearch.snapshots.SnapshotInfo;
+import org.opensearch.snapshots.SnapshotState;
 import org.opensearch.threadpool.ThreadPoolStats;
 import org.opensearch.transport.TransportStats;
 
@@ -54,19 +57,23 @@ public class PrometheusMetricsCollector {
 
     private boolean isPrometheusClusterSettings;
     private boolean isPrometheusIndices;
+    private boolean isPrometheusSnapshots;
     private PrometheusMetricsCatalog catalog;
 
     /**
      * A constructor.
      * @param catalog {@link PrometheusMetricsCatalog}
      * @param isPrometheusIndices boolean flag for index level metric
+     * @param isPrometheusSnapshots boolean flag for snapshots related metrics
      * @param isPrometheusClusterSettings boolean flag cluster settings metrics
      */
     public PrometheusMetricsCollector(PrometheusMetricsCatalog catalog,
                                       boolean isPrometheusIndices,
+                                      boolean isPrometheusSnapshots,
                                       boolean isPrometheusClusterSettings) {
         this.isPrometheusClusterSettings = isPrometheusClusterSettings;
         this.isPrometheusIndices = isPrometheusIndices;
+        this.isPrometheusSnapshots = isPrometheusSnapshots;
         this.catalog = catalog;
     }
 
@@ -80,6 +87,7 @@ public void registerMetrics() {
         registerNodeMetrics();
         registerIndicesMetrics();
         registerPerIndexMetrics();
+        registerSnapshotMetrics();
         registerTransportMetrics();
         registerHTTPMetrics();
         registerThreadPoolMetrics();
@@ -465,6 +473,30 @@ private void updatePerIndexMetrics(@Nullable ClusterHealthResponse chr, @Nullabl
         }
     }
 
+    @SuppressWarnings("checkstyle:LineLength")
+    private void registerSnapshotMetrics() {
+        catalog.registerClusterGauge("min_snapshot_age", "Time elapsed in milliseconds since the most recent successful snapshot's start time", "sm_policy");
+    }
+
+    private void updateSnapshotsMetrics(@Nullable SnapshotsResponse snapshotsResponse) {
+        if (snapshotsResponse == null) {
+            return;
+        }
+        Map<String, Long> smPolicyMinSnapshotAge = new HashMap<>();
+        for (SnapshotInfo snapshotInfo : snapshotsResponse.getSnapshotInfos()) {
+            // emit min_snapshot_age metric only for successful snapshots
+            if (snapshotInfo.state() != SnapshotState.SUCCESS) {
+                continue;
+            }
+            String smPolicy = snapshotInfo.userMetadata() == null ? "adhoc" : snapshotInfo.userMetadata().getOrDefault("sm_policy", "adhoc").toString();
+            long snapshotAge = System.currentTimeMillis() - snapshotInfo.startTime();
+            smPolicyMinSnapshotAge.compute(smPolicy, (key, oldValue) -> oldValue == null ? snapshotAge : Math.min(oldValue, snapshotAge));
+        }
+        for(Map.Entry<String, Long> entry : smPolicyMinSnapshotAge.entrySet()) {
+            catalog.setClusterGauge("min_snapshot_age", entry.getValue(), entry.getKey());
+        }
+    }
+
     @SuppressWarnings("checkstyle:LineLength")
     private void updatePerIndexContextMetrics(String indexName, String context, CommonStats idx) {
         catalog.setClusterGauge("index_doc_number", idx.getDocs().getCount(), indexName, context);
@@ -920,12 +952,14 @@ private void updateESSettings(@Nullable ClusterStatsData stats) {
      * @param nodeStats                 NodeStats filtered using nodes filter
      * @param indicesStats              IndicesStatsResponse
      * @param clusterStatsData          ClusterStatsData
+     * @param snapshotsResponse         SnapshotsResponse
      */
     public void updateMetrics(String originNodeName, String originNodeId,
                               @Nullable ClusterHealthResponse clusterHealthResponse,
                               NodeStats[] nodeStats,
                               @Nullable IndicesStatsResponse indicesStats,
-                              @Nullable ClusterStatsData clusterStatsData) {
+                              @Nullable ClusterStatsData clusterStatsData,
+                              @Nullable SnapshotsResponse snapshotsResponse) {
         Summary.Timer timer = catalog.startSummaryTimer(
                 new Tuple<>(originNodeName, originNodeId),
                 "metrics_generate_time_seconds");
@@ -956,7 +990,9 @@ public void updateMetrics(String originNodeName, String originNodeId,
         if (isPrometheusClusterSettings) {
             updateESSettings(clusterStatsData);
         }
-
+        if (isPrometheusSnapshots) {
+            updateSnapshotsMetrics(snapshotsResponse);
+        }
         timer.observeDuration();
     }
 

src/main/java/org/compuscene/metrics/prometheus/PrometheusSettings.java

@@ -49,26 +49,35 @@ public enum INDEX_FILTER_OPTIONS {
 
     static String PROMETHEUS_CLUSTER_SETTINGS_KEY = "prometheus.cluster.settings";
     static String PROMETHEUS_INDICES_KEY = "prometheus.indices";
+    static String PROMETHEUS_SNAPSHOTS_KEY = "prometheus.snapshots";
     static String PROMETHEUS_NODES_FILTER_KEY = "prometheus.nodes.filter";
     static String PROMETHEUS_SELECTED_INDICES_KEY = "prometheus.indices_filter.selected_indices";
     static String PROMETHEUS_SELECTED_OPTION_KEY = "prometheus.indices_filter.selected_option";
 
     /**
-     * This setting is used configure weather to expose cluster settings metrics or not. The default value is true.
+     * This setting is used configure whether to expose cluster settings metrics or not. The default value is true.
      * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_CLUSTER_SETTINGS_KEY}.
      */
     public static final Setting<Boolean> PROMETHEUS_CLUSTER_SETTINGS =
             Setting.boolSetting(PROMETHEUS_CLUSTER_SETTINGS_KEY, true,
                     Setting.Property.Dynamic, Setting.Property.NodeScope);
 
     /**
-     * This setting is used configure weather to expose low level index metrics or not. The default value is true.
+     * This setting is used configure whether to expose low level index metrics or not. The default value is true.
      * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_INDICES_KEY}.
      */
     public static final Setting<Boolean> PROMETHEUS_INDICES =
             Setting.boolSetting(PROMETHEUS_INDICES_KEY, true,
                     Setting.Property.Dynamic, Setting.Property.NodeScope);
 
+    /**
+     * This setting is used configure whether to expose snapshot metrics or not. The default value is false.
+     * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_SNAPSHOTS_KEY}.
+     */
+    public static final Setting<Boolean> PROMETHEUS_SNAPSHOTS =
+            Setting.boolSetting(PROMETHEUS_SNAPSHOTS_KEY, false,
+                    Setting.Property.Dynamic, Setting.Property.NodeScope);
+
     /**
      * This setting is used configure which cluster nodes to gather metrics from. The default value is _local.
      * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_NODES_FILTER_KEY}.
@@ -97,6 +106,7 @@ public enum INDEX_FILTER_OPTIONS {
 
     private volatile boolean clusterSettings;
     private volatile boolean indices;
+    private volatile boolean snapshots;
     private volatile String nodesFilter;
     private volatile String selectedIndices;
     private volatile INDEX_FILTER_OPTIONS selectedOption;
@@ -109,11 +119,13 @@ public enum INDEX_FILTER_OPTIONS {
     public PrometheusSettings(Settings settings, ClusterSettings clusterSettings) {
         setPrometheusClusterSettings(PROMETHEUS_CLUSTER_SETTINGS.get(settings));
         setPrometheusIndices(PROMETHEUS_INDICES.get(settings));
+        setPrometheusSnapshots(PROMETHEUS_SNAPSHOTS.get(settings));
         setPrometheusNodesFilter(PROMETHEUS_NODES_FILTER.get(settings));
         setPrometheusSelectedIndices(PROMETHEUS_SELECTED_INDICES.get(settings));
         setPrometheusSelectedOption(PROMETHEUS_SELECTED_OPTION.get(settings));
         clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_CLUSTER_SETTINGS, this::setPrometheusClusterSettings);
         clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_INDICES, this::setPrometheusIndices);
+        clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_SNAPSHOTS, this::setPrometheusSnapshots);
         clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_NODES_FILTER, this::setPrometheusNodesFilter);
         clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_SELECTED_INDICES, this::setPrometheusSelectedIndices);
         clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_SELECTED_OPTION, this::setPrometheusSelectedOption);
@@ -127,6 +139,10 @@ private void setPrometheusIndices(boolean flag) {
         this.indices = flag;
     }
 
+    private void setPrometheusSnapshots(boolean flag) {
+        this.snapshots = flag;
+    }
+
     private void setPrometheusNodesFilter(String filter) { this.nodesFilter = filter; }
 
     private void setPrometheusSelectedIndices(String selectedIndices) {
@@ -153,6 +169,14 @@ public boolean getPrometheusIndices() {
         return this.indices;
     }
 
+    /**
+     * Get value of settings key {@link #PROMETHEUS_SNAPSHOTS_KEY}.
+     * @return boolean value of the key
+     */
+    public boolean getPrometheusSnapshots() {
+        return this.snapshots;
+    }
+
     /**
      * Get value of settings key {@link #PROMETHEUS_NODES_FILTER_KEY}.
      * @return boolean value of the key

src/main/java/org/opensearch/action/NodePrometheusMetricsResponse.java

@@ -17,6 +17,7 @@
 
 package org.opensearch.action;
 
+import org.opensearch.Version;
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.opensearch.action.admin.cluster.node.stats.NodeStats;
@@ -29,6 +30,7 @@
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.search.pipeline.SearchPipelineStats;
 
 import java.io.IOException;
 
@@ -43,6 +45,7 @@ public class NodePrometheusMetricsResponse extends ActionResponse {
     private final NodeStats[] nodeStats;
     @Nullable private final IndicesStatsResponse indicesStats;
     private ClusterStatsData clusterStatsData = null;
+    @Nullable private final SnapshotsResponse snapshotsResponse;
 
     /**
      * A constructor that materialize the instance from inputStream.
@@ -56,6 +59,11 @@ public NodePrometheusMetricsResponse(StreamInput in) throws IOException {
         nodeStats = in.readArray(NodeStats::new, NodeStats[]::new);
         indicesStats = PackageAccessHelper.createIndicesStatsResponse(in);
         clusterStatsData = new ClusterStatsData(in);
+        if (in.getVersion().onOrAfter(Version.V_2_17_1)) {
+            snapshotsResponse = new SnapshotsResponse(in);
+        } else {
+            snapshotsResponse = null;
+        }
     }
 
     /**
@@ -65,6 +73,7 @@ public NodePrometheusMetricsResponse(StreamInput in) throws IOException {
      * @param nodesStats NodesStats
      * @param indicesStats IndicesStats
      * @param clusterStateResponse ClusterStateResponse
+     * @param snapshotsResponse SnapshotsResponse
      * @param settings Settings
      * @param clusterSettings ClusterSettings
      */
@@ -73,6 +82,7 @@ public NodePrometheusMetricsResponse(ClusterHealthResponse clusterHealth,
                                          NodeStats[] nodesStats,
                                          @Nullable IndicesStatsResponse indicesStats,
                                          @Nullable ClusterStateResponse clusterStateResponse,
+                                         @Nullable SnapshotsResponse snapshotsResponse,
                                          Settings settings,
                                          ClusterSettings clusterSettings) {
         this.clusterHealth = clusterHealth;
@@ -82,6 +92,7 @@ public NodePrometheusMetricsResponse(ClusterHealthResponse clusterHealth,
         if (clusterStateResponse != null) {
             this.clusterStatsData = new ClusterStatsData(clusterStateResponse, settings, clusterSettings);
         }
+        this.snapshotsResponse = snapshotsResponse;
     }
 
     /**
@@ -106,6 +117,15 @@ public NodeStats[] getNodeStats() {
         return this.nodeStats;
     }
 
+    /**
+     * Get internal {@link SnapshotsResponse} object.
+     * @return SnapshotsResponse object
+     */
+    @Nullable
+    public SnapshotsResponse getSnapshotsResponse() {
+        return this.snapshotsResponse;
+    }
+
     /**
      * Get internal {@link IndicesStatsResponse} object.
      * @return IndicesStatsResponse object
@@ -131,5 +151,8 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeArray(nodeStats);
         out.writeOptionalWriteable(indicesStats);
         clusterStatsData.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_2_17_1)) {
+            snapshotsResponse.writeTo(out);
+        }
     }
 }

src/main/java/org/opensearch/action/SnapshotsResponse.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright [2018] [Vincent VAN HOLLEBEKE]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.opensearch.action;
+
+import org.opensearch.core.action.ActionResponse;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.snapshots.SnapshotInfo;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Represents a container class for holding response data related to snapshots.
+ */
+public class SnapshotsResponse extends ActionResponse {
+    private final List<SnapshotInfo> snapshotInfos;
+
+    /**
+     * A constructor.
+     * @param in A streamInput to materialize the instance from
+     * @throws IOException if there is an exception reading from inputStream
+     */
+    public SnapshotsResponse(StreamInput in) throws IOException {
+        super(in);
+        snapshotInfos = Collections.synchronizedList(in.readList(SnapshotInfo::new));
+    }
+
+    /**
+     * A constructor.
+     */
+    public SnapshotsResponse() {
+        this.snapshotInfos = Collections.synchronizedList(new ArrayList<>());
+    }
+
+    /**
+     * Writes the instance into {@link StreamOutput}.
+     *
+     * @param out the output stream to which the instance is to be written
+     * @throws IOException if there is an exception writing to the output stream
+     */
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeCollection(snapshotInfos);
+    }
+
+    /**
+     * Getter for {@code snapshotInfos} list.
+     *
+     * @return the list of {@link SnapshotInfo} objects
+     */
+    public List<SnapshotInfo> getSnapshotInfos() {
+        return snapshotInfos;
+    }
+
+    /**
+     * Adds {@code snapshotInfosToAdd} to the {@code snapshotInfos} list.
+     *
+     * @param snapshotInfosToAdd {@link List<SnapshotInfo>} to be added
+     */
+    public void addSnapshotInfos(List<SnapshotInfo> snapshotInfosToAdd) {
+        snapshotInfos.addAll(snapshotInfosToAdd);
+    }
+}

src/main/java/org/opensearch/action/TransportNodePrometheusMetricsAction.java

@@ -28,6 +28,10 @@
 import org.opensearch.action.admin.cluster.node.stats.NodeStats;
 import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
+import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.opensearch.action.admin.cluster.state.ClusterStateRequest;
 import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
 import org.opensearch.action.admin.indices.stats.IndicesStatsRequest;
@@ -36,6 +40,7 @@
 import org.opensearch.action.support.HandledTransportAction;
 import org.opensearch.client.Client;
 import org.opensearch.client.Requests;
+import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.settings.ClusterSettings;
@@ -44,6 +49,10 @@
 import org.opensearch.tasks.Task;
 import org.opensearch.transport.TransportService;
 
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.stream.Collectors;
+
 /**
  * Transport action class for Prometheus Exporter plugin.
  *
@@ -94,15 +103,18 @@ private class AsyncAction {
         private final NodesStatsRequest nodesStatsRequest;
         private final IndicesStatsRequest indicesStatsRequest;
         private final ClusterStateRequest clusterStateRequest;
-
+        private final GetRepositoriesRequest repositoriesRequest;
         private ClusterHealthResponse clusterHealthResponse = null;
         private NodesInfoResponse localNodesInfoResponse = null;
         private NodesStatsResponse nodesStatsResponse = null;
         private IndicesStatsResponse indicesStatsResponse = null;
         private ClusterStateResponse clusterStateResponse = null;
+        private final Queue<String> snapshotRepositories = new ConcurrentLinkedQueue<>();
+        private final SnapshotsResponse snapshotsResponse = new SnapshotsResponse();
 
         // read the state of prometheus dynamic settings only once at the beginning of the async request
         private final boolean isPrometheusIndices = prometheusSettings.getPrometheusIndices();
+        private final boolean isPrometheusSnapshots = prometheusSettings.getPrometheusSnapshots();
         private final boolean isPrometheusClusterSettings = prometheusSettings.getPrometheusClusterSettings();
         private final String prometheusNodesFilter = prometheusSettings.getNodesFilter();
 
@@ -142,6 +154,12 @@ private AsyncAction(ActionListener<NodePrometheusMetricsResponse> listener) {
                 this.indicesStatsRequest = null;
             }
 
+            if (isPrometheusSnapshots) {
+                this.repositoriesRequest = new GetRepositoriesRequest();
+            } else {
+                this.repositoriesRequest = null;
+            }
+
             // Cluster settings are get via ClusterStateRequest (see elasticsearch RestClusterGetSettingsAction for details)
             // We prefer to send it to master node (hence local=false; it should be set by default but we want to be sure).
             this.clusterStateRequest = isPrometheusClusterSettings ? Requests.clusterStateRequest()
@@ -150,15 +168,59 @@ private AsyncAction(ActionListener<NodePrometheusMetricsResponse> listener) {
 
         private void gatherRequests() {
             listener.onResponse(buildResponse(clusterHealthResponse, localNodesInfoResponse, nodesStatsResponse, indicesStatsResponse,
-                    clusterStateResponse));
+                    clusterStateResponse, snapshotsResponse));
         }
 
+        private final ActionListener<GetRepositoriesResponse> repositoriesResponseActionListener =
+            new ActionListener<GetRepositoriesResponse>() {
+                @Override
+                public void onResponse(GetRepositoriesResponse response) {
+                    if (response.repositories().isEmpty()) {
+                        gatherRequests();
+                        return;
+                    }
+                    snapshotRepositories.addAll(response.repositories().stream()
+                            .map(RepositoryMetadata::name).collect(Collectors.toList()));
+                    String snapshotRepository = snapshotRepositories.poll();
+                    client.admin().cluster().getSnapshots(new GetSnapshotsRequest(snapshotRepository), snapshotsResponseActionListener);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    listener.onFailure(new OpenSearchException("Get repositories request failed", e));
+                }
+            };
+
+        private final ActionListener<GetSnapshotsResponse> snapshotsResponseActionListener =
+            new ActionListener<GetSnapshotsResponse>() {
+                @Override
+                public void onResponse(GetSnapshotsResponse response) {
+                    snapshotsResponse.addSnapshotInfos(response.getSnapshots());
+                    if (snapshotRepositories.isEmpty()) {
+                        gatherRequests();
+                        return;
+                    }
+                    // Fetch the snapshots for the next repository in the queue
+                    String snapshotRepository = snapshotRepositories.poll();
+                    client.admin().cluster().getSnapshots(new GetSnapshotsRequest(snapshotRepository), snapshotsResponseActionListener);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    listener.onFailure(new OpenSearchException("Get snapshots request failed", e));
+                }
+            };
+
         private final ActionListener<ClusterStateResponse> clusterStateResponseActionListener =
             new ActionListener<ClusterStateResponse>() {
                 @Override
                 public void onResponse(ClusterStateResponse response) {
                     clusterStateResponse = response;
-                    gatherRequests();
+                    if (isPrometheusSnapshots) {
+                        client.admin().cluster().getRepositories(repositoriesRequest, repositoriesResponseActionListener);
+                    } else {
+                        gatherRequests();
+                    }
                 }
 
                 @Override
@@ -175,7 +237,7 @@ public void onResponse(IndicesStatsResponse response) {
                     if (isPrometheusClusterSettings) {
                         client.admin().cluster().state(clusterStateRequest, clusterStateResponseActionListener);
                     } else {
-                        gatherRequests();
+                        clusterStateResponseActionListener.onResponse(null);
                     }
                 }
 
@@ -239,12 +301,13 @@ protected NodePrometheusMetricsResponse buildResponse(ClusterHealthResponse clus
                                                               NodesInfoResponse localNodesInfoResponse,
                                                               NodesStatsResponse nodesStats,
                                                               @Nullable IndicesStatsResponse indicesStats,
-                                                              @Nullable ClusterStateResponse clusterStateResponse) {
+                                                              @Nullable ClusterStateResponse clusterStateResponse,
+                                                              @Nullable SnapshotsResponse snapshotsResponse) {
             NodePrometheusMetricsResponse response = new NodePrometheusMetricsResponse(
                     clusterHealth,
                     localNodesInfoResponse,
                     nodesStats.getNodes().toArray(new NodeStats[0]),
-                    indicesStats, clusterStateResponse,
+                    indicesStats, clusterStateResponse, snapshotsResponse,
                     settings, clusterSettings);
             if (logger.isTraceEnabled()) {
                 logger.trace("Return response: [{}]", response);

src/main/java/org/opensearch/plugin/prometheus/PrometheusExporterPlugin.java

@@ -75,6 +75,7 @@ public List<Setting<?>> getSettings() {
         List<Setting<?>> settings = Arrays.asList(
                 PrometheusSettings.PROMETHEUS_CLUSTER_SETTINGS,
                 PrometheusSettings.PROMETHEUS_INDICES,
+                PrometheusSettings.PROMETHEUS_SNAPSHOTS,
                 PrometheusSettings.PROMETHEUS_NODES_FILTER,
                 PrometheusSettings.PROMETHEUS_SELECTED_INDICES,
                 PrometheusSettings.PROMETHEUS_SELECTED_OPTION,

src/main/java/org/opensearch/rest/prometheus/RestPrometheusMetricsAction.java

@@ -91,8 +91,8 @@ public String getName() {
         return "prometheus_metrics_action";
     }
 
-     // This method does not throw any IOException because there are no request parameters to be parsed
-     // and processed. This may change in the future.
+    // This method does not throw any IOException because there are no request parameters to be parsed
+    // and processed. This may change in the future.
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
         if (logger.isTraceEnabled()) {
@@ -125,12 +125,13 @@ public RestResponse buildResponse(NodePrometheusMetricsResponse response) throws
                             collector = new PrometheusMetricsCollector(
                                     catalog,
                                     prometheusSettings.getPrometheusIndices(),
+                                    prometheusSettings.getPrometheusSnapshots(),
                                     prometheusSettings.getPrometheusClusterSettings()
                             );
                             collector.registerMetrics();
                             collector.updateMetrics(
                                     nodeName, nodeId, response.getClusterHealth(), response.getNodeStats(),
-                                    response.getIndicesStats(), response.getClusterStatsData());
+                                    response.getIndicesStats(), response.getClusterStatsData(), response.getSnapshotsResponse());
                             textContent = collector.getTextContent();
                         } catch (Exception ex) {
                             // We use try-catch block to catch exception from Prometheus catalog and collector processing

src/yamlRestTest/resources/rest-api-spec/test/70_10_snapshots_metrics.yml

@@ -0,0 +1,50 @@
+---
+"Verify snapshots metrics enabled":
+
+  # Create 'fs' snapshot repository
+  - do:
+      snapshot.create_repository:
+        repository: test_repo_get_1
+        body:
+          type: fs
+          settings:
+            location: "test_repo_get_1_loc"
+
+  - do:
+      snapshot.get_repository: { }
+
+  - is_true: test_repo_get_1
+
+  # Enable snapshots metrics
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            prometheus.snapshots: "true"
+        flat_settings: true
+
+  - match: { persistent: { prometheus.snapshots: "true" } }
+
+  # Create snapshot
+  - do:
+      snapshot.create:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_1
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: test_snapshot_1 }
+  - match: { snapshot.state : SUCCESS }
+
+  # Fetch and verify metrics
+  - do:
+      prometheus.metrics: {}
+
+  - match:
+      $body: |
+        /.*
+        \# \s* HELP \s+ opensearch_min_snapshot_age \s+.*\n
+        \# \s* TYPE \s+ opensearch_min_snapshot_age \s+ gauge\n
+        opensearch_min_snapshot_age\{
+          cluster="yamlRestTest",sm_policy="adhoc",
+        \}\s+\d+\.\d+\n
+        .*/