Skip to content

Commit 9059456

Browse files
author
Jay Deng
committed
WIP
1 parent 90aa335 commit 9059456

File tree

3 files changed

+143
-1
lines changed

3 files changed

+143
-1
lines changed

‎src/main/java/org/opensearch/knn/plugin/stats/KNNRemoteIndexBuildValue.java

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.knn.plugin.stats;
7+
8+
import lombok.Getter;
9+
10+
import java.util.concurrent.atomic.LongAdder;
11+
12+
public enum KNNRemoteIndexBuildValue {
13+
14+
REPOSITORY_WRITE_SUCCESS_COUNT("repository_write_success_count"),
15+
REPOSITORY_WRITE_FAILURE_COUNT("repository_write_failure_count"),
16+
REPOSITORY_WRITE_SIZE("repository_write_size_in_bytes"),
17+
REPOSITORY_WRITE_TIME("repository_write_time_in_millis"),
18+
REPOSITORY_READ_SUCCESS_COUNT("repository_read_success_count"),
19+
REPOSITORY_READ_FAILURE_COUNT("repository_read_failure_count"),
20+
REPOSITORY_READ_SIZE("repository_read_size_in_bytes"),
21+
REPOSITORY_READ_TIME("repository_read_time_in_millis"),
22+
REPOSITORY_PARALLEL_UPLOAD_SUCCESS_COUNT("repository_parallel_upload_success_count"),
23+
REPOSITORY_PARALLEL_UPLOAD_FAILURE_COUNT("repository_parallel_upload_failure_count"),
24+
REPOSITORY_SEQUENTIAL_UPLOAD_SUCCESS_COUNT("repository_sequential_upload_success_count"),
25+
REPOSITORY_SEQUENTIAL_UPLOAD_FAILURE_COUNT("repository_sequential_upload_failure_count"),
26+
27+
BUILD_REQUEST_SUCCESS_COUNT("build_request_success_count"),
28+
BUILD_REQUEST_FAILURE_COUNT("build_request_failure_count"),
29+
STATUS_REQUEST_SUCCESS_COUNT("status_request_success_count"),
30+
STATUS_REQUEST_FAILURE_COUNT("status_request_failure_count"),
31+
INDEX_BUILD_SUCCESS_COUNT("index_build_success_count"),
32+
INDEX_BUILD_FAILURE_COUNT("index_build_failure_count"),
33+
34+
REMOTE_FLUSH_TIME("remote_flush_time_in_millis"),
35+
REMOTE_MERGE_TIME("remote_merge_time_in_millis");
36+
37+
@Getter
38+
private final String name;
39+
private final LongAdder value;
40+
41+
/**
42+
* Constructor
43+
*
44+
* @param name name of the graph value
45+
*/
46+
KNNRemoteIndexBuildValue(String name) {
47+
this.name = name;
48+
this.value = new LongAdder();
49+
}
50+
51+
/**
52+
* Get the graph value
53+
*
54+
* @return value
55+
*/
56+
public Long getValue() {
57+
return value.longValue();
58+
}
59+
60+
/**
61+
* Increment the graph value
62+
*/
63+
public void increment() {
64+
value.increment();
65+
}
66+
67+
/**
68+
* Decrement the graph value
69+
*/
70+
public void decrement() {
71+
value.decrement();
72+
}
73+
74+
/**
75+
* Increment the graph value by a specified amount
76+
*
77+
* @param delta The amount to increment
78+
*/
79+
public void incrementBy(long delta) {
80+
value.add(delta);
81+
}
82+
83+
/**
84+
* Decrement the graph value by a specified amount
85+
*
86+
* @param delta The amount to decrement
87+
*/
88+
public void decrementBy(long delta) {
89+
value.add(delta * -1);
90+
}
91+
}

‎src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java

+49-1
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@
88
import com.google.common.cache.CacheStats;
99
import com.google.common.collect.ImmutableMap;
1010
import org.opensearch.knn.common.KNNConstants;
11-
import org.opensearch.knn.index.memory.NativeMemoryCacheManager;
11+
import org.opensearch.knn.common.featureflags.KNNFeatureFlags;
1212
import org.opensearch.knn.index.engine.KNNEngine;
13+
import org.opensearch.knn.index.memory.NativeMemoryCacheManager;
1314
import org.opensearch.knn.indices.ModelCache;
1415
import org.opensearch.knn.indices.ModelDao;
1516
import org.opensearch.knn.plugin.stats.suppliers.EventOccurredWithinThresholdSupplier;
@@ -86,6 +87,10 @@ private Map<String, KNNStat<?>> buildStatsMap() {
8687
addScriptStats(builder);
8788
addModelStats(builder);
8889
addGraphStats(builder);
90+
if (KNNFeatureFlags.isKNNRemoteVectorBuildEnabled())
91+
{
92+
addRemoteIndexBuildStats(builder);
93+
}
8994
return builder.build();
9095
}
9196

@@ -210,12 +215,55 @@ private Map<String, Map<String, Object>> createGraphStatsMap() {
210215
mergeMap.put(KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.getName(), KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.getValue());
211216
mergeMap.put(KNNGraphValue.MERGE_TOTAL_DOCS.getName(), KNNGraphValue.MERGE_TOTAL_DOCS.getValue());
212217
mergeMap.put(KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getName(), KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue());
218+
if (KNNFeatureFlags.isKNNRemoteVectorBuildEnabled())
219+
{
220+
mergeMap.put(KNNRemoteIndexBuildValue.REMOTE_MERGE_TIME.getName(), KNNRemoteIndexBuildValue.REMOTE_MERGE_TIME.getValue());
221+
}
213222
Map<String, Object> refreshMap = new HashMap<>();
214223
refreshMap.put(KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getName(), KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue());
215224
refreshMap.put(KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getName(), KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue());
225+
if (KNNFeatureFlags.isKNNRemoteVectorBuildEnabled())
226+
{
227+
mergeMap.put(KNNRemoteIndexBuildValue.REMOTE_FLUSH_TIME.getName(), KNNRemoteIndexBuildValue.REMOTE_FLUSH_TIME.getValue());
228+
}
216229
Map<String, Map<String, Object>> graphStatsMap = new HashMap<>();
217230
graphStatsMap.put(StatNames.MERGE.getName(), mergeMap);
218231
graphStatsMap.put(StatNames.REFRESH.getName(), refreshMap);
219232
return graphStatsMap;
220233
}
234+
235+
236+
private void addRemoteIndexBuildStats(ImmutableMap.Builder<String, KNNStat<?>> builder) {
237+
builder.put(StatNames.REMOTE_VECTOR_INDEX_BUILD_STATS.getName(), new KNNStat<>(false, this::createRemoteIndexStatsMap));
238+
}
239+
240+
private Map<String, Map<String, Object>> createRemoteIndexStatsMap() {
241+
Map<String, Object> clientStatsMap = new HashMap<>();
242+
clientStatsMap.put(KNNRemoteIndexBuildValue.BUILD_REQUEST_SUCCESS_COUNT.getName(), KNNRemoteIndexBuildValue.BUILD_REQUEST_SUCCESS_COUNT.getValue());
243+
clientStatsMap.put(KNNRemoteIndexBuildValue.BUILD_REQUEST_FAILURE_COUNT.getName(), KNNRemoteIndexBuildValue.BUILD_REQUEST_FAILURE_COUNT.getValue());
244+
clientStatsMap.put(KNNRemoteIndexBuildValue.STATUS_REQUEST_SUCCESS_COUNT.getName(), KNNRemoteIndexBuildValue.STATUS_REQUEST_SUCCESS_COUNT.getValue());
245+
clientStatsMap.put(KNNRemoteIndexBuildValue.STATUS_REQUEST_FAILURE_COUNT.getName(), KNNRemoteIndexBuildValue.STATUS_REQUEST_FAILURE_COUNT.getValue());
246+
clientStatsMap.put(KNNRemoteIndexBuildValue.INDEX_BUILD_SUCCESS_COUNT.getName(), KNNRemoteIndexBuildValue.INDEX_BUILD_SUCCESS_COUNT.getValue());
247+
clientStatsMap.put(KNNRemoteIndexBuildValue.INDEX_BUILD_FAILURE_COUNT.getName(), KNNRemoteIndexBuildValue.INDEX_BUILD_FAILURE_COUNT.getValue());
248+
249+
Map<String, Object> repoStatsMap = new HashMap<>();
250+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_WRITE_SUCCESS_COUNT.getName(), KNNRemoteIndexBuildValue.REPOSITORY_WRITE_SUCCESS_COUNT.getValue());
251+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_WRITE_FAILURE_COUNT.getName(), KNNRemoteIndexBuildValue.REPOSITORY_WRITE_FAILURE_COUNT.getValue());
252+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_WRITE_SIZE.getName(), KNNRemoteIndexBuildValue.REPOSITORY_WRITE_SIZE.getValue());
253+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_WRITE_TIME.getName(), KNNRemoteIndexBuildValue.REPOSITORY_WRITE_TIME.getValue());
254+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_READ_SUCCESS_COUNT.getName(), KNNRemoteIndexBuildValue.REPOSITORY_READ_SUCCESS_COUNT.getValue());
255+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_READ_FAILURE_COUNT.getName(), KNNRemoteIndexBuildValue.REPOSITORY_READ_FAILURE_COUNT.getValue());
256+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_READ_SIZE.getName(), KNNRemoteIndexBuildValue.REPOSITORY_READ_SIZE.getValue());
257+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_READ_TIME.getName(), KNNRemoteIndexBuildValue.REPOSITORY_READ_TIME.getValue());
258+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_PARALLEL_UPLOAD_SUCCESS_COUNT.getName(), KNNRemoteIndexBuildValue.REPOSITORY_PARALLEL_UPLOAD_SUCCESS_COUNT.getValue());
259+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_PARALLEL_UPLOAD_FAILURE_COUNT.getName(), KNNRemoteIndexBuildValue.REPOSITORY_PARALLEL_UPLOAD_FAILURE_COUNT.getValue());
260+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_SEQUENTIAL_UPLOAD_SUCCESS_COUNT.getName(), KNNRemoteIndexBuildValue.REPOSITORY_SEQUENTIAL_UPLOAD_SUCCESS_COUNT.getValue());
261+
repoStatsMap.put(KNNRemoteIndexBuildValue.REPOSITORY_SEQUENTIAL_UPLOAD_FAILURE_COUNT.getName(), KNNRemoteIndexBuildValue.REPOSITORY_SEQUENTIAL_UPLOAD_FAILURE_COUNT.getValue());
262+
263+
Map<String, Map<String, Object>> remoteIndexBuildStatsMap = new HashMap<>();
264+
remoteIndexBuildStatsMap.put(StatNames.REMOTE_VECTOR_INDEX_BUILD_CLIENT_STATS.getName(), clientStatsMap);
265+
remoteIndexBuildStatsMap.put(StatNames.REMOTE_VECTOR_INDEX_BUILD_REPO_STATS.getName(), repoStatsMap);
266+
267+
return remoteIndexBuildStatsMap;
268+
}
221269
}

‎src/main/java/org/opensearch/knn/plugin/stats/StatNames.java

+3
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ public enum StatNames {
4545
GRAPH_STATS("graph_stats"),
4646
REFRESH("refresh"),
4747
MERGE("merge"),
48+
REMOTE_VECTOR_INDEX_BUILD_STATS("remote_vector_index_build_stats"),
49+
REMOTE_VECTOR_INDEX_BUILD_CLIENT_STATS("remote_vector_index_build_client_stats"),
50+
REMOTE_VECTOR_INDEX_BUILD_REPO_STATS("remote_vector_index_build_repo_stats"),
4851
MIN_SCORE_QUERY_REQUESTS(KNNCounter.MIN_SCORE_QUERY_REQUESTS.getName()),
4952
MIN_SCORE_QUERY_WITH_FILTER_REQUESTS(KNNCounter.MIN_SCORE_QUERY_WITH_FILTER_REQUESTS.getName()),
5053
MAX_DISTANCE_QUERY_REQUESTS(KNNCounter.MAX_DISTANCE_QUERY_REQUESTS.getName()),

0 commit comments

Comments
 (0)