Skip to content

Commit 04a417a

Browse files
ansjcyansjcy
authored
consume query level cpu and memory usage in query insights (opensearch-project#13739)
* consume query level cpu and memory usage in query insights Signed-off-by: Chenyang Ji <cyji@amazon.com> * handle failed requests metrics in query insights Signed-off-by: Chenyang Ji <cyji@amazon.com> * refactor the code to make it more maintainable Signed-off-by: Chenyang Ji <cyji@amazon.com> --------- Signed-off-by: Chenyang Ji <cyji@amazon.com>
1 parent 8b1a4b3 commit 04a417a

15 files changed

+415
-92
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2424
- [Query Insights] Add X-Opaque-Id to search request metadata for top n queries ([#13374](https://github.com/opensearch-project/OpenSearch/pull/13374))
2525
- Add support for query level resource usage tracking ([#13172](https://github.com/opensearch-project/OpenSearch/pull/13172))
2626
- Move Remote Store Migration from DocRep to GA and modify remote migration settings name ([#14100](https://github.com/opensearch-project/OpenSearch/pull/14100))
27+
- [Query Insights] Add cpu and memory metrics to top n queries ([#13739](https://github.com/opensearch-project/OpenSearch/pull/13739))
2728

2829
### Dependencies
2930
- Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))

‎plugins/query-insights/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,15 @@ public List<Setting<?>> getSettings() {
111111
QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED,
112112
QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE,
113113
QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE,
114-
QueryInsightsSettings.TOP_N_LATENCY_EXPORTER_SETTINGS
114+
QueryInsightsSettings.TOP_N_LATENCY_EXPORTER_SETTINGS,
115+
QueryInsightsSettings.TOP_N_CPU_QUERIES_ENABLED,
116+
QueryInsightsSettings.TOP_N_CPU_QUERIES_SIZE,
117+
QueryInsightsSettings.TOP_N_CPU_QUERIES_WINDOW_SIZE,
118+
QueryInsightsSettings.TOP_N_CPU_EXPORTER_SETTINGS,
119+
QueryInsightsSettings.TOP_N_MEMORY_QUERIES_ENABLED,
120+
QueryInsightsSettings.TOP_N_MEMORY_QUERIES_SIZE,
121+
QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE,
122+
QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS
115123
);
116124
}
117125
}

‎plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactory.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import java.util.Locale;
2020
import java.util.Set;
2121

22-
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_N_LATENCY_QUERIES_INDEX_PATTERN;
22+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_N_QUERIES_INDEX_PATTERN;
2323
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_QUERIES_EXPORTER_TYPE;
2424
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORTER_TYPE;
2525
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORT_INDEX;
@@ -71,7 +71,7 @@ public void validateExporterConfig(final Settings settings) throws IllegalArgume
7171
}
7272
switch (type) {
7373
case LOCAL_INDEX:
74-
final String indexPattern = settings.get(EXPORT_INDEX, DEFAULT_TOP_N_LATENCY_QUERIES_INDEX_PATTERN);
74+
final String indexPattern = settings.get(EXPORT_INDEX, DEFAULT_TOP_N_QUERIES_INDEX_PATTERN);
7575
if (indexPattern.length() == 0) {
7676
throw new IllegalArgumentException("Empty index pattern configured for the exporter");
7777
}

‎plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java

+67-22
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,10 @@
1414
import org.opensearch.action.search.SearchRequest;
1515
import org.opensearch.action.search.SearchRequestContext;
1616
import org.opensearch.action.search.SearchRequestOperationsListener;
17+
import org.opensearch.action.search.SearchTask;
1718
import org.opensearch.cluster.service.ClusterService;
1819
import org.opensearch.common.inject.Inject;
20+
import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo;
1921
import org.opensearch.core.xcontent.ToXContent;
2022
import org.opensearch.plugin.insights.core.service.QueryInsightsService;
2123
import org.opensearch.plugin.insights.rules.model.Attribute;
@@ -25,13 +27,14 @@
2527

2628
import java.util.Collections;
2729
import java.util.HashMap;
30+
import java.util.List;
2831
import java.util.Locale;
2932
import java.util.Map;
3033
import java.util.concurrent.TimeUnit;
3134

32-
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED;
33-
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE;
34-
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE;
35+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNEnabledSetting;
36+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNSizeSetting;
37+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNWindowSizeSetting;
3538

3639
/**
3740
* The listener for query insights services.
@@ -46,6 +49,7 @@ public final class QueryInsightsListener extends SearchRequestOperationsListener
4649
private static final Logger log = LogManager.getLogger(QueryInsightsListener.class);
4750

4851
private final QueryInsightsService queryInsightsService;
52+
private final ClusterService clusterService;
4953

5054
/**
5155
* Constructor for QueryInsightsListener
@@ -55,26 +59,32 @@ public final class QueryInsightsListener extends SearchRequestOperationsListener
5559
*/
5660
@Inject
5761
public QueryInsightsListener(final ClusterService clusterService, final QueryInsightsService queryInsightsService) {
62+
this.clusterService = clusterService;
5863
this.queryInsightsService = queryInsightsService;
59-
clusterService.getClusterSettings()
60-
.addSettingsUpdateConsumer(TOP_N_LATENCY_QUERIES_ENABLED, v -> this.setEnableTopQueries(MetricType.LATENCY, v));
61-
clusterService.getClusterSettings()
62-
.addSettingsUpdateConsumer(
63-
TOP_N_LATENCY_QUERIES_SIZE,
64-
v -> this.queryInsightsService.getTopQueriesService(MetricType.LATENCY).setTopNSize(v),
65-
v -> this.queryInsightsService.getTopQueriesService(MetricType.LATENCY).validateTopNSize(v)
66-
);
67-
clusterService.getClusterSettings()
68-
.addSettingsUpdateConsumer(
69-
TOP_N_LATENCY_QUERIES_WINDOW_SIZE,
70-
v -> this.queryInsightsService.getTopQueriesService(MetricType.LATENCY).setWindowSize(v),
71-
v -> this.queryInsightsService.getTopQueriesService(MetricType.LATENCY).validateWindowSize(v)
72-
);
73-
this.setEnableTopQueries(MetricType.LATENCY, clusterService.getClusterSettings().get(TOP_N_LATENCY_QUERIES_ENABLED));
74-
this.queryInsightsService.getTopQueriesService(MetricType.LATENCY)
75-
.setTopNSize(clusterService.getClusterSettings().get(TOP_N_LATENCY_QUERIES_SIZE));
76-
this.queryInsightsService.getTopQueriesService(MetricType.LATENCY)
77-
.setWindowSize(clusterService.getClusterSettings().get(TOP_N_LATENCY_QUERIES_WINDOW_SIZE));
64+
// Setting endpoints set up for top n queries, including enabling top n queries, window size and top n size
65+
// Expected metricTypes are Latency, CPU and Memory.
66+
for (MetricType type : MetricType.allMetricTypes()) {
67+
clusterService.getClusterSettings()
68+
.addSettingsUpdateConsumer(getTopNEnabledSetting(type), v -> this.setEnableTopQueries(type, v));
69+
clusterService.getClusterSettings()
70+
.addSettingsUpdateConsumer(
71+
getTopNSizeSetting(type),
72+
v -> this.queryInsightsService.setTopNSize(type, v),
73+
v -> this.queryInsightsService.validateTopNSize(type, v)
74+
);
75+
clusterService.getClusterSettings()
76+
.addSettingsUpdateConsumer(
77+
getTopNWindowSizeSetting(type),
78+
v -> this.queryInsightsService.setWindowSize(type, v),
79+
v -> this.queryInsightsService.validateWindowSize(type, v)
80+
);
81+
82+
this.setEnableTopQueries(type, clusterService.getClusterSettings().get(getTopNEnabledSetting(type)));
83+
this.queryInsightsService.validateTopNSize(type, clusterService.getClusterSettings().get(getTopNSizeSetting(type)));
84+
this.queryInsightsService.setTopNSize(type, clusterService.getClusterSettings().get(getTopNSizeSetting(type)));
85+
this.queryInsightsService.validateWindowSize(type, clusterService.getClusterSettings().get(getTopNWindowSizeSetting(type)));
86+
this.queryInsightsService.setWindowSize(type, clusterService.getClusterSettings().get(getTopNWindowSizeSetting(type)));
87+
}
7888
}
7989

8090
/**
@@ -124,6 +134,27 @@ public void onRequestStart(SearchRequestContext searchRequestContext) {}
124134

125135
@Override
126136
public void onRequestEnd(final SearchPhaseContext context, final SearchRequestContext searchRequestContext) {
137+
constructSearchQueryRecord(context, searchRequestContext);
138+
}
139+
140+
@Override
141+
public void onRequestFailure(final SearchPhaseContext context, final SearchRequestContext searchRequestContext) {
142+
constructSearchQueryRecord(context, searchRequestContext);
143+
}
144+
145+
private void constructSearchQueryRecord(final SearchPhaseContext context, final SearchRequestContext searchRequestContext) {
146+
SearchTask searchTask = context.getTask();
147+
List<TaskResourceInfo> tasksResourceUsages = searchRequestContext.getPhaseResourceUsage();
148+
tasksResourceUsages.add(
149+
new TaskResourceInfo(
150+
searchTask.getAction(),
151+
searchTask.getId(),
152+
searchTask.getParentTaskId().getId(),
153+
clusterService.localNode().getId(),
154+
searchTask.getTotalResourceStats()
155+
)
156+
);
157+
127158
final SearchRequest request = context.getRequest();
128159
try {
129160
Map<MetricType, Number> measurements = new HashMap<>();
@@ -133,12 +164,25 @@ public void onRequestEnd(final SearchPhaseContext context, final SearchRequestCo
133164
TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - searchRequestContext.getAbsoluteStartNanos())
134165
);
135166
}
167+
if (queryInsightsService.isCollectionEnabled(MetricType.CPU)) {
168+
measurements.put(
169+
MetricType.CPU,
170+
tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getCpuTimeInNanos()).mapToLong(Long::longValue).sum()
171+
);
172+
}
173+
if (queryInsightsService.isCollectionEnabled(MetricType.MEMORY)) {
174+
measurements.put(
175+
MetricType.MEMORY,
176+
tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getMemoryInBytes()).mapToLong(Long::longValue).sum()
177+
);
178+
}
136179
Map<Attribute, Object> attributes = new HashMap<>();
137180
attributes.put(Attribute.SEARCH_TYPE, request.searchType().toString().toLowerCase(Locale.ROOT));
138181
attributes.put(Attribute.SOURCE, request.source().toString(FORMAT_PARAMS));
139182
attributes.put(Attribute.TOTAL_SHARDS, context.getNumShards());
140183
attributes.put(Attribute.INDICES, request.indices());
141184
attributes.put(Attribute.PHASE_LATENCY_MAP, searchRequestContext.phaseTookMap());
185+
attributes.put(Attribute.TASK_RESOURCE_USAGES, tasksResourceUsages);
142186

143187
Map<String, Object> labels = new HashMap<>();
144188
// Retrieve user provided label if exists
@@ -154,4 +198,5 @@ public void onRequestEnd(final SearchPhaseContext context, final SearchRequestCo
154198
log.error(String.format(Locale.ROOT, "fail to ingest query insight data, error: %s", e));
155199
}
156200
}
201+
157202
}

‎plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java

+82-6
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
import org.opensearch.common.inject.Inject;
1313
import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
1414
import org.opensearch.common.settings.ClusterSettings;
15+
import org.opensearch.common.settings.Settings;
16+
import org.opensearch.common.unit.TimeValue;
1517
import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporterFactory;
1618
import org.opensearch.plugin.insights.rules.model.MetricType;
1719
import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
@@ -27,7 +29,7 @@
2729
import java.util.Map;
2830
import java.util.concurrent.LinkedBlockingQueue;
2931

30-
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_LATENCY_EXPORTER_SETTINGS;
32+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getExporterSettings;
3133

3234
/**
3335
* Service responsible for gathering, analyzing, storing and exporting
@@ -86,11 +88,13 @@ public QueryInsightsService(final ClusterSettings clusterSettings, final ThreadP
8688
enableCollect.put(metricType, false);
8789
topQueriesServices.put(metricType, new TopQueriesService(metricType, threadPool, queryInsightsExporterFactory));
8890
}
89-
clusterSettings.addSettingsUpdateConsumer(
90-
TOP_N_LATENCY_EXPORTER_SETTINGS,
91-
(settings -> getTopQueriesService(MetricType.LATENCY).setExporter(settings)),
92-
(settings -> getTopQueriesService(MetricType.LATENCY).validateExporterConfig(settings))
93-
);
91+
for (MetricType type : MetricType.allMetricTypes()) {
92+
clusterSettings.addSettingsUpdateConsumer(
93+
getExporterSettings(type),
94+
(settings -> setExporter(type, settings)),
95+
(settings -> validateExporterConfig(type, settings))
96+
);
97+
}
9498
}
9599

96100
/**
@@ -177,6 +181,78 @@ public boolean isEnabled() {
177181
return false;
178182
}
179183

184+
/**
185+
* Validate the window size config for a metricType
186+
*
187+
* @param type {@link MetricType}
188+
* @param windowSize {@link TimeValue}
189+
*/
190+
public void validateWindowSize(final MetricType type, final TimeValue windowSize) {
191+
if (topQueriesServices.containsKey(type)) {
192+
topQueriesServices.get(type).validateWindowSize(windowSize);
193+
}
194+
}
195+
196+
/**
197+
* Set window size for a metricType
198+
*
199+
* @param type {@link MetricType}
200+
* @param windowSize {@link TimeValue}
201+
*/
202+
public void setWindowSize(final MetricType type, final TimeValue windowSize) {
203+
if (topQueriesServices.containsKey(type)) {
204+
topQueriesServices.get(type).setWindowSize(windowSize);
205+
}
206+
}
207+
208+
/**
209+
* Validate the top n size config for a metricType
210+
*
211+
* @param type {@link MetricType}
212+
* @param topNSize top n size
213+
*/
214+
public void validateTopNSize(final MetricType type, final int topNSize) {
215+
if (topQueriesServices.containsKey(type)) {
216+
topQueriesServices.get(type).validateTopNSize(topNSize);
217+
}
218+
}
219+
220+
/**
221+
* Set the top n size config for a metricType
222+
*
223+
* @param type {@link MetricType}
224+
* @param topNSize top n size
225+
*/
226+
public void setTopNSize(final MetricType type, final int topNSize) {
227+
if (topQueriesServices.containsKey(type)) {
228+
topQueriesServices.get(type).setTopNSize(topNSize);
229+
}
230+
}
231+
232+
/**
233+
* Set the exporter config for a metricType
234+
*
235+
* @param type {@link MetricType}
236+
* @param settings exporter settings
237+
*/
238+
public void setExporter(final MetricType type, final Settings settings) {
239+
if (topQueriesServices.containsKey(type)) {
240+
topQueriesServices.get(type).setExporter(settings);
241+
}
242+
}
243+
244+
/**
245+
* Validate the exporter config for a metricType
246+
*
247+
* @param type {@link MetricType}
248+
* @param settings exporter settings
249+
*/
250+
public void validateExporterConfig(final MetricType type, final Settings settings) {
251+
if (topQueriesServices.containsKey(type)) {
252+
topQueriesServices.get(type).validateExporterConfig(settings);
253+
}
254+
}
255+
180256
@Override
181257
protected void doStart() {
182258
if (isEnabled()) {

‎plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java

+3-6
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
import java.util.stream.Collectors;
3636
import java.util.stream.Stream;
3737

38-
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_N_LATENCY_QUERIES_INDEX_PATTERN;
38+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_N_QUERIES_INDEX_PATTERN;
3939
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_QUERIES_EXPORTER_TYPE;
4040
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORTER_TYPE;
4141
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORT_INDEX;
@@ -218,10 +218,7 @@ public void setExporter(final Settings settings) {
218218
if (settings.get(EXPORTER_TYPE) != null) {
219219
SinkType expectedType = SinkType.parse(settings.get(EXPORTER_TYPE, DEFAULT_TOP_QUERIES_EXPORTER_TYPE));
220220
if (exporter != null && expectedType == SinkType.getSinkTypeFromExporter(exporter)) {
221-
queryInsightsExporterFactory.updateExporter(
222-
exporter,
223-
settings.get(EXPORT_INDEX, DEFAULT_TOP_N_LATENCY_QUERIES_INDEX_PATTERN)
224-
);
221+
queryInsightsExporterFactory.updateExporter(exporter, settings.get(EXPORT_INDEX, DEFAULT_TOP_N_QUERIES_INDEX_PATTERN));
225222
} else {
226223
try {
227224
queryInsightsExporterFactory.closeExporter(this.exporter);
@@ -230,7 +227,7 @@ public void setExporter(final Settings settings) {
230227
}
231228
this.exporter = queryInsightsExporterFactory.createExporter(
232229
SinkType.parse(settings.get(EXPORTER_TYPE, DEFAULT_TOP_QUERIES_EXPORTER_TYPE)),
233-
settings.get(EXPORT_INDEX, DEFAULT_TOP_N_LATENCY_QUERIES_INDEX_PATTERN)
230+
settings.get(EXPORT_INDEX, DEFAULT_TOP_N_QUERIES_INDEX_PATTERN)
234231
);
235232
}
236233
} else {

‎plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java

+4
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ public enum Attribute {
4444
* The node id for this request
4545
*/
4646
NODE_ID,
47+
/**
48+
* Tasks level resource usages in this request
49+
*/
50+
TASK_RESOURCE_USAGES,
4751
/**
4852
* Custom search request labels
4953
*/

‎plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java

+5-7
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public enum MetricType implements Comparator<Number> {
3535
/**
3636
* JVM heap usage metric type
3737
*/
38-
JVM;
38+
MEMORY;
3939

4040
/**
4141
* Read a MetricType from a StreamInput
@@ -93,10 +93,9 @@ public static Set<MetricType> allMetricTypes() {
9393
public int compare(final Number a, final Number b) {
9494
switch (this) {
9595
case LATENCY:
96-
return Long.compare(a.longValue(), b.longValue());
97-
case JVM:
9896
case CPU:
99-
return Double.compare(a.doubleValue(), b.doubleValue());
97+
case MEMORY:
98+
return Long.compare(a.longValue(), b.longValue());
10099
}
101100
return -1;
102101
}
@@ -110,10 +109,9 @@ public int compare(final Number a, final Number b) {
110109
Number parseValue(final Object o) {
111110
switch (this) {
112111
case LATENCY:
113-
return (Long) o;
114-
case JVM:
115112
case CPU:
116-
return (Double) o;
113+
case MEMORY:
114+
return (Long) o;
117115
default:
118116
return (Number) o;
119117
}

0 commit comments

Comments
 (0)