Skip to content

Commit 692482c

Browse files
ansjcyansjcy
committed
Move exporter config to query insights level
Signed-off-by: Chenyang Ji <cyji@amazon.com>
1 parent bd6debd commit 692482c

20 files changed

+282
-324
lines changed

‎src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -131,21 +131,19 @@ public List<Setting<?>> getSettings() {
131131
QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED,
132132
QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE,
133133
QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE,
134-
QueryInsightsSettings.TOP_N_LATENCY_EXPORTER_SETTINGS,
135134
QueryInsightsSettings.TOP_N_CPU_QUERIES_ENABLED,
136135
QueryInsightsSettings.TOP_N_CPU_QUERIES_SIZE,
137136
QueryInsightsSettings.TOP_N_CPU_QUERIES_WINDOW_SIZE,
138-
QueryInsightsSettings.TOP_N_CPU_EXPORTER_SETTINGS,
139137
QueryInsightsSettings.TOP_N_MEMORY_QUERIES_ENABLED,
140138
QueryInsightsSettings.TOP_N_MEMORY_QUERIES_SIZE,
141139
QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE,
142-
QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS,
143140
QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY,
144141
QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N,
145142
QueryInsightsSettings.TOP_N_QUERIES_GROUPING_FIELD_NAME,
146143
QueryInsightsSettings.TOP_N_QUERIES_GROUPING_FIELD_TYPE,
147144
QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING,
148145
QueryInsightsSettings.TOP_N_EXPORTER_DELETE_AFTER,
146+
QueryInsightsSettings.TOP_N_EXPORTER_TYPE,
149147
QueryCategorizationSettings.SEARCH_QUERY_FIELD_TYPE_CACHE_SIZE_KEY
150148
);
151149
}

‎src/main/java/org/opensearch/plugin/insights/core/exporter/DebugExporter.java

+6
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,18 @@ public final class DebugExporter implements QueryInsightsExporter {
2121
* Logger of the debug exporter
2222
*/
2323
private final Logger logger = LogManager.getLogger();
24+
private static final String EXPORTER_ID = "debug_exporter";
2425

2526
/**
2627
* Constructor of DebugExporter
2728
*/
2829
private DebugExporter() {}
2930

31+
@Override
32+
public String getId() {
33+
return EXPORTER_ID;
34+
}
35+
3036
private static class InstanceHolder {
3137
private static final DebugExporter INSTANCE = new DebugExporter();
3238
}

‎src/main/java/org/opensearch/plugin/insights/core/exporter/LocalIndexExporter.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -46,17 +46,24 @@ public final class LocalIndexExporter implements QueryInsightsExporter {
4646
private final Client client;
4747
private DateTimeFormatter indexPattern;
4848
private int deleteAfter;
49+
private final String id;
4950

5051
/**
5152
* Constructor of LocalIndexExporter
5253
*
5354
* @param client OS client
5455
* @param indexPattern the pattern of index to export to
5556
*/
56-
public LocalIndexExporter(final Client client, final DateTimeFormatter indexPattern) {
57+
public LocalIndexExporter(final Client client, final DateTimeFormatter indexPattern, final String id) {
5758
this.indexPattern = indexPattern;
5859
this.client = client;
5960
this.deleteAfter = DEFAULT_DELETE_AFTER_VALUE;
61+
this.id = id;
62+
}
63+
64+
@Override
65+
public String getId() {
66+
return id;
6067
}
6168

6269
/**

‎src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporter.java

+2
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,6 @@ public interface QueryInsightsExporter extends Closeable {
2222
* @param records list of {@link SearchQueryRecord}
2323
*/
2424
void export(final List<SearchQueryRecord> records);
25+
26+
String getId();
2527
}

‎src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactory.java

+23-24
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,14 @@
88

99
package org.opensearch.plugin.insights.core.exporter;
1010

11-
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_QUERIES_EXPORTER_TYPE;
12-
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORTER_TYPE;
13-
1411
import java.io.IOException;
1512
import java.time.format.DateTimeFormatter;
16-
import java.util.HashSet;
13+
import java.util.HashMap;
1714
import java.util.Locale;
18-
import java.util.Set;
15+
import java.util.Map;
1916
import org.apache.logging.log4j.LogManager;
2017
import org.apache.logging.log4j.Logger;
2118
import org.opensearch.client.Client;
22-
import org.opensearch.common.settings.Settings;
2319
import org.opensearch.plugin.insights.core.metrics.OperationalMetric;
2420
import org.opensearch.plugin.insights.core.metrics.OperationalMetricsCounter;
2521

@@ -32,7 +28,7 @@ public class QueryInsightsExporterFactory {
3228
*/
3329
private final Logger logger = LogManager.getLogger();
3430
final private Client client;
35-
final private Set<QueryInsightsExporter> exporters;
31+
final private Map<String, QueryInsightsExporter> exporters;
3632

3733
/**
3834
* Constructor of QueryInsightsExporterFactory
@@ -41,32 +37,26 @@ public class QueryInsightsExporterFactory {
4137
*/
4238
public QueryInsightsExporterFactory(final Client client) {
4339
this.client = client;
44-
this.exporters = new HashSet<>();
40+
this.exporters = new HashMap<>();
4541
}
4642

4743
/**
4844
* Validate exporter sink config
4945
*
50-
* @param settings exporter sink config {@link Settings}
46+
* @param exporterType exporter sink type
5147
* @throws IllegalArgumentException if provided exporter sink config settings are invalid
5248
*/
53-
public void validateExporterConfig(final Settings settings) throws IllegalArgumentException {
49+
public void validateExporterType(final String exporterType) throws IllegalArgumentException {
5450
// Disable exporter if the EXPORTER_TYPE setting is null
55-
if (settings.get(EXPORTER_TYPE) == null) {
51+
if (exporterType == null) {
5652
return;
5753
}
58-
SinkType type;
5954
try {
60-
type = SinkType.parse(settings.get(EXPORTER_TYPE, DEFAULT_TOP_QUERIES_EXPORTER_TYPE));
55+
SinkType.parse(exporterType);
6156
} catch (IllegalArgumentException e) {
6257
OperationalMetricsCounter.getInstance().incrementCounter(OperationalMetric.INVALID_EXPORTER_TYPE_FAILURES);
6358
throw new IllegalArgumentException(
64-
String.format(
65-
Locale.ROOT,
66-
"Invalid exporter type [%s], type should be one of %s",
67-
settings.get(EXPORTER_TYPE),
68-
SinkType.allSinkTypes()
69-
)
59+
String.format(Locale.ROOT, "Invalid exporter type [%s], type should be one of %s", exporterType, SinkType.allSinkTypes())
7060
);
7161
}
7262
}
@@ -78,10 +68,10 @@ public void validateExporterConfig(final Settings settings) throws IllegalArgume
7868
* @param indexPattern the index pattern if creating a index exporter
7969
* @return QueryInsightsExporter the created exporter sink
8070
*/
81-
public QueryInsightsExporter createExporter(SinkType type, String indexPattern) {
71+
public QueryInsightsExporter createExporter(String id, SinkType type, String indexPattern) {
8272
if (SinkType.LOCAL_INDEX.equals(type)) {
83-
QueryInsightsExporter exporter = new LocalIndexExporter(client, DateTimeFormatter.ofPattern(indexPattern, Locale.ROOT));
84-
this.exporters.add(exporter);
73+
QueryInsightsExporter exporter = new LocalIndexExporter(client, DateTimeFormatter.ofPattern(indexPattern, Locale.ROOT), id);
74+
this.exporters.put(id, exporter);
8575
return exporter;
8676
}
8777
return DebugExporter.getInstance();
@@ -101,6 +91,15 @@ public QueryInsightsExporter updateExporter(QueryInsightsExporter exporter, Stri
10191
return exporter;
10292
}
10393

94+
/**
95+
* Get a exporter by id
96+
* @param id The id of the exporter
97+
* @return QueryInsightsReader the Reader
98+
*/
99+
public QueryInsightsExporter getExporter(String id) {
100+
return this.exporters.get(id);
101+
}
102+
104103
/**
105104
* Close an exporter
106105
*
@@ -110,7 +109,7 @@ public QueryInsightsExporter updateExporter(QueryInsightsExporter exporter, Stri
110109
public void closeExporter(QueryInsightsExporter exporter) throws IOException {
111110
if (exporter != null) {
112111
exporter.close();
113-
this.exporters.remove(exporter);
112+
this.exporters.remove(exporter.getId());
114113
}
115114
}
116115

@@ -119,7 +118,7 @@ public void closeExporter(QueryInsightsExporter exporter) throws IOException {
119118
*
120119
*/
121120
public void closeAllExporters() {
122-
for (QueryInsightsExporter exporter : exporters) {
121+
for (QueryInsightsExporter exporter : exporters.values()) {
123122
try {
124123
closeExporter(exporter);
125124
} catch (IOException e) {

‎src/main/java/org/opensearch/plugin/insights/core/exporter/SinkType.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
* Type of supported sinks
1818
*/
1919
public enum SinkType {
20+
/** no exporter */
21+
NONE("none"),
2022
/** debug exporter */
2123
DEBUG("debug"),
2224
/** local index exporter */
@@ -60,7 +62,9 @@ public static Set<SinkType> allSinkTypes() {
6062
public static SinkType getSinkTypeFromExporter(QueryInsightsExporter exporter) {
6163
if (exporter.getClass().equals(LocalIndexExporter.class)) {
6264
return SinkType.LOCAL_INDEX;
65+
} else if (exporter.getClass().equals(DebugExporter.class)) {
66+
return SinkType.DEBUG;
6367
}
64-
return SinkType.DEBUG;
68+
return SinkType.NONE;
6569
}
6670
}

‎src/main/java/org/opensearch/plugin/insights/core/reader/LocalIndexReader.java

+13-1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public final class LocalIndexReader implements QueryInsightsReader {
4646
private final Client client;
4747
private DateTimeFormatter indexPattern;
4848
private final NamedXContentRegistry namedXContentRegistry;
49+
private final String id;
4950

5051
/**
5152
* Constructor of LocalIndexReader
@@ -54,12 +55,23 @@ public final class LocalIndexReader implements QueryInsightsReader {
5455
* @param indexPattern the pattern of index to read from
5556
* @param namedXContentRegistry for parsing purposes
5657
*/
57-
public LocalIndexReader(final Client client, final DateTimeFormatter indexPattern, final NamedXContentRegistry namedXContentRegistry) {
58+
public LocalIndexReader(
59+
final Client client,
60+
final DateTimeFormatter indexPattern,
61+
final NamedXContentRegistry namedXContentRegistry,
62+
final String id
63+
) {
5864
this.indexPattern = indexPattern;
5965
this.client = client;
66+
this.id = id;
6067
this.namedXContentRegistry = namedXContentRegistry;
6168
}
6269

70+
@Override
71+
public String getId() {
72+
return id;
73+
}
74+
6375
/**
6476
* Getter of indexPattern
6577
*

‎src/main/java/org/opensearch/plugin/insights/core/reader/QueryInsightsReader.java

+2
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,6 @@ public interface QueryInsightsReader extends Closeable {
2525
* @return List of SearchQueryRecord
2626
*/
2727
List<SearchQueryRecord> read(final String from, final String to, final String id);
28+
29+
String getId();
2830
}

‎src/main/java/org/opensearch/plugin/insights/core/reader/QueryInsightsReaderFactory.java

+36-26
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@
1010

1111
import java.io.IOException;
1212
import java.time.format.DateTimeFormatter;
13-
import java.util.HashSet;
13+
import java.util.HashMap;
1414
import java.util.Locale;
15-
import java.util.Set;
15+
import java.util.Map;
1616
import org.apache.logging.log4j.LogManager;
1717
import org.apache.logging.log4j.Logger;
1818
import org.opensearch.client.Client;
@@ -27,7 +27,7 @@ public class QueryInsightsReaderFactory {
2727
*/
2828
private final Logger logger = LogManager.getLogger();
2929
final private Client client;
30-
final private Set<QueryInsightsReader> Readers;
30+
final private Map<String, QueryInsightsReader> readers;
3131

3232
/**
3333
* Constructor of QueryInsightsReaderFactory
@@ -36,7 +36,7 @@ public class QueryInsightsReaderFactory {
3636
*/
3737
public QueryInsightsReaderFactory(final Client client) {
3838
this.client = client;
39-
this.Readers = new HashSet<>();
39+
this.readers = new HashMap<>();
4040
}
4141

4242
/**
@@ -46,39 +46,49 @@ public QueryInsightsReaderFactory(final Client client) {
4646
* @param namedXContentRegistry for parsing purposes
4747
* @return QueryInsightsReader the created Reader
4848
*/
49-
public QueryInsightsReader createReader(String indexPattern, NamedXContentRegistry namedXContentRegistry) {
50-
QueryInsightsReader Reader = new LocalIndexReader(
49+
public QueryInsightsReader createReader(String id, String indexPattern, NamedXContentRegistry namedXContentRegistry) {
50+
QueryInsightsReader reader = new LocalIndexReader(
5151
client,
5252
DateTimeFormatter.ofPattern(indexPattern, Locale.ROOT),
53-
namedXContentRegistry
53+
namedXContentRegistry,
54+
id
5455
);
55-
this.Readers.add(Reader);
56-
return Reader;
56+
this.readers.put(id, reader);
57+
return reader;
5758
}
5859

5960
/**
60-
* Update a Reader based on provided parameters
61+
* Update a reader based on provided parameters
6162
*
62-
* @param Reader The Reader to update
63-
* @param indexPattern the index pattern if creating an index Reader
64-
* @return QueryInsightsReader the updated Reader sink
63+
* @param reader The reader to update
64+
* @param indexPattern the index pattern if creating an index reader
65+
* @return QueryInsightsReader the updated reader sink
6566
*/
66-
public QueryInsightsReader updateReader(QueryInsightsReader Reader, String indexPattern) {
67-
if (Reader.getClass() == LocalIndexReader.class) {
68-
((LocalIndexReader) Reader).setIndexPattern(DateTimeFormatter.ofPattern(indexPattern, Locale.ROOT));
67+
public QueryInsightsReader updateReader(QueryInsightsReader reader, String indexPattern) {
68+
if (reader.getClass() == LocalIndexReader.class) {
69+
((LocalIndexReader) reader).setIndexPattern(DateTimeFormatter.ofPattern(indexPattern, Locale.ROOT));
6970
}
70-
return Reader;
71+
return reader;
72+
}
73+
74+
/**
75+
* Get a reader by id
76+
* @param id The id of the reader
77+
* @return QueryInsightsReader the Reader
78+
*/
79+
public QueryInsightsReader getReader(String id) {
80+
return this.readers.get(id);
7181
}
7282

7383
/**
74-
* Close a Reader
84+
* Close a reader
7585
*
76-
* @param Reader the Reader to close
86+
* @param reader the Reader to close
7787
*/
78-
public void closeReader(QueryInsightsReader Reader) throws IOException {
79-
if (Reader != null) {
80-
Reader.close();
81-
this.Readers.remove(Reader);
88+
public void closeReader(QueryInsightsReader reader) throws IOException {
89+
if (reader != null) {
90+
reader.close();
91+
this.readers.remove(reader.getId());
8292
}
8393
}
8494

@@ -87,11 +97,11 @@ public void closeReader(QueryInsightsReader Reader) throws IOException {
8797
*
8898
*/
8999
public void closeAllReaders() {
90-
for (QueryInsightsReader Reader : Readers) {
100+
for (QueryInsightsReader reader : readers.values()) {
91101
try {
92-
closeReader(Reader);
102+
closeReader(reader);
93103
} catch (IOException e) {
94-
logger.error("Fail to close query insights Reader, error: ", e);
104+
logger.error("Fail to close query insights reader, error: ", e);
95105
}
96106
}
97107
}

0 commit comments

Comments
 (0)