Skip to content

Commit 61762b3

Browse files
sarthakaggarwal97sarthakaggarwal97
committed
validating QAT hardware support before making codecs available
Signed-off-by: Sarthak Aggarwal <sarthagg@amazon.com>
1 parent d18505e commit 61762b3

File tree

5 files changed

+71
-12
lines changed

5 files changed

+71
-12
lines changed

‎src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java

+23
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
2121
import org.apache.hc.core5.reactor.ssl.TlsDetails;
2222
import org.apache.hc.core5.ssl.SSLContextBuilder;
23+
import org.opensearch.client.ResponseException;
2324
import org.opensearch.client.RestClient;
2425
import org.opensearch.client.RestClientBuilder;
2526
import org.opensearch.cluster.metadata.IndexMetadata;
@@ -67,6 +68,28 @@ public void testCreateIndexWithZstdCodec() throws IOException {
6768
}
6869
}
6970

71+
public void testCreateIndexWithQatCodecWithQatHardwareUnavailable() throws IOException {
72+
73+
final String index = "custom-codecs-test-index";
74+
75+
if (!QatZipperFactory.isQatAvailable()) {
76+
// creating index
77+
final ResponseException e = expectThrows(
78+
ResponseException.class,
79+
() -> createIndex(
80+
index,
81+
Settings.builder()
82+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
83+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
84+
.put("index.codec", randomFrom(QAT_DEFLATE_CODEC, QAT_LZ4_CODEC))
85+
.put("index.codec.compression_level", randomIntBetween(1, 6))
86+
.build()
87+
)
88+
);
89+
assertTrue(e.getResponse().toString().contains("400 Bad Request"));
90+
}
91+
}
92+
7093
public void testCreateIndexWithQatCodec() throws IOException {
7194
assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true));
7295

‎src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java

+16-12
Original file line numberDiff line numberDiff line change
@@ -51,21 +51,25 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin
5151
if (mapperService == null) {
5252
codecs.put(ZSTD_CODEC, new Zstd99Codec(compressionLevel));
5353
codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(compressionLevel));
54-
codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(compressionLevel, () -> {
55-
return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
56-
}));
57-
codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(compressionLevel, () -> {
58-
return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
59-
}));
54+
if (QatZipperFactory.isQatAvailable()) {
55+
codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(compressionLevel, () -> {
56+
return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
57+
}));
58+
codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(compressionLevel, () -> {
59+
return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
60+
}));
61+
}
6062
} else {
6163
codecs.put(ZSTD_CODEC, new Zstd99Codec(mapperService, logger, compressionLevel));
6264
codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(mapperService, logger, compressionLevel));
63-
codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(mapperService, logger, compressionLevel, () -> {
64-
return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
65-
}));
66-
codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(mapperService, logger, compressionLevel, () -> {
67-
return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
68-
}));
65+
if (QatZipperFactory.isQatAvailable()) {
66+
codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(mapperService, logger, compressionLevel, () -> {
67+
return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
68+
}));
69+
codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(mapperService, logger, compressionLevel, () -> {
70+
return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
71+
}));
72+
}
6973
}
7074
this.codecs = codecs.immutableMap();
7175
}

‎src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java

+3
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ public boolean supports(Setting<?> setting) {
8686

8787
@Override
8888
public Set<String> aliases() {
89+
if (!QatZipperFactory.isQatAvailable()) {
90+
return Set.of();
91+
}
8992
return Mode.QAT_DEFLATE.getAliases();
9093
}
9194
}

‎src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java

+3
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ public boolean supports(Setting<?> setting) {
8686

8787
@Override
8888
public Set<String> aliases() {
89+
if (!QatZipperFactory.isQatAvailable()) {
90+
return Set.of();
91+
}
8992
return Mode.QAT_LZ4.getAliases();
9093
}
9194
}

‎src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java

+26
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@
6565
import java.util.Collections;
6666
import java.util.Optional;
6767

68+
import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_DEFLATE_CODEC;
69+
import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_LZ4_CODEC;
6870
import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_CODEC;
6971
import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_NO_DICT_CODEC;
7072
import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING;
@@ -176,6 +178,30 @@ public void testZstdNoDictMapperServiceNull() throws Exception {
176178
assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel());
177179
}
178180

181+
public void testQatCodecsNotAvailable() throws IOException {
182+
if (!QatZipperFactory.isQatAvailable()) {
183+
assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_lz4"));
184+
assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_deflate"));
185+
186+
QatLz499Codec qatLz499Codec = new QatLz499Codec();
187+
assertTrue(qatLz499Codec.aliases().isEmpty());
188+
189+
QatDeflate99Codec qatDeflate99Codec = new QatDeflate99Codec();
190+
assertTrue(qatDeflate99Codec.aliases().isEmpty());
191+
}
192+
}
193+
194+
public void testCodecServiceFactoryQatUnavailable() throws IOException {
195+
if (!QatZipperFactory.isQatAvailable()) {
196+
Settings nodeSettings = Settings.builder()
197+
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
198+
.put("index.codec", randomFrom(QAT_DEFLATE_CODEC, QAT_LZ4_CODEC))
199+
.build();
200+
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
201+
assertThrows(IllegalArgumentException.class, () -> plugin.getCustomCodecServiceFactory(indexSettings));
202+
}
203+
}
204+
179205
// write some docs with it, inspect .si to see this was the used compression
180206
private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Codec actual) throws Exception {
181207
SegmentReader sr = getSegmentReader(actual);

0 commit comments

Comments
 (0)