Skip to content

Commit 6dbb079

Browse files
dzane17reta
dzane17
and
reta
authored
Remove mmap.extensions setting (opensearch-project#9392)
* Remove mmap.extensions setting in favor of nio.extensions Signed-off-by: David Zane <davizane@amazon.com> * Update CHANGELOG-3.0.md Co-authored-by: Andriy Redko <drreta@gmail.com> Signed-off-by: David Zane <38449481+dzane17@users.noreply.github.com> --------- Signed-off-by: David Zane <davizane@amazon.com> Signed-off-by: David Zane <38449481+dzane17@users.noreply.github.com> Co-authored-by: Andriy Redko <drreta@gmail.com>
1 parent e26608b commit 6dbb079

File tree

5 files changed

+5
-172
lines changed

5 files changed

+5
-172
lines changed

‎CHANGELOG-3.0.md

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4343
- Remove LegacyESVersion.V_7_10_ Constants ([#5018](https://github.com/opensearch-project/OpenSearch/pull/5018))
4444
- Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021))
4545
- Remove custom Map, List and Set collection classes ([#6871](https://github.com/opensearch-project/OpenSearch/pull/6871))
46+
- Remove `index.store.hybrid.mmap.extensions` setting in favor of `index.store.hybrid.nio.extensions` setting ([#9392](https://github.com/opensearch-project/OpenSearch/pull/9392))
4647

4748
### Fixed
4849
- Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))

‎server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

-1
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
191191
BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
192192
IndexModule.INDEX_STORE_TYPE_SETTING,
193193
IndexModule.INDEX_STORE_PRE_LOAD_SETTING,
194-
IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS,
195194
IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS,
196195
IndexModule.INDEX_RECOVERY_TYPE_SETTING,
197196
IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING,

‎server/src/main/java/org/opensearch/index/IndexModule.java

+1-76
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,6 @@
9797
import java.util.Collections;
9898
import java.util.HashMap;
9999
import java.util.HashSet;
100-
import java.util.Iterator;
101100
import java.util.List;
102101
import java.util.Map;
103102
import java.util.Objects;
@@ -183,52 +182,7 @@ public final class IndexModule {
183182
Property.PrivateIndex
184183
);
185184

186-
/** Which lucene file extensions to load with the mmap directory when using hybridfs store. This settings is ignored if {@link #INDEX_STORE_HYBRID_NIO_EXTENSIONS} is set.
187-
* This is an expert setting.
188-
* @see <a href="https://lucene.apache.org/core/9_9_0/core/org/apache/lucene/codecs/lucene99/package-summary.html#file-names">Lucene File Extensions</a>.
189-
*
190-
* @deprecated This setting will be removed in OpenSearch 3.x. Use {@link #INDEX_STORE_HYBRID_NIO_EXTENSIONS} instead.
191-
*/
192-
@Deprecated
193-
public static final Setting<List<String>> INDEX_STORE_HYBRID_MMAP_EXTENSIONS = Setting.listSetting(
194-
"index.store.hybrid.mmap.extensions",
195-
List.of("nvd", "dvd", "tim", "tip", "dim", "kdd", "kdi", "cfs", "doc"),
196-
Function.identity(),
197-
new Setting.Validator<List<String>>() {
198-
199-
@Override
200-
public void validate(final List<String> value) {}
201-
202-
@Override
203-
public void validate(final List<String> value, final Map<Setting<?>, Object> settings) {
204-
if (value.equals(INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getDefault(Settings.EMPTY)) == false) {
205-
final List<String> nioExtensions = (List<String>) settings.get(INDEX_STORE_HYBRID_NIO_EXTENSIONS);
206-
final List<String> defaultNioExtensions = INDEX_STORE_HYBRID_NIO_EXTENSIONS.getDefault(Settings.EMPTY);
207-
if (nioExtensions.equals(defaultNioExtensions) == false) {
208-
throw new IllegalArgumentException(
209-
"Settings "
210-
+ INDEX_STORE_HYBRID_NIO_EXTENSIONS.getKey()
211-
+ " & "
212-
+ INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getKey()
213-
+ " cannot both be set. Use "
214-
+ INDEX_STORE_HYBRID_NIO_EXTENSIONS.getKey()
215-
+ " only."
216-
);
217-
}
218-
}
219-
}
220-
221-
@Override
222-
public Iterator<Setting<?>> settings() {
223-
return List.<Setting<?>>of(INDEX_STORE_HYBRID_NIO_EXTENSIONS).iterator();
224-
}
225-
},
226-
Property.IndexScope,
227-
Property.NodeScope,
228-
Property.Deprecated
229-
);
230-
231-
/** Which lucene file extensions to load with nio. All others will default to mmap. Takes precedence over {@link #INDEX_STORE_HYBRID_MMAP_EXTENSIONS}.
185+
/** Which lucene file extensions to load with nio. All others will default to mmap.
232186
* This is an expert setting.
233187
* @see <a href="https://lucene.apache.org/core/9_9_0/core/org/apache/lucene/codecs/lucene99/package-summary.html#file-names">Lucene File Extensions</a>.
234188
*/
@@ -253,35 +207,6 @@ public Iterator<Setting<?>> settings() {
253207
"vem"
254208
),
255209
Function.identity(),
256-
new Setting.Validator<List<String>>() {
257-
258-
@Override
259-
public void validate(final List<String> value) {}
260-
261-
@Override
262-
public void validate(final List<String> value, final Map<Setting<?>, Object> settings) {
263-
if (value.equals(INDEX_STORE_HYBRID_NIO_EXTENSIONS.getDefault(Settings.EMPTY)) == false) {
264-
final List<String> mmapExtensions = (List<String>) settings.get(INDEX_STORE_HYBRID_MMAP_EXTENSIONS);
265-
final List<String> defaultMmapExtensions = INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getDefault(Settings.EMPTY);
266-
if (mmapExtensions.equals(defaultMmapExtensions) == false) {
267-
throw new IllegalArgumentException(
268-
"Settings "
269-
+ INDEX_STORE_HYBRID_NIO_EXTENSIONS.getKey()
270-
+ " & "
271-
+ INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getKey()
272-
+ " cannot both be set. Use "
273-
+ INDEX_STORE_HYBRID_NIO_EXTENSIONS.getKey()
274-
+ " only."
275-
);
276-
}
277-
}
278-
}
279-
280-
@Override
281-
public Iterator<Setting<?>> settings() {
282-
return List.<Setting<?>>of(INDEX_STORE_HYBRID_MMAP_EXTENSIONS).iterator();
283-
}
284-
},
285210
Property.IndexScope,
286211
Property.NodeScope
287212
);

‎server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java

+1-18
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545
import org.apache.lucene.store.SimpleFSLockFactory;
4646
import org.opensearch.common.settings.Setting;
4747
import org.opensearch.common.settings.Setting.Property;
48-
import org.opensearch.common.settings.Settings;
4948
import org.opensearch.common.util.io.IOUtils;
5049
import org.opensearch.index.IndexModule;
5150
import org.opensearch.index.IndexSettings;
@@ -57,8 +56,6 @@
5756
import java.nio.file.Path;
5857
import java.util.HashSet;
5958
import java.util.Set;
60-
import java.util.stream.Collectors;
61-
import java.util.stream.Stream;
6259

6360
/**
6461
* Factory for a filesystem directory
@@ -100,21 +97,7 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
10097
case HYBRIDFS:
10198
// Use Lucene defaults
10299
final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
103-
final Set<String> nioExtensions;
104-
final Set<String> mmapExtensions = Set.copyOf(indexSettings.getValue(IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS));
105-
if (mmapExtensions.equals(
106-
new HashSet(IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getDefault(Settings.EMPTY))
107-
) == false) {
108-
// If the mmap extension setting was defined, then compute nio extensions by subtracting out the
109-
// mmap extensions from the set of all extensions.
110-
nioExtensions = Stream.concat(
111-
IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS.getDefault(Settings.EMPTY).stream(),
112-
IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getDefault(Settings.EMPTY).stream()
113-
).filter(e -> mmapExtensions.contains(e) == false).collect(Collectors.toUnmodifiableSet());
114-
} else {
115-
// Otherwise, get the list of nio extensions from the nio setting
116-
nioExtensions = Set.copyOf(indexSettings.getValue(IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS));
117-
}
100+
final Set<String> nioExtensions = new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS));
118101
if (primaryDirectory instanceof MMapDirectory) {
119102
MMapDirectory mMapDirectory = (MMapDirectory) primaryDirectory;
120103
return new HybridDirectory(lockFactory, setPreload(mMapDirectory, lockFactory, preLoadExtensions), nioExtensions);

‎server/src/test/java/org/opensearch/index/store/FsDirectoryFactoryTests.java

+2-77
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ public void testPreload() throws IOException {
9696
build = Settings.builder()
9797
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.HYBRIDFS.name().toLowerCase(Locale.ROOT))
9898
.putList(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), "nvd", "dvd", "cfs")
99-
.putList(IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS.getKey(), "tip", "dim", "kdd", "kdi", "cfs", "doc")
99+
.putList(IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS.getKey(), "tip", "dim", "kdd", "kdi", "cfs", "doc", "new")
100100
.build();
101101
try (Directory directory = newDirectory(build)) {
102102
assertTrue(FsDirectoryFactory.isHybridFs(directory));
@@ -108,7 +108,7 @@ public void testPreload() throws IOException {
108108
assertTrue(hybridDirectory.useDelegate("foo.tim"));
109109
assertTrue(hybridDirectory.useDelegate("foo.pos"));
110110
assertTrue(hybridDirectory.useDelegate("foo.pay"));
111-
assertTrue(hybridDirectory.useDelegate("foo.new"));
111+
assertFalse(hybridDirectory.useDelegate("foo.new"));
112112
assertFalse(hybridDirectory.useDelegate("foo.tip"));
113113
assertFalse(hybridDirectory.useDelegate("foo.dim"));
114114
assertFalse(hybridDirectory.useDelegate("foo.kdd"));
@@ -123,63 +123,6 @@ public void testPreload() throws IOException {
123123
assertTrue(preLoadMMapDirectory.useDelegate("foo.cfs"));
124124
assertTrue(preLoadMMapDirectory.useDelegate("foo.nvd"));
125125
}
126-
build = Settings.builder()
127-
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.HYBRIDFS.name().toLowerCase(Locale.ROOT))
128-
.putList(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), "nvd", "dvd", "cfs")
129-
.putList(IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getKey(), "nvd", "dvd", "tim", "pos")
130-
.build();
131-
try (Directory directory = newDirectory(build)) {
132-
assertTrue(FsDirectoryFactory.isHybridFs(directory));
133-
FsDirectoryFactory.HybridDirectory hybridDirectory = (FsDirectoryFactory.HybridDirectory) directory;
134-
// test custom hybrid mmap extensions
135-
// true->mmap, false->nio
136-
assertTrue(hybridDirectory.useDelegate("foo.nvd"));
137-
assertTrue(hybridDirectory.useDelegate("foo.dvd"));
138-
assertTrue(hybridDirectory.useDelegate("foo.tim"));
139-
assertTrue(hybridDirectory.useDelegate("foo.pos"));
140-
assertTrue(hybridDirectory.useDelegate("foo.new"));
141-
assertFalse(hybridDirectory.useDelegate("foo.pay"));
142-
assertFalse(hybridDirectory.useDelegate("foo.tip"));
143-
assertFalse(hybridDirectory.useDelegate("foo.dim"));
144-
assertFalse(hybridDirectory.useDelegate("foo.kdd"));
145-
assertFalse(hybridDirectory.useDelegate("foo.kdi"));
146-
assertFalse(hybridDirectory.useDelegate("foo.cfs"));
147-
assertFalse(hybridDirectory.useDelegate("foo.doc"));
148-
MMapDirectory delegate = hybridDirectory.getDelegate();
149-
assertThat(delegate, Matchers.instanceOf(FsDirectoryFactory.PreLoadMMapDirectory.class));
150-
assertWarnings(
151-
"[index.store.hybrid.mmap.extensions] setting was deprecated in OpenSearch and will be removed in a future release!"
152-
+ " See the breaking changes documentation for the next major version."
153-
);
154-
}
155-
build = Settings.builder()
156-
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.HYBRIDFS.name().toLowerCase(Locale.ROOT))
157-
.putList(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), "nvd", "dvd", "cfs")
158-
.putList(IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getKey(), "nvd", "dvd", "tim", "pos")
159-
.putList(IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS.getKey(), "nvd", "dvd", "tim", "pos")
160-
.build();
161-
try {
162-
newDirectory(build);
163-
} catch (final Exception e) {
164-
assertEquals(
165-
"Settings index.store.hybrid.nio.extensions & index.store.hybrid.mmap.extensions cannot both be set. Use index.store.hybrid.nio.extensions only.",
166-
e.getMessage()
167-
);
168-
}
169-
build = Settings.builder()
170-
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.HYBRIDFS.name().toLowerCase(Locale.ROOT))
171-
.putList(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), "nvd", "dvd", "cfs")
172-
.putList(IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS.getKey(), "nvd", "dvd", "tim", "pos")
173-
.putList(IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getKey(), "nvd", "dvd", "tim", "pos")
174-
.build();
175-
try {
176-
newDirectory(build);
177-
} catch (final Exception e) {
178-
assertEquals(
179-
"Settings index.store.hybrid.nio.extensions & index.store.hybrid.mmap.extensions cannot both be set. Use index.store.hybrid.nio.extensions only.",
180-
e.getMessage()
181-
);
182-
}
183126
build = Settings.builder()
184127
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.HYBRIDFS.name().toLowerCase(Locale.ROOT))
185128
.putList(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), "nvd", "dvd", "cfs")
@@ -198,24 +141,6 @@ public void testPreload() throws IOException {
198141
MMapDirectory delegate = hybridDirectory.getDelegate();
199142
assertThat(delegate, Matchers.instanceOf(FsDirectoryFactory.PreLoadMMapDirectory.class));
200143
}
201-
build = Settings.builder()
202-
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.HYBRIDFS.name().toLowerCase(Locale.ROOT))
203-
.putList(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), "nvd", "dvd", "cfs")
204-
.putList(IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getKey())
205-
.build();
206-
try (Directory directory = newDirectory(build)) {
207-
assertTrue(FsDirectoryFactory.isHybridFs(directory));
208-
FsDirectoryFactory.HybridDirectory hybridDirectory = (FsDirectoryFactory.HybridDirectory) directory;
209-
// test custom hybrid mmap extensions
210-
// true->mmap, false->nio
211-
assertTrue(hybridDirectory.useDelegate("foo.new"));
212-
assertFalse(hybridDirectory.useDelegate("foo.nvd"));
213-
assertFalse(hybridDirectory.useDelegate("foo.dvd"));
214-
assertFalse(hybridDirectory.useDelegate("foo.cfs"));
215-
assertFalse(hybridDirectory.useDelegate("foo.doc"));
216-
MMapDirectory delegate = hybridDirectory.getDelegate();
217-
assertThat(delegate, Matchers.instanceOf(FsDirectoryFactory.PreLoadMMapDirectory.class));
218-
}
219144
}
220145

221146
private Directory newDirectory(Settings settings) throws IOException {

0 commit comments

Comments
 (0)