Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Centralize codes related to 'master_timeout' deprecation for eaiser removal - in CAT Nodes API #2670

Merged
31 changes: 31 additions & 0 deletions server/src/main/java/org/opensearch/rest/BaseRestHandler.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,12 @@
import org.apache.logging.log4j.Logger;
import org.apache.lucene.search.spell.LevenshteinDistance;
import org.apache.lucene.util.CollectionUtil;
import org.opensearch.OpenSearchParseException;
import org.opensearch.action.support.master.MasterNodeRequest;
import org.opensearch.client.node.NodeClient;
import org.opensearch.common.CheckedConsumer;
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.logging.DeprecationLogger;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.plugins.ActionPlugin;
Expand Down Expand Up @@ -200,6 +203,34 @@ protected Set<String> responseParams() {
return Collections.emptySet();
}

/**
* Parse the deprecated request parameter 'master_timeout', and add deprecated log if the parameter is used.
* It also validates whether the two parameters 'master_timeout' and 'cluster_manager_timeout' are not assigned together.
* The method is temporarily added in 2.0 duing applying inclusive language. Remove the method along with MASTER_ROLE.
* @param mnr the action request
* @param request the REST request to handle
* @param logger the logger that logs deprecation notices
* @param logMsgKeyPrefix the key prefix of a deprecation message to avoid duplicate messages.
*/
protected static void parseDeprecatedMasterTimeoutParameter(
MasterNodeRequest mnr,
RestRequest request,
DeprecationLogger logger,
String logMsgKeyPrefix
) {
final String MASTER_TIMEOUT_DEPRECATED_MESSAGE =
"Deprecated parameter [master_timeout] used. To promote inclusive language, please use [cluster_manager_timeout] instead. It will be unsupported in a future major version.";
final String DUPLICATE_PARAMETER_ERROR_MESSAGE =
"Please only use one of the request parameters [master_timeout, cluster_manager_timeout].";
if (request.hasParam("master_timeout")) {
logger.deprecate(logMsgKeyPrefix + "_master_timeout_parameter", MASTER_TIMEOUT_DEPRECATED_MESSAGE);
if (request.hasParam("cluster_manager_timeout")) {
throw new OpenSearchParseException(DUPLICATE_PARAMETER_ERROR_MESSAGE);
}
mnr.masterNodeTimeout(request.paramAsTime("master_timeout", mnr.masterNodeTimeout()));
}
}

public static class Wrapper extends BaseRestHandler {

protected final BaseRestHandler delegate;
Expand Down
27 changes: 0 additions & 27 deletions server/src/main/java/org/opensearch/rest/RestRequest.java
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
Expand Down Expand Up @@ -579,32 +578,6 @@ public static XContentType parseContentType(List<String> header) {
throw new IllegalArgumentException("empty Content-Type header");
}

/**
* The method is only used to validate whether the values of the 2 request parameters "master_timeout" and "cluster_manager_timeout" is the same value or not.
* If the 2 values are not the same, throw an {@link OpenSearchParseException}.
* @param keys Names of the request parameters.
* @deprecated The method will be removed along with the request parameter "master_timeout".
*/
@Deprecated
public void validateParamValuesAreEqual(String... keys) {
// Track the last seen value and ensure that every subsequent value matches it.
// The value to be tracked is the non-empty values of the parameters with the key.
String lastSeenValue = null;
for (String key : keys) {
String value = param(key);
if (!Strings.isNullOrEmpty(value)) {
if (lastSeenValue == null || value.equals(lastSeenValue)) {
lastSeenValue = value;
} else {
throw new OpenSearchParseException(
"The values of the request parameters: {} are required to be equal, otherwise please only assign value to one of the request parameters.",
Arrays.toString(keys)
);
}
}
}
}

public static class ContentTypeHeaderException extends RuntimeException {

ContentTypeHeaderException(final IllegalArgumentException cause) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,6 @@ public class RestNodesAction extends AbstractCatAction {
private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestNodesAction.class);
static final String LOCAL_DEPRECATED_MESSAGE = "Deprecated parameter [local] used. This parameter does not cause this API to act "
+ "locally, and should not be used. It will be unsupported in version 8.0.";
static final String MASTER_TIMEOUT_DEPRECATED_MESSAGE =
"Deprecated parameter [master_timeout] used. To promote inclusive language, please use [cluster_manager_timeout] instead. It will be unsupported in a future major version.";

@Override
public List<Route> routes() {
Expand All @@ -113,7 +111,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli
}
clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
clusterStateRequest.masterNodeTimeout(request.paramAsTime("cluster_manager_timeout", clusterStateRequest.masterNodeTimeout()));
parseDeprecatedMasterTimeoutParameter(clusterStateRequest, request);
parseDeprecatedMasterTimeoutParameter(clusterStateRequest, request, deprecationLogger, getName());
final boolean fullId = request.paramAsBoolean("full_id", false);
return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
@Override
Expand Down Expand Up @@ -529,20 +527,4 @@ Table buildTable(
private short calculatePercentage(long used, long max) {
return max <= 0 ? 0 : (short) ((100d * used) / max);
}

/**
* Parse the deprecated request parameter 'master_timeout', and add deprecated log if the parameter is used.
* It also validates whether the value of 'master_timeout' is the same with 'cluster_manager_timeout'.
* Remove the method along with MASTER_ROLE.
* @deprecated As of 2.0, because promoting inclusive language.
*/
@Deprecated
private void parseDeprecatedMasterTimeoutParameter(ClusterStateRequest clusterStateRequest, RestRequest request) {
final String deprecatedTimeoutParam = "master_timeout";
if (request.hasParam(deprecatedTimeoutParam)) {
deprecationLogger.deprecate("cat_nodes_master_timeout_parameter", MASTER_TIMEOUT_DEPRECATED_MESSAGE);
request.validateParamValuesAreEqual(deprecatedTimeoutParam, "cluster_manager_timeout");
clusterStateRequest.masterNodeTimeout(request.paramAsTime(deprecatedTimeoutParam, clusterStateRequest.masterNodeTimeout()));
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.action;

import org.junit.AfterClass;
import org.opensearch.OpenSearchParseException;
import org.opensearch.client.node.NodeClient;
import org.opensearch.common.settings.Settings;
import org.opensearch.rest.action.cat.RestNodesAction;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.test.rest.FakeRestRequest;
import org.opensearch.threadpool.TestThreadPool;

import static org.hamcrest.Matchers.containsString;

/**
* As of 2.0, the request parameter 'master_timeout' in all applicable REST APIs is deprecated,
* and alternative parameter 'cluster_manager_timeout' is added.
* The tests are used to validate the behavior about the renamed request parameter.
* Remove the test after removing MASTER_ROLE and 'master_timeout'.
*/
public class RenamedTimeoutRequestParameterTests extends OpenSearchTestCase {
private static final TestThreadPool threadPool = new TestThreadPool(RenamedTimeoutRequestParameterTests.class.getName());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this have to be static?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for this suggestion! I will learn from it.
It doesn't, the variable can has got any value as long as it's an instance of the same class.
The reason I use static is just usually heard others saying "to define a constant, use static final".
I will remove static keyword.

Copy link
Collaborator Author

@tlfeng tlfeng Mar 31, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, because the method with @AfterClass annotation has to be static, and I wrote a method to terminate the ThreadPool after every test in Line 37, I can only make the threadPool variable static.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

to define a constant, use static final

Yeah, the issue here is that a thread pool can't really be a "constant" because it is a stateful and mutable object (hence the need to terminate it). For any scalar values or immutable objects it would be fine to make them static final constants.

A potentially valid reason for making the thread pool static would be that if it was expensive to create and tear down, so you only wanted to do it once and run multiple tests with it. That carries the risk of tests interfering with one another since they would share mutable static state, so that should only be done in cases where it makes a big difference in test run time.

Copy link
Member

@andrross andrross Mar 31, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, because the method with @AfterClass annotation has to be static

Just use the @After annotation on a non-static method and it will be run after every test case.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you so much for telling me the detail! 👍 👍 (String is immutable, so it's fine to be static 😄)
Sorry my mistake, I wanted to say the threadPool can be terminated after all test cases run and can be shared with test cases, because I think the interfering is not a problem, because my test cases only care about the request parameters, and doesn't check the result.
So looks fine to keep the threadpool it static.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right that the current implementation will likely work fine with a static thread pool. But in absence of a strong reason to use shared static state, you should prefer to avoid shared static state. You never know what kind of test the next developer will add to this test class :)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I updated the threadpool to be non-static, and terminate the threadpool after every test cases, hope it will not expensive to tear down. 😄 Thank you!

private final NodeClient client = new NodeClient(Settings.EMPTY, threadPool);

private static final String DUPLICATE_PARAMETER_ERROR_MESSAGE =
"Please only use one of the request parameters [master_timeout, cluster_manager_timeout].";
private static final String MASTER_TIMEOUT_DEPRECATED_MESSAGE =
"Deprecated parameter [master_timeout] used. To promote inclusive language, please use [cluster_manager_timeout] instead. It will be unsupported in a future major version.";

@AfterClass
public static void terminateThreadPool() {
terminate(threadPool);
}

/**
* Validate both parameters 'cluster_manager_timeout' and its predecessor can be parsed correctly.
*/
public void testCatAllocation() {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should probably break this into 3 separate test methods, something like:

  • testNoWarningsForNewParam
  • testDeprecationWarningForOldParam
  • testBothParamsNotValid

Then you don't need the comments because the test methods are self-explanatory, and if there was any failure it would point to a more specific cause.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You never know what kind of test the next developer will add to this test class :)

Haha, looks like I should explicitly state do not adding new tests to this class, I'm joking. Though it is a temporary and tedious test case.

Copy link
Collaborator Author

@tlfeng tlfeng Mar 31, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, the method testCatAllocation() is not a unit test for the method RenamedTimeoutRequestParameterTests(), but testing the CAT Nodes API can parse the 2 parameters properly. The problem is that there are 60 other REST APIs to be adding new request parameter cluster_manager_timeout, if I break that method to 3 separate ones, it will result 180 new method to be added 🤯.
Instead, I think I'd better add unit test for the method RenamedTimeoutRequestParameterTests(), and create separate test methods as you suggested. And only cover testNoWarningsForNewParam() for every specific REST Action class (for every REST API)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm just suggesting that you break this test method down into 3 separate tests instead of 1, since it is actually testing 3 separate things. I think it's fine to test this behavior in this one class and was not suggesting that you add tests to all other APIs.

Copy link
Collaborator Author

@tlfeng tlfeng Mar 31, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see 😄. I agree to test the behavior in this class, and I will modify the unit tests.
I was thinking if applying 3 test cases to test the behavior for 1 API, then there is no reason not to apply the same to other APIs. 😁

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for your suggestions! I modified to test one behavior in a single test case. For the other REST APIs that has got "master_timeout" parameter, I will add one test case for each.

RestNodesAction action = new RestNodesAction();
// Request with only new parameter will be parsed without warning and exception.
action.doCatRequest(getRestRequestWithNewParam(), client);
// Request with only deprecated parameter will result deprecation warning.
action.doCatRequest(getRestRequestWithDeprecatedParam(), client);
assertWarnings(MASTER_TIMEOUT_DEPRECATED_MESSAGE);
// Request with both new and deprecated parameters and different values will result exception.
// It should have warning, but the same deprecation warning won't be logged again.
Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(getRestRequestWithWrongValues(), client));
assertThat(e.getMessage(), containsString(DUPLICATE_PARAMETER_ERROR_MESSAGE));
}

private FakeRestRequest getRestRequestWithWrongValues() {
FakeRestRequest request = new FakeRestRequest();
request.params().put("cluster_manager_timeout", "1h");
request.params().put("master_timeout", "3s");
return request;
}

private FakeRestRequest getRestRequestWithDeprecatedParam() {
FakeRestRequest request = new FakeRestRequest();
request.params().put("master_timeout", "3s");
return request;
}

private FakeRestRequest getRestRequestWithNewParam() {
FakeRestRequest request = new FakeRestRequest();
request.params().put("cluster_manager_timeout", "2m");
return request;
}
}
37 changes: 0 additions & 37 deletions server/src/test/java/org/opensearch/rest/RestRequestTests.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@

import org.opensearch.OpenSearchParseException;
import org.opensearch.common.CheckedConsumer;
import org.opensearch.common.Strings;
import org.opensearch.common.bytes.BytesArray;
import org.opensearch.common.bytes.BytesReference;
import org.opensearch.common.collect.MapBuilder;
Expand All @@ -51,13 +50,11 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;

import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.mockito.Mockito.mock;
Expand Down Expand Up @@ -283,40 +280,6 @@ public void testRequiredContent() {
assertEquals("unknown content type", e.getMessage());
}

/*
* The test is added in 2.0 when the request parameter "cluster_manager_timeout" is introduced.
* Remove the test along with the removal of the non-inclusive terminology "master_timeout".
*/
public void testValidateParamValuesAreEqualWhenTheyAreEqual() {
FakeRestRequest request = new FakeRestRequest();
String valueForKey1 = randomFrom("value1", "", null);
String valueForKey2 = "value1";
request.params().put("key1", valueForKey1);
request.params().put("key2", valueForKey2);
request.validateParamValuesAreEqual("key1", "key2");
assertTrue(
String.format(
Locale.ROOT,
"The 2 values should be equal, or having 1 null/empty value. Value of key1: %s. Value of key2: %s",
valueForKey1,
valueForKey2
),
Strings.isNullOrEmpty(valueForKey1) || valueForKey1.equals(valueForKey2)
);
}

/*
* The test is added in 2.0 when the request parameter "cluster_manager_timeout" is introduced.
* Remove the test along with the removal of the non-inclusive terminology "master_timeout".
*/
public void testValidateParamValuesAreEqualWhenTheyAreNotEqual() {
FakeRestRequest request = new FakeRestRequest();
request.params().put("key1", "value1");
request.params().put("key2", "value2");
Exception e = assertThrows(OpenSearchParseException.class, () -> request.validateParamValuesAreEqual("key1", "key2"));
assertThat(e.getMessage(), containsString("The values of the request parameters: [key1, key2] are required to be equal"));
}

private static RestRequest contentRestRequest(String content, Map<String, String> params) {
Map<String, List<String>> headers = new HashMap<>();
headers.put("Content-Type", Collections.singletonList("application/json"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@

package org.opensearch.rest.action.cat;

import org.opensearch.OpenSearchParseException;
import org.opensearch.Version;
import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
Expand All @@ -52,7 +51,6 @@

import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.CoreMatchers.containsString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

Expand Down Expand Up @@ -91,20 +89,4 @@ public void testCatNodesWithLocalDeprecationWarning() {

terminate(threadPool);
}

/**
* Validate both cluster_manager_timeout and its predecessor can be parsed correctly.
* Remove the test along with MASTER_ROLE. It's added in version 2.0.0.
*/
public void testCatNodesWithClusterManagerTimeout() {
TestThreadPool threadPool = new TestThreadPool(RestNodesActionTests.class.getName());
NodeClient client = new NodeClient(Settings.EMPTY, threadPool);
FakeRestRequest request = new FakeRestRequest();
request.params().put("cluster_manager_timeout", randomFrom("1h", "2m"));
request.params().put("master_timeout", "3s");
Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(request, client));
assertThat(e.getMessage(), containsString("[master_timeout, cluster_manager_timeout] are required to be equal"));
assertWarnings(RestNodesAction.MASTER_TIMEOUT_DEPRECATED_MESSAGE);
terminate(threadPool);
}
}