From aa41e446d289d78624de8556578b300fd079b9d5 Mon Sep 17 00:00:00 2001 From: JackyYangPassion Date: Tue, 2 Jul 2024 14:43:34 +0800 Subject: [PATCH 1/6] filter dynamice path(PUT/GET/DELETE) with params cause OOM --- .../hugegraph/api/filter/AccessLogFilter.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java index d429db4d9b..ff246d7603 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java @@ -25,8 +25,9 @@ import java.io.IOException; import java.net.URI; +import java.util.regex.Pattern; -import org.apache.hugegraph.auth.HugeAuthenticator; +import jakarta.ws.rs.core.UriInfo; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.config.ServerOptions; import org.apache.hugegraph.core.GraphManager; @@ -54,6 +55,9 @@ public class AccessLogFilter implements ContainerResponseFilter { private static final String GREMLIN = "gremlin"; private static final String CYPHER = "cypher"; + private static final Pattern ID_PATTERN = Pattern.compile("\"\\d+:\\w+\""); + private static final Pattern QUOTED_STRING_PATTERN = Pattern.compile("\"\\w+\""); + @Context private jakarta.inject.Provider configProvider; @@ -78,6 +82,17 @@ private String join(String path1, String path2) { return String.join(DELIMITER, path1, path2); } + private String normalizePath(String path, String method) { + // Replace variable parts of the path with placeholders + //TODO: Jersey Filter Determine if the method parameter is on the path. + if (method.equals("PUT") || method.equals("GET") || method.equals("DELETE")) { + path = ID_PATTERN.matcher(path).replaceAll(method); + path = QUOTED_STRING_PATTERN.matcher(path).replaceAll(method); + } + + return path; + } + /** * Use filter to log request info * @@ -89,8 +104,10 @@ public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) throws IOException { // Grab corresponding request / response info from context; URI uri = requestContext.getUriInfo().getRequestUri(); - String path = uri.getRawPath(); String method = requestContext.getMethod(); + UriInfo uriInfo = requestContext.getUriInfo(); + + String path = normalizePath(uriInfo.getPath(),method); String metricsName = join(path, method); MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_TOTAL_COUNTER)).inc(); From d55382da9f046324a44154f0accc1069579d96e1 Mon Sep 17 00:00:00 2001 From: JackyYangPassion Date: Thu, 11 Jul 2024 17:35:48 +0800 Subject: [PATCH 2/6] fix metrics OOM --- .../hugegraph/api/filter/AccessLogFilter.java | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java index ff246d7603..28598b8037 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java @@ -25,9 +25,10 @@ import java.io.IOException; import java.net.URI; +import java.util.Map; import java.util.regex.Pattern; -import jakarta.ws.rs.core.UriInfo; +import jakarta.ws.rs.core.MultivaluedMap; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.config.ServerOptions; import org.apache.hugegraph.core.GraphManager; @@ -82,15 +83,25 @@ private String join(String path1, String path2) { return String.join(DELIMITER, path1, path2); } - private String normalizePath(String path, String method) { + private String normalizePath(ContainerRequestContext requestContext) { // Replace variable parts of the path with placeholders - //TODO: Jersey Filter Determine if the method parameter is on the path. - if (method.equals("PUT") || method.equals("GET") || method.equals("DELETE")) { - path = ID_PATTERN.matcher(path).replaceAll(method); - path = QUOTED_STRING_PATTERN.matcher(path).replaceAll(method); + String requestPath = requestContext.getUriInfo().getPath(); + // get uri params + MultivaluedMap pathParameters = requestContext.getUriInfo().getPathParameters(); + + String newPath = requestPath; + for (Map.Entry> entry : pathParameters.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue().get(0); + if(key.equals("graph")){ + newPath = newPath.replace(key, value); + } + newPath = newPath.replace(value, key); } - return path; + LOG.debug("Original Path: " + requestPath + " New Path: " + newPath); + + return newPath; } /** @@ -105,16 +116,24 @@ public void filter(ContainerRequestContext requestContext, // Grab corresponding request / response info from context; URI uri = requestContext.getUriInfo().getRequestUri(); String method = requestContext.getMethod(); - UriInfo uriInfo = requestContext.getUriInfo(); - - String path = normalizePath(uriInfo.getPath(),method); + String path = normalizePath(requestContext); String metricsName = join(path, method); + int status = responseContext.getStatus(); + + if (status != 500 && status != 415) { + MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_TOTAL_COUNTER)).inc(); + } - MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_TOTAL_COUNTER)).inc(); if (statusOk(responseContext.getStatus())) { MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_SUCCESS_COUNTER)).inc(); + } else { - MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_FAILED_COUNTER)).inc(); + //TODO: The return codes for compatibility need to be further detailed. + LOG.debug("Failed Status: "+status); + if (status != 500 && status != 415) { + MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_FAILED_COUNTER)).inc(); + } + } Object requestTime = requestContext.getProperty(REQUEST_TIME); @@ -123,8 +142,10 @@ public void filter(ContainerRequestContext requestContext, long start = (Long) requestTime; long executeTime = now - start; - MetricsUtil.registerHistogram(join(metricsName, METRICS_PATH_RESPONSE_TIME_HISTOGRAM)) - .update(executeTime); + if (status != 500 && status != 415) { + MetricsUtil.registerHistogram(join(metricsName, METRICS_PATH_RESPONSE_TIME_HISTOGRAM)) + .update(executeTime); + } HugeConfig config = configProvider.get(); long timeThreshold = config.get(ServerOptions.SLOW_QUERY_LOG_TIME_THRESHOLD); From ecd0944b11822bb197ec05cd3fddf3af0f64e974 Mon Sep 17 00:00:00 2001 From: JackyYangPassion Date: Thu, 11 Jul 2024 17:41:55 +0800 Subject: [PATCH 3/6] fix metrics OOM --- .../java/org/apache/hugegraph/api/filter/AccessLogFilter.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java index 28598b8037..d006fcb61f 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java @@ -56,8 +56,6 @@ public class AccessLogFilter implements ContainerResponseFilter { private static final String GREMLIN = "gremlin"; private static final String CYPHER = "cypher"; - private static final Pattern ID_PATTERN = Pattern.compile("\"\\d+:\\w+\""); - private static final Pattern QUOTED_STRING_PATTERN = Pattern.compile("\"\\w+\""); @Context private jakarta.inject.Provider configProvider; From bf3bfc3fe7514ede78d6a78ede3c77398b1b0bba Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 20 Jul 2024 23:53:12 +0800 Subject: [PATCH 4/6] tiny fix & change to trace level --- .../hugegraph/api/filter/AccessLogFilter.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java index d006fcb61f..5c9c7420fd 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java @@ -26,9 +26,7 @@ import java.io.IOException; import java.net.URI; import java.util.Map; -import java.util.regex.Pattern; -import jakarta.ws.rs.core.MultivaluedMap; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.config.ServerOptions; import org.apache.hugegraph.core.GraphManager; @@ -42,6 +40,7 @@ import jakarta.ws.rs.container.ContainerResponseContext; import jakarta.ws.rs.container.ContainerResponseFilter; import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MultivaluedMap; import jakarta.ws.rs.ext.Provider; // TODO: should add test for this class @@ -56,7 +55,6 @@ public class AccessLogFilter implements ContainerResponseFilter { private static final String GREMLIN = "gremlin"; private static final String CYPHER = "cypher"; - @Context private jakarta.inject.Provider configProvider; @@ -85,20 +83,20 @@ private String normalizePath(ContainerRequestContext requestContext) { // Replace variable parts of the path with placeholders String requestPath = requestContext.getUriInfo().getPath(); // get uri params - MultivaluedMap pathParameters = requestContext.getUriInfo().getPathParameters(); + MultivaluedMap pathParameters = requestContext.getUriInfo() + .getPathParameters(); String newPath = requestPath; for (Map.Entry> entry : pathParameters.entrySet()) { String key = entry.getKey(); String value = entry.getValue().get(0); - if(key.equals("graph")){ + if ("graph".equals(key)) { newPath = newPath.replace(key, value); } newPath = newPath.replace(value, key); } - LOG.debug("Original Path: " + requestPath + " New Path: " + newPath); - + LOG.trace("Original Path: {} New Path: {}", requestPath, newPath); return newPath; } @@ -116,22 +114,20 @@ public void filter(ContainerRequestContext requestContext, String method = requestContext.getMethod(); String path = normalizePath(requestContext); String metricsName = join(path, method); - int status = responseContext.getStatus(); + int status = responseContext.getStatus(); if (status != 500 && status != 415) { MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_TOTAL_COUNTER)).inc(); } if (statusOk(responseContext.getStatus())) { MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_SUCCESS_COUNTER)).inc(); - } else { //TODO: The return codes for compatibility need to be further detailed. - LOG.debug("Failed Status: "+status); + LOG.trace("Failed Status: {}", status); if (status != 500 && status != 415) { MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_FAILED_COUNTER)).inc(); } - } Object requestTime = requestContext.getProperty(REQUEST_TIME); @@ -141,7 +137,8 @@ public void filter(ContainerRequestContext requestContext, long executeTime = now - start; if (status != 500 && status != 415) { - MetricsUtil.registerHistogram(join(metricsName, METRICS_PATH_RESPONSE_TIME_HISTOGRAM)) + MetricsUtil.registerHistogram(join(metricsName, + METRICS_PATH_RESPONSE_TIME_HISTOGRAM)) .update(executeTime); } From 5b087d6c0e5007999f001e3df7142ce3b571be57 Mon Sep 17 00:00:00 2001 From: yangjiaqi Date: Mon, 28 Oct 2024 10:06:45 +0800 Subject: [PATCH 5/6] fix code style --- .../org/apache/hugegraph/api/filter/AccessLogFilter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java index ff0eff28d4..c344b32202 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java @@ -79,7 +79,7 @@ private String join(String path1, String path2) { return String.join(DELIMITER, path1, path2); } - private String normalizePath(ContainerRequestContext requestContext) { + private static String normalizePath(ContainerRequestContext requestContext) { // Replace variable parts of the path with placeholders String requestPath = requestContext.getUriInfo().getPath(); // get uri params @@ -96,7 +96,7 @@ private String normalizePath(ContainerRequestContext requestContext) { newPath = newPath.replace(value, key); } - LOG.trace("Original Path: {} New Path: {}", requestPath, newPath); + LOG.trace("normalize path, original path: '{}', new path: '{}'", requestPath, newPath); return newPath; } @@ -123,7 +123,7 @@ public void filter(ContainerRequestContext requestContext, if (statusOk(responseContext.getStatus())) { MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_SUCCESS_COUNTER)).inc(); } else { - //TODO: The return codes for compatibility need to be further detailed. + // TODO: The return codes for compatibility need to be further detailed. LOG.trace("Failed Status: {}", status); if (status != 500 && status != 415) { MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_FAILED_COUNTER)).inc(); From e49de4415c92d09f8fe142c4393f5c9aca2cc7fd Mon Sep 17 00:00:00 2001 From: yangjiaqi Date: Mon, 4 Nov 2024 20:47:31 +0800 Subject: [PATCH 6/6] enhance code logic --- .../org/apache/hugegraph/api/filter/AccessLogFilter.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java index c344b32202..194a0c45e4 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java @@ -116,7 +116,7 @@ public void filter(ContainerRequestContext requestContext, String metricsName = join(path, method); int status = responseContext.getStatus(); - if (status != 500 && status != 415) { + if (isStatusXx(status)) { MetricsUtil.registerCounter(join(metricsName, METRICS_PATH_TOTAL_COUNTER)).inc(); } @@ -164,4 +164,8 @@ public void filter(ContainerRequestContext requestContext, private boolean statusOk(int status) { return status >= 200 && status < 300; } + + private boolean isStatusXx(int status) { + return status != 500 && status != 415; + } }