Limit the max log size for schema in warning. (#568)

linkedin · Feb 26, 2025 · b0cb48a · b0cb48a
1 parent 17a828a
commit b0cb48a
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 2 deletions.
diff --git a/...erde/avro-fastserde-tests-common/src/test/java/com/linkedin/avro/fastserde/UtilsTest.java b/...erde/avro-fastserde-tests-common/src/test/java/com/linkedin/avro/fastserde/UtilsTest.java
@@ -4,6 +4,7 @@
 
 import javax.lang.model.SourceVersion;
 
+import org.apache.avro.Schema;
 import org.testng.Assert;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
@@ -73,4 +74,44 @@ void shouldFailGeneratingValidJavaIdentifier(String invalidProposal) {
     // NPE expected
     Utils.toValidJavaIdentifier(invalidProposal);
   }
+
+  @Test
+  void testGetTruncateSchemaForWarningNull() {
+    Schema schema = null;
+    Assert.assertEquals("null", Utils.getTruncateSchemaForWarning(schema));
+  }
+
+  @Test
+  void testGetTruncateSchemaForWarningSmall() {
+    String schemaJson = "{"
+            + "\"type\": \"record\","
+            + "\"name\": \"User\","
+            + "\"fields\": ["
+            + "   {\"name\": \"name\", \"type\": \"string\"}"
+            + "]"
+            + "}";
+    Schema schema = Schema.parse(schemaJson);
+    Assert.assertTrue(schema.toString().length() <= Utils.MAX_SCHEMA_LENGTH_IN_WARNING);
+    Assert.assertEquals(schema.toString(), Utils.getTruncateSchemaForWarning(schema));
+  }
+
+  @Test
+  void testGetTruncateSchemaForWarningLarge() {
+    String schemaJson = "{"
+            + "\"type\": \"record\","
+            + "\"name\": \"User\","
+            + "\"namespace\": \"com.example.avro\","
+            + "\"fields\": ["
+            + "   {\"name\": \"name\", \"type\": \"string\"},"
+            + "   {\"name\": \"age\", \"type\": \"int\"},"
+            + "   {\"name\": \"email\", \"type\": [\"null\", \"string\"], \"default\": null}"
+            + "]"
+            + "}";
+    Schema schema = Schema.parse(schemaJson);
+    Assert.assertTrue(schema.toString().length() > Utils.MAX_SCHEMA_LENGTH_IN_WARNING);
+    String truncatedSchema = Utils.getTruncateSchemaForWarning(schema);
+    Assert.assertEquals(truncatedSchema.length(), Utils.MAX_SCHEMA_LENGTH_IN_WARNING + 3);
+    Assert.assertTrue(truncatedSchema.endsWith("..."));
+    Assert.assertTrue(schema.toString().startsWith(truncatedSchema.substring(0, Utils.MAX_SCHEMA_LENGTH_IN_WARNING)));
+  }
 }
diff --git a/...erde/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastGenericDatumReader.java b/...erde/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastGenericDatumReader.java
@@ -116,8 +116,21 @@ public T read(T reuse, Decoder in) throws IOException {
             */
           cachedFastDeserializer.compareAndSet(null,
               getRegularAvroImplWhenGenerationFail(writerSchema, readerSchema, modelData, customization));
-          LOGGER.warn("FastGenericDeserializer generation fails, and will cache cold deserializer "
-              + "for reader schema: [" + readerSchema + "], writer schema: [" + writerSchema + "]");
+
+          LOGGER.warn("FastGenericDeserializer generation fails, and will cache cold deserializer for "
+              + "reader schema: [" + Utils.getTruncateSchemaForWarning(readerSchema) + "],"
+              + "writer schema: [" + Utils.getTruncateSchemaForWarning(writerSchema) + "].");
+
+          if (LOGGER.isDebugEnabled()) {
+            String readerSchemaStr = readerSchema.toString();
+            String writerSchemaStr = writerSchema.toString();
+            if (readerSchemaStr.length() > Utils.MAX_SCHEMA_LENGTH_IN_WARNING ||
+                writerSchemaStr.length() > Utils.MAX_SCHEMA_LENGTH_IN_WARNING) {
+              LOGGER.debug("FastGenericDeserializer generation fails, and will cache cold deserializer for "
+                  + "reader schema: [" + readerSchemaStr + "],"
+                  + "writer schema: [" + writerSchemaStr + "].");
+            }
+          }
         }
         fastDeserializer = cachedFastDeserializer.get();
       } else {

diff --git a/fastserde/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/Utils.java b/fastserde/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/Utils.java
@@ -55,6 +55,9 @@ public class Utils {
   // Cache the mapping between Schema and the corresponding fingerprint
   private static final Map<Schema, Integer> SCHEMA_IDS_CACHE = new ConcurrentHashMap<>();
 
+  // Limit max schema length in WARNING logs.
+  static final int MAX_SCHEMA_LENGTH_IN_WARNING = 100;
+
   private Utils() {
   }
 
@@ -259,4 +262,14 @@ public static String toValidJavaIdentifier(String javaIdentifier) {
 
     return javaIdentifier;
   }
+
+  static String getTruncateSchemaForWarning(Schema schema) {
+    if (schema == null) {
+      return "null";
+    }
+    String schemaString = schema.toString();
+    return (schemaString.length() > MAX_SCHEMA_LENGTH_IN_WARNING)
+        ? schemaString.substring(0, MAX_SCHEMA_LENGTH_IN_WARNING) + "..."
+        : schemaString;
+  }
 }