Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding avro doc comparison to compare versions #570

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
+114 −5
Draft

Adding avro doc comparison to compare versions #570

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions ...mon/src/main/java/com/linkedin/avroutil1/compatibility/SchemaComparisonConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,16 @@ public class SchemaComparisonConfiguration {
* behaves like avro <= 1.7.2 - non-string props on fields or types are ignored
*/
public static final SchemaComparisonConfiguration PRE_1_7_3 = new SchemaComparisonConfiguration(
true, false, false, false, true, false, Collections.emptySet()
true, false, false, false, true, false, false, Collections.emptySet()
);
/**
* same as {@link #STRICT} but allows int default values to match (round) float default values
*/
public static final SchemaComparisonConfiguration LOOSE_NUMERICS = new SchemaComparisonConfiguration(
true, true, true, true, true, true, Collections.emptySet()
true, true, true, true, true, true, true, Collections.emptySet()
);
public static final SchemaComparisonConfiguration STRICT = new SchemaComparisonConfiguration(
true, true, true, false, true, true, Collections.emptySet()
true, true, true, false, true, true, true, Collections.emptySet()
);

private final boolean compareStringJsonProps;
Expand All @@ -33,6 +33,7 @@ public class SchemaComparisonConfiguration {
private final boolean compareIntToFloatDefaults;
private final boolean compareFieldOrder;
private final boolean compareFieldLogicalTypes;
private final boolean compareFieldDocs;
private final Set<String> jsonPropNamesToIgnore;

public SchemaComparisonConfiguration(
Expand All @@ -42,6 +43,7 @@ public SchemaComparisonConfiguration(
boolean compareIntToFloatDefaults,
boolean compareFieldOrder,
boolean compareFieldLogicalTypes,
boolean compareFieldDocs,
Set<String> jsonPropNamesToIgnore
) {
this.compareStringJsonProps = compareStringJsonProps;
Expand All @@ -50,6 +52,7 @@ public SchemaComparisonConfiguration(
this.compareIntToFloatDefaults = compareIntToFloatDefaults;
this.compareFieldOrder = compareFieldOrder;
this.compareFieldLogicalTypes = compareFieldLogicalTypes;
this.compareFieldDocs = compareFieldDocs;
this.jsonPropNamesToIgnore = jsonPropNamesToIgnore;
}

Expand Down Expand Up @@ -89,6 +92,7 @@ public SchemaComparisonConfiguration compareStringJsonProps(boolean compare) {
compareIntToFloatDefaults,
compareFieldOrder,
compareFieldLogicalTypes,
compareFieldDocs,
jsonPropNamesToIgnore
);
}
Expand All @@ -101,6 +105,7 @@ public SchemaComparisonConfiguration compareNonStringJsonProps(boolean compare)
compareIntToFloatDefaults,
compareFieldOrder,
compareFieldLogicalTypes,
compareFieldDocs,
jsonPropNamesToIgnore
);
}
Expand All @@ -113,6 +118,7 @@ public SchemaComparisonConfiguration compareAliases(boolean compare) {
compareIntToFloatDefaults,
compareFieldOrder,
compareFieldLogicalTypes,
compareFieldDocs,
jsonPropNamesToIgnore
);
}
Expand All @@ -125,6 +131,7 @@ public SchemaComparisonConfiguration compareIntToFloatDefaults(boolean compare)
compare,
compareFieldOrder,
compareFieldLogicalTypes,
compareFieldDocs,
jsonPropNamesToIgnore
);
}
Expand All @@ -137,6 +144,7 @@ public SchemaComparisonConfiguration compareFieldOrder(boolean compare) {
compareIntToFloatDefaults,
compare,
compareFieldLogicalTypes,
compareFieldDocs,
jsonPropNamesToIgnore
);
}
Expand All @@ -149,6 +157,7 @@ public SchemaComparisonConfiguration compareFieldLogicalTypes(boolean compare) {
compareIntToFloatDefaults,
compareFieldOrder,
compare,
compareFieldDocs,
jsonPropNamesToIgnore
);
}
Expand All @@ -161,9 +170,14 @@ public SchemaComparisonConfiguration jsonPropNamesToIgnore(Set<String> jsonPropN
compareIntToFloatDefaults,
compareFieldOrder,
compareFieldLogicalTypes,
compareFieldDocs,
jsonPropNamesToIgnore
);
}

public boolean isCompareFieldDocs() {
return compareFieldDocs;
}
}


9 changes: 9 additions & 0 deletions ...lper/src/main/java/com/linkedin/avroutil1/compatibility/ConfigurableSchemaComparator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,18 @@ private static boolean equals(Schema a, Schema b, SchemaComparisonConfiguration
boolean considerJsonNonStringProps = config.isCompareNonStringJsonProps();
boolean considerAliases = config.isCompareAliases();
boolean considerJsonProps = considerJsonStringProps || considerJsonNonStringProps;
boolean considerDocs = config.isCompareFieldDocs();
Set<String> jsonPropNamesToIgnore = config.getJsonPropNamesToIgnore();

if (considerJsonProps && !hasSameObjectProps(a, b, considerJsonStringProps, considerJsonNonStringProps,
jsonPropNamesToIgnore)) {
return false;
}

if (considerDocs && !Objects.equals(a.getDoc(), b.getDoc())) {
return false;
}

switch (type) {
//all of these have nothing more to compare by beyond their type and props
case NULL:
Expand Down Expand Up @@ -125,6 +130,7 @@ private static boolean recordSchemaEquals(Schema a, Schema b, SchemaComparisonCo
boolean considerAliases = config.isCompareAliases();
boolean considerJsonProps = considerJsonStringProps || considerJsonNonStringProps;
boolean compareIntToFloatDefaults = config.isCompareIntToFloatDefaults();
boolean compareDocs = config.isCompareFieldDocs();
Set<String> jsonPropNamesToIgnore = config.getJsonPropNamesToIgnore();

try {
Expand Down Expand Up @@ -171,6 +177,9 @@ private static boolean recordSchemaEquals(Schema a, Schema b, SchemaComparisonCo
if (considerAliases && !hasSameAliases(aField, bField)) {
return false;
}
if (compareDocs && !aField.doc().equals(bField.doc())) {
return false;
}
}
return true;
} finally {
Expand Down
88 changes: 87 additions & 1 deletion .../src/test/java/com/linkedin/avroutil1/compatibility/ConfigurableSchemaComparatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ public void testExceptionOnTryingToCompareNonStringsUnderOldAvro() throws Except
true,
true,
true,
false,
Collections.emptySet()
);
switch (runtimeAvroVersion) {
Expand Down Expand Up @@ -131,6 +132,91 @@ public void testCompareOnlyStringProps() throws Exception {
}
}

@Test
public void testCompareSchemaDoc() throws Exception {
AvroVersion runtimeAvroVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
String avscA = "{\n"
+ " \"type\": \"record\",\n"
+ " \"name\": \"Bob\",\n"
+ " \"schemaStrProp\": \"val1\",\n"
+ " \"schemaIntProp\": 1,\n"
+ " \"doc\": \"doc1\",\n"
+ " \"fields\": [\n"
+ " {\n"
+ " \"name\": \"f\",\n"
+ " \"type\": \"string\",\n"
+ " \"fieldStrProp\": \"val2\",\n"
+ " \"fieldIntProp\": 2\n"
+ " }\n"
+ " ]\n"
+ "}";
String avscB = "{\n"
+ " \"type\": \"record\",\n"
+ " \"name\": \"Bob\",\n"
+ " \"schemaStrProp\": \"val1\",\n"
+ " \"schemaIntProp\": 1,\n"
+ " \"doc\": null,\n"
+ " \"fields\": [\n"
+ " {\n"
+ " \"name\": \"f\",\n"
+ " \"type\": \"string\",\n"
+ " \"fieldStrProp\": \"val2\",\n"
+ " \"fieldIntProp\": 2\n"
+ " }\n"
+ " ]\n"
+ "}";
Schema a = Schema.parse(avscA);
Schema b = Schema.parse(avscB);
Assert.assertTrue(ConfigurableSchemaComparator.equals(a, b, SchemaComparisonConfiguration.PRE_1_7_3));
if (runtimeAvroVersion.laterThan(AvroVersion.AVRO_1_8)) {
//complex props are only supported by avro >= 1.8.0
Assert.assertFalse(ConfigurableSchemaComparator.equals(a, b, SchemaComparisonConfiguration.STRICT));
}
}


@Test
public void testCompareSchemaFieldDoc() throws Exception {
AvroVersion runtimeAvroVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
String avscA = "{\n"
+ " \"type\": \"record\",\n"
+ " \"name\": \"Bob\",\n"
+ " \"schemaStrProp\": \"val1\",\n"
+ " \"schemaIntProp\": 1,\n"
+ " \"fields\": [\n"
+ " {\n"
+ " \"name\": \"f\",\n"
+ " \"type\": \"string\",\n"
+ " \"doc\": \"doc1\",\n"
+ " \"fieldStrProp\": \"val2\",\n"
+ " \"fieldIntProp\": 2\n"
+ " }\n"
+ " ]\n"
+ "}";
String avscB = "{\n"
+ " \"type\": \"record\",\n"
+ " \"name\": \"Bob\",\n"
+ " \"schemaStrProp\": \"val1\",\n"
+ " \"schemaIntProp\": 1,\n"
+ " \"fields\": [\n"
+ " {\n"
+ " \"name\": \"f\",\n"
+ " \"type\": \"string\",\n"
+ " \"doc\": null,\n"
+ " \"fieldStrProp\": \"val2\",\n"
+ " \"fieldIntProp\": 2\n"
+ " }\n"
+ " ]\n"
+ "}";
Schema a = Schema.parse(avscA);
Schema b = Schema.parse(avscB);
Assert.assertTrue(ConfigurableSchemaComparator.equals(a, b, SchemaComparisonConfiguration.PRE_1_7_3));
if (runtimeAvroVersion.laterThan(AvroVersion.AVRO_1_8)) {
//complex props are only supported by avro >= 1.8.0
Assert.assertFalse(ConfigurableSchemaComparator.equals(a, b, SchemaComparisonConfiguration.STRICT));
}
}

@Test
public void testLooseNumericDefaults() throws Exception {
AvroVersion runtimeAvroVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
Expand Down Expand Up @@ -393,7 +479,7 @@ public void testDefaultValuesModernAvro() {
default:
Assert.assertTrue(ConfigurableSchemaComparator.equals(schemaA, schemaB,
new SchemaComparisonConfiguration(false, false, true,
SchemaComparisonConfiguration.STRICT.isCompareIntToFloatDefaults(), true, true,
SchemaComparisonConfiguration.STRICT.isCompareIntToFloatDefaults(), true, true, false,
Collections.emptySet())));
}

Expand Down
2 changes: 1 addition & 1 deletion parser/src/test/java/com/linkedin/avroutil1/util/ConfigurableAvroSchemaComparatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public void testEquals(String path1, String path2, boolean expectedResult) throw
AvroRecordSchema recordSchema2 = (AvroRecordSchema) validateAndGetAvroRecordSchema(path2);

SchemaComparisonConfiguration comparisonConfiguration =
new SchemaComparisonConfiguration(true, false, false, false, true, false, new HashSet(
new SchemaComparisonConfiguration(true, false, false, false, true, false, false, new HashSet(
Arrays.asList("li.data.proto.numberFieldType", "li.data.proto.fieldNumber",
"li.data.proto.fullyQualifiedName", "li.data.proto.enumValueNumbers")));

Expand Down
Loading