From df3b66cc901ccf1c6a23bd3d4437205c48e0e3c4 Mon Sep 17 00:00:00 2001 From: anthonymckale-6point6 Date: Tue, 25 Apr 2023 13:05:41 +0100 Subject: [PATCH 1/2] #61 updating get_python_type to lookup class by final class name without namespace --- src/pydantic_avro/avro_to_pydantic.py | 6 ++++- tests/test_from_avro.py | 33 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/pydantic_avro/avro_to_pydantic.py b/src/pydantic_avro/avro_to_pydantic.py index 9f1c66a..e7c7aa0 100644 --- a/src/pydantic_avro/avro_to_pydantic.py +++ b/src/pydantic_avro/avro_to_pydantic.py @@ -32,7 +32,11 @@ def get_python_type(t: Union[str, dict]) -> str: elif t in classes: py_type = t else: - raise NotImplementedError(f"Type {t} not supported yet") + t_without_namespace = t.split('.')[-1] + if t_without_namespace in classes: + py_type = t_without_namespace + else: + raise NotImplementedError(f"Type {t} not supported yet") elif isinstance(t, list): if "null" in t and len(t) == 2: optional = True diff --git a/tests/test_from_avro.py b/tests/test_from_avro.py index 10b6a6c..e51f911 100644 --- a/tests/test_from_avro.py +++ b/tests/test_from_avro.py @@ -68,6 +68,39 @@ def test_avsc_to_pydantic_map_nested_object(): assert "class Nested(BaseModel):\n" " col1: str" in pydantic_code +def test_avsc_to_pydantic_namespaced_object_reuse(): + pydantic_code = avsc_to_pydantic( + { + "name": "Test", + "type": "record", + "fields": [ + { + "name": "col1", + "type": { + "type": "record", + "name": "Nested", + "namespace": "com.pydantic", + "fields": [{"name": "col1", "type": "string"}]}, + }, + { + "name": "col2", + "type": "com.pydantic.Nested", + }, + ], + } + ) + expected_code: str = """ +class Nested(BaseModel): + col1: str + + +class Test(BaseModel): + col1: Nested + col2: Nested +""" + assert expected_code in pydantic_code + + def test_avsc_to_pydantic_map_nested_array(): pydantic_code = avsc_to_pydantic( { From eb655281b40e65680c3989c4d921679ba535b359 Mon Sep 17 00:00:00 2001 From: anthonymckale-6point6 Date: Tue, 2 May 2023 16:33:16 +0100 Subject: [PATCH 2/2] #65 adding graph ql functionality --- src/pydantic_avro/__main__.py | 22 +- src/pydantic_avro/avro_to_graphql.py | 230 ++++++++++++ src/pydantic_avro/avro_to_pydantic.py | 22 +- src/pydantic_avro/base.py | 18 +- tests/test_from_avro.py | 67 +++- tests/test_graphql_from_avro.py | 504 ++++++++++++++++++++++++++ tests/test_to_avro.py | 40 +- 7 files changed, 876 insertions(+), 27 deletions(-) create mode 100644 src/pydantic_avro/avro_to_graphql.py create mode 100644 tests/test_graphql_from_avro.py diff --git a/src/pydantic_avro/__main__.py b/src/pydantic_avro/__main__.py index 067852b..4557315 100644 --- a/src/pydantic_avro/__main__.py +++ b/src/pydantic_avro/__main__.py @@ -2,7 +2,9 @@ import sys from typing import List -from pydantic_avro.avro_to_pydantic import convert_file + +from pydantic_avro import avro_to_graphql +from pydantic_avro import avro_to_pydantic def main(input_args: List[str]): @@ -13,10 +15,26 @@ def main(input_args: List[str]): parser_cache.add_argument("--asvc", type=str, dest="avsc", required=True) parser_cache.add_argument("--output", type=str, dest="output") + parser_cache = subparsers.add_parser("avro_to_graphql") + parser_cache.add_argument("--asvc", type=str, dest="avsc", required=True) + parser_cache.add_argument("--output", type=str, dest="output") + parser_cache.add_argument("--config", type=str, dest="config") + + parser_cache = subparsers.add_parser("avro_folder_to_graphql") + parser_cache.add_argument("--asvc", type=str, dest="avsc", required=True) + parser_cache.add_argument("--output", type=str, dest="output") + parser_cache.add_argument("--config", type=str, dest="config") + args = parser.parse_args(input_args) if args.sub_command == "avro_to_pydantic": - convert_file(args.avsc, args.output) + avro_to_pydantic.convert_file(args.avsc, args.output) + + if args.sub_command == "avro_to_graphql": + avro_to_graphql.convert_file(args.avsc, args.output, args.config) + + if args.sub_command == "avro_folder_to_graphql": + avro_to_graphql.convert_files(args.avsc, args.output, args.config) def root_main(): diff --git a/src/pydantic_avro/avro_to_graphql.py b/src/pydantic_avro/avro_to_graphql.py new file mode 100644 index 0000000..50eb11f --- /dev/null +++ b/src/pydantic_avro/avro_to_graphql.py @@ -0,0 +1,230 @@ +import glob +import json +import os +from typing import Optional, Union +from re import sub + + +def camel_type(s): + """very simple camel caser""" + camelled_type = sub(r"(_|-)+", " ", s).title() + camelled_type = sub(r"[ !\[\]]+", "", camelled_type) + if s[-1] != "!": + camelled_type = "Optional" + camelled_type + return camelled_type + + +def avsc_to_graphql(schema: dict, config: dict = None) -> dict: + """Generate python code of pydantic of given Avro Schema""" + if "type" not in schema or schema["type"] != "record": + raise AttributeError("Type not supported") + if "name" not in schema: + raise AttributeError("Name is required") + if "fields" not in schema: + raise AttributeError("fields are required") + + classes: dict = {} + + def add_optional(py_type: str, optional) -> str: + if optional: + # if non-optional type but optional by union remove '!' + if py_type[-1] == "!": + return py_type[0:-1] + return py_type + else: + return py_type + f"!" + + def get_directive_str(type_name: str, field_name: str, config: dict) -> str: + if not config: + return "" + directive_str: str = "" + if field_name in config["field_directives"]: + directive_str += " " + config["field_directives"][field_name] + if type_name in config["type_directives"]: + type_directives = config["type_directives"][type_name] + if field_name in type_directives: + directive_str += " " + type_directives[field_name] + return directive_str + + def get_graphql_type(t: Union[str, dict], force_optional: bool = False) -> str: + """Returns python type for given avro type""" + optional = force_optional + optional_handled = False + if isinstance(t, str): + if t == "string": + py_type = "String" + elif t == "int": + py_type = "Int" + elif t == "long": + py_type = "Float" + elif t == "boolean": + py_type = "Boolean" + elif t == "double" or t == "float": + py_type = "Float" + elif t == "bytes": + py_type = "String" + elif t in classes: + py_type = t + else: + t_without_namespace = t.split(".")[-1] + if t_without_namespace in classes: + py_type = t_without_namespace + else: + raise NotImplementedError(f"Type {t} not supported yet") + elif isinstance(t, list): + optional_handled = True + if "null" in t and len(t) == 2: + c = t.copy() + c.remove("null") + py_type = get_graphql_type(c[0], True) + else: + if "null" in t: + optional = True + py_type = f"{' | '.join([ get_graphql_type(e, optional) for e in t if e != 'null'])}" + elif t.get("logicalType") == "uuid": + py_type = "ID" + elif t.get("logicalType") == "decimal": + py_type = "Float" + elif ( + t.get("logicalType") == "timestamp-millis" + or t.get("logicalType") == "timestamp-micros" + ): + py_type = "Int" + elif ( + t.get("logicalType") == "time-millis" + or t.get("logicalType") == "time-micros" + ): + py_type = "Int" + elif t.get("logicalType") == "date": + py_type = "String" + elif t.get("type") == "enum": + enum_name = t.get("name") + if enum_name not in classes: + enum_class = f"enum {enum_name} " + "{\n" + for s in t.get("symbols"): + enum_class += f" {s}\n" + enum_class += "}\n" + classes[enum_name] = enum_class + py_type = enum_name + elif t.get("type") == "string": + py_type = "str" + elif t.get("type") == "array": + sub_type = get_graphql_type(t.get("items")) + py_type = f"List[{sub_type}]" + elif t.get("type") == "record": + record_type_to_graphql(t) + py_type = t.get("name") + elif t.get("type") == "map": + value_type = get_graphql_type(t.get("values")) + tuple_type = camel_type(value_type) + "MapTuple" + if tuple_type not in classes: + tuple_class = f"""type {tuple_type} {{ + key: String + value: [{value_type}] +}}\n""" + classes[tuple_type] = tuple_class + py_type = f"[{tuple_type}]" + else: + raise NotImplementedError( + f"Type {t} not supported yet, " + f"please report this at https://github.com/godatadriven/pydantic-avro/issues" + ) + if optional_handled: + return py_type + py_type = add_optional(py_type, optional) + return py_type + + def record_type_to_graphql(schema: dict, config: dict = None): + """Convert a single avro record type to a pydantic class""" + type_name = schema["name"] + current = f"type {type_name} " + "{\n" + + for field in schema["fields"]: + field_name = field["name"] + field_type = get_graphql_type(field["type"]) + field_directives = get_directive_str(type_name, field_name, config) + default = field.get("default") + if ( + field["type"] == "int" + and "default" in field + and isinstance(default, (bool, type(None))) + ): + current += f" # use 'default' in queries, defaults not supported in graphql schemas\n" + current += f" {field_name}: {field_type}{field_directives}\n" + elif field["type"] == "int" and "default" in field: + current += f" # use '{json.dumps(default)}' in queries, defaults not supported in graphql schemas\n" + current += f" {field_name}: {field_type}{field_directives}\n" + elif field["type"] == "int": + current += f" {field_name}: {field_type}{field_directives}\n" + elif "default" not in field: + current += f" {field_name}: {field_type}{field_directives}\n" + elif isinstance(default, type(None)): + current += f" {field_name}: {field_type}{field_directives}\n" + elif isinstance(default, bool): + current += f" # use '{default}' in queries, defaults not supported in graphql schemas\n" + current += f" {field_name}: {field_type}{field_directives}\n" + else: + current += f" # use '{json.dumps(default)}' in queries, defaults not supported in graphql schemas\n" + current += f" {field_name}: {field_type}{field_directives}\n" + if len(schema["fields"]) == 0: + current += " _void: String\n" + + current += "}\n" + + classes[type_name] = current + + record_type_to_graphql(schema, config) + + return classes + + +def classes_to_graphql_str(classes: dict) -> str: + file_content = "# GENERATED GRAPHQL USING graphql_avro, DO NOT MANUALLY EDIT\n\n" + file_content += "\n\n".join(sorted(classes.values())) + + return file_content + + +def get_config(config_json: Optional[str] = None) -> dict: + if not config_json: + return None + with open(config_json, "r") as file_handler: + return json.load(file_handler) + + +def convert_file( + avsc_path: str, output_path: Optional[str] = None, config_json: Optional[str] = None +): + config = get_config(config_json) + with open(avsc_path, "r") as file_handler: + avsc_dict = json.load(file_handler) + file_content = avsc_to_graphql(avsc_dict, config=config) + if output_path is None: + print(file_content) + else: + with open(output_path, "w") as file_handler: + file_handler.write(file_content) + + +def convert_files( + avsc_folder: str, + output_path: Optional[str] = None, + config_json: Optional[str] = None, +): + config = get_config(config_json) + avsc_files: list = glob.glob("*.avsc", root_dir=avsc_folder, recursive=True) + all_graphql_classes = {} + for avsc_file in avsc_files: + avsc_filepath = os.path.join(avsc_folder, avsc_file) + with open(avsc_filepath, "r") as file_handle: + avsc_dict = json.load(file_handle) + if "type" in avsc_dict and avsc_dict["type"] == "enum": + continue + graphql_classes = avsc_to_graphql(avsc_dict, config=config) + all_graphql_classes.update(graphql_classes) + file_content = classes_to_graphql_str(all_graphql_classes) + if output_path is None: + print(file_content) + else: + with open(output_path, "w") as file_handle: + file_handle.write(file_content) diff --git a/src/pydantic_avro/avro_to_pydantic.py b/src/pydantic_avro/avro_to_pydantic.py index e7c7aa0..8baadef 100644 --- a/src/pydantic_avro/avro_to_pydantic.py +++ b/src/pydantic_avro/avro_to_pydantic.py @@ -32,7 +32,7 @@ def get_python_type(t: Union[str, dict]) -> str: elif t in classes: py_type = t else: - t_without_namespace = t.split('.')[-1] + t_without_namespace = t.split(".")[-1] if t_without_namespace in classes: py_type = t_without_namespace else: @@ -52,9 +52,15 @@ def get_python_type(t: Union[str, dict]) -> str: py_type = "UUID" elif t.get("logicalType") == "decimal": py_type = "Decimal" - elif t.get("logicalType") == "timestamp-millis" or t.get("logicalType") == "timestamp-micros": + elif ( + t.get("logicalType") == "timestamp-millis" + or t.get("logicalType") == "timestamp-micros" + ): py_type = "datetime" - elif t.get("logicalType") == "time-millis" or t.get("logicalType") == "time-micros": + elif ( + t.get("logicalType") == "time-millis" + or t.get("logicalType") == "time-micros" + ): py_type = "time" elif t.get("logicalType") == "date": py_type = "date" @@ -96,8 +102,14 @@ def record_type_to_pydantic(schema: dict): n = field["name"] t = get_python_type(field["type"]) default = field.get("default") - if field["type"] == "int" and "default" in field and isinstance(default, (bool, type(None))): - current += f" {n}: {t} = Field({default}, ge=-2**31, le=(2**31 - 1))\n" + if ( + field["type"] == "int" + and "default" in field + and isinstance(default, (bool, type(None))) + ): + current += ( + f" {n}: {t} = Field({default}, ge=-2**31, le=(2**31 - 1))\n" + ) elif field["type"] == "int" and "default" in field: current += f" {n}: {t} = Field({json.dumps(default)}, ge=-2**31, le=(2**31 - 1))\n" elif field["type"] == "int": diff --git a/src/pydantic_avro/base.py b/src/pydantic_avro/base.py index 2c6c0cc..18131ab 100644 --- a/src/pydantic_avro/base.py +++ b/src/pydantic_avro/base.py @@ -7,7 +7,9 @@ class AvroBase(BaseModel): """This is base pydantic class that will add some methods""" @classmethod - def avro_schema(cls, by_alias: bool = True, namespace: Optional[str] = None) -> dict: + def avro_schema( + cls, by_alias: bool = True, namespace: Optional[str] = None + ) -> dict: """ Return the avro schema for the pydantic class @@ -121,7 +123,12 @@ def get_type(value: dict) -> dict: avro_type_dict["type"] = "double" elif t == "integer": # integer in python can be a long, only if minimum and maximum value is set a int can be used - if minimum is not None and minimum >= -(2**31) and maximum is not None and maximum <= (2**31 - 1): + if ( + minimum is not None + and minimum >= -(2**31) + and maximum is not None + and maximum <= (2**31 - 1) + ): avro_type_dict["type"] = "int" else: avro_type_dict["type"] = "long" @@ -163,4 +170,9 @@ def get_fields(s: dict) -> List[dict]: fields = get_fields(schema) - return {"type": "record", "namespace": namespace, "name": schema["title"], "fields": fields} + return { + "type": "record", + "namespace": namespace, + "name": schema["title"], + "fields": fields, + } diff --git a/tests/test_from_avro.py b/tests/test_from_avro.py index e51f911..f96da44 100644 --- a/tests/test_from_avro.py +++ b/tests/test_from_avro.py @@ -40,7 +40,10 @@ def test_avsc_to_pydantic_map(): "name": "Test", "type": "record", "fields": [ - {"name": "col1", "type": {"type": "map", "values": "string", "default": {}}}, + { + "name": "col1", + "type": {"type": "map", "values": "string", "default": {}}, + }, ], } ) @@ -57,7 +60,11 @@ def test_avsc_to_pydantic_map_nested_object(): "name": "col1", "type": { "type": "map", - "values": {"type": "record", "name": "Nested", "fields": [{"name": "col1", "type": "string"}]}, + "values": { + "type": "record", + "name": "Nested", + "fields": [{"name": "col1", "type": "string"}], + }, "default": {}, }, }, @@ -80,7 +87,8 @@ def test_avsc_to_pydantic_namespaced_object_reuse(): "type": "record", "name": "Nested", "namespace": "com.pydantic", - "fields": [{"name": "col1", "type": "string"}]}, + "fields": [{"name": "col1", "type": "string"}], + }, }, { "name": "col2", @@ -212,8 +220,16 @@ def test_default(): "fields": [ {"name": "col1", "type": "string", "default": "test"}, {"name": "col2_1", "type": ["null", "string"], "default": None}, - {"name": "col2_2", "type": ["string", "null"], "default": "default_str"}, - {"name": "col3", "type": {"type": "map", "values": "string"}, "default": {"key": "value"}}, + { + "name": "col2_2", + "type": ["string", "null"], + "default": "default_str", + }, + { + "name": "col3", + "type": {"type": "map", "values": "string"}, + "default": {"key": "value"}, + }, {"name": "col4", "type": "boolean", "default": True}, {"name": "col5", "type": "boolean", "default": False}, ], @@ -236,14 +252,25 @@ def test_enums(): "name": "Test", "type": "record", "fields": [ - {"name": "c1", "type": {"type": "enum", "symbols": ["passed", "failed"], "name": "Status"}}, + { + "name": "c1", + "type": { + "type": "enum", + "symbols": ["passed", "failed"], + "name": "Status", + }, + }, ], } ) assert "class Test(BaseModel):\n" " c1: Status" in pydantic_code - assert "class Status(str, Enum):\n" ' passed = "passed"\n' ' failed = "failed"' in pydantic_code + assert ( + "class Status(str, Enum):\n" + ' passed = "passed"\n' + ' failed = "failed"' in pydantic_code + ) def test_enums_reuse(): @@ -252,15 +279,28 @@ def test_enums_reuse(): "name": "Test", "type": "record", "fields": [ - {"name": "c1", "type": {"type": "enum", "symbols": ["passed", "failed"], "name": "Status"}}, + { + "name": "c1", + "type": { + "type": "enum", + "symbols": ["passed", "failed"], + "name": "Status", + }, + }, {"name": "c2", "type": "Status"}, ], } ) - assert "class Test(BaseModel):\n" " c1: Status\n" " c2: Status" in pydantic_code + assert ( + "class Test(BaseModel):\n" " c1: Status\n" " c2: Status" in pydantic_code + ) - assert "class Status(str, Enum):\n" ' passed = "passed"\n' ' failed = "failed"' in pydantic_code + assert ( + "class Status(str, Enum):\n" + ' passed = "passed"\n' + ' failed = "failed"' in pydantic_code + ) def test_unions(): @@ -278,7 +318,12 @@ def test_unions(): { "type": "record", "name": "ARecord", - "fields": [{"name": "values", "type": {"type": "map", "values": "string"}}], + "fields": [ + { + "name": "values", + "type": {"type": "map", "values": "string"}, + } + ], }, ], }, diff --git a/tests/test_graphql_from_avro.py b/tests/test_graphql_from_avro.py new file mode 100644 index 0000000..f05d867 --- /dev/null +++ b/tests/test_graphql_from_avro.py @@ -0,0 +1,504 @@ +import pytest + +from pydantic_avro.avro_to_graphql import ( + avsc_to_graphql, + camel_type, + classes_to_graphql_str, +) + +camel_type_tests = [ + ["string", "OptionalString"], + ["String", "OptionalString"], + ["string!", "String"], + ["String!", "String"], + ["List[String!]!", "ListString"], + ["List[String!]", "OptionalListString"], +] + + +@pytest.mark.parametrize("input_type,expected_type", camel_type_tests) +def test_camel_type(input_type: str, expected_type: str): + assert camel_type(input_type) == expected_type + + +def test_avsc_to_graphql_empty(): + graphql_classes = avsc_to_graphql({"name": "Test", "type": "record", "fields": []}) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_code = """ +type Test { + _void: String +}""" + assert expected_code in graphql_code + + +def test_avsc_to_graphql_primitive(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + {"name": "col1", "type": "string"}, + {"name": "col2", "type": "int"}, + {"name": "col3", "type": "long"}, + {"name": "col4", "type": "double"}, + {"name": "col5", "type": "float"}, + {"name": "col6", "type": "boolean"}, + {"name": "col7", "type": "bytes"}, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_code = """ +type Test { + col1: String! + col2: Int! + col3: Float! + col4: Float! + col5: Float! + col6: Boolean! + col7: String! +}""" + assert expected_code in graphql_code + + +def test_avsc_to_graphql_map(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + { + "name": "col1", + "type": {"type": "map", "values": "string", "default": {}}, + }, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_tuple = """ +type StringMapTuple { + key: String + value: [String!] +}""" + expected_type = """type Test { + col1: [StringMapTuple]! +}""" + assert expected_tuple in graphql_code + assert expected_type in graphql_code + + +def test_avsc_to_graphql_map_nested_object(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + { + "name": "col1", + "type": { + "type": "map", + "values": { + "type": "record", + "name": "Nested", + "fields": [{"name": "col1", "type": "string"}], + }, + "default": {}, + }, + }, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_nested_type = """type Nested { + col1: String! +}""" + expected_tuple = """type NestedMapTuple { + key: String + value: [Nested!] +}""" + expected_type = """type Test { + col1: [NestedMapTuple]! +}""" + assert expected_nested_type in graphql_code + assert expected_tuple in graphql_code + assert expected_type in graphql_code + + +def test_avsc_to_graphql_namespaced_object_reuse(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + { + "name": "col1", + "type": { + "type": "record", + "name": "Nested", + "namespace": "com.pydantic", + "fields": [{"name": "col1", "type": "string"}], + }, + }, + { + "name": "col2", + "type": "com.pydantic.Nested", + }, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_nested_type: str = """type Nested { + col1: String! +}""" + expected_type: str = """type Test { + col1: Nested! + col2: Nested! +}""" + assert expected_nested_type in graphql_code + assert expected_type in graphql_code + + +def test_avsc_to_graphql_map_nested_array(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + { + "name": "col1", + "type": { + "type": "map", + "values": { + "type": "array", + "items": "string", + }, + "default": {}, + }, + }, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_nested_type: str = """type ListStringMapTuple { + key: String + value: [List[String!]!] +}""" + expected_type: str = """type Test { + col1: [ListStringMapTuple]! +}""" + assert expected_nested_type in graphql_code + assert expected_type in graphql_code + + +def test_avsc_to_graphql_logical(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + { + "name": "col1", + "type": {"type": "int", "logicalType": "date"}, + }, + { + "name": "col2", + "type": {"type": "long", "logicalType": "time-micros"}, + }, + { + "name": "col3", + "type": {"type": "long", "logicalType": "time-millis"}, + }, + { + "name": "col4", + "type": {"type": "long", "logicalType": "timestamp-micros"}, + }, + { + "name": "col5", + "type": {"type": "long", "logicalType": "timestamp-millis"}, + }, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_type: str = """ +type Test { + col1: String! + col2: Int! + col3: Int! + col4: Int! + col5: Int! +}""" + assert expected_type in graphql_code + + +def test_avsc_to_graphql_complex(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + { + "name": "col1", + "type": { + "name": "Nested", + "type": "record", + "fields": [], + }, + }, + { + "name": "col2", + "type": { + "type": "array", + "items": "int", + }, + }, + { + "name": "col3", + "type": { + "type": "array", + "items": "Nested", + }, + }, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_nested_type: str = """type Nested { + _void: String +}""" + expected_type: str = """type Test { + col1: Nested! + col2: List[Int!]! + col3: List[Nested!]! +}""" + assert expected_type in graphql_code + + assert expected_nested_type in graphql_code + + +def test_default_optional(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + {"name": "col1", "type": ["null", "string"], "default": None}, + {"name": "col2", "type": ["string", "null"], "default": "default_str"}, + {"name": "col3", "type": "string"}, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_type: str = """type Test { + col1: String + # use \'"default_str"\' in queries, defaults not supported in graphql schemas + col2: String + col3: String! +}""" + assert expected_type in graphql_code + + +def test_default(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + {"name": "col1", "type": "string", "default": "test"}, + {"name": "col2_1", "type": ["null", "string"], "default": None}, + { + "name": "col2_2", + "type": ["string", "null"], + "default": "default_str", + }, + { + "name": "col3", + "type": {"type": "map", "values": "string"}, + "default": {"key": "value"}, + }, + {"name": "col4", "type": "boolean", "default": True}, + {"name": "col5", "type": "boolean", "default": False}, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_map_type: str = """type StringMapTuple { + key: String + value: [String!] +}""" + expected_type: str = """type Test { + # use '"test"' in queries, defaults not supported in graphql schemas + col1: String! + col2_1: String + # use '"default_str"' in queries, defaults not supported in graphql schemas + col2_2: String + # use '{"key": "value"}' in queries, defaults not supported in graphql schemas + col3: [StringMapTuple]! + # use 'True' in queries, defaults not supported in graphql schemas + col4: Boolean! + # use 'False' in queries, defaults not supported in graphql schemas + col5: Boolean! +}""" + assert expected_map_type in graphql_code + assert expected_type in graphql_code + + +def test_enums(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + { + "name": "c1", + "type": { + "type": "enum", + "symbols": ["passed", "failed"], + "name": "Status", + }, + }, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_enum = """enum Status { + passed + failed +}""" + expected_type = """type Test { + c1: Status! +}""" + + assert expected_enum in graphql_code + assert expected_type in graphql_code + + +def test_enums_reuse(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + { + "name": "c1", + "type": { + "type": "enum", + "symbols": ["passed", "failed"], + "name": "Status", + }, + }, + {"name": "c2", "type": "Status"}, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_enum = """enum Status { + passed + failed +}""" + expected_type = """type Test { + c1: Status! + c2: Status! +}""" + + assert expected_enum in graphql_code + assert expected_type in graphql_code + + +def test_unions(): + graphql_classes = avsc_to_graphql( + { + "type": "record", + "name": "Test", + "fields": [ + { + "name": "a_union", + "type": [ + "null", + "long", + "string", + { + "type": "record", + "name": "ARecord", + "fields": [ + { + "name": "values", + "type": {"type": "map", "values": "string"}, + } + ], + }, + ], + }, + { + "name": "b_union", + "type": [ + "long", + "string", + "ARecord", + ], + }, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_tuple_type = """type StringMapTuple { + key: String + value: [String!] +}""" + expected_sub_type = """type ARecord { + values: [StringMapTuple]! +}""" + expected_type = """type Test { + a_union: Float | String | ARecord + b_union: Float! | String! | ARecord! +}""" + + assert expected_tuple_type in graphql_code + assert expected_sub_type in graphql_code + assert expected_type in graphql_code + + +def test_int(): + graphql_classes = avsc_to_graphql( + { + "type": "record", + "name": "Test", + "fields": [ + { + "name": "c1", + "type": "int", + }, + ], + } + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_type = """type Test { + c1: Int! +}""" + assert expected_type in graphql_code + + +def test_avsc_to_graphql_directives(): + graphql_classes = avsc_to_graphql( + { + "name": "Test", + "type": "record", + "fields": [ + {"name": "col1", "type": "string"}, + {"name": "col2", "type": "string"}, + {"name": "col3", "type": "string"}, + ], + }, + { + "field_directives": {"col1": "@field-dir", "col3": "@field-dir"}, + "type_directives": {"Test": {"col2": "@type-dir", "col3": "@type-dir"}}, + }, + ) + graphql_code = classes_to_graphql_str(graphql_classes) + expected_code = """ +type Test { + col1: String! @field-dir + col2: String! @type-dir + col3: String! @field-dir @type-dir +}""" + assert expected_code in graphql_code diff --git a/tests/test_to_avro.py b/tests/test_to_avro.py index 064d6bc..d1c2d7e 100644 --- a/tests/test_to_avro.py +++ b/tests/test_to_avro.py @@ -99,7 +99,11 @@ def test_avro(): {"name": "c6", "type": {"type": "long", "logicalType": "time-micros"}}, {"name": "c7", "type": ["null", "string"], "default": None}, {"name": "c8", "type": "boolean"}, - {"name": "c9", "type": {"type": "string", "logicalType": "uuid"}, "doc": "This is UUID"}, + { + "name": "c9", + "type": {"type": "string", "logicalType": "uuid"}, + "doc": "This is UUID", + }, { "name": "c10", "type": ["null", {"type": "string", "logicalType": "uuid"}], @@ -110,7 +114,11 @@ def test_avro(): {"name": "c12", "type": {"type": "map", "values": "string"}}, { "name": "c13", - "type": {"type": "enum", "symbols": ["passed", "failed"], "name": "Status"}, + "type": { + "type": "enum", + "symbols": ["passed", "failed"], + "name": "Status", + }, "doc": "This is Status", }, {"name": "c14", "type": "bytes"}, @@ -168,7 +176,11 @@ def test_reused_object(): "fields": [ { "name": "c1", - "type": {"fields": [{"name": "c111", "type": "string"}], "name": "Nested2Model", "type": "record"}, + "type": { + "fields": [{"name": "c111", "type": "string"}], + "name": "Nested2Model", + "type": "record", + }, }, {"name": "c2", "type": "Nested2Model"}, ], @@ -187,7 +199,11 @@ def test_reused_object_array(): { "name": "c1", "type": { - "items": {"fields": [{"name": "c111", "type": "string"}], "name": "Nested2Model", "type": "record"}, + "items": { + "fields": [{"name": "c111", "type": "string"}], + "name": "Nested2Model", + "type": "record", + }, "type": "array", }, }, @@ -231,7 +247,13 @@ def test_complex_avro(): "type": "array", }, }, - {"name": "c4", "type": {"items": {"logicalType": "timestamp-micros", "type": "long"}, "type": "array"}}, + { + "name": "c4", + "type": { + "items": {"logicalType": "timestamp-micros", "type": "long"}, + "type": "array", + }, + }, {"name": "c5", "type": {"type": "map", "values": "NestedModel"}}, {"name": "c6", "type": ["null", "string", "long", "NestedModel"]}, ], @@ -384,7 +406,13 @@ def test_optional_array(): "type": "record", "namespace": "OptionalArray", "name": "OptionalArray", - "fields": [{"type": ["null", {"type": "array", "items": {"type": "string"}}], "name": "c1", "default": None}], + "fields": [ + { + "type": ["null", {"type": "array", "items": {"type": "string"}}], + "name": "c1", + "default": None, + } + ], }