From 14090efcf89ff8123f0ffe11fdb30c72091c00a7 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Thu, 2 Sep 2021 10:13:10 -0400 Subject: [PATCH] feat: parse Avro name field to custom x-parser-schema-id (#69) --- README.md | 10 ++++++ tests/asyncapi-avro-1.9.0-namespace.yaml | 11 ++++++ tests/parse.test.js | 19 +++++++---- tests/schemas/Person-1.8.2.avsc | 5 ++- tests/schemas/Person-1.9.0-namespace.avsc | 22 ++++++++++++ tests/schemas/Person-1.9.0.avsc | 6 ++-- tests/to-json-schema.test.js | 2 ++ to-json-schema.js | 41 ++++++++++++++++++----- 8 files changed, 95 insertions(+), 21 deletions(-) create mode 100644 tests/asyncapi-avro-1.9.0-namespace.yaml create mode 100644 tests/schemas/Person-1.9.0-namespace.avsc diff --git a/README.md b/README.md index d01f09b4..e15fec1e 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ channels: schemaFormat: 'application/vnd.apache.avro;version=1.9.0' payload: # The following is an Avro schema in YAML format (JSON format is also supported) type: record + name: User + namespace: com.company doc: User information fields: - name: displayName @@ -136,6 +138,14 @@ Additional attributes not defined in the [Avro Specification](https://avro.apach - `example` - Can be used to define the example value from the business domain of given field. Value will be propagated into [examples attribute](https://json-schema.org/draft/2020-12/json-schema-validation.html#rfc.section.9.5) of JSON schema and therefore will be picked for the generated "Example of payload" when using some AsyncAPI documentation generator. +### Support for names and namespaces + +If, at the top level of the Avro schema, the 'name' attribute is defined, it will be copied to the corresponding JSON schema's 'x-parser-schema-id' attribute. If the Avro schema also has the 'namespace' attribute defined, then that schema's fully qualified name will be put into that attribute. The fully qualified name is defined by the namespace, followed by a dot, followed by the name. + +If there are two schemas that resolve to the same fully qualified name, only the last one will be returned by the parser. Make sure names of your schemas are unique. + +If no name attribute is present, the 'x-parser-schema-id' will have a generated unique id with a name like 'anonymous-schema-1' generated by the main parser. 'x-parser-schema-id' is one of the [custom extensions](https://github.com/asyncapi/parser-js/#custom-extensions) supported by the parser. + ## Limitations ### Float and double-precision numbers diff --git a/tests/asyncapi-avro-1.9.0-namespace.yaml b/tests/asyncapi-avro-1.9.0-namespace.yaml new file mode 100644 index 00000000..9d8040e2 --- /dev/null +++ b/tests/asyncapi-avro-1.9.0-namespace.yaml @@ -0,0 +1,11 @@ +asyncapi: 2.0.0 +info: + title: My API + version: '1.0.0' +channels: + mychannel: + publish: + message: + schemaFormat: application/vnd.apache.avro;version=1.9.0 + payload: + $ref: 'schemas/Person-1.9.0-namespace.avsc' \ No newline at end of file diff --git a/tests/parse.test.js b/tests/parse.test.js index 44fd746f..68557317 100644 --- a/tests/parse.test.js +++ b/tests/parse.test.js @@ -3,21 +3,28 @@ const path = require('path'); const avroSchemaParser = require('..'); const parser = require('@asyncapi/parser'); +const inputWithAvro182 = fs.readFileSync(path.resolve(__dirname, './asyncapi-avro-1.8.2.yaml'), 'utf8'); +const outputWithAvro182 = '{"asyncapi":"2.0.0","info":{"title":"My API","version":"1.0.0"},"channels":{"mychannel":{"publish":{"message":{"payload":{"type":"object","properties":{"name":{"type":"string","examples":["Donkey"],"x-parser-schema-id":""},"age":{"oneOf":[{"type":"integer","minimum":-2147483648,"maximum":2147483647,"x-parser-schema-id":""},{"type":"null","x-parser-schema-id":""}],"default":null,"x-parser-schema-id":""},"favoriteProgrammingLanguage":{"type":"string","enum":["JS","Java","Go","Rust","C"],"x-parser-schema-id":""},"address":{"type":"object","properties":{"zipcode":{"type":"integer","minimum":-2147483648,"maximum":2147483647,"examples":[53003],"x-parser-schema-id":""}},"x-parser-schema-id":""}},"x-parser-schema-id":""},"x-parser-original-schema-format":"application/vnd.apache.avro;version=1.8.2","x-parser-original-payload":{"type":"record","fields":[{"name":"name","type":"string","example":"Donkey"},{"name":"age","type":["null","int"],"default":null},{"name":"favoriteProgrammingLanguage","type":{"name":"ProgrammingLanguage","type":"enum","symbols":["JS","Java","Go","Rust","C"]}},{"name":"address","type":{"name":"Address","type":"record","fields":[{"name":"zipcode","type":"int","example":"53003"}]}}]},"schemaFormat":"application/vnd.aai.asyncapi;version=2.0.0","x-parser-message-parsed":true,"x-parser-message-name":""}}}},"x-parser-spec-parsed":true}'; + const inputWithAvro190 = fs.readFileSync(path.resolve(__dirname, './asyncapi-avro-1.9.0.yaml'), 'utf8'); -const outputWithAvro190 = '{"asyncapi":"2.0.0","info":{"title":"My API","version":"1.0.0"},"channels":{"mychannel":{"publish":{"message":{"payload":{"type":"object","properties":{"name":{"type":"string","examples":["Donkey"],"x-parser-schema-id":""},"age":{"oneOf":[{"type":"integer","minimum":-2147483648,"maximum":2147483647,"examples":[123],"x-parser-schema-id":""},{"type":"null","x-parser-schema-id":""}],"default":null,"x-parser-schema-id":""},"favoriteProgrammingLanguage":{"type":"string","enum":["JS","Java","Go","Rust","C"],"default":"JS","x-parser-schema-id":""},"address":{"type":"object","properties":{"zipcode":{"type":"integer","minimum":-2147483648,"maximum":2147483647,"examples":[53003],"x-parser-schema-id":""}},"x-parser-schema-id":""},"someid":{"type":"string","x-parser-schema-id":""}},"x-parser-schema-id":""},"x-parser-original-schema-format":"application/vnd.apache.avro;version=1.9.0","x-parser-original-payload":{"name":"Person","type":"record","fields":[{"name":"name","type":"string","example":"Donkey"},{"name":"age","type":["null","int"],"default":null,"example":"123"},{"name":"favoriteProgrammingLanguage","type":{"name":"ProgrammingLanguage","type":"enum","symbols":["JS","Java","Go","Rust","C"],"default":"JS"}},{"name":"address","type":{"name":"Address","type":"record","fields":[{"name":"zipcode","type":"int","example":"53003"}]}},{"name":"someid","type":"uuid"}]},"schemaFormat":"application/vnd.aai.asyncapi;version=2.0.0","x-parser-message-parsed":true,"x-parser-message-name":""}}}},"x-parser-spec-parsed":true}'; +const outputWithAvro190 = '{"asyncapi":"2.0.0","info":{"title":"My API","version":"1.0.0"},"channels":{"mychannel":{"publish":{"message":{"payload":{"type":"object","properties":{"name":{"type":"string","examples":["Donkey"],"x-parser-schema-id":""},"age":{"oneOf":[{"type":"integer","minimum":-2147483648,"maximum":2147483647,"examples":[123],"x-parser-schema-id":""},{"type":"null","x-parser-schema-id":""}],"default":null,"x-parser-schema-id":""},"favoriteProgrammingLanguage":{"type":"string","enum":["JS","Java","Go","Rust","C"],"default":"JS","x-parser-schema-id":""},"address":{"type":"object","properties":{"zipcode":{"type":"integer","minimum":-2147483648,"maximum":2147483647,"examples":[53003],"x-parser-schema-id":""}},"x-parser-schema-id":""},"someid":{"type":"string","x-parser-schema-id":""}},"x-parser-schema-id":"Person"},"x-parser-original-schema-format":"application/vnd.apache.avro;version=1.9.0","x-parser-original-payload":{"name":"Person","type":"record","fields":[{"name":"name","type":"string","example":"Donkey"},{"name":"age","type":["null","int"],"default":null,"example":"123"},{"name":"favoriteProgrammingLanguage","type":{"name":"ProgrammingLanguage","type":"enum","symbols":["JS","Java","Go","Rust","C"],"default":"JS"}},{"name":"address","type":{"name":"Address","type":"record","fields":[{"name":"zipcode","type":"int","example":"53003"}]}},{"name":"someid","type":"uuid"}]},"schemaFormat":"application/vnd.aai.asyncapi;version=2.0.0","x-parser-message-parsed":true,"x-parser-message-name":""}}}},"x-parser-spec-parsed":true}'; -const inputWithAvro182 = fs.readFileSync(path.resolve(__dirname, './asyncapi-avro-1.8.2.yaml'), 'utf8'); -const outputWithAvro182 = '{"asyncapi":"2.0.0","info":{"title":"My API","version":"1.0.0"},"channels":{"mychannel":{"publish":{"message":{"payload":{"type":"object","properties":{"name":{"type":"string","examples":["Donkey"],"x-parser-schema-id":""},"age":{"oneOf":[{"type":"integer","minimum":-2147483648,"maximum":2147483647,"x-parser-schema-id":""},{"type":"null","x-parser-schema-id":""}],"default":null,"x-parser-schema-id":""},"favoriteProgrammingLanguage":{"type":"string","enum":["JS","Java","Go","Rust","C"],"x-parser-schema-id":""},"address":{"type":"object","properties":{"zipcode":{"type":"integer","minimum":-2147483648,"maximum":2147483647,"examples":[53003],"x-parser-schema-id":""}},"x-parser-schema-id":""}},"x-parser-schema-id":""},"x-parser-original-schema-format":"application/vnd.apache.avro;version=1.8.2","x-parser-original-payload":{"name":"Person","type":"record","fields":[{"name":"name","type":"string","example":"Donkey"},{"name":"age","type":["null","int"],"default":null},{"name":"favoriteProgrammingLanguage","type":{"name":"ProgrammingLanguage","type":"enum","symbols":["JS","Java","Go","Rust","C"]}},{"name":"address","type":{"name":"Address","type":"record","fields":[{"name":"zipcode","type":"int","example":"53003"}]}}]},"schemaFormat":"application/vnd.aai.asyncapi;version=2.0.0","x-parser-message-parsed":true,"x-parser-message-name":""}}}},"x-parser-spec-parsed":true}'; +const inputWithAvro190WithNamespace = fs.readFileSync(path.resolve(__dirname, './asyncapi-avro-1.9.0-namespace.yaml'), 'utf8'); +const outputWithAvro190ithNamespace = '{"asyncapi":"2.0.0","info":{"title":"My API","version":"1.0.0"},"channels":{"mychannel":{"publish":{"message":{"payload":{"type":"object","properties":{"name":{"type":"string","examples":["Donkey"],"x-parser-schema-id":""},"age":{"oneOf":[{"type":"integer","minimum":-2147483648,"maximum":2147483647,"examples":[123],"x-parser-schema-id":""},{"type":"null","x-parser-schema-id":""}],"default":null,"x-parser-schema-id":""},"favoriteProgrammingLanguage":{"type":"string","enum":["JS","Java","Go","Rust","C"],"default":"JS","x-parser-schema-id":""},"address":{"type":"object","properties":{"zipcode":{"type":"integer","minimum":-2147483648,"maximum":2147483647,"examples":[53003],"x-parser-schema-id":""}},"x-parser-schema-id":""},"someid":{"type":"string","x-parser-schema-id":""}},"x-parser-schema-id":"com.company.Person"},"x-parser-original-schema-format":"application/vnd.apache.avro;version=1.9.0","x-parser-original-payload":{"name":"Person","namespace":"com.company","type":"record","fields":[{"name":"name","type":"string","example":"Donkey"},{"name":"age","type":["null","int"],"default":null,"example":"123"},{"name":"favoriteProgrammingLanguage","type":{"name":"ProgrammingLanguage","type":"enum","symbols":["JS","Java","Go","Rust","C"],"default":"JS"}},{"name":"address","type":{"name":"Address","type":"record","fields":[{"name":"zipcode","type":"int","example":"53003"}]}},{"name":"someid","type":"uuid"}]},"schemaFormat":"application/vnd.aai.asyncapi;version=2.0.0","x-parser-message-parsed":true,"x-parser-message-name":""}}}},"x-parser-spec-parsed":true}'; parser.registerSchemaParser(avroSchemaParser); describe('parse()', function() { + it('should parse Avro schema 1.8.2', async function() { + const result = await parser.parse(inputWithAvro182, { path: __filename }); + expect(JSON.stringify(result.json())).toEqual(outputWithAvro182); + }); it('should parse Avro schema 1.9.0', async function() { const result = await parser.parse(inputWithAvro190, { path: __filename }); expect(JSON.stringify(result.json())).toEqual(outputWithAvro190); }); - it('should parse Avro schema 1.8.2', async function() { - const result = await parser.parse(inputWithAvro182, { path: __filename }); - expect(JSON.stringify(result.json())).toEqual(outputWithAvro182); + it('should parse Avro schema 1.9.0 with a namespace', async function() { + const result = await parser.parse(inputWithAvro190WithNamespace, { path: __filename }); + expect(JSON.stringify(result.json())).toEqual(outputWithAvro190ithNamespace); }); }); diff --git a/tests/schemas/Person-1.8.2.avsc b/tests/schemas/Person-1.8.2.avsc index 8d9dcbd5..060494f9 100644 --- a/tests/schemas/Person-1.8.2.avsc +++ b/tests/schemas/Person-1.8.2.avsc @@ -1,8 +1,7 @@ { - "name": "Person", "type": "record", "fields": [ - {"name": "name", "type": "string", example: "Donkey"}, + {"name": "name", "type": "string", "example": "Donkey"}, {"name": "age", "type": ["null", "int"], "default": null}, { "name": "favoriteProgrammingLanguage", @@ -13,7 +12,7 @@ "type": { "name": "Address", "type": "record", - "fields": [{"name": "zipcode", "type": "int", example: "53003"}] + "fields": [{"name": "zipcode", "type": "int", "example": "53003"}] } } ] diff --git a/tests/schemas/Person-1.9.0-namespace.avsc b/tests/schemas/Person-1.9.0-namespace.avsc new file mode 100644 index 00000000..a1e72490 --- /dev/null +++ b/tests/schemas/Person-1.9.0-namespace.avsc @@ -0,0 +1,22 @@ +{ + "name": "Person", + "namespace": "com.company", + "type": "record", + "fields": [ + {"name": "name", "type": "string", "example": "Donkey"}, + {"name": "age", "type": ["null", "int"], "default": null, "example": "123"}, + { + "name": "favoriteProgrammingLanguage", + "type": {"name": "ProgrammingLanguage", "type": "enum", "symbols": ["JS", "Java", "Go", "Rust", "C"], "default": "JS"} + }, + { + "name": "address", + "type": { + "name": "Address", + "type": "record", + "fields": [{"name": "zipcode", "type": "int", "example": "53003"}] + } + }, + {"name": "someid", "type": "uuid"} + ] +} diff --git a/tests/schemas/Person-1.9.0.avsc b/tests/schemas/Person-1.9.0.avsc index ab4660f5..3685ad52 100644 --- a/tests/schemas/Person-1.9.0.avsc +++ b/tests/schemas/Person-1.9.0.avsc @@ -2,8 +2,8 @@ "name": "Person", "type": "record", "fields": [ - {"name": "name", "type": "string", example: "Donkey"}, - {"name": "age", "type": ["null", "int"], "default": null, example: "123"}, + {"name": "name", "type": "string", "example": "Donkey"}, + {"name": "age", "type": ["null", "int"], "default": null, "example": "123"}, { "name": "favoriteProgrammingLanguage", "type": {"name": "ProgrammingLanguage", "type": "enum", "symbols": ["JS", "Java", "Go", "Rust", "C"], "default": "JS"} @@ -13,7 +13,7 @@ "type": { "name": "Address", "type": "record", - "fields": [{"name": "zipcode", "type": "int", example: "53003"}] + "fields": [{"name": "zipcode", "type": "int", "example": "53003"}] } }, {"name": "someid", "type": "uuid"} diff --git a/tests/to-json-schema.test.js b/tests/to-json-schema.test.js index 2802275f..0abbbfee 100644 --- a/tests/to-json-schema.test.js +++ b/tests/to-json-schema.test.js @@ -81,6 +81,7 @@ describe('avroToJsonSchema()', function () { const result = await avroToJsonSchema({ type: 'record', doc: 'My test record', + name: 'MyName', fields: [ { name: 'key1', type: 'long', doc: 'Key1 docs' }, { name: 'key2', type: 'string', default: 'value2', doc: 'Key2 docs' }, @@ -88,6 +89,7 @@ describe('avroToJsonSchema()', function () { }); expect(result).toEqual({ type: 'object', + 'x-parser-schema-id': 'MyName', description: 'My test record', properties: { key1: { diff --git a/to-json-schema.js b/to-json-schema.js index bfb5470c..f3e35a88 100644 --- a/to-json-schema.js +++ b/to-json-schema.js @@ -21,11 +21,30 @@ const typeMappings = { uuid: 'string', }; -const commonAttributesMapping = (avroDefinition, jsonSchema) => { +const commonAttributesMapping = (avroDefinition, jsonSchema, isTopLevel) => { if (avroDefinition.doc) jsonSchema.description = avroDefinition.doc; if (avroDefinition.default !== undefined) jsonSchema.default = avroDefinition.default; + + const fullyQualifiedName = getFullyQualifiedName(avroDefinition); + if (isTopLevel && fullyQualifiedName !== undefined) { + jsonSchema['x-parser-schema-id'] = fullyQualifiedName; + } }; +function getFullyQualifiedName(avroDefinition) { + let name; + + if (avroDefinition.name) { + if (avroDefinition.namespace) { + name = `${avroDefinition.namespace}.${avroDefinition.name}`; + } else { + name = avroDefinition.name; + } + } + + return name; +} + const exampleAttributeMapping = (typeInput, example, jsonSchemaInput) => { let type = typeInput; let jsonSchema = jsonSchemaInput; @@ -51,7 +70,7 @@ const exampleAttributeMapping = (typeInput, example, jsonSchemaInput) => { } }; -module.exports.avroToJsonSchema = async function avroToJsonSchema(avroDefinition) { +async function convertAvroToJsonSchema(avroDefinition, isTopLevel) { const jsonSchema = {}; const isUnion = Array.isArray(avroDefinition); @@ -59,7 +78,7 @@ module.exports.avroToJsonSchema = async function avroToJsonSchema(avroDefinition jsonSchema.oneOf = []; let nullDef = null; for (const avroDef of avroDefinition) { - const def = await avroToJsonSchema(avroDef); + const def = await convertAvroToJsonSchema(avroDef, isTopLevel); // avroDef can be { type: 'int', default: 1 } and this is why avroDef.type has priority here const defType = avroDef.type || avroDef; // To prefer non-null values in the examples skip null definition here and push it as the last element after loop @@ -94,10 +113,10 @@ module.exports.avroToJsonSchema = async function avroToJsonSchema(avroDefinition jsonSchema.maxLength = avroDefinition.size; break; case 'map': - jsonSchema.additionalProperties = await avroToJsonSchema(avroDefinition.values); + jsonSchema.additionalProperties = await convertAvroToJsonSchema(avroDefinition.values, false); break; case 'array': - jsonSchema.items = await avroToJsonSchema(avroDefinition.items); + jsonSchema.items = await convertAvroToJsonSchema(avroDefinition.items, false); break; case 'enum': jsonSchema.enum = avroDefinition.symbols; @@ -105,9 +124,9 @@ module.exports.avroToJsonSchema = async function avroToJsonSchema(avroDefinition case 'record': const propsMap = new Map(); for (const field of avroDefinition.fields) { - const def = await avroToJsonSchema(field.type); + const def = await convertAvroToJsonSchema(field.type, false); - commonAttributesMapping(field, def); + commonAttributesMapping(field, def, false); exampleAttributeMapping(field.type, field.example, def); propsMap.set(field.name, def); @@ -116,8 +135,12 @@ module.exports.avroToJsonSchema = async function avroToJsonSchema(avroDefinition break; } - commonAttributesMapping(avroDefinition, jsonSchema); + commonAttributesMapping(avroDefinition, jsonSchema, isTopLevel); exampleAttributeMapping(type, avroDefinition.example, jsonSchema); return jsonSchema; -}; +} + +module.exports.avroToJsonSchema = async function avroToJsonSchema(avroDefinition) { + return convertAvroToJsonSchema(avroDefinition, true); +}; \ No newline at end of file