diff --git a/.gitignore b/.gitignore index 35b1a4b60be..56e0fdf7459 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,7 @@ openmetadata-ui/src/main/resources/ui/cypress/fixtures #UI - Dereferenced Schemas openmetadata-ui/src/main/resources/ui/src/jsons/connectionSchemas +openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas #UI - Generated TS openmetadata-ui/src/main/resources/ui/src/generated diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json index 98cc2a5e406..f208095c269 100644 --- a/openmetadata-ui/src/main/resources/ui/package.json +++ b/openmetadata-ui/src/main/resources/ui/package.json @@ -112,12 +112,14 @@ "lint": "eslint \"./**/*.{js,jsx,ts,tsx,json}\"", "lint:fix": "eslint './**/*.{js,jsx,ts,tsx,json}' --fix", "pretty": "prettier . --config './.prettierrc.yaml' --ignore-path './.prettierignore' --write", - "build-check": "yarn run json2ts && yarn run js-antlr && yarn run parse-conn-schema", + "build-check": "yarn run json2ts && yarn run js-antlr && yarn run parse-schema", "commit-check": "yarn run pretty && yarn run build", "license-header-check": "license-check-and-add check --config-file .licenseheaderrc.json", "license-header-fix": "license-check-and-add add --config-file .licenseheaderrc.json --regex-replacements $(date +%Y)", "json2ts": "sh json2ts.sh", + "parse-schema": "yarn run parse-conn-schema && yarn run parse-ingestion-schema", "parse-conn-schema": "node parseConnectionSchema && rm -rf connTemp", + "parse-ingestion-schema": "node parseIngestionSchema && rm -rf ingestionTemp", "js-antlr": "PWD=$(echo $PWD) antlr4 -Dlanguage=JavaScript -o src/generated/antlr \"$PWD\"/../../../../../openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4", "cypress:open": "cypress open --e2e", "cypress:run": "cypress run --config-file=cypress.config.ts", diff --git a/openmetadata-ui/src/main/resources/ui/parseIngestionSchema.js b/openmetadata-ui/src/main/resources/ui/parseIngestionSchema.js new file mode 100644 index 00000000000..e6873a9a602 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/parseIngestionSchema.js @@ -0,0 +1,98 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* eslint-disable */ +const $RefParser = require('@apidevtools/json-schema-ref-parser'); +const path = require('path'); +const fs = require('fs'); +const fse = require('fs-extra'); +const process = require('process'); + +const cwd = process.cwd(); + +const schemaDir = + '../../../../../openmetadata-spec/src/main/resources/json/schema'; + +const rootDir = 'ingestionTemp'; +const srcDir = 'schema/metadataIngestion'; +const destDir = 'src/jsons/ingestionSchemas'; + +const playDir = `${rootDir}/${srcDir}`; + +const globalParserOptions = { + continueOnError: true, + dereference: { + circular: true, + }, +}; + +const parser = new $RefParser(globalParserOptions); + +async function parseSchema(filePath, destPath) { + try { + const fileDir = `${cwd}/${path.dirname(filePath)}`; + const fileName = path.basename(filePath); + process.chdir(fileDir); + const parsedSchema = await parser.parse(fileName); + const api = await parser.bundle(parsedSchema); + const dirname = `${cwd}/${path.dirname(destPath)}`; + if (!fs.existsSync(dirname)) { + try { + fs.mkdirSync(dirname, { recursive: true }); + } catch (err) { + console.log(err); + } + } + fs.writeFileSync(`${cwd}/${destPath}`, JSON.stringify(api, null, 2)); + } catch (err) { + console.log(err); + } finally { + process.chdir(cwd); + } +} + +async function traverseDirectory(Directory) { + const Files = fs.readdirSync(Directory); + for (const File of Files) { + const Absolute = path.join(Directory, File); + if (fs.statSync(Absolute).isDirectory()) { + await traverseDirectory(Absolute); + } else { + const name = Absolute.replace(playDir, destDir); + await parseSchema(Absolute, name); + } + } +} + +function copySourceFiles() { + try { + fse.copySync(schemaDir, `${rootDir}/schema`); + } catch (err) { + console.error(err); + } +} + +async function main() { + try { + if (fs.existsSync(destDir)) { + fs.rmSync(destDir, { recursive: true }); + } + fs.mkdirSync(destDir, { recursive: true }); + copySourceFiles(); + + await traverseDirectory(`${playDir}`); + } catch (err) { + console.log(err); + } +} + +main();