chore:(ui): parse ingestion workflow schemas (#12477)

* chore:(ui): parse ingestion workflow schemas

* chore: remove dereference part from script

* chore: address comments

.gitignore

@@ -85,6 +85,7 @@ openmetadata-ui/src/main/resources/ui/cypress/fixtures
 
 #UI - Dereferenced Schemas
 openmetadata-ui/src/main/resources/ui/src/jsons/connectionSchemas
+openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas
 
 #UI - Generated TS
 openmetadata-ui/src/main/resources/ui/src/generated

openmetadata-ui/src/main/resources/ui/package.json

@@ -112,12 +112,14 @@
     "lint": "eslint \"./**/*.{js,jsx,ts,tsx,json}\"",
     "lint:fix": "eslint './**/*.{js,jsx,ts,tsx,json}' --fix",
     "pretty": "prettier . --config './.prettierrc.yaml' --ignore-path './.prettierignore'  --write",
-    "build-check": "yarn run json2ts && yarn run js-antlr && yarn run parse-conn-schema",
+    "build-check": "yarn run json2ts && yarn run js-antlr && yarn run parse-schema",
     "commit-check": "yarn run pretty && yarn run build",
     "license-header-check": "license-check-and-add check --config-file .licenseheaderrc.json",
     "license-header-fix": "license-check-and-add add --config-file .licenseheaderrc.json --regex-replacements $(date +%Y)",
     "json2ts": "sh json2ts.sh",
+    "parse-schema": "yarn run parse-conn-schema && yarn run parse-ingestion-schema",
     "parse-conn-schema": "node parseConnectionSchema && rm -rf connTemp",
+    "parse-ingestion-schema": "node parseIngestionSchema && rm -rf ingestionTemp",
     "js-antlr": "PWD=$(echo $PWD) antlr4 -Dlanguage=JavaScript -o src/generated/antlr \"$PWD\"/../../../../../openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4",
     "cypress:open": "cypress open --e2e",
     "cypress:run": "cypress run --config-file=cypress.config.ts",

openmetadata-ui/src/main/resources/ui/parseIngestionSchema.js

@@ -0,0 +1,98 @@
+/*
+ *  Copyright 2023 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+/* eslint-disable */
+const $RefParser = require('@apidevtools/json-schema-ref-parser');
+const path = require('path');
+const fs = require('fs');
+const fse = require('fs-extra');
+const process = require('process');
+
+const cwd = process.cwd();
+
+const schemaDir =
+  '../../../../../openmetadata-spec/src/main/resources/json/schema';
+
+const rootDir = 'ingestionTemp';
+const srcDir = 'schema/metadataIngestion';
+const destDir = 'src/jsons/ingestionSchemas';
+
+const playDir = `${rootDir}/${srcDir}`;
+
+const globalParserOptions = {
+  continueOnError: true,
+  dereference: {
+    circular: true,
+  },
+};
+
+const parser = new $RefParser(globalParserOptions);
+
+async function parseSchema(filePath, destPath) {
+  try {
+    const fileDir = `${cwd}/${path.dirname(filePath)}`;
+    const fileName = path.basename(filePath);
+    process.chdir(fileDir);
+    const parsedSchema = await parser.parse(fileName);
+    const api = await parser.bundle(parsedSchema);
+    const dirname = `${cwd}/${path.dirname(destPath)}`;
+    if (!fs.existsSync(dirname)) {
+      try {
+        fs.mkdirSync(dirname, { recursive: true });
+      } catch (err) {
+        console.log(err);
+      }
+    }
+    fs.writeFileSync(`${cwd}/${destPath}`, JSON.stringify(api, null, 2));
+  } catch (err) {
+    console.log(err);
+  } finally {
+    process.chdir(cwd);
+  }
+}
+
+async function traverseDirectory(Directory) {
+  const Files = fs.readdirSync(Directory);
+  for (const File of Files) {
+    const Absolute = path.join(Directory, File);
+    if (fs.statSync(Absolute).isDirectory()) {
+      await traverseDirectory(Absolute);
+    } else {
+      const name = Absolute.replace(playDir, destDir);
+      await parseSchema(Absolute, name);
+    }
+  }
+}
+
+function copySourceFiles() {
+  try {
+    fse.copySync(schemaDir, `${rootDir}/schema`);
+  } catch (err) {
+    console.error(err);
+  }
+}
+
+async function main() {
+  try {
+    if (fs.existsSync(destDir)) {
+      fs.rmSync(destDir, { recursive: true });
+    }
+    fs.mkdirSync(destDir, { recursive: true });
+    copySourceFiles();
+
+    await traverseDirectory(`${playDir}`);
+  } catch (err) {
+    console.log(err);
+  }
+}
+
+main();