Optimising generation of typescripts from schema and as prebuild script (#5037)

* Adding json2ts as prebuild script

* Add generated TS to gitignore

* Fixing CI errors

* Fix Sonar CI

* Bumping metadata version

* Fixing sonar CI

* Fix json2ts script

* Revert gitignore

* prettify

* Fix CI build failures

* Removing unnecessary changes

* Add pre-script for coverage

.gitignore

@@ -97,4 +97,4 @@ mlruns/
 /openmetadata-ui/src/main/resources/ui/src/antlr/generated/Fqn.interp
 /openmetadata-ui/src/main/resources/ui/src/antlr/generated/Fqn.tokens
 /openmetadata-ui/src/main/resources/ui/src/antlr/generated/FqnLexer.interp
-/openmetadata-ui/src/main/resources/ui/src/antlr/generated/FqnLexer.tokens
+/openmetadata-ui/src/main/resources/ui/src/antlr/generated/FqnLexer.tokens

catalog-rest-service/src/main/resources/json/schema/entity/type.json

@@ -7,14 +7,11 @@
   "javaType": "org.openmetadata.catalog.entity.Type",
   "javaInterfaces": ["org.openmetadata.catalog.EntityInterface"],
   "definitions": {
-    "category" : {
+    "category": {
-      "description" : "Metadata category to which a type belongs to.",
+      "description": "Metadata category to which a type belongs to.",
       "type": "string",
       "javaType": "org.openmetadata.catalog.entity.type.Category",
-      "enum": [
+      "enum": ["field", "entity"],
-        "field",
-        "entity"
-      ],
       "javaEnums": [
         {
           "name": "Field"
@@ -26,36 +23,32 @@
     },
     "fieldName": {
       "description": "Name of the entity field. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.",
-      "type" : "string",
+      "type": "string",
       "pattern": "^[a-z][a-zA-Z0-9]+$"
     },
     "typeName": {
       "description": "Name of the field or entity types. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.",
-      "type" : "string",
+      "type": "string",
       "pattern": "^[a-z][a-zA-Z0-9]+$"
     },
-    "customField" : {
+    "customField": {
-      "description" : "Type used for adding custom field to an entity to extend it.",
+      "description": "Type used for adding custom field to an entity to extend it.",
-      "type" : "object",
+      "type": "object",
       "javaType": "org.openmetadata.catalog.entity.type.CustomField",
       "properties": {
         "name": {
           "description": "Name of the entity field. Note a field name must be unique for an entity. Field name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.",
-          "$ref" : "#/definitions/fieldName"
+          "$ref": "#/definitions/fieldName"
         },
         "description": {
-          "$ref" : "../type/basic.json#/definitions/markdown"
+          "$ref": "../type/basic.json#/definitions/markdown"
         },
         "fieldType": {
           "description": "Reference to a field type. Only field types are allows and entity types are not allowed as custom fields to extend an existing entity",
           "$ref": "../type/entityReference.json"
         }
       },
-      "required": [
+      "required": ["name", "description", "fieldType"],
-        "name",
-        "description",
-        "fieldType"
-      ],
       "additionalProperties": false
     }
   },
@@ -78,33 +71,25 @@
     },
     "description": {
       "description": "Optional description of entity.",
-      "$ref" : "../type/basic.json#/definitions/markdown"
+      "$ref": "../type/basic.json#/definitions/markdown"
     },
     "nameSpace": {
       "description": "Namespace or group to which this type belongs to. For example, some of the field types commonly used can come from `basic` namespace. Some of the entities such as `table`, `database`, etc. come from `data` namespace.",
       "type": "string",
-      "default" : "custom"
+      "default": "custom"
     },
-    "category" : {
+    "category": {
-      "$ref" : "#/definitions/category"
+      "$ref": "#/definitions/category"
-    },
-    "nameSpace": {
-      "description": "Namespace or group to which this type belongs to. For example, some of the field types commonly used can come from `basic` namespace. Some of the entities such as `table`, `database`, etc. come from `data` namespace.",
-      "type": "string",
-      "default" : "custom"
-    },
-    "category" : {
-      "$ref" : "#/definitions/category"
     },
     "schema": {
       "description": "JSON schema encoded as string that defines the type. This will be used to validate the type values.",
       "$ref": "../type/basic.json#/definitions/jsonSchema"
     },
-    "customFields" : {
+    "customFields": {
       "description": "Custom fields added to extend the entity. Only available for entity type",
-      "type" : "array",
+      "type": "array",
-      "items" : {
+      "items": {
-        "$ref" : "#/definitions/customField"
+        "$ref": "#/definitions/customField"
       }
     },
     "version": {

ingestion-core/src/metadata/_version.py

@@ -7,5 +7,5 @@ Provides metadata version information.
 
 from incremental import Version
 
-__version__ = Version("metadata", 0, 11, 0, dev=7)
+__version__ = Version("metadata", 0, 11, 0, dev=8)
 __all__ = ["__version__"]

openmetadata-ui/pom.xml

@@ -119,20 +119,6 @@
               <arguments>install --frozen-lockfile</arguments>
             </configuration>
           </execution>
-          <execution>
-            <id>yarn run build-check</id>
-            <goals>
-              <goal>yarn</goal>
-            </goals>
-            <!-- optional: the default phase is "generate-resources" -->
-            <phase>prepare-package</phase>
-            <configuration>
-              <!-- optional: if not specified, it will run gulp's default task
-                  (and you can remove this whole <configuration> section.) -->
-              <arguments>run build-check</arguments>
-              <!--arguments>build</arguments-->
-            </configuration>
-          </execution>
           <execution>
             <id>yarn run build-test</id>
             <goals>

openmetadata-ui/src/main/resources/ui/.prettierignore

@@ -40,3 +40,10 @@ swagger.html
 
 # antlr
 src/antlr/generated/
+
+# Generated TS
+src/generated/
+
+# cypress
+**/*.spec.js
+cypress/

openmetadata-ui/src/main/resources/ui/json2ts.sh

@@ -10,34 +10,37 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+schema_directory='catalog-rest-service/src/main/resources/json/schema/'
+om_ui_directory='openmetadata-ui/src/main/resources/ui/src/generated/'
 addLicensing(){
     dir=$1
-    txt=`cat types-licensing.txt; cat "$dir"`
+    txt=`cat openmetadata-ui/src/main/resources/ui/types-licensing.txt; cat "$dir"`
     echo "$txt" > "$dir"
 }
-
+generateType(){
-getTypes(){
+    ./node_modules/.bin/quicktype -s schema $PWD"/${schema_directory}$1" -o $PWD"/"$om_ui_directory$2 --just-types
-    dir=$1
+    if [ -s $om_ui_directory$2 ]
-    for f in $dir/*
-    do
-    filename=$f
-    if [[ -d $f ]];
     then
-        getTypes "$f" "$2" "$3"
+        addLicensing "$om_ui_directory$2"
     else
-        fileTS=${f//.json/.ts}
+        rm -f "$om_ui_directory$2"
-        fileTS=${fileTS//$2/$3}
-        mkdir -p "$(dirname "$fileTS")" && ../../../../../node_modules/.bin/quicktype -s schema "$f" -o "$fileTS" --just-types
-        if [[ -s $fileTS ]]
-        then
-            addLicensing "$fileTS"
-        else
-            rm -f $fileTS
-        fi
     fi
+}
+getTypes(){
+    if [ -d "$om_ui_directory" ]
+    then
+        rm -r $om_ui_directory
+    fi
+
+    for file_with_dir in $(find $schema_directory  -name "*.json" | sed -e 's/catalog-rest-service\/src\/main\/resources\/json\/schema\///g')
+    do
+        mkdir -p "$(dirname "$om_ui_directory$file_with_dir")"
+        fileTS=$(echo "$file_with_dir" | sed "s/.json/.ts/g")
+        generateType "$file_with_dir" "$fileTS" &
     done
 }
 
-rm -r $2
+# Checkout root directory to generate typescript from schema
-mkdir $2
+cd ../../../../..
-getTypes "$1" "$1" "$2"
+getTypes
+wait

openmetadata-ui/src/main/resources/ui/package.json

@@ -97,7 +97,9 @@
     "build": "NODE_ENV=production BABEL_ENV=production webpack --config ./webpack.config.prod.js --env production",
     "preinstall": "cd ../../../../.. && yarn install --frozen-lockfile",
     "mock-api": "node ./mock-api/app.js",
+    "prebuild": "yarn run build-check",
     "pretest": "yarn run build-check",
+    "pretest:coverage": "yarn run pretest",
     "test": "jest --passWithNoTests",
     "test:watch": "jest --passWithNoTests --watch",
     "test:coverage": "jest --passWithNoTests --coverage",
@@ -105,10 +107,10 @@
     "lint": "eslint \"./src/**/*.{js,jsx,ts,tsx}\"",
     "lint:fix": "eslint './src/**/*.{js,jsx,ts,tsx}' --fix",
     "pretty": "prettier . --config './.prettierrc' --write",
-    "build-check": "yarn run lint && yarn run parse-conn-schema",
+    "build-check": "yarn run json2ts && yarn run lint && yarn run parse-conn-schema",
     "build-test": "yarn run test",
-    "commit-check": "yarn run pretty && yarn run build-check && yarn run build",
+    "commit-check": "yarn run pretty && yarn run build",
-    "json2ts": "./json2ts.sh ../../../../../catalog-rest-service/src/main/resources/json/schema ./src/generated && yarn run lint:fix",
+    "json2ts": "sh json2ts.sh",
     "parse-conn-schema": "node parseConnectionSchema && rm -rf connTemp",
     "js-antlr": "PWD=$(echo $PWD) antlr -Dlanguage=JavaScript -o src/antlr/generated $PWD/../../../../../catalog-rest-service/src/main/antlr4/org/openmetadata/catalog/Fqn.g4",
     "cypress:open": "cypress open",

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDatabase.ts

@@ -26,6 +26,10 @@ export interface CreateDatabase {
    * Description of the database instance. What it has and how to use it.
    */
   description?: string;
+  /**
+   * Display Name that identifies this database.
+   */
+  displayName?: string;
   /**
    * Name that identifies this database instance uniquely.
    */

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDatabaseSchema.ts

@@ -24,6 +24,10 @@ export interface CreateDatabaseSchema {
    * Description of the schema instance. What it has and how to use it.
    */
   description?: string;
+  /**
+   * Display Name that identifies this database schema.
+   */
+  displayName?: string;
   /**
    * Name that identifies this database schema instance uniquely.
    */

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createGlossaryTerm.ts

@@ -21,7 +21,7 @@ export interface CreateGlossaryTerm {
    */
   description: string;
   /**
-   * Display Name that identifies this glossary.
+   * Display Name that identifies this glossary term.
    */
   displayName?: string;
   /**

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createLocation.ts

@@ -20,6 +20,10 @@ export interface CreateLocation {
    * Description of the location instance.
    */
   description?: string;
+  /**
+   * Display Name that identifies this location.
+   */
+  displayName?: string;
   locationType?: LocationType;
   /**
    * Name that identifies this Location.

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTable.ts

@@ -28,6 +28,14 @@ export interface CreateTable {
    * Description of entity instance.
    */
   description?: string;
+  /**
+   * Display Name that identifies this table.
+   */
+  displayName?: string;
+  /**
+   * Entity extension data with custom attributes added to the entity.
+   */
+  extension?: any;
   /**
    * Name that identifies the this entity instance uniquely. Same as id if when name is not
    * unique

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTopic.ts

@@ -24,6 +24,10 @@ export interface CreateTopic {
    * Description of the topic instance. What it has and how to use it.
    */
   description?: string;
+  /**
+   * Display Name that identifies this topic.
+   */
+  displayName?: string;
   /**
    * Maximum message size in bytes. For Kafka - `max.message.bytes` configuration.
    */

openmetadata-ui/src/main/resources/ui/src/generated/api/events/createWebhook.ts

@@ -24,6 +24,10 @@ export interface CreateWebhook {
    * Description of the application
    */
   description?: string;
+  /**
+   * Display Name that identifies this webhook.
+   */
+  displayName?: string;
   /**
    * When set to `true`, the webhook event notification is enabled. Set it to `false` to
    * disable the subscription. (Default `true`)

openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDashboardService.ts

@@ -21,6 +21,10 @@ export interface CreateDashboardService {
    * Description of dashboard service entity.
    */
   description?: string;
+  /**
+   * Display Name that identifies this dashboard service.
+   */
+  displayName?: string;
   /**
    * Name that identifies the this entity instance uniquely
    */

openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts

@@ -21,6 +21,10 @@ export interface CreateDatabaseService {
    * Description of Database entity.
    */
   description?: string;
+  /**
+   * Display Name that identifies this database service.
+   */
+  displayName?: string;
   /**
    * Name that identifies the this entity instance uniquely
    */

openmetadata-ui/src/main/resources/ui/src/generated/api/services/createMessagingService.ts

@@ -21,6 +21,11 @@ export interface CreateMessagingService {
    * Description of messaging service entity.
    */
   description?: string;
+  /**
+   * Display Name that identifies this messaging service. It could be title or label from the
+   * source services.
+   */
+  displayName?: string;
   /**
    * Name that identifies the this entity instance uniquely
    */

openmetadata-ui/src/main/resources/ui/src/generated/api/services/createPipelineService.ts

@@ -20,6 +20,10 @@ export interface CreatePipelineService {
    * Description of pipeline service entity.
    */
   description?: string;
+  /**
+   * Display Name that identifies this pipeline service.
+   */
+  displayName?: string;
   /**
    * Schedule for running pipeline ingestion jobs
    */

openmetadata-ui/src/main/resources/ui/src/generated/api/services/createStorageService.ts

@@ -20,6 +20,10 @@ export interface CreateStorageService {
    * Description of Storage entity.
    */
   description?: string;
+  /**
+   * Display Name that identifies this storage service.
+   */
+  displayName?: string;
   /**
    * Name that identifies the this entity instance uniquely
    */

openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts

@@ -22,7 +22,7 @@ export interface CreateIngestionPipeline {
    */
   description?: string;
   /**
-   * Display Name that identifies this pipeline.
+   * Display Name that identifies this ingeestion pipeline.
    */
   displayName?: string;
   /**

openmetadata-ui/src/main/resources/ui/src/generated/api/tags/createTag.ts

@@ -24,5 +24,9 @@ export interface CreateTag {
    * Unique name of the tag category
    */
   description: string;
+  /**
+   * Display Name that identifies this tag.
+   */
+  displayName?: string;
   name: string;
 }

openmetadata-ui/src/main/resources/ui/src/generated/api/tags/createTagCategory.ts

@@ -21,6 +21,10 @@ export interface CreateTagCategory {
    * Description of the tag category
    */
   description: string;
+  /**
+   * Display Name that identifies this tag category.
+   */
+  displayName?: string;
   name: string;
 }
 

openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts

@@ -26,10 +26,6 @@ export interface Table {
    * Columns in this table.
    */
   columns: Column[];
-  /**
-   * Custom attributes added to the entity
-   */
-  customAttributes?: Type[];
   /**
    * Reference to Database that contains this table.
    */
@@ -51,12 +47,15 @@ export interface Table {
    * Description of a table.
    */
   description?: string;
-  displayConfig?: any;
   /**
    * Display Name that identifies this table. It could be title or label from the source
    * services.
    */
   displayName?: string;
+  /**
+   * Entity extension data with custom attributes added to the entity.
+   */
+  extension?: any;
   /**
    * Followers of this table.
    */
@@ -662,62 +661,6 @@ export enum State {
   Suggested = 'Suggested',
 }
 
-/**
- * This schema defines a type entity used for extending an entity with custom attributes.
- */
-export interface Type {
-  /**
-   * Change that lead to this version of the entity.
-   */
-  changeDescription?: ChangeDescription;
-  /**
-   * Optional description of entity.
-   */
-  description: string;
-  /**
-   * Display Name that identifies this type.
-   */
-  displayName?: string;
-  /**
-   * FullyQualifiedName same as `name`.
-   */
-  fullyQualifiedName?: string;
-  /**
-   * Link to this table resource.
-   */
-  href?: string;
-  /**
-   * Unique identifier of the type instance.
-   */
-  id?: string;
-  /**
-   * Unique name that identifies the type.
-   */
-  name: string;
-  /**
-   * Namespace or group to which this type belongs to.
-   */
-  nameSpace: string;
-  /**
-   * JSON schema encoded as string that defines the type. This will be used to validate the
-   * type values.
-   */
-  schema?: string;
-  /**
-   * Last update time corresponding to the new version of the entity in Unix epoch time
-   * milliseconds.
-   */
-  updatedAt?: number;
-  /**
-   * User who made the update.
-   */
-  updatedBy?: string;
-  /**
-   * Metadata version of the entity.
-   */
-  version?: number;
-}
-
 /**
  * This captures information about how the table is modeled. Currently only DBT model is
  * supported.

openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/metadata/openMetadataConnection.ts

@@ -25,6 +25,10 @@ export interface OpenMetadataConnection {
    * the one configured on OpenMetadata server.
    */
   authProvider?: AuthProvider;
+  /**
+   * Validate Openmetadata Server & Client Version.
+   */
+  enableVersionValidation?: boolean;
   /**
    * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api
    */

openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts

@@ -617,6 +617,10 @@ export interface Connection {
    * the one configured on OpenMetadata server.
    */
   authProvider?: AuthProvider;
+  /**
+   * Validate Openmetadata Server & Client Version.
+   */
+  enableVersionValidation?: boolean;
   /**
    * OpenMetadata Client security configuration.
    */

openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts

@@ -220,6 +220,10 @@ export interface OpenMetadataConnection {
    * the one configured on OpenMetadata server.
    */
   authProvider?: AuthProvider;
+  /**
+   * Validate Openmetadata Server & Client Version.
+   */
+  enableVersionValidation?: boolean;
   /**
    * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api
    */
@@ -1081,6 +1085,10 @@ export interface Connection {
    * the one configured on OpenMetadata server.
    */
   authProvider?: AuthProvider;
+  /**
+   * Validate Openmetadata Server & Client Version.
+   */
+  enableVersionValidation?: boolean;
   /**
    * OpenMetadata Client security configuration.
    */

openmetadata-ui/src/main/resources/ui/src/generated/entity/services/metadataService.ts

@@ -229,6 +229,10 @@ export interface Connection {
    * the one configured on OpenMetadata server.
    */
   authProvider?: AuthProvider;
+  /**
+   * Validate Openmetadata Server & Client Version.
+   */
+  enableVersionValidation?: boolean;
   /**
    * OpenMetadata Client security configuration.
    */

openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts

@@ -689,6 +689,10 @@ export interface Connection {
    * the one configured on OpenMetadata server.
    */
   authProvider?: AuthProvider;
+  /**
+   * Validate Openmetadata Server & Client Version.
+   */
+  enableVersionValidation?: boolean;
   /**
    * OpenMetadata Client security configuration.
    */
@@ -1217,6 +1221,10 @@ export interface OpenMetadataConnection {
    * the one configured on OpenMetadata server.
    */
   authProvider?: AuthProvider;
+  /**
+   * Validate Openmetadata Server & Client Version.
+   */
+  enableVersionValidation?: boolean;
   /**
    * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api
    */