From c34abffab4d0d2729ab3086f0e454de2a54284f2 Mon Sep 17 00:00:00 2001 From: darth-coder00 <86726556+darth-coder00@users.noreply.github.com> Date: Mon, 22 Nov 2021 21:55:40 +0530 Subject: [PATCH] Bugfix: Fix json schemas to avoid creating empty TS interface files (#1323) --- .../operations/workflows/createIngestion.json | 2 +- .../json/schema/entity/policies/filters.json | 2 +- .../operations/workflows/ingestion.json | 2 +- .../operations/workflows/createIngestion.ts | 246 +++++++++++++ .../operations/workflows/ingestion.ts | 341 ++++++++++++++++++ 5 files changed, 590 insertions(+), 3 deletions(-) create mode 100644 catalog-rest-service/src/main/resources/ui/src/generated/api/operations/workflows/createIngestion.ts create mode 100644 catalog-rest-service/src/main/resources/ui/src/generated/operations/workflows/ingestion.ts diff --git a/catalog-rest-service/src/main/resources/json/schema/api/operations/workflows/createIngestion.json b/catalog-rest-service/src/main/resources/json/schema/api/operations/workflows/createIngestion.json index 4cf25bf0bbe..3433175ff23 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/operations/workflows/createIngestion.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/operations/workflows/createIngestion.json @@ -1,5 +1,5 @@ { - "$id": "https://open-metadata.org/api/operations/workflows/ingestion.json", + "$id": "https://open-metadata.org/schema/api/operations/workflows/ingestion.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "Create Ingestion Entity Request", "description": "Ingestion Config is used to setup a Airflow Ingestion pipeline.", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/policies/filters.json b/catalog-rest-service/src/main/resources/json/schema/entity/policies/filters.json index a26e569c4af..52f52d3a06a 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/policies/filters.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/policies/filters.json @@ -1,5 +1,5 @@ { - "$id": "https://open-metadata.org/schema/entity/data/policies/filters.json", + "$id": "https://open-metadata.org/schema/entity/policies/filters.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "Filters", "definitions": { diff --git a/catalog-rest-service/src/main/resources/json/schema/operations/workflows/ingestion.json b/catalog-rest-service/src/main/resources/json/schema/operations/workflows/ingestion.json index 35c8391df5f..9015facff0d 100644 --- a/catalog-rest-service/src/main/resources/json/schema/operations/workflows/ingestion.json +++ b/catalog-rest-service/src/main/resources/json/schema/operations/workflows/ingestion.json @@ -1,5 +1,5 @@ { - "$id": "https://open-metadata.org/operations/workflows/ingestion.json", + "$id": "https://open-metadata.org/schema/operations/workflows/ingestion.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "Ingestion", "description": "Ingestion Config is used to setup a Airflow Ingestion pipeline.", diff --git a/catalog-rest-service/src/main/resources/ui/src/generated/api/operations/workflows/createIngestion.ts b/catalog-rest-service/src/main/resources/ui/src/generated/api/operations/workflows/createIngestion.ts new file mode 100644 index 00000000000..1dc1493874a --- /dev/null +++ b/catalog-rest-service/src/main/resources/ui/src/generated/api/operations/workflows/createIngestion.ts @@ -0,0 +1,246 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Ingestion Config is used to setup a Airflow Ingestion pipeline. + */ +export interface CreateIngestion { + /** + * Concurrency of the Pipeline. + */ + concurrency?: number; + connectorConfig: ConnectorConfig; + /** + * Description of the workflow. + */ + description?: string; + /** + * Display Name that identifies this Ingestion. + */ + displayName?: string; + /** + * End Date of the workflow. + */ + endDate?: Date; + /** + * Deploy the workflow by overwriting existing workflow with the same name. + */ + forceDeploy?: boolean; + ingestionType?: IngestionType; + /** + * Name that identifies this ingestion instance uniquely. + */ + name: string; + /** + * Owner of this Ingestion. + */ + owner?: EntityReference; + /** + * pause the workflow from running once the deploy is finished successfully. + */ + pauseWorkflow?: boolean; + /** + * Retry workflow in case of failure + */ + retries?: number; + /** + * Delay between retries in seconds. + */ + retryDelay?: number; + /** + * Scheduler Interval for the Workflow in cron format. + */ + scheduleInterval?: string; + /** + * Link to the database service where this database is hosted in. + */ + service: EntityReference; + /** + * Start date of the workflow. + */ + startDate: Date; + /** + * Tags associated with the Ingestion. + */ + tags?: TagLabel[]; + /** + * Workflow catchup for past executions. + */ + workflowCatchup?: boolean; + /** + * Timeout for the workflow in seconds. + */ + workflowTimeout?: number; + /** + * Timezone in which workflow going to be scheduled. + */ + workflowTimezone?: string; +} + +/** + * This defines the configuration for connector. + */ +export interface ConnectorConfig { + /** + * Database of the data source. + */ + database?: string; + /** + * Run data profiler as part of ingestion to get table profile data. + */ + enableDataProfiler?: boolean; + /** + * Regex exclude tables or databases that matches the pattern. + */ + excludeFilterPattern?: + | string[] + | boolean + | number + | number + | { [key: string]: any } + | null + | string; + /** + * Host and port of the data source. + */ + host?: string; + /** + * Regex to only fetch tables or databases that matches the pattern. + */ + includeFilterPattern?: string[]; + /** + * optional configuration to turn off fetching metadata for views. + */ + includeViews?: boolean; + /** + * password to connect to the data source. + */ + password?: string; + /** + * username to connect to the data source. + */ + username?: string; +} + +/** + * Type of Ingestion - Bigquery, Redshift, Snowflake etc... + */ +export enum IngestionType { + Bigquery = 'bigquery', + BigqueryUsage = 'bigquery-usage', + Hive = 'hive', + Mssql = 'mssql', + Mysql = 'mysql', + Postgres = 'postgres', + Redshift = 'redshift', + RedshiftUsage = 'redshift-usage', + Snowflake = 'snowflake', + SnowflakeUsage = 'snowflake-usage', + Trino = 'trino', + Vertica = 'vertica', +} + +/** + * Owner of this Ingestion. + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + * + * Link to the database service where this database is hosted in. + */ +export interface EntityReference { + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * Name of the entity instance. For entities such as tables, databases where the name is not + * unique, fullyQualifiedName is returned in this field. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `redshift`, `mysql`, + * `bigquery`, `snowflake`... + */ + type: string; +} + +/** + * This schema defines the type for labeling an entity with a Tag. + */ +export interface TagLabel { + /** + * Unique name of the tag category. + */ + description?: string; + /** + * Link to the tag resource. + */ + href?: string; + /** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see TagCategory.json for more details). 'Propagated` indicates a tag label + * was propagated from upstream based on lineage. 'Automated' is used when a tool was used + * to determine the tag label. + */ + labelType: LabelType; + /** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ + state: State; + tagFQN: string; +} + +/** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see TagCategory.json for more details). 'Propagated` indicates a tag label + * was propagated from upstream based on lineage. 'Automated' is used when a tool was used + * to determine the tag label. + */ +export enum LabelType { + Automated = 'Automated', + Derived = 'Derived', + Manual = 'Manual', + Propagated = 'Propagated', +} + +/** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ +export enum State { + Confirmed = 'Confirmed', + Suggested = 'Suggested', +} diff --git a/catalog-rest-service/src/main/resources/ui/src/generated/operations/workflows/ingestion.ts b/catalog-rest-service/src/main/resources/ui/src/generated/operations/workflows/ingestion.ts new file mode 100644 index 00000000000..90320d8f645 --- /dev/null +++ b/catalog-rest-service/src/main/resources/ui/src/generated/operations/workflows/ingestion.ts @@ -0,0 +1,341 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Ingestion Config is used to setup a Airflow Ingestion pipeline. + */ +export interface Ingestion { + /** + * Change that led to this version of the entity. + */ + changeDescription?: ChangeDescription; + /** + * Concurrency of the Pipeline. + */ + concurrency?: number; + connectorConfig: ConnectorConfig; + /** + * Description of the workflow. + */ + description?: string; + /** + * Display Name that identifies this Ingestion. + */ + displayName?: string; + /** + * End Date of the workflow. + */ + endDate?: Date; + /** + * Deploy the workflow by overwriting existing workflow with the same name. + */ + forceDeploy?: boolean; + /** + * Name that uniquely identifies a Ingestion. + */ + fullyQualifiedName?: string; + /** + * Link to this ingestion resource. + */ + href?: string; + /** + * Unique identifier that identifies this Ingestion. + */ + id?: string; + /** + * List of executions and status for the Ingestion Pipeline. + */ + ingestionStatuses?: IngestionStatus[]; + ingestionType?: IngestionType; + /** + * Name that identifies this ingestion instance uniquely. + */ + name: string; + /** + * Next execution date from the underlying workflow platform once the ingestion scheduled. + */ + nextExecutionDate?: Date; + /** + * Owner of this Ingestion. + */ + owner?: EntityReference; + /** + * pause the workflow from running once the deploy is finished successfully. + */ + pauseWorkflow?: boolean; + /** + * Retry workflow in case of failure. + */ + retries?: number; + /** + * Delay between retries in seconds. + */ + retryDelay?: number; + /** + * Scheduler Interval for the Workflow in cron format. + */ + scheduleInterval?: string; + /** + * Link to the database service where this database is hosted in. + */ + service: EntityReference; + /** + * Start date of the workflow. + */ + startDate: Date; + /** + * Tags associated with the Ingestion. + */ + tags?: TagLabel[]; + /** + * Last update time corresponding to the new version of the entity. + */ + updatedAt?: Date; + /** + * User who made the update. + */ + updatedBy?: string; + /** + * Metadata version of the entity. + */ + version?: number; + /** + * Run past executions if the start date is in the past. + */ + workflowCatchup?: boolean; + /** + * Timeout for the workflow in seconds. + */ + workflowTimeout?: number; + /** + * Timezone in which workflow going to be scheduled. + */ + workflowTimezone?: string; +} + +/** + * Change that led to this version of the entity. + * + * Description of the change. + */ +export interface ChangeDescription { + /** + * Names of fields added during the version changes. + */ + fieldsAdded?: FieldChange[]; + /** + * Fields deleted during the version changes with old value before deleted. + */ + fieldsDeleted?: FieldChange[]; + /** + * Fields modified during the version changes with old and new values. + */ + fieldsUpdated?: FieldChange[]; + /** + * When a change did not result in change, this could be same as the current version. + */ + previousVersion?: number; +} + +export interface FieldChange { + /** + * Name of the entity field that changed. + */ + name?: string; + /** + * New value of the field. Note that this is a JSON string and use the corresponding field + * type to deserialize it. + */ + newValue?: any; + /** + * Previous value of the field. Note that this is a JSON string and use the corresponding + * field type to deserialize it. + */ + oldValue?: any; +} + +/** + * This defines the configuration for connector. + */ +export interface ConnectorConfig { + /** + * Database of the data source. + */ + database?: string; + /** + * Run data profiler as part of ingestion to get table profile data. + */ + enableDataProfiler?: boolean; + /** + * Regex exclude tables or databases that matches the pattern. + */ + excludeFilterPattern?: + | string[] + | boolean + | number + | number + | { [key: string]: any } + | null + | string; + /** + * Host and port of the data source. + */ + host?: string; + /** + * Regex to only fetch tables or databases that matches the pattern. + */ + includeFilterPattern?: string[]; + /** + * optional configuration to turn off fetching metadata for views. + */ + includeViews?: boolean; + /** + * password to connect to the data source. + */ + password?: string; + /** + * username to connect to the data source. + */ + username?: string; +} + +/** + * This defines the runtime status of Ingestion. + */ +export interface IngestionStatus { + /** + * endDate of the Ingestion pipeline run for this particular execution. + */ + endDate?: string; + /** + * startDate of the Ingestion Pipeline run for this particular execution. + */ + startDate?: string; + /** + * Workflow status denotes if its failed or succeeded. + */ + state?: string; +} + +/** + * Type of Ingestion - Bigquery, Redshift, Snowflake etc... + */ +export enum IngestionType { + Bigquery = 'bigquery', + BigqueryUsage = 'bigquery-usage', + Hive = 'hive', + Mssql = 'mssql', + Mysql = 'mysql', + Postgres = 'postgres', + Redshift = 'redshift', + RedshiftUsage = 'redshift-usage', + Snowflake = 'snowflake', + SnowflakeUsage = 'snowflake-usage', + Trino = 'trino', + Vertica = 'vertica', +} + +/** + * Owner of this Ingestion. + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + * + * Link to the database service where this database is hosted in. + */ +export interface EntityReference { + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * Name of the entity instance. For entities such as tables, databases where the name is not + * unique, fullyQualifiedName is returned in this field. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `redshift`, `mysql`, + * `bigquery`, `snowflake`... + */ + type: string; +} + +/** + * This schema defines the type for labeling an entity with a Tag. + */ +export interface TagLabel { + /** + * Unique name of the tag category. + */ + description?: string; + /** + * Link to the tag resource. + */ + href?: string; + /** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see TagCategory.json for more details). 'Propagated` indicates a tag label + * was propagated from upstream based on lineage. 'Automated' is used when a tool was used + * to determine the tag label. + */ + labelType: LabelType; + /** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ + state: State; + tagFQN: string; +} + +/** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see TagCategory.json for more details). 'Propagated` indicates a tag label + * was propagated from upstream based on lineage. 'Automated' is used when a tool was used + * to determine the tag label. + */ +export enum LabelType { + Automated = 'Automated', + Derived = 'Derived', + Manual = 'Manual', + Propagated = 'Propagated', +} + +/** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ +export enum State { + Confirmed = 'Confirmed', + Suggested = 'Suggested', +}