From 61cb071ebc0be64ae2100b31e808b58dfd75a90b Mon Sep 17 00:00:00 2001 From: Shailesh Parmar Date: Wed, 20 Apr 2022 18:38:01 +0530 Subject: [PATCH] fix issue: 4177 Implement pipeline status for service page (#4281) --- .../Ingestion/Ingestion.component.tsx | 89 +++-- .../createIngestionPipeline.ts | 139 ++++++-- .../ingestionPipelines/ingestionPipeline.ts | 318 +++++++++++++++--- .../resources/ui/src/pages/service/index.tsx | 2 +- 4 files changed, 440 insertions(+), 108 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx index 432ae1cc763..a7138d2288d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx @@ -168,53 +168,49 @@ const Ingestion: React.FC = ({ : ingestionList; }, [searchText, ingestionList]); - /* eslint-disable max-len */ - // TODO:- once api support status we need below function - // const getStatuses = (ingestion: AirflowPipeline) => { - // const lastFiveIngestions = ingestion.pipelineStatuses - // ?.sort((a, b) => { - // // Turn your strings into millis, and then subtract them - // // to get a value that is either negative, positive, or zero. - // const date1 = new Date(a.startDate || ''); - // const date2 = new Date(b.startDate || ''); + const getStatuses = (ingestion: IngestionPipeline) => { + const lastFiveIngestions = ingestion.pipelineStatuses + ?.sort((a, b) => { + // Turn your strings into millis, and then subtract them + // to get a value that is either negative, positive, or zero. + const date1 = new Date(a.startDate || ''); + const date2 = new Date(b.startDate || ''); - // return date1.getTime() - date2.getTime(); - // }) - // .slice(Math.max(ingestion.pipelineStatuses.length - 5, 0)); + return date1.getTime() - date2.getTime(); + }) + .slice(Math.max(ingestion.pipelineStatuses.length - 5, 0)); - // return lastFiveIngestions?.map((r, i) => { - // return ( - // - // {r.startDate ? ( - //

Start Date: {new Date(r.startDate).toUTCString()}

- // ) : null} - // {r.endDate ? ( - //

End Date: {new Date(r.endDate).toUTCString()}

- // ) : null} - // - // } - // key={i} - // position="bottom" - // theme="light" - // trigger="mouseenter"> - // {i === lastFiveIngestions.length - 1 ? ( - //

- // {capitalize(r.state)} - //

- // ) : ( - //

- // )} - // - // ); - // }); - // }; - - /* eslint-enable max-len */ + return lastFiveIngestions?.map((r, i) => { + return ( + + {r.startDate ? ( +

Start Date: {new Date(r.startDate).toUTCString()}

+ ) : null} + {r.endDate ? ( +

End Date: {new Date(r.endDate).toUTCString()}

+ ) : null} + + } + key={i} + position="bottom" + theme="light" + trigger="mouseenter"> + {i === lastFiveIngestions.length - 1 ? ( +

+ {capitalize(r.state)} +

+ ) : ( +

+ )} + + ); + }); + }; const getIngestionTab = () => { return ( @@ -312,8 +308,7 @@ const Ingestion: React.FC = ({ )} - {/* TODO:- update this once api support pipeline status */} - {/*

{getStatuses(ingestion)}
*/} +
{getStatuses(ingestion)}
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts index 4ad55ff5e96..719a447f0e7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts @@ -65,10 +65,6 @@ export interface AirflowConfig { * Maximum Number of active runs. */ maxActiveRuns?: number; - /** - * Next execution date from the underlying pipeline platform once the pipeline scheduled. - */ - nextExecutionDate?: Date; /** * pause the pipeline from running once the deploy is finished successfully. */ @@ -77,18 +73,10 @@ export interface AirflowConfig { * Run past executions if the start date is in the past. */ pipelineCatchup?: boolean; - /** - * Timeout for the pipeline in seconds. - */ - pipelineTimeout?: number; /** * Timezone in which pipeline going to be scheduled. */ pipelineTimezone?: string; - /** - * File system directory path where managed python operator files are stored. - */ - pythonOperatorLocation?: string; /** * Retry pipeline in case of failure. */ @@ -101,10 +89,6 @@ export interface AirflowConfig { * Scheduler Interval for the pipeline in cron format. */ scheduleInterval?: string; - /** - * python method call back on SLA miss. - */ - slaMissCallback?: string; /** * Start date of the pipeline. */ @@ -188,13 +172,24 @@ export interface SourceConfig { export interface ConfigClass { /** - * DBT catalog file to extract dbt models with their column schemas. + * DBT Catalog file name */ - dbtCatalogFilePath?: string; + dbtCatalogFileName?: string; /** - * DBT manifest file path to extract dbt models and associate with tables. + * DBT configuration. */ - dbtManifestFilePath?: string; + dbtConfig?: LocalHTTPDbtConfig; + /** + * DBT Manifest file name + */ + dbtManifestFileName?: string; + /** + * Method from which the DBT files will be fetched. Accepted values are: 's3'(Required aws + * s3 credentials to be provided), 'gcs'(Required gcs credentials to be provided), + * 'gcs-path'(path of the file containing gcs credentials), 'local'(path of dbt files on + * local system), 'http'(url path of dbt files). + */ + dbtProvider?: DbtProvider; /** * Run data profiler as part of this metadata ingestion to get table profile data. */ @@ -274,3 +269,107 @@ export interface FilterPattern { */ includes?: string[]; } + +/** + * DBT configuration. + * + * Local and HTTP DBT configs. + * + * GCS credentials configs. + * + * AWS S3 credentials configs. + */ +export interface LocalHTTPDbtConfig { + /** + * DBT catalog file to extract dbt models with their column schemas. + */ + dbtCatalogFilePath?: string; + /** + * DBT manifest file path to extract dbt models and associate with tables. + */ + dbtManifestFilePath?: string; + /** + * GCS configs. + */ + gcsConfig?: GCSValues | string; + /** + * AWS Access key ID. + */ + awsAccessKeyId?: string; + /** + * AWS Region + */ + awsRegion?: string; + /** + * AWS Secret Access Key. + */ + awsSecretAccessKey?: string; + /** + * AWS Session Token. + */ + awsSessionToken?: string; + /** + * EndPoint URL for the AWS + */ + endPointURL?: string; +} + +/** + * GCS Credentials. + */ +export interface GCSValues { + /** + * Google Cloud auth provider certificate. + */ + authProviderX509CertUrl?: string; + /** + * Google Cloud auth uri. + */ + authUri?: string; + /** + * Google Cloud email. + */ + clientEmail?: string; + /** + * Google Cloud Client ID. + */ + clientId?: string; + /** + * Google Cloud client certificate uri. + */ + clientX509CertUrl?: string; + /** + * Google Cloud private key. + */ + privateKey?: string; + /** + * Google Cloud private key id. + */ + privateKeyId?: string; + /** + * Google Cloud project id. + */ + projectId?: string; + /** + * Google Cloud token uri. + */ + tokenUri?: string; + /** + * Google Cloud service account type. + */ + type?: string; +} + +/** + * Method from which the DBT files will be fetched. Accepted values are: 's3'(Required aws + * s3 credentials to be provided), 'gcs'(Required gcs credentials to be provided), + * 'gcs-path'(path of the file containing gcs credentials), 'local'(path of dbt files on + * local system), 'http'(url path of dbt files). + */ +export enum DbtProvider { + Gcs = 'gcs', + GcsPath = 'gcs-path', + HTTP = 'http', + Local = 'local', + S3 = 's3', +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts index 525198be8d7..8902c28d464 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts @@ -50,11 +50,19 @@ export interface IngestionPipeline { * Name that identifies this pipeline instance uniquely. */ name: string; + /** + * Next execution date from the underlying pipeline platform once the pipeline scheduled. + */ + nextExecutionDate?: Date; openMetadataServerConnection: OpenMetadataConnection; /** * Owner of this Pipeline. */ owner?: EntityReference; + /** + * List of executions and status for the Pipeline. + */ + pipelineStatuses?: PipelineStatus[]; pipelineType: PipelineType; /** * Link to the database service where this database is hosted in. @@ -100,10 +108,6 @@ export interface AirflowConfig { * Maximum Number of active runs. */ maxActiveRuns?: number; - /** - * Next execution date from the underlying pipeline platform once the pipeline scheduled. - */ - nextExecutionDate?: Date; /** * pause the pipeline from running once the deploy is finished successfully. */ @@ -112,18 +116,10 @@ export interface AirflowConfig { * Run past executions if the start date is in the past. */ pipelineCatchup?: boolean; - /** - * Timeout for the pipeline in seconds. - */ - pipelineTimeout?: number; /** * Timezone in which pipeline going to be scheduled. */ pipelineTimezone?: string; - /** - * File system directory path where managed python operator files are stored. - */ - pythonOperatorLocation?: string; /** * Retry pipeline in case of failure. */ @@ -136,10 +132,6 @@ export interface AirflowConfig { * Scheduler Interval for the pipeline in cron format. */ scheduleInterval?: string; - /** - * python method call back on SLA miss. - */ - slaMissCallback?: string; /** * Start date of the pipeline. */ @@ -216,10 +208,47 @@ export interface OpenMetadataConnection { * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api */ hostPort: string; + /** + * Include Dashboards for Indexing + */ + includeDashboards?: boolean; + /** + * Include Glossary Terms for Indexing + */ + includeGlossaryTerms?: boolean; + /** + * Include Pipelines for Indexing + */ + includePipelines?: boolean; + /** + * Include Tables for Indexing + */ + includeTables?: boolean; + /** + * Include Teams for Indexing + */ + includeTeams?: boolean; + /** + * Include Topics for Indexing + */ + includeTopics?: boolean; + /** + * Include Users for Indexing + */ + includeUsers?: boolean; + /** + * Limit the number of records for Indexing. + */ + limitRecords?: number; /** * OpenMetadata Client security configuration. */ securityConfig?: SsoClientConfig; + supportsMetadataExtraction?: boolean; + /** + * Service Type + */ + type?: OpenmetadataType; } /** @@ -307,6 +336,15 @@ export interface SsoClientConfig { tokenEndpoint?: string; } +/** + * Service Type + * + * OpenMetadata service type + */ +export enum OpenmetadataType { + OpenMetadata = 'OpenMetadata', +} + /** * Owner of this Pipeline. * @@ -355,6 +393,28 @@ export interface EntityReference { type: string; } +/** + * This defines runtime status of Pipeline. + */ +export interface PipelineStatus { + /** + * endDate of the pipeline run for this particular execution. + */ + endDate?: string; + /** + * Pipeline unique run ID. + */ + runId?: string; + /** + * startDate of the pipeline run for this particular execution. + */ + startDate?: string; + /** + * Pipeline status denotes if its failed or succeeded. + */ + state?: string; +} + /** * Type of Pipeline - metadata, usage */ @@ -420,7 +480,7 @@ export interface ServiceConnection { * * Databricks Connection Config * - * DB2 Connection Config + * Db2 Connection Config * * DeltaLake Database Connection Config * @@ -465,6 +525,10 @@ export interface ServiceConnection { * Pulsar Connection Config * * Amundsen Connection Config + * + * Metadata to ElasticSeach Connection Config + * + * OpenMetadata Connection Config */ export interface Connection { /** @@ -478,6 +542,10 @@ export interface Connection { * * Host and Port of Metabase instance. * + * Dashboard URL for the power BI. + * + * URL for the redash instance + * * URL for the superset instance * * Tableau Server @@ -494,10 +562,6 @@ export interface Connection { * * Host and port of the Druid * - * Host and port of the DynamoDB - * - * Host and port of the Glue - * * Host and port of the Hive. * * Host and port of the data source. @@ -513,8 +577,10 @@ export interface Connection { * Host and port of the Redshift. * * Host and port of the Amundsen Neo4j Connection. + * + * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api */ - hostPort?: string; + hostPort?: any; /** * password to connect to the Looker. * @@ -565,7 +631,7 @@ export interface Connection { /** * Service Type */ - type?: AmundsenType; + type?: Type; /** * username to connect to the Looker. This user should have privileges to read all the * metadata in Looker. @@ -649,12 +715,10 @@ export interface Connection { clientSecret?: string; /** * Credentials for the PowerBI. + * + * GCS Credentials */ - credentials?: string; - /** - * Dashboard URL for the power BI. - */ - dashboardURL?: string; + credentials?: GCSCredentials | string; /** * Dashboard redirect URI for the PowerBI. */ @@ -667,10 +731,6 @@ export interface Connection { * API key of the redash instance to access. */ apiKey?: string; - /** - * URL for the redash instance - */ - redashURL?: string; /** * Additional connection options that can be sent to service during the connection. */ @@ -685,6 +745,8 @@ export interface Connection { provider?: string; /** * Tableau API version + * + * OpenMetadata server API version to use. */ apiVersion?: string; /** @@ -795,9 +857,9 @@ export interface Connection { */ partitionQueryDuration?: number; /** - * Google BigQuery project id. + * BigQuery project ID. Inform it here if passing the credentials path. */ - projectID?: string; + projectId?: string; /** * SQLAlchemy driver scheme options. */ @@ -848,7 +910,7 @@ export interface Connection { /** * AWS Access key ID. */ - awsAccessKeyId?: any; + awsAccessKeyId?: string; /** * AWS Secret Access Key. */ @@ -950,6 +1012,105 @@ export interface Connection { * Enable SSL validation for the Amundsen Neo4j Connection. */ validateSSL?: boolean; + /** + * Include Dashboards for Indexing + */ + includeDashboards?: boolean; + /** + * Include Glossary Terms for Indexing + */ + includeGlossaryTerms?: boolean; + /** + * Include Pipelines for Indexing + */ + includePipelines?: boolean; + /** + * Include Tables for Indexing + */ + includeTables?: boolean; + /** + * Include Teams for Indexing + */ + includeTeams?: boolean; + /** + * Include Topics for Indexing + */ + includeTopics?: boolean; + /** + * Include Users for Indexing + */ + includeUsers?: boolean; + /** + * Limit the number of records for Indexing. + */ + limitRecords?: number; + /** + * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as + * the one configured on OpenMetadaata server. + */ + authProvider?: AuthProvider; + /** + * OpenMetadata Client security configuration. + */ + securityConfig?: SsoClientConfig; +} + +/** + * GCS Credentials + * + * GCS credentials configs. + */ +export interface GCSCredentials { + /** + * GCS configs. + */ + gcsConfig: GCSValues | string; +} + +/** + * GCS Credentials. + */ +export interface GCSValues { + /** + * Google Cloud auth provider certificate. + */ + authProviderX509CertUrl?: string; + /** + * Google Cloud auth uri. + */ + authUri?: string; + /** + * Google Cloud email. + */ + clientEmail?: string; + /** + * Google Cloud Client ID. + */ + clientId?: string; + /** + * Google Cloud client certificate uri. + */ + clientX509CertUrl?: string; + /** + * Google Cloud private key. + */ + privateKey?: string; + /** + * Google Cloud private key id. + */ + privateKeyId?: string; + /** + * Google Cloud project id. + */ + projectId?: string; + /** + * Google Cloud token uri. + */ + tokenUri?: string; + /** + * Google Cloud service account type. + */ + type?: string; } /** @@ -1000,8 +1161,12 @@ export enum Scheme { * Pulsar service type * * Amundsen service type + * + * Metadata to Elastic Seach type + * + * OpenMetadata service type */ -export enum AmundsenType { +export enum Type { Amundsen = 'Amundsen', Athena = 'Athena', AzureSQL = 'AzureSQL', @@ -1018,8 +1183,10 @@ export enum AmundsenType { Looker = 'Looker', MariaDB = 'MariaDB', Metabase = 'Metabase', + MetadataES = 'MetadataES', Mssql = 'Mssql', Mysql = 'Mysql', + OpenMetadata = 'OpenMetadata', Oracle = 'Oracle', Postgres = 'Postgres', PowerBI = 'PowerBI', @@ -1047,13 +1214,24 @@ export interface SourceConfig { export interface ConfigClass { /** - * DBT catalog file to extract dbt models with their column schemas. + * DBT Catalog file name */ - dbtCatalogFilePath?: string; + dbtCatalogFileName?: string; /** - * DBT manifest file path to extract dbt models and associate with tables. + * DBT configuration. */ - dbtManifestFilePath?: string; + dbtConfig?: LocalHTTPDbtConfig; + /** + * DBT Manifest file name + */ + dbtManifestFileName?: string; + /** + * Method from which the DBT files will be fetched. Accepted values are: 's3'(Required aws + * s3 credentials to be provided), 'gcs'(Required gcs credentials to be provided), + * 'gcs-path'(path of the file containing gcs credentials), 'local'(path of dbt files on + * local system), 'http'(url path of dbt files). + */ + dbtProvider?: DbtProvider; /** * Run data profiler as part of this metadata ingestion to get table profile data. */ @@ -1133,3 +1311,63 @@ export interface FilterPattern { */ includes?: string[]; } + +/** + * DBT configuration. + * + * Local and HTTP DBT configs. + * + * GCS Credentials + * + * GCS credentials configs. + * + * AWS S3 credentials configs. + */ +export interface LocalHTTPDbtConfig { + /** + * DBT catalog file to extract dbt models with their column schemas. + */ + dbtCatalogFilePath?: string; + /** + * DBT manifest file path to extract dbt models and associate with tables. + */ + dbtManifestFilePath?: string; + /** + * GCS configs. + */ + gcsConfig?: GCSValues | string; + /** + * AWS Access key ID. + */ + awsAccessKeyId?: string; + /** + * AWS Region + */ + awsRegion?: string; + /** + * AWS Secret Access Key. + */ + awsSecretAccessKey?: string; + /** + * AWS Session Token. + */ + awsSessionToken?: string; + /** + * EndPoint URL for the AWS + */ + endPointURL?: string; +} + +/** + * Method from which the DBT files will be fetched. Accepted values are: 's3'(Required aws + * s3 credentials to be provided), 'gcs'(Required gcs credentials to be provided), + * 'gcs-path'(path of the file containing gcs credentials), 'local'(path of dbt files on + * local system), 'http'(url path of dbt files). + */ +export enum DbtProvider { + Gcs = 'gcs', + GcsPath = 'gcs-path', + HTTP = 'http', + Local = 'local', + S3 = 's3', +} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx index d3546d28c91..ae177add2cb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx @@ -216,7 +216,7 @@ const ServicePage: FunctionComponent = () => { const getAllIngestionWorkflows = (paging?: string) => { setIsloading(true); - getIngestionPipelines(['owner'], serviceFQN, paging) + getIngestionPipelines(['owner', 'pipelineStatuses'], serviceFQN, paging) .then((res) => { if (res.data.data) { setIngestions(res.data.data);