yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-02 19:48:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #2569: Custom Pipeline Service defenition (#2824)

Browse Source 
* Fix #2569: Custom Pipeline Service defenition

* Fix #2569: Custom Pipeline Service defenition

* Adding UI support to show all services available from enum

Co-authored-by: darth-coder00 <aashit@getcollate.io>
This commit is contained in:
Sriharsha Chintalapani 2022-02-17 22:15:09 -08:00 committed by GitHub
parent 78ee32f585
commit 42dc42174f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 127 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json
View File

@ -8,7 +8,7 @@
"pipelineServiceType": {
"description": "Type of pipeline service - Airflow or Prefect.",
"type": "string",
"enum": ["Airflow", "Prefect", "Glue"],
"enum": ["Airflow", "Prefect", "Glue", "Generic"],
"javaEnums": [
{
"name": "Airflow"
@ -18,6 +18,9 @@
},
{
"name": "Glue"
},
{
"name": "Generic"
}
]
}

30
openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx
View File

@ -32,17 +32,21 @@ import React, {
import { ONLY_NUMBER_REGEX } from '../../../constants/constants';
import { serviceTypes } from '../../../constants/services.const';
import {
DashboardServiceType,
MessagingServiceType,
// DashboardServiceType,
// MessagingServiceType,
ServiceCategory,
} from '../../../enums/service.enum';
import {
CreateAirflowPipeline,
Schema,
} from '../../../generated/api/operations/pipelines/createAirflowPipeline';
import { DashboardServiceType } from '../../../generated/entity/services/dashboardService';
// import { DashboardService } from '../../../generated/entity/services/dashboardService';
import { DatabaseService } from '../../../generated/entity/services/databaseService';
import { MessagingService } from '../../../generated/entity/services/messagingService';
import {
MessagingService,
MessagingServiceType,
} from '../../../generated/entity/services/messagingService';
import { PipelineService } from '../../../generated/entity/services/pipelineService';
import { PipelineType } from '../../../generated/operations/pipelines/airflowPipeline';
import {
@ -335,7 +339,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
const markdownRef = useRef<EditorContentRef>();
const getBrokerUrlPlaceholder = (): string => {
return selectService === MessagingServiceType.PULSAR
return selectService === MessagingServiceType.Pulsar
? 'hostname:port'
: 'hostname1:port1, hostname2:port2';
};
@ -460,7 +464,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
dataObj = {
...dataObj,
brokers:
selectService === MessagingServiceType.PULSAR
selectService === MessagingServiceType.Pulsar
? [brokers]
: brokers.split(',').map((broker) => broker.trim()),
schemaRegistry: schemaRegistry,
@ -471,7 +475,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
case ServiceCategory.DASHBOARD_SERVICES:
{
switch (selectService) {
case DashboardServiceType.REDASH:
case DashboardServiceType.Redash:
{
dataObj = {
...dataObj,
@ -482,7 +486,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
}
break;
case DashboardServiceType.TABLEAU:
case DashboardServiceType.Tableau:
{
dataObj = {
...dataObj,
@ -638,7 +642,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
case ServiceCategory.DASHBOARD_SERVICES:
{
switch (selectService) {
case DashboardServiceType.REDASH:
case DashboardServiceType.Redash:
{
setMsg = {
...setMsg,
@ -650,7 +654,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
isValid = Boolean(dashboardUrl && apiKey);
break;
case DashboardServiceType.TABLEAU:
case DashboardServiceType.Tableau:
{
setMsg = {
...setMsg,
@ -1010,7 +1014,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
const getDashboardFields = (): JSX.Element => {
let elemFields: JSX.Element;
switch (selectService) {
case DashboardServiceType.REDASH: {
case DashboardServiceType.Redash: {
elemFields = (
<>
<Field>
@ -1048,7 +1052,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
break;
}
case DashboardServiceType.TABLEAU: {
case DashboardServiceType.Tableau: {
elemFields = (
<>
<Field>
@ -1358,7 +1362,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
case ServiceCategory.DASHBOARD_SERVICES:
switch (selectService) {
case DashboardServiceType.REDASH:
case DashboardServiceType.Redash:
data = [
{
key: 'Dashboard Url',
@ -1394,7 +1398,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
break;
case DashboardServiceType.TABLEAU:
case DashboardServiceType.Tableau:
data = [
{
key: 'Site Name',

22
openmetadata-ui/src/main/resources/ui/src/components/ServiceConfig/ServiceConfig.tsx
View File

@ -3,10 +3,12 @@ import { ServicesData } from 'Models';
import React, { useState } from 'react';
import { ONLY_NUMBER_REGEX } from '../../constants/constants';
import {
DashboardServiceType,
MessagingServiceType,
// DashboardServiceType,
// MessagingServiceType,
ServiceCategory,
} from '../../enums/service.enum';
import { DashboardServiceType } from '../../generated/entity/services/dashboardService';
import { MessagingServiceType } from '../../generated/entity/services/messagingService';
import useToastContext from '../../hooks/useToastContext';
import { errorMsg, requiredField } from '../../utils/CommonUtils';
import {
@ -118,7 +120,7 @@ const ServiceConfig = ({
const [env, setEnv] = useState(data?.env || '');
const getBrokerUrlPlaceholder = (): string => {
return data?.serviceType === MessagingServiceType.PULSAR
return data?.serviceType === MessagingServiceType.Pulsar
? 'hostname:port'
: 'hostname1:port1, hostname2:port2';
};
@ -215,7 +217,7 @@ const ServiceConfig = ({
case ServiceCategory.DASHBOARD_SERVICES: {
switch (data?.serviceType) {
case DashboardServiceType.REDASH:
case DashboardServiceType.Redash:
setShowErrorMsg({
...showErrorMsg,
dashboardUrl: !dashboardUrl,
@ -224,7 +226,7 @@ const ServiceConfig = ({
return Boolean(dashboardUrl && apiKey);
case DashboardServiceType.TABLEAU:
case DashboardServiceType.Tableau:
setShowErrorMsg({
...showErrorMsg,
dashboardUrl: !dashboardUrl,
@ -286,14 +288,14 @@ const ServiceConfig = ({
return {
...data,
brokers:
data?.serviceType === MessagingServiceType.PULSAR
data?.serviceType === MessagingServiceType.Pulsar
? [brokers]
: brokers.split(',').map((broker) => broker.trim()),
schemaRegistry: schemaRegistry,
};
case ServiceCategory.DASHBOARD_SERVICES: {
switch (data?.serviceType) {
case DashboardServiceType.REDASH:
case DashboardServiceType.Redash:
return {
...data,
dashboardUrl: dashboardUrl,
@ -301,7 +303,7 @@ const ServiceConfig = ({
api_key: apiKey,
};
case DashboardServiceType.TABLEAU:
case DashboardServiceType.Tableau:
return {
...data,
dashboardUrl: dashboardUrl,
@ -742,7 +744,7 @@ const ServiceConfig = ({
const getDashboardFields = (): JSX.Element => {
let elemFields: JSX.Element;
switch (data?.serviceType) {
case DashboardServiceType.REDASH: {
case DashboardServiceType.Redash: {
elemFields = (
<>
<Field>
@ -794,7 +796,7 @@ const ServiceConfig = ({
break;
}
case DashboardServiceType.TABLEAU: {
case DashboardServiceType.Tableau: {
elemFields = (
<>
<Field>

37
openmetadata-ui/src/main/resources/ui/src/constants/services.const.ts
View File

@ -16,7 +16,7 @@ import noDataFound from '../assets/img/no-data-placeholder.png';
import noService from '../assets/img/no-service.png';
import airflow from '../assets/img/service-icon-airflow.png';
import athena from '../assets/img/service-icon-athena.png';
import serviceDefault from '../assets/img/service-icon-generic.png';
import databaseDefault from '../assets/img/service-icon-generic.png';
import glue from '../assets/img/service-icon-glue.png';
import hive from '../assets/img/service-icon-hive.png';
import kafka from '../assets/img/service-icon-kafka.png';
@ -38,7 +38,14 @@ import superset from '../assets/img/service-icon-superset.png';
import tableau from '../assets/img/service-icon-tableau.png';
import trino from '../assets/img/service-icon-trino.png';
import vertica from '../assets/img/service-icon-vertica.png';
import dashboardDefault from '../assets/svg/dashboard.svg';
import pipelineDefault from '../assets/svg/pipeline.svg';
import plus from '../assets/svg/plus.svg';
import topicDefault from '../assets/svg/topic.svg';
import { DashboardServiceType } from '../generated/entity/services/dashboardService';
import { DatabaseServiceType } from '../generated/entity/services/databaseService';
import { MessagingServiceType } from '../generated/entity/services/messagingService';
import { PipelineServiceType } from '../generated/entity/services/pipelineService';
export const NoDataFoundPlaceHolder = noDataFound;
export const MYSQL = mysql;
@ -65,31 +72,19 @@ export const METABASE = metabase;
export const AIRFLOW = airflow;
export const PREFECT = prefect;
export const SERVICE_DEFAULT = serviceDefault;
export const DATABASE_DEFAULT = databaseDefault;
export const TOPIC_DEFAULT = topicDefault;
export const DASHBOARD_DEFAULT = dashboardDefault;
export const PIPELINE_DEFAULT = pipelineDefault;
export const PLUS = plus;
export const NOSERVICE = noService;
export const serviceTypes: Record<ServiceTypes, Array<string>> = {
databaseServices: [
'BigQuery',
'MySQL',
'Redshift',
'Snowflake',
'Postgres',
'MSSQL',
'Hive',
'Oracle',
'Athena',
'Presto',
'Trino',
'Glue',
'MariaDB',
'Vertica',
],
messagingServices: ['Kafka'],
dashboardServices: ['Superset', 'Looker', 'Tableau', 'Redash', 'Metabase'],
pipelineServices: ['Airflow', 'Prefect', 'Glue'],
databaseServices: Object.values(DatabaseServiceType),
messagingServices: Object.values(MessagingServiceType),
dashboardServices: Object.values(DashboardServiceType),
pipelineServices: Object.values(PipelineServiceType),
};
export const arrServiceTypes: Array<ServiceTypes> = [

35
openmetadata-ui/src/main/resources/ui/src/enums/service.enum.ts
View File

@ -18,41 +18,6 @@ export enum ServiceCategory {
PIPELINE_SERVICES = 'pipelineServices',
}
export enum DatabaseServiceType {
MYSQL = 'MySQL',
REDSHIFT = 'Redshift',
BIGQUERY = 'BigQuery',
HIVE = 'Hive',
POSTGRES = 'Postgres',
ORACLE = 'Oracle',
SNOWFLAKE = 'Snowflake',
MSSQL = 'MSSQL',
ATHENA = 'Athena',
PRESTO = 'Presto',
TRINO = 'Trino',
GLUE = 'Glue',
MARIADB = 'MariaDB',
VERTICA = 'Vertica',
}
export enum MessagingServiceType {
KAFKA = 'Kafka',
PULSAR = 'Pulsar',
}
export enum DashboardServiceType {
SUPERSET = 'Superset',
LOOKER = 'Looker',
TABLEAU = 'Tableau',
REDASH = 'Redash',
METABASE = 'Metabase',
}
export enum PipelineServiceType {
AIRFLOW = 'Airflow',
PREFECT = 'Prefect',
}
export enum IngestionType {
BIGQUERY = 'bigquery',
BIGQUERY_USAGE = 'bigquery-usage',

1
openmetadata-ui/src/main/resources/ui/src/generated/api/services/createPipelineService.ts
View File

@ -98,6 +98,7 @@ export interface EntityReference {
*/
export enum PipelineServiceType {
Airflow = 'Airflow',
Generic = 'Generic',
Glue = 'Glue',
Prefect = 'Prefect',
}

1
openmetadata-ui/src/main/resources/ui/src/generated/entity/data/pipeline.ts
View File

@ -195,6 +195,7 @@ export interface EntityReference {
*/
export enum PipelineServiceType {
Airflow = 'Airflow',
Generic = 'Generic',
Glue = 'Glue',
Prefect = 'Prefect',
}

1
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts
View File

@ -178,6 +178,7 @@ export interface EntityReference {
*/
export enum PipelineServiceType {
Airflow = 'Airflow',
Generic = 'Generic',
Glue = 'Glue',
Prefect = 'Prefect',
}

4
openmetadata-ui/src/main/resources/ui/src/pages/services/index.tsx
View File

@ -269,7 +269,9 @@ const ServicesPage = () => {
data.status === 'rejected' &&
showToast({
variant: 'error',
body: data.reason || 'Something went wrong!',
body:
data.reason?.response?.data?.message ||
'Something went wrong!',
});
});
setIsModalOpen(false);

162
openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.ts
View File

@ -28,6 +28,8 @@ import {
arrServiceTypes,
ATHENA,
BIGQUERY,
DASHBOARD_DEFAULT,
DATABASE_DEFAULT,
GLUE,
HIVE,
KAFKA,
@ -37,6 +39,7 @@ import {
MSSQL,
MYSQL,
ORACLE,
PIPELINE_DEFAULT,
POSTGRES,
PREFECT,
PRESTO,
@ -44,96 +47,105 @@ import {
REDASH,
REDSHIFT,
serviceTypes,
SERVICE_DEFAULT,
SNOWFLAKE,
SUPERSET,
TABLEAU,
TOPIC_DEFAULT,
TRINO,
VERTICA,
} from '../constants/services.const';
import {
DashboardServiceType,
DatabaseServiceType,
IngestionType,
MessagingServiceType,
PipelineServiceType,
ServiceCategory,
} from '../enums/service.enum';
import { IngestionType, ServiceCategory } from '../enums/service.enum';
import { DashboardServiceType } from '../generated/entity/services/dashboardService';
import { DatabaseServiceType } from '../generated/entity/services/databaseService';
import { MessagingServiceType } from '../generated/entity/services/messagingService';
import { PipelineServiceType } from '../generated/entity/services/pipelineService';
import { PipelineType } from '../generated/operations/pipelines/airflowPipeline';
import { ApiData } from '../pages/services';
export const serviceTypeLogo = (type: string) => {
switch (type) {
case DatabaseServiceType.MYSQL:
case DatabaseServiceType.MySQL:
return MYSQL;
case DatabaseServiceType.REDSHIFT:
case DatabaseServiceType.Redshift:
return REDSHIFT;
case DatabaseServiceType.BIGQUERY:
case DatabaseServiceType.BigQuery:
return BIGQUERY;
case DatabaseServiceType.HIVE:
case DatabaseServiceType.Hive:
return HIVE;
case DatabaseServiceType.POSTGRES:
case DatabaseServiceType.Postgres:
return POSTGRES;
case DatabaseServiceType.ORACLE:
case DatabaseServiceType.Oracle:
return ORACLE;
case DatabaseServiceType.SNOWFLAKE:
case DatabaseServiceType.Snowflake:
return SNOWFLAKE;
case DatabaseServiceType.MSSQL:
case DatabaseServiceType.Mssql:
return MSSQL;
case DatabaseServiceType.ATHENA:
case DatabaseServiceType.Athena:
return ATHENA;
case DatabaseServiceType.PRESTO:
case DatabaseServiceType.Presto:
return PRESTO;
case DatabaseServiceType.TRINO:
case DatabaseServiceType.Trino:
return TRINO;
case DatabaseServiceType.GLUE:
case DatabaseServiceType.Glue:
return GLUE;
case DatabaseServiceType.MARIADB:
case DatabaseServiceType.MariaDB:
return MARIADB;
case DatabaseServiceType.VERTICA:
case DatabaseServiceType.Vertica:
return VERTICA;
case MessagingServiceType.KAFKA:
case MessagingServiceType.Kafka:
return KAFKA;
case MessagingServiceType.PULSAR:
case MessagingServiceType.Pulsar:
return PULSAR;
case DashboardServiceType.SUPERSET:
case DashboardServiceType.Superset:
return SUPERSET;
case DashboardServiceType.LOOKER:
case DashboardServiceType.Looker:
return LOOKER;
case DashboardServiceType.TABLEAU:
case DashboardServiceType.Tableau:
return TABLEAU;
case DashboardServiceType.REDASH:
case DashboardServiceType.Redash:
return REDASH;
case DashboardServiceType.METABASE:
case DashboardServiceType.Metabase:
return METABASE;
case PipelineServiceType.AIRFLOW:
case PipelineServiceType.Airflow:
return AIRFLOW;
case PipelineServiceType.PREFECT:
case PipelineServiceType.Prefect:
return PREFECT;
default:
return SERVICE_DEFAULT;
default: {
let logo;
if (serviceTypes.messagingServices.includes(type)) {
logo = TOPIC_DEFAULT;
} else if (serviceTypes.dashboardServices.includes(type)) {
logo = DASHBOARD_DEFAULT;
} else if (serviceTypes.pipelineServices.includes(type)) {
logo = PIPELINE_DEFAULT;
} else if (serviceTypes.databaseServices.includes(type)) {
logo = DATABASE_DEFAULT;
}
return logo;
}
}
};
@ -254,44 +266,14 @@ export const getEntityCountByService = (buckets: Array<Bucket>) => {
pipelineCount: 0,
};
buckets?.forEach((bucket) => {
switch (bucket.key) {
case DatabaseServiceType.ATHENA:
case DatabaseServiceType.BIGQUERY:
case DatabaseServiceType.HIVE:
case DatabaseServiceType.MSSQL:
case DatabaseServiceType.MYSQL:
case DatabaseServiceType.ORACLE:
case DatabaseServiceType.POSTGRES:
case DatabaseServiceType.PRESTO:
case DatabaseServiceType.TRINO:
case DatabaseServiceType.GLUE:
case DatabaseServiceType.MARIADB:
case DatabaseServiceType.VERTICA:
case DatabaseServiceType.REDSHIFT:
case DatabaseServiceType.SNOWFLAKE:
entityCounts.tableCount += bucket.doc_count;
break;
case MessagingServiceType.KAFKA:
case MessagingServiceType.PULSAR:
entityCounts.topicCount += bucket.doc_count;
break;
case DashboardServiceType.SUPERSET:
case DashboardServiceType.LOOKER:
case DashboardServiceType.TABLEAU:
case DashboardServiceType.REDASH:
case DashboardServiceType.METABASE:
entityCounts.dashboardCount += bucket.doc_count;
break;
case PipelineServiceType.AIRFLOW:
case PipelineServiceType.PREFECT:
entityCounts.pipelineCount += bucket.doc_count;
break;
default:
break;
if (serviceTypes.databaseServices.includes(bucket.key)) {
entityCounts.tableCount += bucket.doc_count;
} else if (serviceTypes.messagingServices.includes(bucket.key)) {
entityCounts.topicCount += bucket.doc_count;
} else if (serviceTypes.dashboardServices.includes(bucket.key)) {
entityCounts.dashboardCount += bucket.doc_count;
} else if (serviceTypes.pipelineServices.includes(bucket.key)) {
entityCounts.pipelineCount += bucket.doc_count;
}
});
@ -313,52 +295,52 @@ export const getIngestionTypeList = (
): Array<string> | undefined => {
let ingestionType: Array<string> | undefined;
switch (serviceType) {
case DatabaseServiceType.BIGQUERY:
case DatabaseServiceType.BigQuery:
ingestionType = onlyMetaData
? [IngestionType.BIGQUERY]
: [IngestionType.BIGQUERY, IngestionType.BIGQUERY_USAGE];
break;
case DatabaseServiceType.HIVE:
case DatabaseServiceType.Hive:
ingestionType = [IngestionType.HIVE];
break;
case DatabaseServiceType.MSSQL:
case DatabaseServiceType.Mssql:
ingestionType = [IngestionType.MSSQL];
break;
case DatabaseServiceType.MYSQL:
case DatabaseServiceType.MySQL:
ingestionType = [IngestionType.MYSQL];
break;
case DatabaseServiceType.POSTGRES:
case DatabaseServiceType.Postgres:
ingestionType = [IngestionType.POSTGRES];
break;
case DatabaseServiceType.REDSHIFT:
case DatabaseServiceType.Redshift:
ingestionType = onlyMetaData
? [IngestionType.REDSHIFT]
: [IngestionType.REDSHIFT, IngestionType.REDSHIFT_USAGE];
break;
case DatabaseServiceType.TRINO:
case DatabaseServiceType.Trino:
ingestionType = [IngestionType.TRINO];
break;
case DatabaseServiceType.SNOWFLAKE:
case DatabaseServiceType.Snowflake:
ingestionType = onlyMetaData
? [IngestionType.SNOWFLAKE]
: [IngestionType.SNOWFLAKE, IngestionType.SNOWFLAKE_USAGE];
break;
case DatabaseServiceType.VERTICA:
case DatabaseServiceType.Vertica:
ingestionType = [IngestionType.VERTICA];
break;
@ -378,17 +360,17 @@ export const getAirflowPipelineTypes = (
return [PipelineType.Metadata];
}
switch (serviceType) {
case DatabaseServiceType.REDSHIFT:
case DatabaseServiceType.BIGQUERY:
case DatabaseServiceType.SNOWFLAKE:
case DatabaseServiceType.Redshift:
case DatabaseServiceType.BigQuery:
case DatabaseServiceType.Snowflake:
return [PipelineType.Metadata, PipelineType.QueryUsage];
case DatabaseServiceType.HIVE:
case DatabaseServiceType.MSSQL:
case DatabaseServiceType.MYSQL:
case DatabaseServiceType.POSTGRES:
case DatabaseServiceType.TRINO:
case DatabaseServiceType.VERTICA:
case DatabaseServiceType.Hive:
case DatabaseServiceType.Mssql:
case DatabaseServiceType.MySQL:
case DatabaseServiceType.Postgres:
case DatabaseServiceType.Trino:
case DatabaseServiceType.Vertica:
return [PipelineType.Metadata];
default: