yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-08 06:53:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #2569: Custom Pipeline Service defenition (#2824)

Browse Source 
* Fix #2569: Custom Pipeline Service defenition

* Fix #2569: Custom Pipeline Service defenition

* Adding UI support to show all services available from enum

Co-authored-by: darth-coder00 <aashit@getcollate.io>
This commit is contained in:
Sriharsha Chintalapani 2022-02-17 22:15:09 -08:00 committed by GitHub
parent 78ee32f585
commit 42dc42174f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 127 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json
View File

@ -8,7 +8,7 @@
"pipelineServiceType": { "pipelineServiceType": {
"description": "Type of pipeline service - Airflow or Prefect.", "description": "Type of pipeline service - Airflow or Prefect.",
"type": "string", "type": "string",
"enum": ["Airflow", "Prefect", "Glue"], "enum": ["Airflow", "Prefect", "Glue", "Generic"],
"javaEnums": [ "javaEnums": [
{ {
"name": "Airflow" "name": "Airflow"
@ -18,6 +18,9 @@
}, },
{ {
"name": "Glue" "name": "Glue"
},
{
"name": "Generic"
} }
] ]
} }

30
openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx
View File

@ -32,17 +32,21 @@ import React, {
import { ONLY_NUMBER_REGEX } from '../../../constants/constants'; import { ONLY_NUMBER_REGEX } from '../../../constants/constants';
import { serviceTypes } from '../../../constants/services.const'; import { serviceTypes } from '../../../constants/services.const';
import { import {
DashboardServiceType, // DashboardServiceType,
MessagingServiceType, // MessagingServiceType,
ServiceCategory, ServiceCategory,
} from '../../../enums/service.enum'; } from '../../../enums/service.enum';
import { import {
CreateAirflowPipeline, CreateAirflowPipeline,
Schema, Schema,
} from '../../../generated/api/operations/pipelines/createAirflowPipeline'; } from '../../../generated/api/operations/pipelines/createAirflowPipeline';
import { DashboardServiceType } from '../../../generated/entity/services/dashboardService';
// import { DashboardService } from '../../../generated/entity/services/dashboardService'; // import { DashboardService } from '../../../generated/entity/services/dashboardService';
import { DatabaseService } from '../../../generated/entity/services/databaseService'; import { DatabaseService } from '../../../generated/entity/services/databaseService';
import { MessagingService } from '../../../generated/entity/services/messagingService'; import {
MessagingService,
MessagingServiceType,
} from '../../../generated/entity/services/messagingService';
import { PipelineService } from '../../../generated/entity/services/pipelineService'; import { PipelineService } from '../../../generated/entity/services/pipelineService';
import { PipelineType } from '../../../generated/operations/pipelines/airflowPipeline'; import { PipelineType } from '../../../generated/operations/pipelines/airflowPipeline';
import { import {
@ -335,7 +339,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
const markdownRef = useRef<EditorContentRef>(); const markdownRef = useRef<EditorContentRef>();
const getBrokerUrlPlaceholder = (): string => { const getBrokerUrlPlaceholder = (): string => {
return selectService === MessagingServiceType.PULSAR return selectService === MessagingServiceType.Pulsar
? 'hostname:port' ? 'hostname:port'
: 'hostname1:port1, hostname2:port2'; : 'hostname1:port1, hostname2:port2';
}; };
@ -460,7 +464,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
dataObj = { dataObj = {
...dataObj, ...dataObj,
brokers: brokers:
selectService === MessagingServiceType.PULSAR selectService === MessagingServiceType.Pulsar
? [brokers] ? [brokers]
: brokers.split(',').map((broker) => broker.trim()), : brokers.split(',').map((broker) => broker.trim()),
schemaRegistry: schemaRegistry, schemaRegistry: schemaRegistry,
@ -471,7 +475,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
case ServiceCategory.DASHBOARD_SERVICES: case ServiceCategory.DASHBOARD_SERVICES:
{ {
switch (selectService) { switch (selectService) {
case DashboardServiceType.REDASH: case DashboardServiceType.Redash:
{ {
dataObj = { dataObj = {
...dataObj, ...dataObj,
@ -482,7 +486,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
} }
break; break;
case DashboardServiceType.TABLEAU: case DashboardServiceType.Tableau:
{ {
dataObj = { dataObj = {
...dataObj, ...dataObj,
@ -638,7 +642,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
case ServiceCategory.DASHBOARD_SERVICES: case ServiceCategory.DASHBOARD_SERVICES:
{ {
switch (selectService) { switch (selectService) {
case DashboardServiceType.REDASH: case DashboardServiceType.Redash:
{ {
setMsg = { setMsg = {
...setMsg, ...setMsg,
@ -650,7 +654,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
isValid = Boolean(dashboardUrl && apiKey); isValid = Boolean(dashboardUrl && apiKey);
break; break;
case DashboardServiceType.TABLEAU: case DashboardServiceType.Tableau:
{ {
setMsg = { setMsg = {
...setMsg, ...setMsg,
@ -1010,7 +1014,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
const getDashboardFields = (): JSX.Element => { const getDashboardFields = (): JSX.Element => {
let elemFields: JSX.Element; let elemFields: JSX.Element;
switch (selectService) { switch (selectService) {
case DashboardServiceType.REDASH: { case DashboardServiceType.Redash: {
elemFields = ( elemFields = (
<> <>
<Field> <Field>
@ -1048,7 +1052,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
break; break;
} }
case DashboardServiceType.TABLEAU: { case DashboardServiceType.Tableau: {
elemFields = ( elemFields = (
<> <>
<Field> <Field>
@ -1358,7 +1362,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
case ServiceCategory.DASHBOARD_SERVICES: case ServiceCategory.DASHBOARD_SERVICES:
switch (selectService) { switch (selectService) {
case DashboardServiceType.REDASH: case DashboardServiceType.Redash:
data = [ data = [
{ {
key: 'Dashboard Url', key: 'Dashboard Url',
@ -1394,7 +1398,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
break; break;
case DashboardServiceType.TABLEAU: case DashboardServiceType.Tableau:
data = [ data = [
{ {
key: 'Site Name', key: 'Site Name',

22
openmetadata-ui/src/main/resources/ui/src/components/ServiceConfig/ServiceConfig.tsx
View File

@ -3,10 +3,12 @@ import { ServicesData } from 'Models';
import React, { useState } from 'react'; import React, { useState } from 'react';
import { ONLY_NUMBER_REGEX } from '../../constants/constants'; import { ONLY_NUMBER_REGEX } from '../../constants/constants';
import { import {
DashboardServiceType, // DashboardServiceType,
MessagingServiceType, // MessagingServiceType,
ServiceCategory, ServiceCategory,
} from '../../enums/service.enum'; } from '../../enums/service.enum';
import { DashboardServiceType } from '../../generated/entity/services/dashboardService';
import { MessagingServiceType } from '../../generated/entity/services/messagingService';
import useToastContext from '../../hooks/useToastContext'; import useToastContext from '../../hooks/useToastContext';
import { errorMsg, requiredField } from '../../utils/CommonUtils'; import { errorMsg, requiredField } from '../../utils/CommonUtils';
import { import {
@ -118,7 +120,7 @@ const ServiceConfig = ({
const [env, setEnv] = useState(data?.env || ''); const [env, setEnv] = useState(data?.env || '');
const getBrokerUrlPlaceholder = (): string => { const getBrokerUrlPlaceholder = (): string => {
return data?.serviceType === MessagingServiceType.PULSAR return data?.serviceType === MessagingServiceType.Pulsar
? 'hostname:port' ? 'hostname:port'
: 'hostname1:port1, hostname2:port2'; : 'hostname1:port1, hostname2:port2';
}; };
@ -215,7 +217,7 @@ const ServiceConfig = ({
case ServiceCategory.DASHBOARD_SERVICES: { case ServiceCategory.DASHBOARD_SERVICES: {
switch (data?.serviceType) { switch (data?.serviceType) {
case DashboardServiceType.REDASH: case DashboardServiceType.Redash:
setShowErrorMsg({ setShowErrorMsg({
...showErrorMsg, ...showErrorMsg,
dashboardUrl: !dashboardUrl, dashboardUrl: !dashboardUrl,
@ -224,7 +226,7 @@ const ServiceConfig = ({
return Boolean(dashboardUrl && apiKey); return Boolean(dashboardUrl && apiKey);
case DashboardServiceType.TABLEAU: case DashboardServiceType.Tableau:
setShowErrorMsg({ setShowErrorMsg({
...showErrorMsg, ...showErrorMsg,
dashboardUrl: !dashboardUrl, dashboardUrl: !dashboardUrl,
@ -286,14 +288,14 @@ const ServiceConfig = ({
return { return {
...data, ...data,
brokers: brokers:
data?.serviceType === MessagingServiceType.PULSAR data?.serviceType === MessagingServiceType.Pulsar
? [brokers] ? [brokers]
: brokers.split(',').map((broker) => broker.trim()), : brokers.split(',').map((broker) => broker.trim()),
schemaRegistry: schemaRegistry, schemaRegistry: schemaRegistry,
}; };
case ServiceCategory.DASHBOARD_SERVICES: { case ServiceCategory.DASHBOARD_SERVICES: {
switch (data?.serviceType) { switch (data?.serviceType) {
case DashboardServiceType.REDASH: case DashboardServiceType.Redash:
return { return {
...data, ...data,
dashboardUrl: dashboardUrl, dashboardUrl: dashboardUrl,
@ -301,7 +303,7 @@ const ServiceConfig = ({
api_key: apiKey, api_key: apiKey,
}; };
case DashboardServiceType.TABLEAU: case DashboardServiceType.Tableau:
return { return {
...data, ...data,
dashboardUrl: dashboardUrl, dashboardUrl: dashboardUrl,
@ -742,7 +744,7 @@ const ServiceConfig = ({
const getDashboardFields = (): JSX.Element => { const getDashboardFields = (): JSX.Element => {
let elemFields: JSX.Element; let elemFields: JSX.Element;
switch (data?.serviceType) { switch (data?.serviceType) {
case DashboardServiceType.REDASH: { case DashboardServiceType.Redash: {
elemFields = ( elemFields = (
<> <>
<Field> <Field>
@ -794,7 +796,7 @@ const ServiceConfig = ({
break; break;
} }
case DashboardServiceType.TABLEAU: { case DashboardServiceType.Tableau: {
elemFields = ( elemFields = (
<> <>
<Field> <Field>

37
openmetadata-ui/src/main/resources/ui/src/constants/services.const.ts
View File

@ -16,7 +16,7 @@ import noDataFound from '../assets/img/no-data-placeholder.png';
import noService from '../assets/img/no-service.png'; import noService from '../assets/img/no-service.png';
import airflow from '../assets/img/service-icon-airflow.png'; import airflow from '../assets/img/service-icon-airflow.png';
import athena from '../assets/img/service-icon-athena.png'; import athena from '../assets/img/service-icon-athena.png';
import serviceDefault from '../assets/img/service-icon-generic.png'; import databaseDefault from '../assets/img/service-icon-generic.png';
import glue from '../assets/img/service-icon-glue.png'; import glue from '../assets/img/service-icon-glue.png';
import hive from '../assets/img/service-icon-hive.png'; import hive from '../assets/img/service-icon-hive.png';
import kafka from '../assets/img/service-icon-kafka.png'; import kafka from '../assets/img/service-icon-kafka.png';
@ -38,7 +38,14 @@ import superset from '../assets/img/service-icon-superset.png';
import tableau from '../assets/img/service-icon-tableau.png'; import tableau from '../assets/img/service-icon-tableau.png';
import trino from '../assets/img/service-icon-trino.png'; import trino from '../assets/img/service-icon-trino.png';
import vertica from '../assets/img/service-icon-vertica.png'; import vertica from '../assets/img/service-icon-vertica.png';
import dashboardDefault from '../assets/svg/dashboard.svg';
import pipelineDefault from '../assets/svg/pipeline.svg';
import plus from '../assets/svg/plus.svg'; import plus from '../assets/svg/plus.svg';
import topicDefault from '../assets/svg/topic.svg';
import { DashboardServiceType } from '../generated/entity/services/dashboardService';
import { DatabaseServiceType } from '../generated/entity/services/databaseService';
import { MessagingServiceType } from '../generated/entity/services/messagingService';
import { PipelineServiceType } from '../generated/entity/services/pipelineService';
export const NoDataFoundPlaceHolder = noDataFound; export const NoDataFoundPlaceHolder = noDataFound;
export const MYSQL = mysql; export const MYSQL = mysql;
@ -65,31 +72,19 @@ export const METABASE = metabase;
export const AIRFLOW = airflow; export const AIRFLOW = airflow;
export const PREFECT = prefect; export const PREFECT = prefect;
export const SERVICE_DEFAULT = serviceDefault; export const DATABASE_DEFAULT = databaseDefault;
export const TOPIC_DEFAULT = topicDefault;
export const DASHBOARD_DEFAULT = dashboardDefault;
export const PIPELINE_DEFAULT = pipelineDefault;
export const PLUS = plus; export const PLUS = plus;
export const NOSERVICE = noService; export const NOSERVICE = noService;
export const serviceTypes: Record<ServiceTypes, Array<string>> = { export const serviceTypes: Record<ServiceTypes, Array<string>> = {
databaseServices: [ databaseServices: Object.values(DatabaseServiceType),
'BigQuery', messagingServices: Object.values(MessagingServiceType),
'MySQL', dashboardServices: Object.values(DashboardServiceType),
'Redshift', pipelineServices: Object.values(PipelineServiceType),
'Snowflake',
'Postgres',
'MSSQL',
'Hive',
'Oracle',
'Athena',
'Presto',
'Trino',
'Glue',
'MariaDB',
'Vertica',
],
messagingServices: ['Kafka'],
dashboardServices: ['Superset', 'Looker', 'Tableau', 'Redash', 'Metabase'],
pipelineServices: ['Airflow', 'Prefect', 'Glue'],
}; };
export const arrServiceTypes: Array<ServiceTypes> = [ export const arrServiceTypes: Array<ServiceTypes> = [

35
openmetadata-ui/src/main/resources/ui/src/enums/service.enum.ts
View File

@ -18,41 +18,6 @@ export enum ServiceCategory {
PIPELINE_SERVICES = 'pipelineServices', PIPELINE_SERVICES = 'pipelineServices',
} }
export enum DatabaseServiceType {
MYSQL = 'MySQL',
REDSHIFT = 'Redshift',
BIGQUERY = 'BigQuery',
HIVE = 'Hive',
POSTGRES = 'Postgres',
ORACLE = 'Oracle',
SNOWFLAKE = 'Snowflake',
MSSQL = 'MSSQL',
ATHENA = 'Athena',
PRESTO = 'Presto',
TRINO = 'Trino',
GLUE = 'Glue',
MARIADB = 'MariaDB',
VERTICA = 'Vertica',
}
export enum MessagingServiceType {
KAFKA = 'Kafka',
PULSAR = 'Pulsar',
}
export enum DashboardServiceType {
SUPERSET = 'Superset',
LOOKER = 'Looker',
TABLEAU = 'Tableau',
REDASH = 'Redash',
METABASE = 'Metabase',
}
export enum PipelineServiceType {
AIRFLOW = 'Airflow',
PREFECT = 'Prefect',
}
export enum IngestionType { export enum IngestionType {
BIGQUERY = 'bigquery', BIGQUERY = 'bigquery',
BIGQUERY_USAGE = 'bigquery-usage', BIGQUERY_USAGE = 'bigquery-usage',

1
openmetadata-ui/src/main/resources/ui/src/generated/api/services/createPipelineService.ts
View File

@ -98,6 +98,7 @@ export interface EntityReference {
*/ */
export enum PipelineServiceType { export enum PipelineServiceType {
Airflow = 'Airflow', Airflow = 'Airflow',
Generic = 'Generic',
Glue = 'Glue', Glue = 'Glue',
Prefect = 'Prefect', Prefect = 'Prefect',
} }

1
openmetadata-ui/src/main/resources/ui/src/generated/entity/data/pipeline.ts
View File

@ -195,6 +195,7 @@ export interface EntityReference {
*/ */
export enum PipelineServiceType { export enum PipelineServiceType {
Airflow = 'Airflow', Airflow = 'Airflow',
Generic = 'Generic',
Glue = 'Glue', Glue = 'Glue',
Prefect = 'Prefect', Prefect = 'Prefect',
} }

1
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts
View File

@ -178,6 +178,7 @@ export interface EntityReference {
*/ */
export enum PipelineServiceType { export enum PipelineServiceType {
Airflow = 'Airflow', Airflow = 'Airflow',
Generic = 'Generic',
Glue = 'Glue', Glue = 'Glue',
Prefect = 'Prefect', Prefect = 'Prefect',
} }

4
openmetadata-ui/src/main/resources/ui/src/pages/services/index.tsx
View File

@ -269,7 +269,9 @@ const ServicesPage = () => {
data.status === 'rejected' && data.status === 'rejected' &&
showToast({ showToast({
variant: 'error', variant: 'error',
body: data.reason || 'Something went wrong!', body:
data.reason?.response?.data?.message ||
'Something went wrong!',
}); });
}); });
setIsModalOpen(false); setIsModalOpen(false);

154
openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.ts
View File

@ -28,6 +28,8 @@ import {
arrServiceTypes, arrServiceTypes,
ATHENA, ATHENA,
BIGQUERY, BIGQUERY,
DASHBOARD_DEFAULT,
DATABASE_DEFAULT,
GLUE, GLUE,
HIVE, HIVE,
KAFKA, KAFKA,
@ -37,6 +39,7 @@ import {
MSSQL, MSSQL,
MYSQL, MYSQL,
ORACLE, ORACLE,
PIPELINE_DEFAULT,
POSTGRES, POSTGRES,
PREFECT, PREFECT,
PRESTO, PRESTO,
@ -44,96 +47,105 @@ import {
REDASH, REDASH,
REDSHIFT, REDSHIFT,
serviceTypes, serviceTypes,
SERVICE_DEFAULT,
SNOWFLAKE, SNOWFLAKE,
SUPERSET, SUPERSET,
TABLEAU, TABLEAU,
TOPIC_DEFAULT,
TRINO, TRINO,
VERTICA, VERTICA,
} from '../constants/services.const'; } from '../constants/services.const';
import { import { IngestionType, ServiceCategory } from '../enums/service.enum';
DashboardServiceType, import { DashboardServiceType } from '../generated/entity/services/dashboardService';
DatabaseServiceType, import { DatabaseServiceType } from '../generated/entity/services/databaseService';
IngestionType, import { MessagingServiceType } from '../generated/entity/services/messagingService';
MessagingServiceType, import { PipelineServiceType } from '../generated/entity/services/pipelineService';
PipelineServiceType,
ServiceCategory,
} from '../enums/service.enum';
import { PipelineType } from '../generated/operations/pipelines/airflowPipeline'; import { PipelineType } from '../generated/operations/pipelines/airflowPipeline';
import { ApiData } from '../pages/services'; import { ApiData } from '../pages/services';
export const serviceTypeLogo = (type: string) => { export const serviceTypeLogo = (type: string) => {
switch (type) { switch (type) {
case DatabaseServiceType.MYSQL: case DatabaseServiceType.MySQL:
return MYSQL; return MYSQL;
case DatabaseServiceType.REDSHIFT: case DatabaseServiceType.Redshift:
return REDSHIFT; return REDSHIFT;
case DatabaseServiceType.BIGQUERY: case DatabaseServiceType.BigQuery:
return BIGQUERY; return BIGQUERY;
case DatabaseServiceType.HIVE: case DatabaseServiceType.Hive:
return HIVE; return HIVE;
case DatabaseServiceType.POSTGRES: case DatabaseServiceType.Postgres:
return POSTGRES; return POSTGRES;
case DatabaseServiceType.ORACLE: case DatabaseServiceType.Oracle:
return ORACLE; return ORACLE;
case DatabaseServiceType.SNOWFLAKE: case DatabaseServiceType.Snowflake:
return SNOWFLAKE; return SNOWFLAKE;
case DatabaseServiceType.MSSQL: case DatabaseServiceType.Mssql:
return MSSQL; return MSSQL;
case DatabaseServiceType.ATHENA: case DatabaseServiceType.Athena:
return ATHENA; return ATHENA;
case DatabaseServiceType.PRESTO: case DatabaseServiceType.Presto:
return PRESTO; return PRESTO;
case DatabaseServiceType.TRINO: case DatabaseServiceType.Trino:
return TRINO; return TRINO;
case DatabaseServiceType.GLUE: case DatabaseServiceType.Glue:
return GLUE; return GLUE;
case DatabaseServiceType.MARIADB: case DatabaseServiceType.MariaDB:
return MARIADB; return MARIADB;
case DatabaseServiceType.VERTICA: case DatabaseServiceType.Vertica:
return VERTICA; return VERTICA;
case MessagingServiceType.KAFKA: case MessagingServiceType.Kafka:
return KAFKA; return KAFKA;
case MessagingServiceType.PULSAR: case MessagingServiceType.Pulsar:
return PULSAR; return PULSAR;
case DashboardServiceType.SUPERSET: case DashboardServiceType.Superset:
return SUPERSET; return SUPERSET;
case DashboardServiceType.LOOKER: case DashboardServiceType.Looker:
return LOOKER; return LOOKER;
case DashboardServiceType.TABLEAU: case DashboardServiceType.Tableau:
return TABLEAU; return TABLEAU;
case DashboardServiceType.REDASH: case DashboardServiceType.Redash:
return REDASH; return REDASH;
case DashboardServiceType.METABASE: case DashboardServiceType.Metabase:
return METABASE; return METABASE;
case PipelineServiceType.AIRFLOW: case PipelineServiceType.Airflow:
return AIRFLOW; return AIRFLOW;
case PipelineServiceType.PREFECT: case PipelineServiceType.Prefect:
return PREFECT; return PREFECT;
default: default: {
return SERVICE_DEFAULT; let logo;
if (serviceTypes.messagingServices.includes(type)) {
logo = TOPIC_DEFAULT;
} else if (serviceTypes.dashboardServices.includes(type)) {
logo = DASHBOARD_DEFAULT;
} else if (serviceTypes.pipelineServices.includes(type)) {
logo = PIPELINE_DEFAULT;
} else if (serviceTypes.databaseServices.includes(type)) {
logo = DATABASE_DEFAULT;
}
return logo;
}
} }
}; };
@ -254,44 +266,14 @@ export const getEntityCountByService = (buckets: Array<Bucket>) => {
pipelineCount: 0, pipelineCount: 0,
}; };
buckets?.forEach((bucket) => { buckets?.forEach((bucket) => {
switch (bucket.key) { if (serviceTypes.databaseServices.includes(bucket.key)) {
case DatabaseServiceType.ATHENA:
case DatabaseServiceType.BIGQUERY:
case DatabaseServiceType.HIVE:
case DatabaseServiceType.MSSQL:
case DatabaseServiceType.MYSQL:
case DatabaseServiceType.ORACLE:
case DatabaseServiceType.POSTGRES:
case DatabaseServiceType.PRESTO:
case DatabaseServiceType.TRINO:
case DatabaseServiceType.GLUE:
case DatabaseServiceType.MARIADB:
case DatabaseServiceType.VERTICA:
case DatabaseServiceType.REDSHIFT:
case DatabaseServiceType.SNOWFLAKE:
entityCounts.tableCount += bucket.doc_count; entityCounts.tableCount += bucket.doc_count;
} else if (serviceTypes.messagingServices.includes(bucket.key)) {
break;
case MessagingServiceType.KAFKA:
case MessagingServiceType.PULSAR:
entityCounts.topicCount += bucket.doc_count; entityCounts.topicCount += bucket.doc_count;
} else if (serviceTypes.dashboardServices.includes(bucket.key)) {
break;
case DashboardServiceType.SUPERSET:
case DashboardServiceType.LOOKER:
case DashboardServiceType.TABLEAU:
case DashboardServiceType.REDASH:
case DashboardServiceType.METABASE:
entityCounts.dashboardCount += bucket.doc_count; entityCounts.dashboardCount += bucket.doc_count;
} else if (serviceTypes.pipelineServices.includes(bucket.key)) {
break;
case PipelineServiceType.AIRFLOW:
case PipelineServiceType.PREFECT:
entityCounts.pipelineCount += bucket.doc_count; entityCounts.pipelineCount += bucket.doc_count;
break;
default:
break;
} }
}); });
@ -313,52 +295,52 @@ export const getIngestionTypeList = (
): Array<string> | undefined => { ): Array<string> | undefined => {
let ingestionType: Array<string> | undefined; let ingestionType: Array<string> | undefined;
switch (serviceType) { switch (serviceType) {
case DatabaseServiceType.BIGQUERY: case DatabaseServiceType.BigQuery:
ingestionType = onlyMetaData ingestionType = onlyMetaData
? [IngestionType.BIGQUERY] ? [IngestionType.BIGQUERY]
: [IngestionType.BIGQUERY, IngestionType.BIGQUERY_USAGE]; : [IngestionType.BIGQUERY, IngestionType.BIGQUERY_USAGE];
break; break;
case DatabaseServiceType.HIVE: case DatabaseServiceType.Hive:
ingestionType = [IngestionType.HIVE]; ingestionType = [IngestionType.HIVE];
break; break;
case DatabaseServiceType.MSSQL: case DatabaseServiceType.Mssql:
ingestionType = [IngestionType.MSSQL]; ingestionType = [IngestionType.MSSQL];
break; break;
case DatabaseServiceType.MYSQL: case DatabaseServiceType.MySQL:
ingestionType = [IngestionType.MYSQL]; ingestionType = [IngestionType.MYSQL];
break; break;
case DatabaseServiceType.POSTGRES: case DatabaseServiceType.Postgres:
ingestionType = [IngestionType.POSTGRES]; ingestionType = [IngestionType.POSTGRES];
break; break;
case DatabaseServiceType.REDSHIFT: case DatabaseServiceType.Redshift:
ingestionType = onlyMetaData ingestionType = onlyMetaData
? [IngestionType.REDSHIFT] ? [IngestionType.REDSHIFT]
: [IngestionType.REDSHIFT, IngestionType.REDSHIFT_USAGE]; : [IngestionType.REDSHIFT, IngestionType.REDSHIFT_USAGE];
break; break;
case DatabaseServiceType.TRINO: case DatabaseServiceType.Trino:
ingestionType = [IngestionType.TRINO]; ingestionType = [IngestionType.TRINO];
break; break;
case DatabaseServiceType.SNOWFLAKE: case DatabaseServiceType.Snowflake:
ingestionType = onlyMetaData ingestionType = onlyMetaData
? [IngestionType.SNOWFLAKE] ? [IngestionType.SNOWFLAKE]
: [IngestionType.SNOWFLAKE, IngestionType.SNOWFLAKE_USAGE]; : [IngestionType.SNOWFLAKE, IngestionType.SNOWFLAKE_USAGE];
break; break;
case DatabaseServiceType.VERTICA: case DatabaseServiceType.Vertica:
ingestionType = [IngestionType.VERTICA]; ingestionType = [IngestionType.VERTICA];
break; break;
@ -378,17 +360,17 @@ export const getAirflowPipelineTypes = (
return [PipelineType.Metadata]; return [PipelineType.Metadata];
} }
switch (serviceType) { switch (serviceType) {
case DatabaseServiceType.REDSHIFT: case DatabaseServiceType.Redshift:
case DatabaseServiceType.BIGQUERY: case DatabaseServiceType.BigQuery:
case DatabaseServiceType.SNOWFLAKE: case DatabaseServiceType.Snowflake:
return [PipelineType.Metadata, PipelineType.QueryUsage]; return [PipelineType.Metadata, PipelineType.QueryUsage];
case DatabaseServiceType.HIVE: case DatabaseServiceType.Hive:
case DatabaseServiceType.MSSQL: case DatabaseServiceType.Mssql:
case DatabaseServiceType.MYSQL: case DatabaseServiceType.MySQL:
case DatabaseServiceType.POSTGRES: case DatabaseServiceType.Postgres:
case DatabaseServiceType.TRINO: case DatabaseServiceType.Trino:
case DatabaseServiceType.VERTICA: case DatabaseServiceType.Vertica:
return [PipelineType.Metadata]; return [PipelineType.Metadata];
default: default: