yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-26 09:22:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added new connectors Glue, MariaDB, Vertica from UI (#1289)

Browse Source
This commit is contained in:
darth-coder00 2021-11-20 21:54:47 +05:30 committed by GitHub
parent 42e0c1caa1
commit 95fe21c18f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 123 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
catalog-rest-service/src/main/resources/ui/src/assets/img/service-icon-glue.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 9.2 KiB

BIN
catalog-rest-service/src/main/resources/ui/src/assets/img/service-icon-mariadb.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 33 KiB

BIN
catalog-rest-service/src/main/resources/ui/src/assets/img/service-icon-vertica.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.6 KiB

5
catalog-rest-service/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx
View File

@ -26,6 +26,7 @@ import {
TITLE_FOR_NON_ADMIN_ACTION,
} from '../../constants/constants';
import { NoDataFoundPlaceHolder } from '../../constants/services.const';
import { ServiceCategory } from '../../enums/service.enum';
import { useAuth } from '../../hooks/authHooks';
import { isEven } from '../../utils/CommonUtils';
import { Button } from '../buttons/Button/Button';
@ -287,7 +288,9 @@ const Ingestion: React.FC<Props> = ({
<Link
to={getServiceDetailsPath(
ingestion.service.name as string,
getServiceTypeFromName(ingestion.service.name)
getServiceTypeFromName(ingestion.service.name),
// TODO: Add logic below to select service-cat if necessary
ServiceCategory.DATABASE_SERVICES
)}>
{ingestion.service.name}
</Link>

7
catalog-rest-service/src/main/resources/ui/src/constants/constants.ts
View File

@ -41,6 +41,7 @@ const PLACEHOLDER_ROUTE_DASHBOARD_FQN = ':dashboardFQN';
const PLACEHOLDER_ROUTE_DATABASE_FQN = ':databaseFQN';
const PLACEHOLDER_ROUTE_SERVICE_FQN = ':serviceFQN';
const PLACEHOLDER_ROUTE_SERVICE_TYPE = ':serviceType';
const PLACEHOLDER_ROUTE_SERVICE_CAT = ':serviceCategory';
const PLACEHOLDER_ROUTE_SEARCHQUERY = ':searchQuery';
const PLACEHOLDER_ROUTE_TAB = ':tab';
const PLACEHOLDER_ROUTE_TEAM = ':team';
@ -114,7 +115,7 @@ export const ROUTES = {
STORE: '/store',
FEEDS: '/feeds',
DUMMY: '/dummy',
SERVICE: `/service/${PLACEHOLDER_ROUTE_SERVICE_TYPE}/${PLACEHOLDER_ROUTE_SERVICE_FQN}`,
SERVICE: `/service/${PLACEHOLDER_ROUTE_SERVICE_CAT}/${PLACEHOLDER_ROUTE_SERVICE_TYPE}/${PLACEHOLDER_ROUTE_SERVICE_FQN}`,
SERVICES: '/services',
USERS: '/users',
SCORECARD: '/scorecard',
@ -170,10 +171,12 @@ export const getDatasetTabPath = (datasetFQN: string, tab = 'schema') => {
export const getServiceDetailsPath = (
serviceFQN: string,
serviceType: string
serviceType: string,
serviceCat: string
) => {
let path = ROUTES.SERVICE;
path = path
.replace(PLACEHOLDER_ROUTE_SERVICE_CAT, serviceCat)
.replace(PLACEHOLDER_ROUTE_SERVICE_TYPE, serviceType)
.replace(PLACEHOLDER_ROUTE_SERVICE_FQN, serviceFQN);

11
catalog-rest-service/src/main/resources/ui/src/constants/services.const.ts
View File

@ -21,9 +21,11 @@ import noService from '../assets/img/no-service.png';
import airflow from '../assets/img/service-icon-airflow.png';
import athena from '../assets/img/service-icon-athena.png';
import serviceDefault from '../assets/img/service-icon-generic.png';
import glue from '../assets/img/service-icon-glue.png';
import hive from '../assets/img/service-icon-hive.png';
import kafka from '../assets/img/service-icon-kafka.png';
import looker from '../assets/img/service-icon-looker.png';
import mariadb from '../assets/img/service-icon-mariadb.png';
import mssql from '../assets/img/service-icon-mssql.png';
import oracle from '../assets/img/service-icon-oracle.png';
import postgres from '../assets/img/service-icon-post.png';
@ -38,6 +40,7 @@ import mysql from '../assets/img/service-icon-sql.png';
import superset from '../assets/img/service-icon-superset.png';
import tableau from '../assets/img/service-icon-tableau.png';
import trino from '../assets/img/service-icon-trino.png';
import vertica from '../assets/img/service-icon-vertica.png';
import plus from '../assets/svg/plus.svg';
export const NoDataFoundPlaceHolder = noDataFound;
@ -52,6 +55,9 @@ export const SNOWFLAKE = snowflakes;
export const ATHENA = athena;
export const PRESTO = presto;
export const TRINO = trino;
export const GLUE = glue;
export const MARIADB = mariadb;
export const VERTICA = vertica;
export const KAFKA = kafka;
export const PULSAR = pulsar;
export const SUPERSET = superset;
@ -79,10 +85,13 @@ export const serviceTypes: Record<ServiceTypes, Array<string>> = {
'Athena',
'Presto',
'Trino',
'Glue',
'MariaDB',
'Vertica',
],
messagingServices: ['Kafka'],
dashboardServices: ['Superset', 'Looker', 'Tableau', 'Redash'],
pipelineServices: ['Airflow', 'Prefect'],
pipelineServices: ['Airflow', 'Prefect', 'Glue'],
};
export const arrServiceTypes: Array<ServiceTypes> = [

3
catalog-rest-service/src/main/resources/ui/src/enums/service.enum.ts
View File

@ -34,6 +34,9 @@ export enum DatabaseServiceType {
ATHENA = 'Athena',
PRESTO = 'Presto',
TRINO = 'Trino',
GLUE = 'Glue',
MARIADB = 'MariaDB',
VERTICA = 'Vertica',
}
export enum MessagingServiceType {

4
catalog-rest-service/src/main/resources/ui/src/pages/DashboardDetailsPage/DashboardDetailsPage.component.tsx
View File

@ -20,6 +20,7 @@ import {
getServiceDetailsPath,
} from '../../constants/constants';
import { EntityType } from '../../enums/entity.enum';
import { ServiceCategory } from '../../enums/service.enum';
import { Chart } from '../../generated/entity/data/chart';
import { Dashboard } from '../../generated/entity/data/dashboard';
import { User } from '../../generated/entity/teams/user';
@ -169,7 +170,8 @@ const DashboardDetailsPage = () => {
url: serviceRes.data.name
? getServiceDetailsPath(
serviceRes.data.name,
serviceRes.data.serviceType
serviceRes.data.serviceType,
ServiceCategory.DASHBOARD_SERVICES
)
: '',
imgSrc: serviceRes.data.serviceType

4
catalog-rest-service/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx
View File

@ -41,6 +41,7 @@ import {
getServiceDetailsPath,
} from '../../constants/constants';
import { EntityType } from '../../enums/entity.enum';
import { ServiceCategory } from '../../enums/service.enum';
import {
Table,
TableData,
@ -231,7 +232,8 @@ const DatasetDetailsPage: FunctionComponent = () => {
url: resService.data.name
? getServiceDetailsPath(
resService.data.name,
resService.data.serviceType
resService.data.serviceType,
ServiceCategory.DATABASE_SERVICES
)
: '',
imgSrc: resService.data.serviceType

7
catalog-rest-service/src/main/resources/ui/src/pages/EntityVersionPage/EntityVersionPage.component.tsx
View File

@ -17,6 +17,7 @@ import {
getDatasetVersionPath,
getServiceDetailsPath,
} from '../../constants/constants';
import { ServiceCategory } from '../../enums/service.enum';
import { Table } from '../../generated/entity/data/table';
import { EntityHistory } from '../../generated/type/entityHistory';
import useToastContext from '../../hooks/useToastContext';
@ -72,7 +73,8 @@ const EntityVersionPage: FunctionComponent = () => {
url: resService.data.name
? getServiceDetailsPath(
resService.data.name,
resService.data.serviceType
resService.data.serviceType,
ServiceCategory.DATABASE_SERVICES
)
: '',
imgSrc: resService.data.serviceType
@ -131,7 +133,8 @@ const EntityVersionPage: FunctionComponent = () => {
url: resService.data.name
? getServiceDetailsPath(
resService.data.name,
resService.data.serviceType
resService.data.serviceType,
ServiceCategory.DATABASE_SERVICES
)
: '',
imgSrc: resService.data.serviceType

13
catalog-rest-service/src/main/resources/ui/src/pages/MyDataPage/MyDataPage.component.tsx
View File

@ -25,15 +25,12 @@ import { getIngestionWorkflows } from '../../axiosAPIs/ingestionWorkflowAPI';
import { searchData } from '../../axiosAPIs/miscAPI';
import Loader from '../../components/Loader/Loader';
import MyData from '../../components/MyData/MyData.component';
import { PAGE_SIZE } from '../../constants/constants';
import {
myDataEntityCounts,
myDataSearchIndex,
} from '../../constants/Mydata.constants';
import {
getAllServices,
getEntityCountByService,
} from '../../utils/ServiceUtils';
import { getEntityCountByType } from '../../utils/EntityUtils';
import { getAllServices } from '../../utils/ServiceUtils';
const MyDataPage = () => {
const [error, setError] = useState<string>('');
@ -49,7 +46,7 @@ const MyDataPage = () => {
searchData(
value.queryString,
value.from,
PAGE_SIZE,
0,
value.filters,
value.sortField,
value.sortOrder,
@ -59,8 +56,8 @@ const MyDataPage = () => {
setSearchResult(res);
if (isUndefined(entityCounts)) {
setEntityCounts(
getEntityCountByService(
res.data.aggregations?.['sterms#Service']?.buckets
getEntityCountByType(
res.data.aggregations?.['sterms#EntityType']?.buckets
)
);
}

4
catalog-rest-service/src/main/resources/ui/src/pages/PipelineDetails/PipelineDetailsPage.component.tsx
View File

@ -21,6 +21,7 @@ import {
getServiceDetailsPath,
} from '../../constants/constants';
import { EntityType } from '../../enums/entity.enum';
import { ServiceCategory } from '../../enums/service.enum';
import { Pipeline, Task } from '../../generated/entity/data/pipeline';
import { User } from '../../generated/entity/teams/user';
import { EntityLineage } from '../../generated/type/entityLineage';
@ -147,7 +148,8 @@ const PipelineDetailsPage = () => {
url: serviceRes.data.name
? getServiceDetailsPath(
serviceRes.data.name,
serviceRes.data.serviceType
serviceRes.data.serviceType,
ServiceCategory.PIPELINE_SERVICES
)
: '',
imgSrc: serviceRes.data.serviceType

4
catalog-rest-service/src/main/resources/ui/src/pages/TopicDetails/TopicDetailsPage.component.tsx
View File

@ -20,6 +20,7 @@ import {
getTopicDetailsPath,
} from '../../constants/constants';
import { EntityType } from '../../enums/entity.enum';
import { ServiceCategory } from '../../enums/service.enum';
import { Topic } from '../../generated/entity/data/topic';
import { User } from '../../generated/entity/teams/user';
import useToastContext from '../../hooks/useToastContext';
@ -143,7 +144,8 @@ const TopicDetailsPage: FunctionComponent = () => {
url: serviceRes.data.name
? getServiceDetailsPath(
serviceRes.data.name,
serviceRes.data.serviceType
serviceRes.data.serviceType,
ServiceCategory.MESSAGING_SERVICES
)
: '',
imgSrc: serviceRes.data.serviceType

4
catalog-rest-service/src/main/resources/ui/src/pages/database-details/index.tsx
View File

@ -48,6 +48,7 @@ import {
pagingObject,
TITLE_FOR_NON_ADMIN_ACTION,
} from '../../constants/constants';
import { ServiceCategory } from '../../enums/service.enum';
import { Database } from '../../generated/entity/data/database';
import { Table } from '../../generated/entity/data/table';
import useToastContext from '../../hooks/useToastContext';
@ -125,7 +126,8 @@ const DatabaseDetails: FunctionComponent = () => {
url: resService.data.name
? getServiceDetailsPath(
resService.data.name,
resService.data.serviceType
resService.data.serviceType,
ServiceCategory.DATABASE_SERVICES
)
: '',
imgSrc: resService.data.serviceType

11
catalog-rest-service/src/main/resources/ui/src/pages/service/index.tsx
View File

@ -70,9 +70,12 @@ type ServiceDataObj = { name: string } & Partial<DatabaseService> &
Partial<PipelineService>;
const ServicePage: FunctionComponent = () => {
const { serviceFQN, serviceType } = useParams() as Record<string, string>;
const { serviceFQN, serviceType, serviceCategory } = useParams() as Record<
string,
string
>;
const [serviceName, setServiceName] = useState(
getServiceCategoryFromType(serviceType)
serviceCategory || getServiceCategoryFromType(serviceType)
);
const [slashedTableName, setSlashedTableName] = useState<
TitleBreadcrumbProps['titleLinks']
@ -578,8 +581,8 @@ const ServicePage: FunctionComponent = () => {
};
useEffect(() => {
setServiceName(getServiceCategoryFromType(serviceType));
}, [serviceType]);
setServiceName(serviceCategory || getServiceCategoryFromType(serviceType));
}, [serviceCategory, serviceType]);
useEffect(() => {
getServiceByFQN(serviceName, serviceFQN).then(

3
catalog-rest-service/src/main/resources/ui/src/pages/services/index.tsx
View File

@ -468,7 +468,8 @@ const ServicesPage = () => {
<Link
to={getServiceDetailsPath(
service.name,
service.serviceType || ''
service.serviceType || '',
serviceName
)}>
<button>
<h6

48
catalog-rest-service/src/main/resources/ui/src/utils/EntityUtils.tsx
View File

@ -1,4 +1,5 @@
import { isNil } from 'lodash';
import { Bucket } from 'Models';
import React from 'react';
import TableProfilerGraph from '../components/TableProfiler/TableProfilerGraph.component';
import {
@ -7,6 +8,7 @@ import {
getTeamDetailsPath,
} from '../constants/constants';
import { EntityType } from '../enums/entity.enum';
import { ServiceCategory } from '../enums/service.enum';
import { Dashboard } from '../generated/entity/data/dashboard';
import { Pipeline } from '../generated/entity/data/pipeline';
import { Table } from '../generated/entity/data/table';
@ -79,7 +81,11 @@ export const getEntityOverview = (
{
name: 'Service',
value: service,
url: getServiceDetailsPath(service, serviceType),
url: getServiceDetailsPath(
service,
serviceType,
ServiceCategory.DATABASE_SERVICES
),
isLink: true,
},
{
@ -166,7 +172,11 @@ export const getEntityOverview = (
{
name: 'Service',
value: service?.name as string,
url: getServiceDetailsPath(service?.name as string, serviceType),
url: getServiceDetailsPath(
service?.name as string,
serviceType,
ServiceCategory.PIPELINE_SERVICES
),
isLink: true,
},
{
@ -200,3 +210,37 @@ export const getEntityOverview = (
return [];
}
};
// Note: This method is enhanced from "getEntityCountByService" of ServiceUtils.ts
export const getEntityCountByType = (buckets: Array<Bucket>) => {
const entityCounts = {
tableCount: 0,
topicCount: 0,
dashboardCount: 0,
pipelineCount: 0,
};
buckets?.forEach((bucket) => {
switch (bucket.key) {
case EntityType.TABLE:
entityCounts.tableCount += bucket.doc_count;
break;
case EntityType.TOPIC:
entityCounts.topicCount += bucket.doc_count;
break;
case EntityType.DASHBOARD:
entityCounts.dashboardCount += bucket.doc_count;
break;
case EntityType.PIPELINE:
entityCounts.pipelineCount += bucket.doc_count;
break;
default:
break;
}
});
return entityCounts;
};

21
catalog-rest-service/src/main/resources/ui/src/utils/ServiceUtils.ts
View File

@ -6,9 +6,11 @@ import {
AIRFLOW,
ATHENA,
BIGQUERY,
GLUE,
HIVE,
KAFKA,
LOOKER,
MARIADB,
MSSQL,
MYSQL,
ORACLE,
@ -24,6 +26,7 @@ import {
SUPERSET,
TABLEAU,
TRINO,
VERTICA,
} from '../constants/services.const';
import {
DashboardServiceType,
@ -69,6 +72,15 @@ export const serviceTypeLogo = (type: string) => {
case DatabaseServiceType.TRINO:
return TRINO;
case DatabaseServiceType.GLUE:
return GLUE;
case DatabaseServiceType.MARIADB:
return MARIADB;
case DatabaseServiceType.VERTICA:
return VERTICA;
case MessagingServiceType.KAFKA:
return KAFKA;
@ -194,6 +206,7 @@ export const getServiceCategoryFromType = (
return serviceCategory;
};
// Note: This method is deprecated by "getEntityCountByType" of EntityUtils.ts
export const getEntityCountByService = (buckets: Array<Bucket>) => {
const entityCounts = {
tableCount: 0,
@ -212,6 +225,9 @@ export const getEntityCountByService = (buckets: Array<Bucket>) => {
case DatabaseServiceType.POSTGRES:
case DatabaseServiceType.PRESTO:
case DatabaseServiceType.TRINO:
case DatabaseServiceType.GLUE:
case DatabaseServiceType.MARIADB:
case DatabaseServiceType.VERTICA:
case DatabaseServiceType.REDSHIFT:
case DatabaseServiceType.SNOWFLAKE:
entityCounts.tableCount += bucket.doc_count;
@ -295,6 +311,11 @@ export const getIngestionTypeList = (
break;
case DatabaseServiceType.VERTICA:
ingestionType = [IngestionType.VERTICA];
break;
default:
break;
}