yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-07-28 11:59:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingestion-ui) Add ingestion form for Postgres (#5671)

Browse Source
This commit is contained in:
Ankit keshari 2022-08-24 02:33:16 +05:30 committed by GitHub
parent 505cefef13
commit c2fbd75d2a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 92 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
datahub-web-react/src/app/entity/glossaryNode/ChildrenTab.tsx
View File

@ -1,5 +1,6 @@
import React from 'react';
import { EntityType, GlossaryNode, GlossaryTerm } from '../../../types.generated';
import EmptyGlossarySection from '../../glossary/EmptyGlossarySection';
import GlossaryEntitiesList from '../../glossary/GlossaryEntitiesList';
import { useEntityRegistry } from '../../useEntityRegistry';
import { sortGlossaryTerms } from '../glossaryTerm/utils';
@ -19,12 +20,18 @@ function ChildrenTab() {
.sort((termA, termB) => sortGlossaryTerms(entityRegistry, termA.entity, termB.entity))
.map((child) => child.entity);
return (
<GlossaryEntitiesList
nodes={(childNodes as GlossaryNode[]) || []}
terms={(childTerms as GlossaryTerm[]) || []}
/>
);
const hasTermsOrNodes = !!childNodes?.length || !!childTerms?.length;
if (hasTermsOrNodes) {
return (
<GlossaryEntitiesList
nodes={(childNodes as GlossaryNode[]) || []}
terms={(childTerms as GlossaryTerm[]) || []}
/>
);
}
return <EmptyGlossarySection description="No Terms or Term Groups" />;
}
export default ChildrenTab;

7
datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx
View File

@ -93,7 +93,12 @@ function BusinessGlossaryPage() {
</HeaderWrapper>
{hasTermsOrNodes && <GlossaryEntitiesList nodes={nodes || []} terms={terms || []} />}
{!(termsLoading || nodesLoading) && !hasTermsOrNodes && (
<EmptyGlossarySection refetchForTerms={refetchForTerms} refetchForNodes={refetchForNodes} />
<EmptyGlossarySection
title="Empty Glossary"
description="Create Terms and Term Groups to organize data assets using a shared vocabulary."
refetchForTerms={refetchForTerms}
refetchForNodes={refetchForNodes}
/>
)}
</MainContentWrapper>
</GlossaryWrapper>

10
datahub-web-react/src/app/glossary/EmptyGlossarySection.tsx
View File

@ -19,12 +19,14 @@ const StyledButton = styled(Button)`
`;
interface Props {
title?: string;
description?: string;
refetchForTerms?: () => void;
refetchForNodes?: () => void;
}
function EmptyGlossarySection(props: Props) {
const { refetchForTerms, refetchForNodes } = props;
const { title, description, refetchForTerms, refetchForNodes } = props;
const [isCreateTermModalVisible, setIsCreateTermModalVisible] = useState(false);
const [isCreateNodeModalVisible, setIsCreateNodeModalVisible] = useState(false);
@ -34,10 +36,8 @@ function EmptyGlossarySection(props: Props) {
<StyledEmpty
description={
<>
<Typography.Title level={4}>Empty Glossary</Typography.Title>
<Typography.Paragraph type="secondary">
Create Terms and Term Groups to organize data assets using a shared vocabulary.
</Typography.Paragraph>
<Typography.Title level={4}>{title}</Typography.Title>
<Typography.Paragraph type="secondary">{description}</Typography.Paragraph>
</>
}
>

16
datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts
View File

@ -74,6 +74,8 @@ import {
TOPIC_ALLOW,
TOPIC_DENY,
} from './kafka';
import { POSTGRES } from '../../conf/postgres/postgres';
import { POSTGRES_HOST_PORT, POSTGRES_DATABASE, POSTGRES_USERNAME, POSTGRES_PASSWORD } from './postgres';
import { HIVE } from '../../conf/hive/hive';
import { HIVE_HOST_PORT, HIVE_DATABASE, HIVE_USERNAME, HIVE_PASSWORD } from './hive';
@ -167,6 +169,20 @@ export const RECIPE_FIELDS: RecipeFields = {
filterSectionTooltip:
'Filter out data assets based on allow/deny regex patterns we match against. Deny patterns take precedence over allow patterns.',
},
[POSTGRES]: {
fields: [POSTGRES_HOST_PORT, POSTGRES_DATABASE, POSTGRES_USERNAME, POSTGRES_PASSWORD],
filterFields: [
REDSHIFT_SCHEMA_ALLOW,
REDSHIFT_SCHEMA_DENY,
REDSHIFT_TABLE_ALLOW,
REDSHIFT_TABLE_DENY,
REDSHIFT_VIEW_ALLOW,
REDSHIFT_VIEW_DENY,
],
advancedFields: [STATEFUL_INGESTION_ENABLED, PROFILING_ENABLED],
filterSectionTooltip:
'Filter out data assets based on allow/deny regex patterns we match against. Deny patterns take precedence over allow patterns.',
},
[HIVE]: {
fields: [HIVE_HOST_PORT, HIVE_DATABASE, HIVE_USERNAME, HIVE_PASSWORD],
filterFields: [

37
datahub-web-react/src/app/ingest/source/builder/RecipeForm/postgres.ts Normal file
View File

@ -0,0 +1,37 @@
import { RecipeField, FieldType } from './common';
export const POSTGRES_HOST_PORT: RecipeField = {
name: 'host_port',
label: 'Host Port',
tooltip: 'host URL.',
type: FieldType.TEXT,
fieldPath: 'source.config.host_port',
rules: null,
};
export const POSTGRES_DATABASE: RecipeField = {
name: 'database',
label: 'Database',
tooltip: 'Database (catalog). Optional, if not specified, ingests from all databases.',
type: FieldType.TEXT,
fieldPath: 'source.config.database',
rules: null,
};
export const POSTGRES_USERNAME: RecipeField = {
name: 'username',
label: 'Username',
tooltip: 'Username',
type: FieldType.TEXT,
fieldPath: 'source.config.username',
rules: null,
};
export const POSTGRES_PASSWORD: RecipeField = {
name: 'password',
label: 'Password',
tooltip: 'Password',
type: FieldType.SECRET,
fieldPath: 'source.config.password',
rules: null,
};

18
datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx
View File

@ -23,16 +23,18 @@ describe('DefineRecipeStep', () => {
it('should not render the RecipeBuilder if the type is not in CONNECTORS_WITH_FORM', () => {
const { getByText, queryByText } = render(
<DefineRecipeStep
state={{ type: 'postgres' }}
updateState={() => {}}
goTo={() => {}}
submit={() => {}}
cancel={() => {}}
/>,
<MockedProvider>
<DefineRecipeStep
state={{ type: 'glue' }}
updateState={() => {}}
goTo={() => {}}
submit={() => {}}
cancel={() => {}}
/>
</MockedProvider>,
);
expect(getByText('Configure Postgres Recipe')).toBeInTheDocument();
expect(getByText('Configure Glue Recipe')).toBeInTheDocument();
expect(queryByText('Connection')).toBeNull();
});
});

6
datahub-web-react/src/app/ingest/source/conf/postgres/postgres.ts
View File

@ -21,10 +21,14 @@ source:
# Profiling
profiling:
enabled: false
stateful_ingestion:
enabled: true
`;
export const POSTGRES = 'postgres';
const postgresConfig: SourceConfig = {
type: 'postgres',
type: POSTGRES,
placeholderRecipe,
displayName: 'Postgres',
docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/postgres/',