From c2fbd75d2a2d09ff21fab78f5ecef1738a7d1f9c Mon Sep 17 00:00:00 2001 From: Ankit keshari <86347578+Ankit-Keshari-Vituity@users.noreply.github.com> Date: Wed, 24 Aug 2022 02:33:16 +0530 Subject: [PATCH] feat(ingestion-ui) Add ingestion form for Postgres (#5671) --- .../app/entity/glossaryNode/ChildrenTab.tsx | 19 +++++++--- .../src/app/glossary/BusinessGlossaryPage.tsx | 7 +++- .../src/app/glossary/EmptyGlossarySection.tsx | 10 ++--- .../source/builder/RecipeForm/constants.ts | 16 ++++++++ .../source/builder/RecipeForm/postgres.ts | 37 +++++++++++++++++++ .../__tests__/DefineRecipeStep.test.tsx | 18 +++++---- .../ingest/source/conf/postgres/postgres.ts | 6 ++- 7 files changed, 92 insertions(+), 21 deletions(-) create mode 100644 datahub-web-react/src/app/ingest/source/builder/RecipeForm/postgres.ts diff --git a/datahub-web-react/src/app/entity/glossaryNode/ChildrenTab.tsx b/datahub-web-react/src/app/entity/glossaryNode/ChildrenTab.tsx index 9aa4f40c2c..c9a58d165d 100644 --- a/datahub-web-react/src/app/entity/glossaryNode/ChildrenTab.tsx +++ b/datahub-web-react/src/app/entity/glossaryNode/ChildrenTab.tsx @@ -1,5 +1,6 @@ import React from 'react'; import { EntityType, GlossaryNode, GlossaryTerm } from '../../../types.generated'; +import EmptyGlossarySection from '../../glossary/EmptyGlossarySection'; import GlossaryEntitiesList from '../../glossary/GlossaryEntitiesList'; import { useEntityRegistry } from '../../useEntityRegistry'; import { sortGlossaryTerms } from '../glossaryTerm/utils'; @@ -19,12 +20,18 @@ function ChildrenTab() { .sort((termA, termB) => sortGlossaryTerms(entityRegistry, termA.entity, termB.entity)) .map((child) => child.entity); - return ( - - ); + const hasTermsOrNodes = !!childNodes?.length || !!childTerms?.length; + + if (hasTermsOrNodes) { + return ( + + ); + } + + return ; } export default ChildrenTab; diff --git a/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx b/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx index f0527f298b..3556b3140f 100644 --- a/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx +++ b/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx @@ -93,7 +93,12 @@ function BusinessGlossaryPage() { {hasTermsOrNodes && } {!(termsLoading || nodesLoading) && !hasTermsOrNodes && ( - + )} diff --git a/datahub-web-react/src/app/glossary/EmptyGlossarySection.tsx b/datahub-web-react/src/app/glossary/EmptyGlossarySection.tsx index 0571e808a5..7f1fd0d062 100644 --- a/datahub-web-react/src/app/glossary/EmptyGlossarySection.tsx +++ b/datahub-web-react/src/app/glossary/EmptyGlossarySection.tsx @@ -19,12 +19,14 @@ const StyledButton = styled(Button)` `; interface Props { + title?: string; + description?: string; refetchForTerms?: () => void; refetchForNodes?: () => void; } function EmptyGlossarySection(props: Props) { - const { refetchForTerms, refetchForNodes } = props; + const { title, description, refetchForTerms, refetchForNodes } = props; const [isCreateTermModalVisible, setIsCreateTermModalVisible] = useState(false); const [isCreateNodeModalVisible, setIsCreateNodeModalVisible] = useState(false); @@ -34,10 +36,8 @@ function EmptyGlossarySection(props: Props) { - Empty Glossary - - Create Terms and Term Groups to organize data assets using a shared vocabulary. - + {title} + {description} } > diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts index 60c4c90116..4b62307b00 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts @@ -74,6 +74,8 @@ import { TOPIC_ALLOW, TOPIC_DENY, } from './kafka'; +import { POSTGRES } from '../../conf/postgres/postgres'; +import { POSTGRES_HOST_PORT, POSTGRES_DATABASE, POSTGRES_USERNAME, POSTGRES_PASSWORD } from './postgres'; import { HIVE } from '../../conf/hive/hive'; import { HIVE_HOST_PORT, HIVE_DATABASE, HIVE_USERNAME, HIVE_PASSWORD } from './hive'; @@ -167,6 +169,20 @@ export const RECIPE_FIELDS: RecipeFields = { filterSectionTooltip: 'Filter out data assets based on allow/deny regex patterns we match against. Deny patterns take precedence over allow patterns.', }, + [POSTGRES]: { + fields: [POSTGRES_HOST_PORT, POSTGRES_DATABASE, POSTGRES_USERNAME, POSTGRES_PASSWORD], + filterFields: [ + REDSHIFT_SCHEMA_ALLOW, + REDSHIFT_SCHEMA_DENY, + REDSHIFT_TABLE_ALLOW, + REDSHIFT_TABLE_DENY, + REDSHIFT_VIEW_ALLOW, + REDSHIFT_VIEW_DENY, + ], + advancedFields: [STATEFUL_INGESTION_ENABLED, PROFILING_ENABLED], + filterSectionTooltip: + 'Filter out data assets based on allow/deny regex patterns we match against. Deny patterns take precedence over allow patterns.', + }, [HIVE]: { fields: [HIVE_HOST_PORT, HIVE_DATABASE, HIVE_USERNAME, HIVE_PASSWORD], filterFields: [ diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/postgres.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/postgres.ts new file mode 100644 index 0000000000..b1d32d08e1 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/postgres.ts @@ -0,0 +1,37 @@ +import { RecipeField, FieldType } from './common'; + +export const POSTGRES_HOST_PORT: RecipeField = { + name: 'host_port', + label: 'Host Port', + tooltip: 'host URL.', + type: FieldType.TEXT, + fieldPath: 'source.config.host_port', + rules: null, +}; + +export const POSTGRES_DATABASE: RecipeField = { + name: 'database', + label: 'Database', + tooltip: 'Database (catalog). Optional, if not specified, ingests from all databases.', + type: FieldType.TEXT, + fieldPath: 'source.config.database', + rules: null, +}; + +export const POSTGRES_USERNAME: RecipeField = { + name: 'username', + label: 'Username', + tooltip: 'Username', + type: FieldType.TEXT, + fieldPath: 'source.config.username', + rules: null, +}; + +export const POSTGRES_PASSWORD: RecipeField = { + name: 'password', + label: 'Password', + tooltip: 'Password', + type: FieldType.SECRET, + fieldPath: 'source.config.password', + rules: null, +}; diff --git a/datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx b/datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx index 1cb38b6c87..f89c6fec4a 100644 --- a/datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx @@ -23,16 +23,18 @@ describe('DefineRecipeStep', () => { it('should not render the RecipeBuilder if the type is not in CONNECTORS_WITH_FORM', () => { const { getByText, queryByText } = render( - {}} - goTo={() => {}} - submit={() => {}} - cancel={() => {}} - />, + + {}} + goTo={() => {}} + submit={() => {}} + cancel={() => {}} + /> + , ); - expect(getByText('Configure Postgres Recipe')).toBeInTheDocument(); + expect(getByText('Configure Glue Recipe')).toBeInTheDocument(); expect(queryByText('Connection')).toBeNull(); }); }); diff --git a/datahub-web-react/src/app/ingest/source/conf/postgres/postgres.ts b/datahub-web-react/src/app/ingest/source/conf/postgres/postgres.ts index 93c4969988..301afe8178 100644 --- a/datahub-web-react/src/app/ingest/source/conf/postgres/postgres.ts +++ b/datahub-web-react/src/app/ingest/source/conf/postgres/postgres.ts @@ -21,10 +21,14 @@ source: # Profiling profiling: enabled: false + stateful_ingestion: + enabled: true `; +export const POSTGRES = 'postgres'; + const postgresConfig: SourceConfig = { - type: 'postgres', + type: POSTGRES, placeholderRecipe, displayName: 'Postgres', docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/postgres/',