feat(ingestion-ui) Add ingestion form for Postgres (#5671)

datahub-web-react/src/app/entity/glossaryNode/ChildrenTab.tsx

@@ -1,5 +1,6 @@
 import React from 'react';
 import { EntityType, GlossaryNode, GlossaryTerm } from '../../../types.generated';
+import EmptyGlossarySection from '../../glossary/EmptyGlossarySection';
 import GlossaryEntitiesList from '../../glossary/GlossaryEntitiesList';
 import { useEntityRegistry } from '../../useEntityRegistry';
 import { sortGlossaryTerms } from '../glossaryTerm/utils';
@@ -19,12 +20,18 @@ function ChildrenTab() {
         .sort((termA, termB) => sortGlossaryTerms(entityRegistry, termA.entity, termB.entity))
         .map((child) => child.entity);
 
-    return (
+    const hasTermsOrNodes = !!childNodes?.length || !!childTerms?.length;
-        <GlossaryEntitiesList
+
-            nodes={(childNodes as GlossaryNode[]) || []}
+    if (hasTermsOrNodes) {
-            terms={(childTerms as GlossaryTerm[]) || []}
+        return (
-        />
+            <GlossaryEntitiesList
-    );
+                nodes={(childNodes as GlossaryNode[]) || []}
+                terms={(childTerms as GlossaryTerm[]) || []}
+            />
+        );
+    }
+
+    return <EmptyGlossarySection description="No Terms or Term Groups" />;
 }
 
 export default ChildrenTab;

datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx

@@ -93,7 +93,12 @@ function BusinessGlossaryPage() {
                     </HeaderWrapper>
                     {hasTermsOrNodes && <GlossaryEntitiesList nodes={nodes || []} terms={terms || []} />}
                     {!(termsLoading || nodesLoading) && !hasTermsOrNodes && (
-                        <EmptyGlossarySection refetchForTerms={refetchForTerms} refetchForNodes={refetchForNodes} />
+                        <EmptyGlossarySection
+                            title="Empty Glossary"
+                            description="Create Terms and Term Groups to organize data assets using a shared vocabulary."
+                            refetchForTerms={refetchForTerms}
+                            refetchForNodes={refetchForNodes}
+                        />
                     )}
                 </MainContentWrapper>
             </GlossaryWrapper>

datahub-web-react/src/app/glossary/EmptyGlossarySection.tsx

@@ -19,12 +19,14 @@ const StyledButton = styled(Button)`
 `;
 
 interface Props {
+    title?: string;
+    description?: string;
     refetchForTerms?: () => void;
     refetchForNodes?: () => void;
 }
 
 function EmptyGlossarySection(props: Props) {
-    const { refetchForTerms, refetchForNodes } = props;
+    const { title, description, refetchForTerms, refetchForNodes } = props;
 
     const [isCreateTermModalVisible, setIsCreateTermModalVisible] = useState(false);
     const [isCreateNodeModalVisible, setIsCreateNodeModalVisible] = useState(false);
@@ -34,10 +36,8 @@ function EmptyGlossarySection(props: Props) {
             <StyledEmpty
                 description={
                     <>
-                        <Typography.Title level={4}>Empty Glossary</Typography.Title>
+                        <Typography.Title level={4}>{title}</Typography.Title>
-                        <Typography.Paragraph type="secondary">
+                        <Typography.Paragraph type="secondary">{description}</Typography.Paragraph>
-                            Create Terms and Term Groups to organize data assets using a shared vocabulary.
-                        </Typography.Paragraph>
                     </>
                 }
             >

datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts

@@ -74,6 +74,8 @@ import {
     TOPIC_ALLOW,
     TOPIC_DENY,
 } from './kafka';
+import { POSTGRES } from '../../conf/postgres/postgres';
+import { POSTGRES_HOST_PORT, POSTGRES_DATABASE, POSTGRES_USERNAME, POSTGRES_PASSWORD } from './postgres';
 import { HIVE } from '../../conf/hive/hive';
 import { HIVE_HOST_PORT, HIVE_DATABASE, HIVE_USERNAME, HIVE_PASSWORD } from './hive';
 
@@ -167,6 +169,20 @@ export const RECIPE_FIELDS: RecipeFields = {
         filterSectionTooltip:
             'Filter out data assets based on allow/deny regex patterns we match against. Deny patterns take precedence over allow patterns.',
     },
+    [POSTGRES]: {
+        fields: [POSTGRES_HOST_PORT, POSTGRES_DATABASE, POSTGRES_USERNAME, POSTGRES_PASSWORD],
+        filterFields: [
+            REDSHIFT_SCHEMA_ALLOW,
+            REDSHIFT_SCHEMA_DENY,
+            REDSHIFT_TABLE_ALLOW,
+            REDSHIFT_TABLE_DENY,
+            REDSHIFT_VIEW_ALLOW,
+            REDSHIFT_VIEW_DENY,
+        ],
+        advancedFields: [STATEFUL_INGESTION_ENABLED, PROFILING_ENABLED],
+        filterSectionTooltip:
+            'Filter out data assets based on allow/deny regex patterns we match against. Deny patterns take precedence over allow patterns.',
+    },
     [HIVE]: {
         fields: [HIVE_HOST_PORT, HIVE_DATABASE, HIVE_USERNAME, HIVE_PASSWORD],
         filterFields: [

datahub-web-react/src/app/ingest/source/builder/RecipeForm/postgres.ts

@@ -0,0 +1,37 @@
+import { RecipeField, FieldType } from './common';
+
+export const POSTGRES_HOST_PORT: RecipeField = {
+    name: 'host_port',
+    label: 'Host Port',
+    tooltip: 'host URL.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.host_port',
+    rules: null,
+};
+
+export const POSTGRES_DATABASE: RecipeField = {
+    name: 'database',
+    label: 'Database',
+    tooltip: 'Database (catalog). Optional, if not specified, ingests from all databases.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.database',
+    rules: null,
+};
+
+export const POSTGRES_USERNAME: RecipeField = {
+    name: 'username',
+    label: 'Username',
+    tooltip: 'Username',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.username',
+    rules: null,
+};
+
+export const POSTGRES_PASSWORD: RecipeField = {
+    name: 'password',
+    label: 'Password',
+    tooltip: 'Password',
+    type: FieldType.SECRET,
+    fieldPath: 'source.config.password',
+    rules: null,
+};

datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx

@@ -23,16 +23,18 @@ describe('DefineRecipeStep', () => {
 
     it('should not render the RecipeBuilder if the type is not in CONNECTORS_WITH_FORM', () => {
         const { getByText, queryByText } = render(
-            <DefineRecipeStep
+            <MockedProvider>
-                state={{ type: 'postgres' }}
+                <DefineRecipeStep
-                updateState={() => {}}
+                    state={{ type: 'glue' }}
-                goTo={() => {}}
+                    updateState={() => {}}
-                submit={() => {}}
+                    goTo={() => {}}
-                cancel={() => {}}
+                    submit={() => {}}
-            />,
+                    cancel={() => {}}
+                />
+            </MockedProvider>,
         );
 
-        expect(getByText('Configure Postgres Recipe')).toBeInTheDocument();
+        expect(getByText('Configure Glue Recipe')).toBeInTheDocument();
         expect(queryByText('Connection')).toBeNull();
     });
 });

datahub-web-react/src/app/ingest/source/conf/postgres/postgres.ts

@@ -21,10 +21,14 @@ source:
         # Profiling
         profiling:
             enabled: false
+        stateful_ingestion:
+            enabled: true    
 `;
 
+export const POSTGRES = 'postgres';
+
 const postgresConfig: SourceConfig = {
-    type: 'postgres',
+    type: POSTGRES,
     placeholderRecipe,
     displayName: 'Postgres',
     docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/postgres/',