yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-07-24 10:00:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(ui): Improving Kafka UI Ingestion Form, Create Domain, Create Secret Modals (#6588)

Browse Source
This commit is contained in:
John Joyce 2022-12-01 15:25:52 -08:00 committed by GitHub
parent 10deee7333
commit df96e89557
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 133 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
datahub-web-react/src/app/domain/CreateDomainModal.tsx
View File

@ -29,15 +29,9 @@ const DESCRIPTION_FIELD_NAME = 'description';
export default function CreateDomainModal({ onClose, onCreate }: Props) {
const [createDomainMutation] = useCreateDomainMutation();
const [createButtonEnabled, setCreateButtonEnabled] = useState(true);
const [createButtonEnabled, setCreateButtonEnabled] = useState(false);
const [form] = Form.useForm();
const setStagedName = (name) => {
form.setFieldsValue({
name,
});
};
const onCreateDomain = () => {
createDomainMutation({
variables: {
@ -88,7 +82,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) {
<Button onClick={onClose} type="text">
Cancel
</Button>
<Button id="createDomainButton" onClick={onCreateDomain} disabled={createButtonEnabled}>
<Button id="createDomainButton" onClick={onCreateDomain} disabled={!createButtonEnabled}>
Create
</Button>
</>
@ -98,9 +92,9 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) {
form={form}
initialValues={{}}
layout="vertical"
onFieldsChange={() =>
setCreateButtonEnabled(form.getFieldsError().some((field) => field.errors.length > 0))
}
onFieldsChange={() => {
setCreateButtonEnabled(!form.getFieldsError().some((field) => field.errors.length > 0));
}}
>
<Form.Item label={<Typography.Text strong>Name</Typography.Text>}>
<Typography.Paragraph>Give your new Domain a name. </Typography.Paragraph>
@ -121,7 +115,15 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) {
<SuggestedNamesGroup>
{SUGGESTED_DOMAIN_NAMES.map((name) => {
return (
<ClickableTag key={name} onClick={() => setStagedName(name)}>
<ClickableTag
key={name}
onClick={() => {
form.setFieldsValue({
name,
});
setCreateButtonEnabled(true);
}}
>
{name}
</ClickableTag>
);
@ -137,7 +139,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) {
rules={[{ whitespace: true }, { min: 1, max: 500 }]}
hasFeedback
>
<Input placeholder="A description for your domain" />
<Input.TextArea placeholder="A description for your domain" />
</Form.Item>
</Form.Item>
<Collapse ghost>

76
datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx
View File

@ -3,6 +3,10 @@ import React, { useState } from 'react';
import { useEnterKeyListener } from '../../shared/useEnterKeyListener';
import { SecretBuilderState } from './types';
const NAME_FIELD_NAME = 'name';
const DESCRIPTION_FIELD_NAME = 'description';
const VALUE_FIELD_NAME = 'value';
type Props = {
initialState?: SecretBuilderState;
visible: boolean;
@ -11,38 +15,15 @@ type Props = {
};
export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }: Props) => {
const [secretBuilderState, setSecretBuilderState] = useState<SecretBuilderState>(initialState || {});
const [createButtonEnabled, setCreateButtonEnabled] = useState(true);
const [createButtonEnabled, setCreateButtonEnabled] = useState(false);
const [form] = Form.useForm();
const setName = (name: string) => {
setSecretBuilderState({
...secretBuilderState,
name,
});
};
const setValue = (value: string) => {
setSecretBuilderState({
...secretBuilderState,
value,
});
};
const setDescription = (description: string) => {
setSecretBuilderState({
...secretBuilderState,
description,
});
};
// Handle the Enter press
useEnterKeyListener({
querySelectorToExecuteClick: '#createSecretButton',
});
function resetValues() {
setSecretBuilderState({});
form.resetFields();
}
@ -60,8 +41,17 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
</Button>
<Button
id="createSecretButton"
onClick={() => onSubmit?.(secretBuilderState, resetValues)}
disabled={createButtonEnabled}
onClick={() =>
onSubmit?.(
{
name: form.getFieldValue(NAME_FIELD_NAME),
description: form.getFieldValue(DESCRIPTION_FIELD_NAME),
value: form.getFieldValue(VALUE_FIELD_NAME),
},
resetValues,
)
}
disabled={!createButtonEnabled}
>
Create
</Button>
@ -73,7 +63,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
initialValues={initialState}
layout="vertical"
onFieldsChange={() =>
setCreateButtonEnabled(form.getFieldsError().some((field) => field.errors.length > 0))
setCreateButtonEnabled(!form.getFieldsError().some((field) => field.errors.length > 0))
}
>
<Form.Item label={<Typography.Text strong>Name</Typography.Text>}>
@ -81,22 +71,19 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
Give your secret a name. This is what you&apos;ll use to reference the secret from your recipes.
</Typography.Paragraph>
<Form.Item
name="name"
name={NAME_FIELD_NAME}
rules={[
{
required: true,
message: 'Enter a name.',
},
{ whitespace: true },
{ whitespace: false },
{ min: 1, max: 50 },
{ pattern: /^[^\s\t${}\\,'"]+$/, message: 'This secret name is not allowed.' },
]}
hasFeedback
>
<Input
placeholder="A name for your secret"
value={secretBuilderState.name}
onChange={(event) => setName(event.target.value)}
/>
<Input placeholder="A name for your secret" />
</Form.Item>
</Form.Item>
<Form.Item label={<Typography.Text strong>Value</Typography.Text>}>
@ -104,7 +91,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
The value of your secret, which will be encrypted and stored securely within DataHub.
</Typography.Paragraph>
<Form.Item
name="value"
name={VALUE_FIELD_NAME}
rules={[
{
required: true,
@ -115,24 +102,19 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
]}
hasFeedback
>
<Input.TextArea
placeholder="The value of your secret"
value={secretBuilderState.value}
onChange={(event) => setValue(event.target.value)}
autoComplete="false"
/>
<Input.TextArea placeholder="The value of your secret" autoComplete="false" />
</Form.Item>
</Form.Item>
<Form.Item label={<Typography.Text strong>Description</Typography.Text>}>
<Typography.Paragraph>
An optional description to help keep track of your secret.
</Typography.Paragraph>
<Form.Item name="description" rules={[{ whitespace: true }, { min: 1, max: 500 }]} hasFeedback>
<Input
placeholder="The value of your secret"
value={secretBuilderState.description}
onChange={(event) => setDescription(event.target.value)}
/>
<Form.Item
name={DESCRIPTION_FIELD_NAME}
rules={[{ whitespace: true }, { min: 1, max: 500 }]}
hasFeedback
>
<Input.TextArea placeholder="A description for your secret" />
</Form.Item>
</Form.Item>
</Form>

1
datahub-web-react/src/app/ingest/source/builder/RecipeForm/DictField.tsx
View File

@ -71,6 +71,7 @@ export default function DictField({ field, removeMargin }: Props) {
{field.keyField && (
<StyledFormItem
{...restField}
required={field.required}
name={[name, field.keyField.name]}
initialValue=""
label={field.keyField.label}

21
datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx
View File

@ -1,5 +1,5 @@
import React from 'react';
import { Checkbox, DatePicker, Form, Input, Select, Tooltip } from 'antd';
import { Checkbox, DatePicker, Form, Input, Select, Tooltip, FormInstance } from 'antd';
import styled from 'styled-components/macro';
import Button from 'antd/lib/button';
import { MinusCircleOutlined, PlusOutlined } from '@ant-design/icons';
@ -56,6 +56,7 @@ function ListField({ field, removeMargin }: CommonFieldProps) {
function SelectField({ field, removeMargin }: CommonFieldProps) {
return (
<StyledFormItem
required={field.required}
name={field.name}
label={field.label}
tooltip={field.tooltip}
@ -63,7 +64,7 @@ function SelectField({ field, removeMargin }: CommonFieldProps) {
rules={field.rules || undefined}
>
{field.options && (
<Select placeholder={field.placeholder}>
<Select placeholder={field.placeholder} allowClear={!field.required}>
{field.options.map((option) => (
<Select.Option value={option.value}>{option.label}</Select.Option>
))}
@ -76,6 +77,7 @@ function SelectField({ field, removeMargin }: CommonFieldProps) {
function DateField({ field, removeMargin }: CommonFieldProps) {
return (
<StyledFormItem
required={field.required}
name={field.name}
label={field.label}
tooltip={field.tooltip}
@ -92,10 +94,11 @@ interface Props {
secrets: Secret[];
refetchSecrets: () => void;
removeMargin?: boolean;
form: FormInstance<any>;
}
function FormField(props: Props) {
const { field, secrets, refetchSecrets, removeMargin } = props;
const { field, secrets, refetchSecrets, removeMargin, form } = props;
if (field.type === FieldType.LIST) return <ListField field={field} removeMargin={removeMargin} />;
@ -105,7 +108,13 @@ function FormField(props: Props) {
if (field.type === FieldType.SECRET)
return (
<SecretField field={field} secrets={secrets} removeMargin={removeMargin} refetchSecrets={refetchSecrets} />
<SecretField
field={field}
secrets={secrets}
removeMargin={removeMargin}
refetchSecrets={refetchSecrets}
form={form}
/>
);
if (field.type === FieldType.DICT) return <DictField field={field} />;
@ -113,12 +122,14 @@ function FormField(props: Props) {
const isBoolean = field.type === FieldType.BOOLEAN;
let input = <Input placeholder={field.placeholder} />;
if (isBoolean) input = <Checkbox />;
if (field.type === FieldType.TEXTAREA) input = <Input.TextArea placeholder={field.placeholder} />;
if (field.type === FieldType.TEXTAREA)
input = <Input.TextArea required={field.required} placeholder={field.placeholder} />;
const valuePropName = isBoolean ? 'checked' : 'value';
const getValueFromEvent = isBoolean ? undefined : (e) => (e.target.value === '' ? null : e.target.value);
return (
<StyledFormItem
required={field.required}
style={isBoolean ? { flexDirection: 'row', alignItems: 'center' } : {}}
label={field.label}
name={field.name}

5
datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx
View File

@ -115,6 +115,7 @@ function RecipeForm(props: Props) {
});
const secrets =
data?.listSecrets?.secrets.sort((secretA, secretB) => secretA.name.localeCompare(secretB.name)) || [];
const [form] = Form.useForm();
function updateFormValues(changedValues: any, allValues: any) {
let updatedValues = YAML.parse(displayRecipe);
@ -137,6 +138,7 @@ function RecipeForm(props: Props) {
layout="vertical"
initialValues={getInitialValues(displayRecipe, allFields)}
onFinish={onClickNext}
form={form}
onValuesChange={updateFormValues}
>
<StyledCollapse defaultActiveKey="0">
@ -147,6 +149,7 @@ function RecipeForm(props: Props) {
secrets={secrets}
refetchSecrets={refetchSecrets}
removeMargin={i === fields.length - 1}
form={form}
/>
))}
{CONNECTORS_WITH_TEST_CONNECTION.has(type as string) && (
@ -184,6 +187,7 @@ function RecipeForm(props: Props) {
secrets={secrets}
refetchSecrets={refetchSecrets}
removeMargin={i === filterFields.length - 1}
form={form}
/>
</MarginWrapper>
</>
@ -209,6 +213,7 @@ function RecipeForm(props: Props) {
secrets={secrets}
refetchSecrets={refetchSecrets}
removeMargin={i === advancedFields.length - 1}
form={form}
/>
))}
</Collapse.Panel>

8
datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx
View File

@ -24,10 +24,11 @@ const CreateButton = styled(Button)`
`;
interface Props {
onSubmit?: (state: SecretBuilderState) => void;
refetchSecrets: () => void;
}
function CreateSecretButton({ refetchSecrets }: Props) {
function CreateSecretButton({ onSubmit, refetchSecrets }: Props) {
const [isCreateModalVisible, setIsCreateModalVisible] = useState(false);
const [createSecretMutation] = useCreateSecretMutation();
@ -42,12 +43,11 @@ function CreateSecretButton({ refetchSecrets }: Props) {
},
})
.then(() => {
onSubmit?.(state);
setIsCreateModalVisible(false);
resetBuilderState();
message.success({ content: `Created secret!` });
setTimeout(() => refetchSecrets(), 3000);
message.loading({ content: `Loading...`, duration: 3 }).then(() => {
message.success({ content: `Successfully created Secret!` });
});
})
.catch((e) => {
message.destroy();

34
datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx
View File

@ -1,5 +1,5 @@
import React, { ReactNode } from 'react';
import { AutoComplete, Divider, Form } from 'antd';
import { AutoComplete, Divider, Form, FormInstance } from 'antd';
import styled from 'styled-components/macro';
import { Secret } from '../../../../../../types.generated';
import CreateSecretButton from './CreateSecretButton';
@ -52,6 +52,7 @@ interface SecretFieldProps {
secrets: Secret[];
removeMargin?: boolean;
refetchSecrets: () => void;
form: FormInstance<any>;
}
function SecretFieldTooltip({ tooltipLabel }: { tooltipLabel?: string | ReactNode }) {
@ -79,11 +80,16 @@ function SecretFieldTooltip({ tooltipLabel }: { tooltipLabel?: string | ReactNod
);
}
function SecretField({ field, secrets, removeMargin, refetchSecrets }: SecretFieldProps) {
const options = secrets.map((secret) => ({ value: `\${${secret.name}}`, label: secret.name }));
const encodeSecret = (secretName: string) => {
return `\${${secretName}}`;
};
function SecretField({ field, secrets, removeMargin, form, refetchSecrets }: SecretFieldProps) {
const options = secrets.map((secret) => ({ value: encodeSecret(secret.name), label: secret.name }));
return (
<StyledFormItem
required={field.required}
name={field.name}
label={field.label}
rules={field.rules || undefined}
@ -94,14 +100,22 @@ function SecretField({ field, secrets, removeMargin, refetchSecrets }: SecretFie
<AutoComplete
placeholder={field.placeholder}
filterOption={(input, option) => !!option?.value.toLowerCase().includes(input.toLowerCase())}
notFoundContent={<>No secrets found</>}
options={options}
dropdownRender={(menu) => (
<>
{menu}
<StyledDivider />
<CreateSecretButton refetchSecrets={refetchSecrets} />
</>
)}
dropdownRender={(menu) => {
return (
<>
{menu}
<StyledDivider />
<CreateSecretButton
onSubmit={(state) =>
form.setFields([{ name: field.name, value: encodeSecret(state.name as string) }])
}
refetchSecrets={refetchSecrets}
/>
</>
);
}}
/>
</StyledFormItem>
);

21
datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx
View File

@ -25,6 +25,7 @@ export interface RecipeField {
type: FieldType;
fieldPath: string | string[];
rules: any[] | null;
required?: boolean; // Today, Only makes a difference on Selects
section?: string;
options?: Option[];
buttonLabel?: string;
@ -55,13 +56,11 @@ function clearFieldAndParents(recipe: any, fieldPath: string | string[]) {
}
export function setFieldValueOnRecipe(recipe: any, value: any, fieldPath: string | string[]) {
const updatedRecipe = { ...recipe };
if (value !== undefined) {
if (value === null || value === '') {
clearFieldAndParents(updatedRecipe, fieldPath);
return updatedRecipe;
}
set(updatedRecipe, fieldPath, value);
if (value === null || value === '' || value === undefined) {
clearFieldAndParents(updatedRecipe, fieldPath);
return updatedRecipe;
}
set(updatedRecipe, fieldPath, value);
return updatedRecipe;
}
@ -267,7 +266,7 @@ export const INCLUDE_TABLE_LINEAGE: RecipeField = {
export const PROFILING_ENABLED: RecipeField = {
name: 'profiling.enabled',
label: 'Enable Profiling',
tooltip: 'Whether profiling should be done.',
tooltip: 'Whether profiling should be performed on the assets extracted from the ingestion source.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.profiling.enabled',
rules: null,
@ -276,7 +275,7 @@ export const PROFILING_ENABLED: RecipeField = {
export const STATEFUL_INGESTION_ENABLED: RecipeField = {
name: 'stateful_ingestion.enabled',
label: 'Enable Stateful Ingestion',
tooltip: 'Remove stale datasets from datahub once they have been deleted in the source.',
tooltip: 'Remove stale assets from DataHub once they have been deleted in the ingestion source.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.stateful_ingestion.enabled',
rules: null,
@ -322,7 +321,7 @@ export const TABLE_LINEAGE_MODE: RecipeField = {
export const INGEST_TAGS: RecipeField = {
name: 'ingest_tags',
label: 'Ingest Tags',
tooltip: 'Ingest Tags from source. This will override Tags entered from UI',
tooltip: 'Ingest Tags from the source. Be careful: This can override Tags entered by users of DataHub.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.ingest_tags',
rules: null,
@ -331,7 +330,7 @@ export const INGEST_TAGS: RecipeField = {
export const INGEST_OWNER: RecipeField = {
name: 'ingest_owner',
label: 'Ingest Owner',
tooltip: 'Ingest Owner from source. This will override Owner info entered from UI',
tooltip: 'Ingest Owner from source. Be careful: This cah override Owners added by users of DataHub.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.ingest_owner',
rules: null,
@ -391,7 +390,7 @@ export const START_TIME: RecipeField = {
name: 'start_time',
label: 'Start Time',
tooltip:
'Earliest date of audit logs to process for usage, lineage etc. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) for your first ingestion run, and then uncheck it for future runs.',
'Earliest date used when processing audit logs for lineage, usage, and more. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) to bootstrap your first ingestion run, and then reduce for subsequent runs.',
placeholder: 'Select date and time',
type: FieldType.DATE,
fieldPath: startTimeFieldPath,

39
datahub-web-react/src/app/ingest/source/builder/RecipeForm/kafka.ts
View File

@ -1,10 +1,15 @@
import { RecipeField, FieldType, setListValuesOnRecipe } from './common';
// TODO: Currently platform_instance is required to be present for stateful ingestion to work
// We need to solve this prior to enabling by default here.
const saslUsernameFieldPath = ['source', 'config', 'connection', 'consumer_config', 'sasl.username'];
export const KAFKA_SASL_USERNAME: RecipeField = {
name: 'connection.consumer_config.sasl.username',
label: 'Username',
tooltip: 'SASL username. You can get (in the Confluent UI) from your cluster -> Data Integration -> API Keys.',
placeholder: 'datahub-client',
tooltip:
'The SASL username. Required if the Security Protocol is SASL based. In the Confluent Control Center, you can find this in Cluster > Data Integration > API Keys.',
type: FieldType.TEXT,
fieldPath: saslUsernameFieldPath,
rules: null,
@ -14,7 +19,9 @@ const saslPasswordFieldPath = ['source', 'config', 'connection', 'consumer_confi
export const KAFKA_SASL_PASSWORD: RecipeField = {
name: 'connection.consumer_config.sasl.password',
label: 'Password',
tooltip: 'SASL password. You can get (in the Confluent UI) from your cluster -> Data Integration -> API Keys.',
placeholder: 'datahub-client-password',
tooltip:
'The SASL Password. Required if the Security Protocol is SASL based. In the Confluent Control Center, you can find this in Cluster > Data Integration > API Keys.',
type: FieldType.SECRET,
fieldPath: saslPasswordFieldPath,
rules: null,
@ -22,8 +29,10 @@ export const KAFKA_SASL_PASSWORD: RecipeField = {
export const KAFKA_BOOTSTRAP: RecipeField = {
name: 'connection.bootstrap',
label: 'Connection Bootstrap',
tooltip: 'Bootstrap URL.',
label: 'Bootstrap Servers',
required: true,
tooltip:
'The ‘host[:port]’ string (or list of ‘host[:port]’ strings) that we should contact to bootstrap initial cluster metadata.',
placeholder: 'abc-defg.eu-west-1.aws.confluent.cloud:9092',
type: FieldType.TEXT,
fieldPath: 'source.config.connection.bootstrap',
@ -33,7 +42,8 @@ export const KAFKA_BOOTSTRAP: RecipeField = {
export const KAFKA_SCHEMA_REGISTRY_URL: RecipeField = {
name: 'connection.schema_registry_url',
label: 'Schema Registry URL',
tooltip: 'URL where your Confluent Cloud Schema Registry is hosted.',
tooltip:
'The URL where the schema Schema Registry is hosted. If provided, DataHub will attempt to extract Avro and Protobuf topic schemas from the registry.',
placeholder: 'https://abc-defgh.us-east-2.aws.confluent.cloud',
type: FieldType.TEXT,
fieldPath: 'source.config.connection.schema_registry_url',
@ -51,7 +61,7 @@ export const KAFKA_SCHEMA_REGISTRY_USER_CREDENTIAL: RecipeField = {
name: 'schema_registry_config.basic.auth.user.info',
label: 'Schema Registry Credentials',
tooltip:
'API credentials for Confluent schema registry which you get (in Confluent UI) from Schema Registry -> API credentials.',
'API credentials for the Schema Registry. In Confluent Control Center, you can find these under Schema Registry > API Credentials.',
// eslint-disable-next-line no-template-curly-in-string
placeholder: '${REGISTRY_API_KEY_ID}:${REGISTRY_API_KEY_SECRET}',
type: FieldType.TEXT,
@ -63,13 +73,16 @@ const securityProtocolFieldPath = ['source', 'config', 'connection', 'consumer_c
export const KAFKA_SECURITY_PROTOCOL: RecipeField = {
name: 'security.protocol',
label: 'Security Protocol',
tooltip: 'Security Protocol',
tooltip: 'The Security Protocol used for authentication.',
type: FieldType.SELECT,
required: true,
fieldPath: securityProtocolFieldPath,
rules: null,
options: [
{ label: 'PLAINTEXT', value: 'PLAINTEXT' },
{ label: 'SASL_SSL', value: 'SASL_SSL' },
{ label: 'SASL_PLAINTEXT', value: 'SASL_PLAINTEXT' },
{ label: 'SSL', value: 'SSL' },
],
};
@ -77,9 +90,11 @@ const saslMechanismFieldPath = ['source', 'config', 'connection', 'consumer_conf
export const KAFKA_SASL_MECHANISM: RecipeField = {
name: 'sasl.mechanism',
label: 'SASL Mechanism',
tooltip: 'SASL Mechanism',
tooltip:
'The SASL mechanism used for authentication. This field is required if the selected Security Protocol is SASL based.',
type: FieldType.SELECT,
fieldPath: saslMechanismFieldPath,
placeholder: 'None',
rules: null,
options: [
{ label: 'PLAIN', value: 'PLAIN' },
@ -92,12 +107,12 @@ const topicAllowFieldPath = 'source.config.topic_patterns.allow';
export const TOPIC_ALLOW: RecipeField = {
name: 'topic_patterns.allow',
label: 'Allow Patterns',
tooltip: 'Use regex here.',
tooltip: 'Provide an optional Regular Expresssion (REGEX) to include specific Kafka Topic names in ingestion.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: topicAllowFieldPath,
rules: null,
section: 'Topics',
section: 'Filter by Topic',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, topicAllowFieldPath),
};
@ -106,12 +121,12 @@ const topicDenyFieldPath = 'source.config.topic_patterns.deny';
export const TOPIC_DENY: RecipeField = {
name: 'topic_patterns.deny',
label: 'Deny Patterns',
tooltip: 'Use regex here.',
tooltip: 'Provide an optional Regular Expresssion (REGEX) to exclude specific Kafka Topic names from ingestion.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: topicDenyFieldPath,
rules: null,
section: 'Topics',
section: 'Filter by Topic',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, topicDenyFieldPath),
};

2
datahub-web-react/src/app/ingest/source/builder/sources.json
View File

@ -25,7 +25,7 @@
"name": "kafka",
"displayName": "Kafka",
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/kafka/",
"recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"SASL_SSL\"\n sasl.mechanism: \"PLAIN\"\n stateful_ingestion:\n enabled: true'"
"recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"PLAINTEXT\"\n stateful_ingestion:\n enabled: false"
},
{
"urn": "urn:li:dataPlatform:looker",

1
datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts
View File

@ -11,6 +11,7 @@ source:
sasl.mechanism: "PLAIN"
stateful_ingestion:
enabled: true
`;
export const KAFKA = 'kafka';