diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx index 75436a463f..629298e479 100644 --- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx +++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx @@ -29,15 +29,9 @@ const DESCRIPTION_FIELD_NAME = 'description'; export default function CreateDomainModal({ onClose, onCreate }: Props) { const [createDomainMutation] = useCreateDomainMutation(); - const [createButtonEnabled, setCreateButtonEnabled] = useState(true); + const [createButtonEnabled, setCreateButtonEnabled] = useState(false); const [form] = Form.useForm(); - const setStagedName = (name) => { - form.setFieldsValue({ - name, - }); - }; - const onCreateDomain = () => { createDomainMutation({ variables: { @@ -88,7 +82,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { - @@ -98,9 +92,9 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { form={form} initialValues={{}} layout="vertical" - onFieldsChange={() => - setCreateButtonEnabled(form.getFieldsError().some((field) => field.errors.length > 0)) - } + onFieldsChange={() => { + setCreateButtonEnabled(!form.getFieldsError().some((field) => field.errors.length > 0)); + }} > Name}> Give your new Domain a name. @@ -121,7 +115,15 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { {SUGGESTED_DOMAIN_NAMES.map((name) => { return ( - setStagedName(name)}> + { + form.setFieldsValue({ + name, + }); + setCreateButtonEnabled(true); + }} + > {name} ); @@ -137,7 +139,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { rules={[{ whitespace: true }, { min: 1, max: 500 }]} hasFeedback > - + diff --git a/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx b/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx index 92bdce6d52..539eef9726 100644 --- a/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx +++ b/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx @@ -3,6 +3,10 @@ import React, { useState } from 'react'; import { useEnterKeyListener } from '../../shared/useEnterKeyListener'; import { SecretBuilderState } from './types'; +const NAME_FIELD_NAME = 'name'; +const DESCRIPTION_FIELD_NAME = 'description'; +const VALUE_FIELD_NAME = 'value'; + type Props = { initialState?: SecretBuilderState; visible: boolean; @@ -11,38 +15,15 @@ type Props = { }; export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }: Props) => { - const [secretBuilderState, setSecretBuilderState] = useState(initialState || {}); - const [createButtonEnabled, setCreateButtonEnabled] = useState(true); + const [createButtonEnabled, setCreateButtonEnabled] = useState(false); const [form] = Form.useForm(); - const setName = (name: string) => { - setSecretBuilderState({ - ...secretBuilderState, - name, - }); - }; - - const setValue = (value: string) => { - setSecretBuilderState({ - ...secretBuilderState, - value, - }); - }; - - const setDescription = (description: string) => { - setSecretBuilderState({ - ...secretBuilderState, - description, - }); - }; - // Handle the Enter press useEnterKeyListener({ querySelectorToExecuteClick: '#createSecretButton', }); function resetValues() { - setSecretBuilderState({}); form.resetFields(); } @@ -60,8 +41,17 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel } @@ -73,7 +63,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel } initialValues={initialState} layout="vertical" onFieldsChange={() => - setCreateButtonEnabled(form.getFieldsError().some((field) => field.errors.length > 0)) + setCreateButtonEnabled(!form.getFieldsError().some((field) => field.errors.length > 0)) } > Name}> @@ -81,22 +71,19 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel } Give your secret a name. This is what you'll use to reference the secret from your recipes. - setName(event.target.value)} - /> + Value}> @@ -104,7 +91,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel } The value of your secret, which will be encrypted and stored securely within DataHub. - setValue(event.target.value)} - autoComplete="false" - /> + Description}> An optional description to help keep track of your secret. - - setDescription(event.target.value)} - /> + + diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/DictField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/DictField.tsx index f378e9d109..e15a83b08e 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/DictField.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/DictField.tsx @@ -71,6 +71,7 @@ export default function DictField({ field, removeMargin }: Props) { {field.keyField && ( {field.options && ( - {field.options.map((option) => ( {option.label} ))} @@ -76,6 +77,7 @@ function SelectField({ field, removeMargin }: CommonFieldProps) { function DateField({ field, removeMargin }: CommonFieldProps) { return ( void; removeMargin?: boolean; + form: FormInstance; } function FormField(props: Props) { - const { field, secrets, refetchSecrets, removeMargin } = props; + const { field, secrets, refetchSecrets, removeMargin, form } = props; if (field.type === FieldType.LIST) return ; @@ -105,7 +108,13 @@ function FormField(props: Props) { if (field.type === FieldType.SECRET) return ( - + ); if (field.type === FieldType.DICT) return ; @@ -113,12 +122,14 @@ function FormField(props: Props) { const isBoolean = field.type === FieldType.BOOLEAN; let input = ; if (isBoolean) input = ; - if (field.type === FieldType.TEXTAREA) input = ; + if (field.type === FieldType.TEXTAREA) + input = ; const valuePropName = isBoolean ? 'checked' : 'value'; const getValueFromEvent = isBoolean ? undefined : (e) => (e.target.value === '' ? null : e.target.value); return ( secretA.name.localeCompare(secretB.name)) || []; + const [form] = Form.useForm(); function updateFormValues(changedValues: any, allValues: any) { let updatedValues = YAML.parse(displayRecipe); @@ -137,6 +138,7 @@ function RecipeForm(props: Props) { layout="vertical" initialValues={getInitialValues(displayRecipe, allFields)} onFinish={onClickNext} + form={form} onValuesChange={updateFormValues} > @@ -147,6 +149,7 @@ function RecipeForm(props: Props) { secrets={secrets} refetchSecrets={refetchSecrets} removeMargin={i === fields.length - 1} + form={form} /> ))} {CONNECTORS_WITH_TEST_CONNECTION.has(type as string) && ( @@ -184,6 +187,7 @@ function RecipeForm(props: Props) { secrets={secrets} refetchSecrets={refetchSecrets} removeMargin={i === filterFields.length - 1} + form={form} /> @@ -209,6 +213,7 @@ function RecipeForm(props: Props) { secrets={secrets} refetchSecrets={refetchSecrets} removeMargin={i === advancedFields.length - 1} + form={form} /> ))} diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx index 01ca495900..31024512cb 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx @@ -24,10 +24,11 @@ const CreateButton = styled(Button)` `; interface Props { + onSubmit?: (state: SecretBuilderState) => void; refetchSecrets: () => void; } -function CreateSecretButton({ refetchSecrets }: Props) { +function CreateSecretButton({ onSubmit, refetchSecrets }: Props) { const [isCreateModalVisible, setIsCreateModalVisible] = useState(false); const [createSecretMutation] = useCreateSecretMutation(); @@ -42,12 +43,11 @@ function CreateSecretButton({ refetchSecrets }: Props) { }, }) .then(() => { + onSubmit?.(state); setIsCreateModalVisible(false); resetBuilderState(); + message.success({ content: `Created secret!` }); setTimeout(() => refetchSecrets(), 3000); - message.loading({ content: `Loading...`, duration: 3 }).then(() => { - message.success({ content: `Successfully created Secret!` }); - }); }) .catch((e) => { message.destroy(); diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx index ac1fe2166f..08213f8912 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx @@ -1,5 +1,5 @@ import React, { ReactNode } from 'react'; -import { AutoComplete, Divider, Form } from 'antd'; +import { AutoComplete, Divider, Form, FormInstance } from 'antd'; import styled from 'styled-components/macro'; import { Secret } from '../../../../../../types.generated'; import CreateSecretButton from './CreateSecretButton'; @@ -52,6 +52,7 @@ interface SecretFieldProps { secrets: Secret[]; removeMargin?: boolean; refetchSecrets: () => void; + form: FormInstance; } function SecretFieldTooltip({ tooltipLabel }: { tooltipLabel?: string | ReactNode }) { @@ -79,11 +80,16 @@ function SecretFieldTooltip({ tooltipLabel }: { tooltipLabel?: string | ReactNod ); } -function SecretField({ field, secrets, removeMargin, refetchSecrets }: SecretFieldProps) { - const options = secrets.map((secret) => ({ value: `\${${secret.name}}`, label: secret.name })); +const encodeSecret = (secretName: string) => { + return `\${${secretName}}`; +}; + +function SecretField({ field, secrets, removeMargin, form, refetchSecrets }: SecretFieldProps) { + const options = secrets.map((secret) => ({ value: encodeSecret(secret.name), label: secret.name })); return ( !!option?.value.toLowerCase().includes(input.toLowerCase())} + notFoundContent={<>No secrets found} options={options} - dropdownRender={(menu) => ( - <> - {menu} - - - - )} + dropdownRender={(menu) => { + return ( + <> + {menu} + + + form.setFields([{ name: field.name, value: encodeSecret(state.name as string) }]) + } + refetchSecrets={refetchSecrets} + /> + + ); + }} /> ); diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx index 56a22f832a..359ca217c9 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx @@ -25,6 +25,7 @@ export interface RecipeField { type: FieldType; fieldPath: string | string[]; rules: any[] | null; + required?: boolean; // Today, Only makes a difference on Selects section?: string; options?: Option[]; buttonLabel?: string; @@ -55,13 +56,11 @@ function clearFieldAndParents(recipe: any, fieldPath: string | string[]) { } export function setFieldValueOnRecipe(recipe: any, value: any, fieldPath: string | string[]) { const updatedRecipe = { ...recipe }; - if (value !== undefined) { - if (value === null || value === '') { - clearFieldAndParents(updatedRecipe, fieldPath); - return updatedRecipe; - } - set(updatedRecipe, fieldPath, value); + if (value === null || value === '' || value === undefined) { + clearFieldAndParents(updatedRecipe, fieldPath); + return updatedRecipe; } + set(updatedRecipe, fieldPath, value); return updatedRecipe; } @@ -267,7 +266,7 @@ export const INCLUDE_TABLE_LINEAGE: RecipeField = { export const PROFILING_ENABLED: RecipeField = { name: 'profiling.enabled', label: 'Enable Profiling', - tooltip: 'Whether profiling should be done.', + tooltip: 'Whether profiling should be performed on the assets extracted from the ingestion source.', type: FieldType.BOOLEAN, fieldPath: 'source.config.profiling.enabled', rules: null, @@ -276,7 +275,7 @@ export const PROFILING_ENABLED: RecipeField = { export const STATEFUL_INGESTION_ENABLED: RecipeField = { name: 'stateful_ingestion.enabled', label: 'Enable Stateful Ingestion', - tooltip: 'Remove stale datasets from datahub once they have been deleted in the source.', + tooltip: 'Remove stale assets from DataHub once they have been deleted in the ingestion source.', type: FieldType.BOOLEAN, fieldPath: 'source.config.stateful_ingestion.enabled', rules: null, @@ -322,7 +321,7 @@ export const TABLE_LINEAGE_MODE: RecipeField = { export const INGEST_TAGS: RecipeField = { name: 'ingest_tags', label: 'Ingest Tags', - tooltip: 'Ingest Tags from source. This will override Tags entered from UI', + tooltip: 'Ingest Tags from the source. Be careful: This can override Tags entered by users of DataHub.', type: FieldType.BOOLEAN, fieldPath: 'source.config.ingest_tags', rules: null, @@ -331,7 +330,7 @@ export const INGEST_TAGS: RecipeField = { export const INGEST_OWNER: RecipeField = { name: 'ingest_owner', label: 'Ingest Owner', - tooltip: 'Ingest Owner from source. This will override Owner info entered from UI', + tooltip: 'Ingest Owner from source. Be careful: This cah override Owners added by users of DataHub.', type: FieldType.BOOLEAN, fieldPath: 'source.config.ingest_owner', rules: null, @@ -391,7 +390,7 @@ export const START_TIME: RecipeField = { name: 'start_time', label: 'Start Time', tooltip: - 'Earliest date of audit logs to process for usage, lineage etc. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) for your first ingestion run, and then uncheck it for future runs.', + 'Earliest date used when processing audit logs for lineage, usage, and more. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) to bootstrap your first ingestion run, and then reduce for subsequent runs.', placeholder: 'Select date and time', type: FieldType.DATE, fieldPath: startTimeFieldPath, diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/kafka.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/kafka.ts index 2b137bcf4d..7b30fda0b7 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/kafka.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/kafka.ts @@ -1,10 +1,15 @@ import { RecipeField, FieldType, setListValuesOnRecipe } from './common'; +// TODO: Currently platform_instance is required to be present for stateful ingestion to work +// We need to solve this prior to enabling by default here. + const saslUsernameFieldPath = ['source', 'config', 'connection', 'consumer_config', 'sasl.username']; export const KAFKA_SASL_USERNAME: RecipeField = { name: 'connection.consumer_config.sasl.username', label: 'Username', - tooltip: 'SASL username. You can get (in the Confluent UI) from your cluster -> Data Integration -> API Keys.', + placeholder: 'datahub-client', + tooltip: + 'The SASL username. Required if the Security Protocol is SASL based. In the Confluent Control Center, you can find this in Cluster > Data Integration > API Keys.', type: FieldType.TEXT, fieldPath: saslUsernameFieldPath, rules: null, @@ -14,7 +19,9 @@ const saslPasswordFieldPath = ['source', 'config', 'connection', 'consumer_confi export const KAFKA_SASL_PASSWORD: RecipeField = { name: 'connection.consumer_config.sasl.password', label: 'Password', - tooltip: 'SASL password. You can get (in the Confluent UI) from your cluster -> Data Integration -> API Keys.', + placeholder: 'datahub-client-password', + tooltip: + 'The SASL Password. Required if the Security Protocol is SASL based. In the Confluent Control Center, you can find this in Cluster > Data Integration > API Keys.', type: FieldType.SECRET, fieldPath: saslPasswordFieldPath, rules: null, @@ -22,8 +29,10 @@ export const KAFKA_SASL_PASSWORD: RecipeField = { export const KAFKA_BOOTSTRAP: RecipeField = { name: 'connection.bootstrap', - label: 'Connection Bootstrap', - tooltip: 'Bootstrap URL.', + label: 'Bootstrap Servers', + required: true, + tooltip: + 'The ‘host[:port]’ string (or list of ‘host[:port]’ strings) that we should contact to bootstrap initial cluster metadata.', placeholder: 'abc-defg.eu-west-1.aws.confluent.cloud:9092', type: FieldType.TEXT, fieldPath: 'source.config.connection.bootstrap', @@ -33,7 +42,8 @@ export const KAFKA_BOOTSTRAP: RecipeField = { export const KAFKA_SCHEMA_REGISTRY_URL: RecipeField = { name: 'connection.schema_registry_url', label: 'Schema Registry URL', - tooltip: 'URL where your Confluent Cloud Schema Registry is hosted.', + tooltip: + 'The URL where the schema Schema Registry is hosted. If provided, DataHub will attempt to extract Avro and Protobuf topic schemas from the registry.', placeholder: 'https://abc-defgh.us-east-2.aws.confluent.cloud', type: FieldType.TEXT, fieldPath: 'source.config.connection.schema_registry_url', @@ -51,7 +61,7 @@ export const KAFKA_SCHEMA_REGISTRY_USER_CREDENTIAL: RecipeField = { name: 'schema_registry_config.basic.auth.user.info', label: 'Schema Registry Credentials', tooltip: - 'API credentials for Confluent schema registry which you get (in Confluent UI) from Schema Registry -> API credentials.', + 'API credentials for the Schema Registry. In Confluent Control Center, you can find these under Schema Registry > API Credentials.', // eslint-disable-next-line no-template-curly-in-string placeholder: '${REGISTRY_API_KEY_ID}:${REGISTRY_API_KEY_SECRET}', type: FieldType.TEXT, @@ -63,13 +73,16 @@ const securityProtocolFieldPath = ['source', 'config', 'connection', 'consumer_c export const KAFKA_SECURITY_PROTOCOL: RecipeField = { name: 'security.protocol', label: 'Security Protocol', - tooltip: 'Security Protocol', + tooltip: 'The Security Protocol used for authentication.', type: FieldType.SELECT, + required: true, fieldPath: securityProtocolFieldPath, rules: null, options: [ + { label: 'PLAINTEXT', value: 'PLAINTEXT' }, { label: 'SASL_SSL', value: 'SASL_SSL' }, { label: 'SASL_PLAINTEXT', value: 'SASL_PLAINTEXT' }, + { label: 'SSL', value: 'SSL' }, ], }; @@ -77,9 +90,11 @@ const saslMechanismFieldPath = ['source', 'config', 'connection', 'consumer_conf export const KAFKA_SASL_MECHANISM: RecipeField = { name: 'sasl.mechanism', label: 'SASL Mechanism', - tooltip: 'SASL Mechanism', + tooltip: + 'The SASL mechanism used for authentication. This field is required if the selected Security Protocol is SASL based.', type: FieldType.SELECT, fieldPath: saslMechanismFieldPath, + placeholder: 'None', rules: null, options: [ { label: 'PLAIN', value: 'PLAIN' }, @@ -92,12 +107,12 @@ const topicAllowFieldPath = 'source.config.topic_patterns.allow'; export const TOPIC_ALLOW: RecipeField = { name: 'topic_patterns.allow', label: 'Allow Patterns', - tooltip: 'Use regex here.', + tooltip: 'Provide an optional Regular Expresssion (REGEX) to include specific Kafka Topic names in ingestion.', type: FieldType.LIST, buttonLabel: 'Add pattern', fieldPath: topicAllowFieldPath, rules: null, - section: 'Topics', + section: 'Filter by Topic', setValueOnRecipeOverride: (recipe: any, values: string[]) => setListValuesOnRecipe(recipe, values, topicAllowFieldPath), }; @@ -106,12 +121,12 @@ const topicDenyFieldPath = 'source.config.topic_patterns.deny'; export const TOPIC_DENY: RecipeField = { name: 'topic_patterns.deny', label: 'Deny Patterns', - tooltip: 'Use regex here.', + tooltip: 'Provide an optional Regular Expresssion (REGEX) to exclude specific Kafka Topic names from ingestion.', type: FieldType.LIST, buttonLabel: 'Add pattern', fieldPath: topicDenyFieldPath, rules: null, - section: 'Topics', + section: 'Filter by Topic', setValueOnRecipeOverride: (recipe: any, values: string[]) => setListValuesOnRecipe(recipe, values, topicDenyFieldPath), }; diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 6f70b40901..10e3176b41 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -25,7 +25,7 @@ "name": "kafka", "displayName": "Kafka", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/kafka/", - "recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"SASL_SSL\"\n sasl.mechanism: \"PLAIN\"\n stateful_ingestion:\n enabled: true'" + "recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"PLAINTEXT\"\n stateful_ingestion:\n enabled: false" }, { "urn": "urn:li:dataPlatform:looker", diff --git a/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts b/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts index 70a9f8bdae..6926d54a03 100644 --- a/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts +++ b/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts @@ -11,6 +11,7 @@ source: sasl.mechanism: "PLAIN" stateful_ingestion: enabled: true + `; export const KAFKA = 'kafka';