diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx
index 75436a463f..629298e479 100644
--- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx
+++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx
@@ -29,15 +29,9 @@ const DESCRIPTION_FIELD_NAME = 'description';
export default function CreateDomainModal({ onClose, onCreate }: Props) {
const [createDomainMutation] = useCreateDomainMutation();
- const [createButtonEnabled, setCreateButtonEnabled] = useState(true);
+ const [createButtonEnabled, setCreateButtonEnabled] = useState(false);
const [form] = Form.useForm();
- const setStagedName = (name) => {
- form.setFieldsValue({
- name,
- });
- };
-
const onCreateDomain = () => {
createDomainMutation({
variables: {
@@ -88,7 +82,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) {
Cancel
-
+
Create
>
@@ -98,9 +92,9 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) {
form={form}
initialValues={{}}
layout="vertical"
- onFieldsChange={() =>
- setCreateButtonEnabled(form.getFieldsError().some((field) => field.errors.length > 0))
- }
+ onFieldsChange={() => {
+ setCreateButtonEnabled(!form.getFieldsError().some((field) => field.errors.length > 0));
+ }}
>
Name}>
Give your new Domain a name.
@@ -121,7 +115,15 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) {
{SUGGESTED_DOMAIN_NAMES.map((name) => {
return (
- setStagedName(name)}>
+ {
+ form.setFieldsValue({
+ name,
+ });
+ setCreateButtonEnabled(true);
+ }}
+ >
{name}
);
@@ -137,7 +139,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) {
rules={[{ whitespace: true }, { min: 1, max: 500 }]}
hasFeedback
>
-
+
diff --git a/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx b/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx
index 92bdce6d52..539eef9726 100644
--- a/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx
+++ b/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx
@@ -3,6 +3,10 @@ import React, { useState } from 'react';
import { useEnterKeyListener } from '../../shared/useEnterKeyListener';
import { SecretBuilderState } from './types';
+const NAME_FIELD_NAME = 'name';
+const DESCRIPTION_FIELD_NAME = 'description';
+const VALUE_FIELD_NAME = 'value';
+
type Props = {
initialState?: SecretBuilderState;
visible: boolean;
@@ -11,38 +15,15 @@ type Props = {
};
export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }: Props) => {
- const [secretBuilderState, setSecretBuilderState] = useState(initialState || {});
- const [createButtonEnabled, setCreateButtonEnabled] = useState(true);
+ const [createButtonEnabled, setCreateButtonEnabled] = useState(false);
const [form] = Form.useForm();
- const setName = (name: string) => {
- setSecretBuilderState({
- ...secretBuilderState,
- name,
- });
- };
-
- const setValue = (value: string) => {
- setSecretBuilderState({
- ...secretBuilderState,
- value,
- });
- };
-
- const setDescription = (description: string) => {
- setSecretBuilderState({
- ...secretBuilderState,
- description,
- });
- };
-
// Handle the Enter press
useEnterKeyListener({
querySelectorToExecuteClick: '#createSecretButton',
});
function resetValues() {
- setSecretBuilderState({});
form.resetFields();
}
@@ -60,8 +41,17 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
onSubmit?.(secretBuilderState, resetValues)}
- disabled={createButtonEnabled}
+ onClick={() =>
+ onSubmit?.(
+ {
+ name: form.getFieldValue(NAME_FIELD_NAME),
+ description: form.getFieldValue(DESCRIPTION_FIELD_NAME),
+ value: form.getFieldValue(VALUE_FIELD_NAME),
+ },
+ resetValues,
+ )
+ }
+ disabled={!createButtonEnabled}
>
Create
@@ -73,7 +63,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
initialValues={initialState}
layout="vertical"
onFieldsChange={() =>
- setCreateButtonEnabled(form.getFieldsError().some((field) => field.errors.length > 0))
+ setCreateButtonEnabled(!form.getFieldsError().some((field) => field.errors.length > 0))
}
>
Name}>
@@ -81,22 +71,19 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
Give your secret a name. This is what you'll use to reference the secret from your recipes.
- setName(event.target.value)}
- />
+
Value}>
@@ -104,7 +91,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
The value of your secret, which will be encrypted and stored securely within DataHub.
- setValue(event.target.value)}
- autoComplete="false"
- />
+
Description}>
An optional description to help keep track of your secret.
-
- setDescription(event.target.value)}
- />
+
+
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/DictField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/DictField.tsx
index f378e9d109..e15a83b08e 100644
--- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/DictField.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/DictField.tsx
@@ -71,6 +71,7 @@ export default function DictField({ field, removeMargin }: Props) {
{field.keyField && (
{field.options && (
-
+
{field.options.map((option) => (
{option.label}
))}
@@ -76,6 +77,7 @@ function SelectField({ field, removeMargin }: CommonFieldProps) {
function DateField({ field, removeMargin }: CommonFieldProps) {
return (
void;
removeMargin?: boolean;
+ form: FormInstance;
}
function FormField(props: Props) {
- const { field, secrets, refetchSecrets, removeMargin } = props;
+ const { field, secrets, refetchSecrets, removeMargin, form } = props;
if (field.type === FieldType.LIST) return ;
@@ -105,7 +108,13 @@ function FormField(props: Props) {
if (field.type === FieldType.SECRET)
return (
-
+
);
if (field.type === FieldType.DICT) return ;
@@ -113,12 +122,14 @@ function FormField(props: Props) {
const isBoolean = field.type === FieldType.BOOLEAN;
let input = ;
if (isBoolean) input = ;
- if (field.type === FieldType.TEXTAREA) input = ;
+ if (field.type === FieldType.TEXTAREA)
+ input = ;
const valuePropName = isBoolean ? 'checked' : 'value';
const getValueFromEvent = isBoolean ? undefined : (e) => (e.target.value === '' ? null : e.target.value);
return (
secretA.name.localeCompare(secretB.name)) || [];
+ const [form] = Form.useForm();
function updateFormValues(changedValues: any, allValues: any) {
let updatedValues = YAML.parse(displayRecipe);
@@ -137,6 +138,7 @@ function RecipeForm(props: Props) {
layout="vertical"
initialValues={getInitialValues(displayRecipe, allFields)}
onFinish={onClickNext}
+ form={form}
onValuesChange={updateFormValues}
>
@@ -147,6 +149,7 @@ function RecipeForm(props: Props) {
secrets={secrets}
refetchSecrets={refetchSecrets}
removeMargin={i === fields.length - 1}
+ form={form}
/>
))}
{CONNECTORS_WITH_TEST_CONNECTION.has(type as string) && (
@@ -184,6 +187,7 @@ function RecipeForm(props: Props) {
secrets={secrets}
refetchSecrets={refetchSecrets}
removeMargin={i === filterFields.length - 1}
+ form={form}
/>
>
@@ -209,6 +213,7 @@ function RecipeForm(props: Props) {
secrets={secrets}
refetchSecrets={refetchSecrets}
removeMargin={i === advancedFields.length - 1}
+ form={form}
/>
))}
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx
index 01ca495900..31024512cb 100644
--- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx
@@ -24,10 +24,11 @@ const CreateButton = styled(Button)`
`;
interface Props {
+ onSubmit?: (state: SecretBuilderState) => void;
refetchSecrets: () => void;
}
-function CreateSecretButton({ refetchSecrets }: Props) {
+function CreateSecretButton({ onSubmit, refetchSecrets }: Props) {
const [isCreateModalVisible, setIsCreateModalVisible] = useState(false);
const [createSecretMutation] = useCreateSecretMutation();
@@ -42,12 +43,11 @@ function CreateSecretButton({ refetchSecrets }: Props) {
},
})
.then(() => {
+ onSubmit?.(state);
setIsCreateModalVisible(false);
resetBuilderState();
+ message.success({ content: `Created secret!` });
setTimeout(() => refetchSecrets(), 3000);
- message.loading({ content: `Loading...`, duration: 3 }).then(() => {
- message.success({ content: `Successfully created Secret!` });
- });
})
.catch((e) => {
message.destroy();
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx
index ac1fe2166f..08213f8912 100644
--- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx
@@ -1,5 +1,5 @@
import React, { ReactNode } from 'react';
-import { AutoComplete, Divider, Form } from 'antd';
+import { AutoComplete, Divider, Form, FormInstance } from 'antd';
import styled from 'styled-components/macro';
import { Secret } from '../../../../../../types.generated';
import CreateSecretButton from './CreateSecretButton';
@@ -52,6 +52,7 @@ interface SecretFieldProps {
secrets: Secret[];
removeMargin?: boolean;
refetchSecrets: () => void;
+ form: FormInstance;
}
function SecretFieldTooltip({ tooltipLabel }: { tooltipLabel?: string | ReactNode }) {
@@ -79,11 +80,16 @@ function SecretFieldTooltip({ tooltipLabel }: { tooltipLabel?: string | ReactNod
);
}
-function SecretField({ field, secrets, removeMargin, refetchSecrets }: SecretFieldProps) {
- const options = secrets.map((secret) => ({ value: `\${${secret.name}}`, label: secret.name }));
+const encodeSecret = (secretName: string) => {
+ return `\${${secretName}}`;
+};
+
+function SecretField({ field, secrets, removeMargin, form, refetchSecrets }: SecretFieldProps) {
+ const options = secrets.map((secret) => ({ value: encodeSecret(secret.name), label: secret.name }));
return (
!!option?.value.toLowerCase().includes(input.toLowerCase())}
+ notFoundContent={<>No secrets found>}
options={options}
- dropdownRender={(menu) => (
- <>
- {menu}
-
-
- >
- )}
+ dropdownRender={(menu) => {
+ return (
+ <>
+ {menu}
+
+
+ form.setFields([{ name: field.name, value: encodeSecret(state.name as string) }])
+ }
+ refetchSecrets={refetchSecrets}
+ />
+ >
+ );
+ }}
/>
);
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx
index 56a22f832a..359ca217c9 100644
--- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx
@@ -25,6 +25,7 @@ export interface RecipeField {
type: FieldType;
fieldPath: string | string[];
rules: any[] | null;
+ required?: boolean; // Today, Only makes a difference on Selects
section?: string;
options?: Option[];
buttonLabel?: string;
@@ -55,13 +56,11 @@ function clearFieldAndParents(recipe: any, fieldPath: string | string[]) {
}
export function setFieldValueOnRecipe(recipe: any, value: any, fieldPath: string | string[]) {
const updatedRecipe = { ...recipe };
- if (value !== undefined) {
- if (value === null || value === '') {
- clearFieldAndParents(updatedRecipe, fieldPath);
- return updatedRecipe;
- }
- set(updatedRecipe, fieldPath, value);
+ if (value === null || value === '' || value === undefined) {
+ clearFieldAndParents(updatedRecipe, fieldPath);
+ return updatedRecipe;
}
+ set(updatedRecipe, fieldPath, value);
return updatedRecipe;
}
@@ -267,7 +266,7 @@ export const INCLUDE_TABLE_LINEAGE: RecipeField = {
export const PROFILING_ENABLED: RecipeField = {
name: 'profiling.enabled',
label: 'Enable Profiling',
- tooltip: 'Whether profiling should be done.',
+ tooltip: 'Whether profiling should be performed on the assets extracted from the ingestion source.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.profiling.enabled',
rules: null,
@@ -276,7 +275,7 @@ export const PROFILING_ENABLED: RecipeField = {
export const STATEFUL_INGESTION_ENABLED: RecipeField = {
name: 'stateful_ingestion.enabled',
label: 'Enable Stateful Ingestion',
- tooltip: 'Remove stale datasets from datahub once they have been deleted in the source.',
+ tooltip: 'Remove stale assets from DataHub once they have been deleted in the ingestion source.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.stateful_ingestion.enabled',
rules: null,
@@ -322,7 +321,7 @@ export const TABLE_LINEAGE_MODE: RecipeField = {
export const INGEST_TAGS: RecipeField = {
name: 'ingest_tags',
label: 'Ingest Tags',
- tooltip: 'Ingest Tags from source. This will override Tags entered from UI',
+ tooltip: 'Ingest Tags from the source. Be careful: This can override Tags entered by users of DataHub.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.ingest_tags',
rules: null,
@@ -331,7 +330,7 @@ export const INGEST_TAGS: RecipeField = {
export const INGEST_OWNER: RecipeField = {
name: 'ingest_owner',
label: 'Ingest Owner',
- tooltip: 'Ingest Owner from source. This will override Owner info entered from UI',
+ tooltip: 'Ingest Owner from source. Be careful: This cah override Owners added by users of DataHub.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.ingest_owner',
rules: null,
@@ -391,7 +390,7 @@ export const START_TIME: RecipeField = {
name: 'start_time',
label: 'Start Time',
tooltip:
- 'Earliest date of audit logs to process for usage, lineage etc. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) for your first ingestion run, and then uncheck it for future runs.',
+ 'Earliest date used when processing audit logs for lineage, usage, and more. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) to bootstrap your first ingestion run, and then reduce for subsequent runs.',
placeholder: 'Select date and time',
type: FieldType.DATE,
fieldPath: startTimeFieldPath,
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/kafka.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/kafka.ts
index 2b137bcf4d..7b30fda0b7 100644
--- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/kafka.ts
+++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/kafka.ts
@@ -1,10 +1,15 @@
import { RecipeField, FieldType, setListValuesOnRecipe } from './common';
+// TODO: Currently platform_instance is required to be present for stateful ingestion to work
+// We need to solve this prior to enabling by default here.
+
const saslUsernameFieldPath = ['source', 'config', 'connection', 'consumer_config', 'sasl.username'];
export const KAFKA_SASL_USERNAME: RecipeField = {
name: 'connection.consumer_config.sasl.username',
label: 'Username',
- tooltip: 'SASL username. You can get (in the Confluent UI) from your cluster -> Data Integration -> API Keys.',
+ placeholder: 'datahub-client',
+ tooltip:
+ 'The SASL username. Required if the Security Protocol is SASL based. In the Confluent Control Center, you can find this in Cluster > Data Integration > API Keys.',
type: FieldType.TEXT,
fieldPath: saslUsernameFieldPath,
rules: null,
@@ -14,7 +19,9 @@ const saslPasswordFieldPath = ['source', 'config', 'connection', 'consumer_confi
export const KAFKA_SASL_PASSWORD: RecipeField = {
name: 'connection.consumer_config.sasl.password',
label: 'Password',
- tooltip: 'SASL password. You can get (in the Confluent UI) from your cluster -> Data Integration -> API Keys.',
+ placeholder: 'datahub-client-password',
+ tooltip:
+ 'The SASL Password. Required if the Security Protocol is SASL based. In the Confluent Control Center, you can find this in Cluster > Data Integration > API Keys.',
type: FieldType.SECRET,
fieldPath: saslPasswordFieldPath,
rules: null,
@@ -22,8 +29,10 @@ export const KAFKA_SASL_PASSWORD: RecipeField = {
export const KAFKA_BOOTSTRAP: RecipeField = {
name: 'connection.bootstrap',
- label: 'Connection Bootstrap',
- tooltip: 'Bootstrap URL.',
+ label: 'Bootstrap Servers',
+ required: true,
+ tooltip:
+ 'The ‘host[:port]’ string (or list of ‘host[:port]’ strings) that we should contact to bootstrap initial cluster metadata.',
placeholder: 'abc-defg.eu-west-1.aws.confluent.cloud:9092',
type: FieldType.TEXT,
fieldPath: 'source.config.connection.bootstrap',
@@ -33,7 +42,8 @@ export const KAFKA_BOOTSTRAP: RecipeField = {
export const KAFKA_SCHEMA_REGISTRY_URL: RecipeField = {
name: 'connection.schema_registry_url',
label: 'Schema Registry URL',
- tooltip: 'URL where your Confluent Cloud Schema Registry is hosted.',
+ tooltip:
+ 'The URL where the schema Schema Registry is hosted. If provided, DataHub will attempt to extract Avro and Protobuf topic schemas from the registry.',
placeholder: 'https://abc-defgh.us-east-2.aws.confluent.cloud',
type: FieldType.TEXT,
fieldPath: 'source.config.connection.schema_registry_url',
@@ -51,7 +61,7 @@ export const KAFKA_SCHEMA_REGISTRY_USER_CREDENTIAL: RecipeField = {
name: 'schema_registry_config.basic.auth.user.info',
label: 'Schema Registry Credentials',
tooltip:
- 'API credentials for Confluent schema registry which you get (in Confluent UI) from Schema Registry -> API credentials.',
+ 'API credentials for the Schema Registry. In Confluent Control Center, you can find these under Schema Registry > API Credentials.',
// eslint-disable-next-line no-template-curly-in-string
placeholder: '${REGISTRY_API_KEY_ID}:${REGISTRY_API_KEY_SECRET}',
type: FieldType.TEXT,
@@ -63,13 +73,16 @@ const securityProtocolFieldPath = ['source', 'config', 'connection', 'consumer_c
export const KAFKA_SECURITY_PROTOCOL: RecipeField = {
name: 'security.protocol',
label: 'Security Protocol',
- tooltip: 'Security Protocol',
+ tooltip: 'The Security Protocol used for authentication.',
type: FieldType.SELECT,
+ required: true,
fieldPath: securityProtocolFieldPath,
rules: null,
options: [
+ { label: 'PLAINTEXT', value: 'PLAINTEXT' },
{ label: 'SASL_SSL', value: 'SASL_SSL' },
{ label: 'SASL_PLAINTEXT', value: 'SASL_PLAINTEXT' },
+ { label: 'SSL', value: 'SSL' },
],
};
@@ -77,9 +90,11 @@ const saslMechanismFieldPath = ['source', 'config', 'connection', 'consumer_conf
export const KAFKA_SASL_MECHANISM: RecipeField = {
name: 'sasl.mechanism',
label: 'SASL Mechanism',
- tooltip: 'SASL Mechanism',
+ tooltip:
+ 'The SASL mechanism used for authentication. This field is required if the selected Security Protocol is SASL based.',
type: FieldType.SELECT,
fieldPath: saslMechanismFieldPath,
+ placeholder: 'None',
rules: null,
options: [
{ label: 'PLAIN', value: 'PLAIN' },
@@ -92,12 +107,12 @@ const topicAllowFieldPath = 'source.config.topic_patterns.allow';
export const TOPIC_ALLOW: RecipeField = {
name: 'topic_patterns.allow',
label: 'Allow Patterns',
- tooltip: 'Use regex here.',
+ tooltip: 'Provide an optional Regular Expresssion (REGEX) to include specific Kafka Topic names in ingestion.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: topicAllowFieldPath,
rules: null,
- section: 'Topics',
+ section: 'Filter by Topic',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, topicAllowFieldPath),
};
@@ -106,12 +121,12 @@ const topicDenyFieldPath = 'source.config.topic_patterns.deny';
export const TOPIC_DENY: RecipeField = {
name: 'topic_patterns.deny',
label: 'Deny Patterns',
- tooltip: 'Use regex here.',
+ tooltip: 'Provide an optional Regular Expresssion (REGEX) to exclude specific Kafka Topic names from ingestion.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: topicDenyFieldPath,
rules: null,
- section: 'Topics',
+ section: 'Filter by Topic',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, topicDenyFieldPath),
};
diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json
index 6f70b40901..10e3176b41 100644
--- a/datahub-web-react/src/app/ingest/source/builder/sources.json
+++ b/datahub-web-react/src/app/ingest/source/builder/sources.json
@@ -25,7 +25,7 @@
"name": "kafka",
"displayName": "Kafka",
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/kafka/",
- "recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"SASL_SSL\"\n sasl.mechanism: \"PLAIN\"\n stateful_ingestion:\n enabled: true'"
+ "recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"PLAINTEXT\"\n stateful_ingestion:\n enabled: false"
},
{
"urn": "urn:li:dataPlatform:looker",
diff --git a/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts b/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts
index 70a9f8bdae..6926d54a03 100644
--- a/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts
+++ b/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts
@@ -11,6 +11,7 @@ source:
sasl.mechanism: "PLAIN"
stateful_ingestion:
enabled: true
+
`;
export const KAFKA = 'kafka';