mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-25 00:48:45 +00:00
feat(ingestion) Build out UI form for Snowflake Managed Ingestion (#5391)
This commit is contained in:
Chris Collins
GitHub
parent
b2edd44b6a
commit
fa07dc69eb
@ -6,6 +6,8 @@ import { getSourceConfigs, jsonToYaml, yamlToJson } from '../utils';
|
||||
import { YamlEditor } from './YamlEditor';
|
||||
import { ANTD_GRAY } from '../../../entity/shared/constants';
|
||||
import { IngestionSourceBuilderStep } from './steps';
|
||||
import RecipeBuilder from './RecipeBuilder';
|
||||
import { CONNECTORS_WITH_FORM } from './RecipeForm/utils';
|
||||
|
||||
const LOOKML_DOC_LINK = 'https://datahubproject.io/docs/generated/ingestion/sources/looker#module-lookml';
|
||||
|
||||
@ -37,17 +39,19 @@ const ControlsContainer = styled.div`
|
||||
export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps) => {
|
||||
const existingRecipeJson = state.config?.recipe;
|
||||
const existingRecipeYaml = existingRecipeJson && jsonToYaml(existingRecipeJson);
|
||||
const { type } = state;
|
||||
const sourceConfigs = getSourceConfigs(type as string);
|
||||
|
||||
const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || '');
|
||||
const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || sourceConfigs.placeholderRecipe);
|
||||
|
||||
useEffect(() => {
|
||||
setStagedRecipeYml(existingRecipeYaml || '');
|
||||
if (existingRecipeYaml) {
|
||||
setStagedRecipeYml(existingRecipeYaml);
|
||||
}
|
||||
}, [existingRecipeYaml]);
|
||||
|
||||
const [stepComplete, setStepComplete] = useState(false);
|
||||
|
||||
const { type } = state;
|
||||
const sourceConfigs = getSourceConfigs(type as string);
|
||||
const isEditing: boolean = prev === undefined;
|
||||
const displayRecipe = stagedRecipeYml || sourceConfigs.placeholderRecipe;
|
||||
const sourceDisplayName = sourceConfigs.displayName;
|
||||
@ -85,6 +89,19 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps)
|
||||
goTo(IngestionSourceBuilderStep.CREATE_SCHEDULE);
|
||||
};
|
||||
|
||||
if (type && CONNECTORS_WITH_FORM.has(type)) {
|
||||
return (
|
||||
<RecipeBuilder
|
||||
type={type}
|
||||
isEditing={isEditing}
|
||||
displayRecipe={displayRecipe}
|
||||
setStagedRecipe={setStagedRecipeYml}
|
||||
onClickNext={onClickNext}
|
||||
goToPrevious={prev}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<Section>
|
||||
|
||||
@ -0,0 +1,103 @@
|
||||
import { Button, message } from 'antd';
|
||||
import React, { useState } from 'react';
|
||||
import YAML from 'yamljs';
|
||||
import { CodeOutlined, FormOutlined } from '@ant-design/icons';
|
||||
import styled from 'styled-components/macro';
|
||||
import { ANTD_GRAY } from '../../../entity/shared/constants';
|
||||
import { YamlEditor } from './YamlEditor';
|
||||
import RecipeForm from './RecipeForm/RecipeForm';
|
||||
|
||||
export const ControlsContainer = styled.div`
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
margin-top: 8px;
|
||||
`;
|
||||
|
||||
const BorderedSection = styled.div`
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding-bottom: 16px;
|
||||
border: solid ${ANTD_GRAY[4]} 0.5px;
|
||||
`;
|
||||
|
||||
const StyledButton = styled(Button)<{ isSelected: boolean }>`
|
||||
${(props) =>
|
||||
props.isSelected &&
|
||||
`
|
||||
color: #1890ff;
|
||||
&:focus {
|
||||
color: #1890ff;
|
||||
}
|
||||
`}
|
||||
`;
|
||||
|
||||
const ButtonsWrapper = styled.div`
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
margin-bottom: 10px;
|
||||
`;
|
||||
|
||||
interface Props {
|
||||
type: string;
|
||||
isEditing: boolean;
|
||||
displayRecipe: string;
|
||||
setStagedRecipe: (recipe: string) => void;
|
||||
onClickNext: () => void;
|
||||
goToPrevious?: () => void;
|
||||
}
|
||||
|
||||
function RecipeBuilder(props: Props) {
|
||||
const { type, isEditing, displayRecipe, setStagedRecipe, onClickNext, goToPrevious } = props;
|
||||
|
||||
const [isViewingForm, setIsViewingForm] = useState(true);
|
||||
|
||||
function switchViews(isFormView: boolean) {
|
||||
try {
|
||||
YAML.parse(displayRecipe);
|
||||
setIsViewingForm(isFormView);
|
||||
} catch (e) {
|
||||
const messageText = (e as any).parsedLine
|
||||
? `Fix line ${(e as any).parsedLine} in your recipe`
|
||||
: 'Please fix your recipe';
|
||||
message.warn(`Found invalid YAML. ${messageText} in order to switch views.`);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<ButtonsWrapper>
|
||||
<StyledButton type="text" isSelected={isViewingForm} onClick={() => switchViews(true)}>
|
||||
<FormOutlined /> Form
|
||||
</StyledButton>
|
||||
<StyledButton type="text" isSelected={!isViewingForm} onClick={() => switchViews(false)}>
|
||||
<CodeOutlined /> YAML
|
||||
</StyledButton>
|
||||
</ButtonsWrapper>
|
||||
{isViewingForm && (
|
||||
<RecipeForm
|
||||
type={type}
|
||||
isEditing={isEditing}
|
||||
displayRecipe={displayRecipe}
|
||||
setStagedRecipe={setStagedRecipe}
|
||||
onClickNext={onClickNext}
|
||||
goToPrevious={goToPrevious}
|
||||
/>
|
||||
)}
|
||||
{!isViewingForm && (
|
||||
<>
|
||||
<BorderedSection>
|
||||
<YamlEditor initialText={displayRecipe} onChange={setStagedRecipe} />
|
||||
</BorderedSection>
|
||||
<ControlsContainer>
|
||||
<Button disabled={isEditing} onClick={goToPrevious}>
|
||||
Previous
|
||||
</Button>
|
||||
<Button onClick={onClickNext}>Next</Button>
|
||||
</ControlsContainer>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default RecipeBuilder;
|
||||
@ -0,0 +1,115 @@
|
||||
import { Button, Checkbox, Form, Input, Tooltip } from 'antd';
|
||||
import React from 'react';
|
||||
import styled from 'styled-components/macro';
|
||||
import { MinusCircleOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons';
|
||||
import { FieldType, RecipeField } from './utils';
|
||||
import { ANTD_GRAY } from '../../../../entity/shared/constants';
|
||||
|
||||
const Label = styled.div`
|
||||
font-weight: bold;
|
||||
padding-bottom: 8px;
|
||||
`;
|
||||
|
||||
const StyledButton = styled(Button)`
|
||||
color: ${ANTD_GRAY[7]};
|
||||
width: 80%;
|
||||
`;
|
||||
|
||||
const StyledQuestion = styled(QuestionCircleOutlined)`
|
||||
color: rgba(0, 0, 0, 0.45);
|
||||
margin-left: 4px;
|
||||
`;
|
||||
|
||||
const StyledRemoveIcon = styled(MinusCircleOutlined)`
|
||||
font-size: 14px;
|
||||
margin-left: 10px;
|
||||
`;
|
||||
|
||||
const StyledFormItem = styled(Form.Item)<{ alignLeft: boolean; removeMargin: boolean }>`
|
||||
margin-bottom: ${(props) => (props.removeMargin ? '0' : '16px')};
|
||||
|
||||
${(props) =>
|
||||
props.alignLeft &&
|
||||
`
|
||||
.ant-form-item {
|
||||
flex-direction: row;
|
||||
|
||||
}
|
||||
|
||||
.ant-form-item-label {
|
||||
padding: 0;
|
||||
margin-right: 10px;
|
||||
}
|
||||
`}
|
||||
`;
|
||||
|
||||
const ListWrapper = styled.div<{ removeMargin: boolean }>`
|
||||
margin-bottom: ${(props) => (props.removeMargin ? '0' : '16px')};
|
||||
`;
|
||||
|
||||
interface ListFieldProps {
|
||||
field: RecipeField;
|
||||
removeMargin?: boolean;
|
||||
}
|
||||
|
||||
function ListField({ field, removeMargin }: ListFieldProps) {
|
||||
return (
|
||||
<Form.List name={field.name}>
|
||||
{(fields, { add, remove }) => (
|
||||
<ListWrapper removeMargin={!!removeMargin}>
|
||||
<Label>
|
||||
{field.label}
|
||||
<Tooltip overlay={field.tooltip}>
|
||||
<StyledQuestion />
|
||||
</Tooltip>
|
||||
</Label>
|
||||
{fields.map((item) => (
|
||||
<Form.Item key={item.fieldKey} style={{ marginBottom: '10px' }}>
|
||||
<Form.Item {...item} noStyle>
|
||||
<Input style={{ width: '80%' }} />
|
||||
</Form.Item>
|
||||
<StyledRemoveIcon onClick={() => remove(item.name)} />
|
||||
</Form.Item>
|
||||
))}
|
||||
<StyledButton type="dashed" onClick={() => add()} style={{ width: '80%' }} icon={<PlusOutlined />}>
|
||||
Add pattern
|
||||
</StyledButton>
|
||||
</ListWrapper>
|
||||
)}
|
||||
</Form.List>
|
||||
);
|
||||
}
|
||||
|
||||
interface Props {
|
||||
field: RecipeField;
|
||||
removeMargin?: boolean;
|
||||
}
|
||||
|
||||
function FormField(props: Props) {
|
||||
const { field, removeMargin } = props;
|
||||
|
||||
if (field.type === FieldType.LIST) return <ListField field={field} removeMargin={removeMargin} />;
|
||||
|
||||
const isBoolean = field.type === FieldType.BOOLEAN;
|
||||
const input = isBoolean ? <Checkbox /> : <Input />;
|
||||
const valuePropName = isBoolean ? 'checked' : 'value';
|
||||
const getValueFromEvent = isBoolean ? undefined : (e) => (e.target.value === '' ? null : e.target.value);
|
||||
|
||||
return (
|
||||
<StyledFormItem
|
||||
style={isBoolean ? { flexDirection: 'row', alignItems: 'center' } : {}}
|
||||
label={field.label}
|
||||
name={field.name}
|
||||
tooltip={field.tooltip}
|
||||
rules={field.rules || undefined}
|
||||
valuePropName={valuePropName}
|
||||
getValueFromEvent={getValueFromEvent}
|
||||
alignLeft={isBoolean}
|
||||
removeMargin={!!removeMargin}
|
||||
>
|
||||
{input}
|
||||
</StyledFormItem>
|
||||
);
|
||||
}
|
||||
|
||||
export default FormField;
|
||||
@ -0,0 +1,148 @@
|
||||
import { Button, Collapse, Form, message, Typography } from 'antd';
|
||||
import React from 'react';
|
||||
import { get } from 'lodash';
|
||||
import YAML from 'yamljs';
|
||||
import { ApiOutlined, FilterOutlined, SettingOutlined } from '@ant-design/icons';
|
||||
import styled from 'styled-components/macro';
|
||||
import { jsonToYaml } from '../../utils';
|
||||
import { RecipeField, RECIPE_FIELDS, setFieldValueOnRecipe } from './utils';
|
||||
import FormField from './FormField';
|
||||
|
||||
export const ControlsContainer = styled.div`
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
margin-top: 12px;
|
||||
`;
|
||||
|
||||
const StyledCollapse = styled(Collapse)`
|
||||
margin-bottom: 16px;
|
||||
|
||||
.ant-collapse-header {
|
||||
font-size: 14px;
|
||||
font-weight: bold;
|
||||
padding: 12px 0;
|
||||
}
|
||||
`;
|
||||
|
||||
const HeaderTitle = styled.span`
|
||||
margin-left: 8px;
|
||||
`;
|
||||
|
||||
const MarginWrapper = styled.div`
|
||||
margin-left: 20px;
|
||||
`;
|
||||
|
||||
function getInitialValues(displayRecipe: string, allFields: any[]) {
|
||||
const initialValues = {};
|
||||
let recipeObj;
|
||||
try {
|
||||
recipeObj = YAML.parse(displayRecipe);
|
||||
} catch (e) {
|
||||
message.warn('Found invalid YAML. Please check your recipe configuration.');
|
||||
return {};
|
||||
}
|
||||
if (recipeObj) {
|
||||
allFields.forEach((field) => {
|
||||
initialValues[field.name] =
|
||||
field.getValueFromRecipeOverride?.(recipeObj) || get(recipeObj, field.fieldPath);
|
||||
});
|
||||
}
|
||||
|
||||
return initialValues;
|
||||
}
|
||||
|
||||
function SectionHeader({ icon, text }: { icon: any; text: string }) {
|
||||
return (
|
||||
<span>
|
||||
{icon}
|
||||
<HeaderTitle>{text}</HeaderTitle>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function shouldRenderFilterSectionHeader(field: RecipeField, index: number, filterFields: RecipeField[]) {
|
||||
if (index === 0 && field.section) return true;
|
||||
if (field.section && filterFields[index - 1].section !== field.section) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
type: string;
|
||||
isEditing: boolean;
|
||||
displayRecipe: string;
|
||||
setStagedRecipe: (recipe: string) => void;
|
||||
onClickNext: () => void;
|
||||
goToPrevious?: () => void;
|
||||
}
|
||||
|
||||
function RecipeForm(props: Props) {
|
||||
const { type, isEditing, displayRecipe, setStagedRecipe, onClickNext, goToPrevious } = props;
|
||||
const { fields, advancedFields, filterFields } = RECIPE_FIELDS[type];
|
||||
const allFields = [...fields, ...advancedFields, ...filterFields];
|
||||
|
||||
function updateFormValues(changedValues: any) {
|
||||
let updatedValues = YAML.parse(displayRecipe);
|
||||
|
||||
Object.keys(changedValues).forEach((fieldName) => {
|
||||
const recipeField = allFields.find((f) => f.name === fieldName);
|
||||
if (recipeField) {
|
||||
updatedValues =
|
||||
recipeField.setValueOnRecipeOverride?.(updatedValues, changedValues[fieldName]) ||
|
||||
setFieldValueOnRecipe(updatedValues, changedValues[fieldName], recipeField.fieldPath);
|
||||
}
|
||||
});
|
||||
|
||||
const stagedRecipe = jsonToYaml(JSON.stringify(updatedValues));
|
||||
setStagedRecipe(stagedRecipe);
|
||||
}
|
||||
|
||||
return (
|
||||
<Form
|
||||
layout="vertical"
|
||||
initialValues={getInitialValues(displayRecipe, allFields)}
|
||||
onFinish={onClickNext}
|
||||
onValuesChange={updateFormValues}
|
||||
>
|
||||
<StyledCollapse defaultActiveKey="0">
|
||||
<Collapse.Panel forceRender header={<SectionHeader icon={<ApiOutlined />} text="Connection" />} key="0">
|
||||
{fields.map((field, i) => (
|
||||
<FormField field={field} removeMargin={i === fields.length - 1} />
|
||||
))}
|
||||
</Collapse.Panel>
|
||||
</StyledCollapse>
|
||||
<StyledCollapse>
|
||||
<Collapse.Panel forceRender header={<SectionHeader icon={<FilterOutlined />} text="Filter" />} key="1">
|
||||
{filterFields.map((field, i) => (
|
||||
<>
|
||||
{shouldRenderFilterSectionHeader(field, i, filterFields) && (
|
||||
<Typography.Title level={4}>{field.section}</Typography.Title>
|
||||
)}
|
||||
<MarginWrapper>
|
||||
<FormField field={field} removeMargin={i === filterFields.length - 1} />
|
||||
</MarginWrapper>
|
||||
</>
|
||||
))}
|
||||
</Collapse.Panel>
|
||||
</StyledCollapse>
|
||||
<StyledCollapse>
|
||||
<Collapse.Panel
|
||||
forceRender
|
||||
header={<SectionHeader icon={<SettingOutlined />} text="Advanced" />}
|
||||
key="2"
|
||||
>
|
||||
{advancedFields.map((field, i) => (
|
||||
<FormField field={field} removeMargin={i === advancedFields.length - 1} />
|
||||
))}
|
||||
</Collapse.Panel>
|
||||
</StyledCollapse>
|
||||
<ControlsContainer>
|
||||
<Button disabled={isEditing} onClick={goToPrevious}>
|
||||
Previous
|
||||
</Button>
|
||||
<Button htmlType="submit">Next</Button>
|
||||
</ControlsContainer>
|
||||
</Form>
|
||||
);
|
||||
}
|
||||
|
||||
export default RecipeForm;
|
||||
@ -0,0 +1,120 @@
|
||||
import { setFieldValueOnRecipe, setListValuesOnRecipe } from '../utils';
|
||||
|
||||
describe('setFieldValueOnRecipe', () => {
|
||||
const accountIdFieldPath = 'source.config.account_id';
|
||||
const profilingEnabledFieldPath = 'source.config.profiling.enabled';
|
||||
|
||||
it('should set the field value on a recipe object when it was not defined', () => {
|
||||
const recipe = { source: { config: {} } };
|
||||
const updatedRecipe = setFieldValueOnRecipe(recipe, 'test', accountIdFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { account_id: 'test' } } });
|
||||
});
|
||||
|
||||
it('should update the field value on a recipe object when it was defined', () => {
|
||||
const recipe = { source: { config: { account_id: 'test' } } };
|
||||
const updatedRecipe = setFieldValueOnRecipe(recipe, 'edited!', accountIdFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { account_id: 'edited!' } } });
|
||||
});
|
||||
|
||||
it('should update the field value on a recipe without changing any other fields', () => {
|
||||
const recipe = { source: { config: { test: 'test' } } };
|
||||
const updatedRecipe = setFieldValueOnRecipe(recipe, 'edited!', accountIdFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test', account_id: 'edited!' } } });
|
||||
});
|
||||
|
||||
it('should clear the key: value pair when passing in null', () => {
|
||||
const recipe = { source: { config: { existingField: true, account_id: 'test' } } };
|
||||
const updatedRecipe = setFieldValueOnRecipe(recipe, null, accountIdFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { existingField: true } } });
|
||||
});
|
||||
|
||||
it('should return the original recipe when passing in undefined', () => {
|
||||
const recipe = { source: { config: { test: 'test' } } };
|
||||
const updatedRecipe = setFieldValueOnRecipe(recipe, undefined, accountIdFieldPath);
|
||||
expect(updatedRecipe).toMatchObject(recipe);
|
||||
});
|
||||
|
||||
it('should set the field value on a recipe object when it was not defined and has a parent', () => {
|
||||
const recipe = { source: { config: {} } };
|
||||
const updatedRecipe = setFieldValueOnRecipe(recipe, true, profilingEnabledFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { profiling: { enabled: true } } } });
|
||||
});
|
||||
|
||||
it('should update the field value on a recipe object when it was defined and has a parent', () => {
|
||||
const recipe = { source: { config: { profiling: { enabled: true } } } };
|
||||
const updatedRecipe = setFieldValueOnRecipe(recipe, false, profilingEnabledFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { profiling: { enabled: false } } } });
|
||||
});
|
||||
|
||||
it('should update the field value with a parent on a recipe without changing any other fields', () => {
|
||||
const recipe = { source: { config: { test: 'test' } } };
|
||||
const updatedRecipe = setFieldValueOnRecipe(recipe, false, profilingEnabledFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test', profiling: { enabled: false } } } });
|
||||
});
|
||||
|
||||
it('should clear the field and its parent when passing in null and field is only child of parent', () => {
|
||||
const recipe = { source: { config: { test: 'test', profiling: { enabled: true } } } };
|
||||
const updatedRecipe = setFieldValueOnRecipe(recipe, null, profilingEnabledFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test' } } });
|
||||
});
|
||||
|
||||
it('should clear the field but not its parent when passing in null and parent has other children', () => {
|
||||
const recipe = { source: { config: { test: 'test', profiling: { enabled: true, testing: 'hello' } } } };
|
||||
const updatedRecipe = setFieldValueOnRecipe(recipe, null, 'source.config.profiling.testing');
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test', profiling: { enabled: true } } } });
|
||||
});
|
||||
});
|
||||
|
||||
describe('setListValuesOnRecipe', () => {
|
||||
const tableAllowFieldPath = 'source.config.table_pattern.allow';
|
||||
|
||||
it('should update list values on a recipe when it was not defined', () => {
|
||||
const recipe = { source: { config: {} } };
|
||||
const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern'], tableAllowFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { table_pattern: { allow: ['*test_pattern'] } } } });
|
||||
});
|
||||
|
||||
it('should update list values on a recipe when it was defined', () => {
|
||||
const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'] } } } };
|
||||
const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern(edit)'], tableAllowFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({
|
||||
source: { config: { table_pattern: { allow: ['*test_pattern(edit)'] } } },
|
||||
});
|
||||
});
|
||||
|
||||
it('should append list values on a recipe', () => {
|
||||
const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'] } } } };
|
||||
const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern', 'new'], tableAllowFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({
|
||||
source: { config: { table_pattern: { allow: ['*test_pattern', 'new'] } } },
|
||||
});
|
||||
});
|
||||
|
||||
it('should remove list values on a recipe', () => {
|
||||
const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern', 'remove_me'] } } } };
|
||||
const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern'], tableAllowFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({
|
||||
source: { config: { table_pattern: { allow: ['*test_pattern'] } } },
|
||||
});
|
||||
});
|
||||
|
||||
it('should remove empty values from the list when updating a recipe', () => {
|
||||
const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'] } } } };
|
||||
const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern', '', '', ''], tableAllowFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({
|
||||
source: { config: { table_pattern: { allow: ['*test_pattern'] } } },
|
||||
});
|
||||
});
|
||||
|
||||
it('should clear the value and its parent when passing in empty list and parent has no other children', () => {
|
||||
const recipe = { source: { config: { existingField: true, table_pattern: { allow: ['*test_pattern'] } } } };
|
||||
const updatedRecipe = setListValuesOnRecipe(recipe, [], tableAllowFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { existingField: true } } });
|
||||
});
|
||||
|
||||
it('should clear the value but not its parent when passing in empty list and parent has other children', () => {
|
||||
const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'], deny: ['test_deny'] } } } };
|
||||
const updatedRecipe = setListValuesOnRecipe(recipe, [], tableAllowFieldPath);
|
||||
expect(updatedRecipe).toMatchObject({ source: { config: { table_pattern: { deny: ['test_deny'] } } } });
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,287 @@
|
||||
import { set, get } from 'lodash';
|
||||
import { SNOWFLAKE } from '../../conf/snowflake/snowflake';
|
||||
|
||||
export enum FieldType {
|
||||
TEXT,
|
||||
BOOLEAN,
|
||||
LIST,
|
||||
}
|
||||
|
||||
export interface RecipeField {
|
||||
name: string;
|
||||
label: string;
|
||||
tooltip: string;
|
||||
type: FieldType;
|
||||
fieldPath: string;
|
||||
rules: any[] | null;
|
||||
section?: string;
|
||||
getValueFromRecipeOverride?: (recipe: any) => any;
|
||||
setValueOnRecipeOverride?: (recipe: any, value: any) => any;
|
||||
}
|
||||
|
||||
function clearFieldAndParents(recipe: any, fieldPath: string) {
|
||||
set(recipe, fieldPath, undefined);
|
||||
|
||||
const updatedFieldPath = fieldPath.split('.').slice(0, -1).join('.'); // remove last item from fieldPath
|
||||
if (updatedFieldPath) {
|
||||
const parentKeys = Object.keys(get(recipe, updatedFieldPath));
|
||||
|
||||
// only child left is what we just set as undefined
|
||||
if (parentKeys.length === 1) {
|
||||
clearFieldAndParents(recipe, updatedFieldPath);
|
||||
}
|
||||
}
|
||||
return recipe;
|
||||
}
|
||||
|
||||
export function setFieldValueOnRecipe(recipe: any, value: any, fieldPath: string) {
|
||||
const updatedRecipe = { ...recipe };
|
||||
if (value !== undefined) {
|
||||
if (value === null) {
|
||||
clearFieldAndParents(updatedRecipe, fieldPath);
|
||||
return updatedRecipe;
|
||||
}
|
||||
set(updatedRecipe, fieldPath, value);
|
||||
}
|
||||
return updatedRecipe;
|
||||
}
|
||||
|
||||
export function setListValuesOnRecipe(recipe: any, values: string[] | undefined, fieldPath: string) {
|
||||
const updatedRecipe = { ...recipe };
|
||||
if (values !== undefined) {
|
||||
const filteredValues: string[] | undefined = values.filter((v) => !!v);
|
||||
return filteredValues.length
|
||||
? setFieldValueOnRecipe(updatedRecipe, filteredValues, fieldPath)
|
||||
: setFieldValueOnRecipe(updatedRecipe, null, fieldPath);
|
||||
}
|
||||
return updatedRecipe;
|
||||
}
|
||||
|
||||
export const SNOWFLAKE_ACCOUNT_ID: RecipeField = {
|
||||
name: 'account_id',
|
||||
label: 'Account ID',
|
||||
tooltip: 'Snowflake account. e.g. abc48144',
|
||||
type: FieldType.TEXT,
|
||||
fieldPath: 'source.config.account_id',
|
||||
rules: null,
|
||||
};
|
||||
|
||||
export const SNOWFLAKE_WAREHOUSE: RecipeField = {
|
||||
name: 'warehouse',
|
||||
label: 'Warehouse',
|
||||
tooltip: 'Snowflake warehouse.',
|
||||
type: FieldType.TEXT,
|
||||
fieldPath: 'source.config.warehouse',
|
||||
rules: null,
|
||||
};
|
||||
|
||||
export const SNOWFLAKE_USERNAME: RecipeField = {
|
||||
name: 'username',
|
||||
label: 'Username',
|
||||
tooltip: 'Snowflake username.',
|
||||
type: FieldType.TEXT,
|
||||
fieldPath: 'source.config.username',
|
||||
rules: null,
|
||||
};
|
||||
|
||||
export const SNOWFLAKE_PASSWORD: RecipeField = {
|
||||
name: 'password',
|
||||
label: 'Password',
|
||||
tooltip: 'Snowflake password.',
|
||||
type: FieldType.TEXT,
|
||||
fieldPath: 'source.config.password',
|
||||
rules: null,
|
||||
};
|
||||
|
||||
export const SNOWFLAKE_ROLE: RecipeField = {
|
||||
name: 'role',
|
||||
label: 'Role',
|
||||
tooltip: 'Snowflake role.',
|
||||
type: FieldType.TEXT,
|
||||
fieldPath: 'source.config.role',
|
||||
rules: null,
|
||||
};
|
||||
|
||||
const includeLineageFieldPathA = 'source.config.include_table_lineage';
|
||||
const includeLineageFieldPathB = 'source.config.include_view_lineage';
|
||||
export const INCLUDE_LINEAGE: RecipeField = {
|
||||
name: 'include_lineage',
|
||||
label: 'Include Lineage',
|
||||
tooltip: 'Include Table and View lineage in your ingestion.',
|
||||
type: FieldType.BOOLEAN,
|
||||
fieldPath: 'source.config.include_table_lineage',
|
||||
rules: null,
|
||||
getValueFromRecipeOverride: (recipe: any) =>
|
||||
get(recipe, includeLineageFieldPathA) && get(recipe, includeLineageFieldPathB),
|
||||
setValueOnRecipeOverride: (recipe: any, value: boolean) => {
|
||||
let updatedRecipe = setFieldValueOnRecipe(recipe, value, includeLineageFieldPathA);
|
||||
updatedRecipe = setFieldValueOnRecipe(updatedRecipe, value, includeLineageFieldPathB);
|
||||
return updatedRecipe;
|
||||
},
|
||||
};
|
||||
|
||||
export const IGNORE_START_TIME_LINEAGE: RecipeField = {
|
||||
name: 'ignore_start_time_lineage',
|
||||
label: 'Ignore Start Time Lineage',
|
||||
tooltip: 'Get all lineage by ignoring the start_time field. It is suggested to set to true initially.',
|
||||
type: FieldType.BOOLEAN,
|
||||
fieldPath: 'source.config.ignore_start_time_lineage',
|
||||
rules: null,
|
||||
};
|
||||
|
||||
export const CHECK_ROLE_GRANTS: RecipeField = {
|
||||
name: 'check_role_grants',
|
||||
label: 'Check Role Grants',
|
||||
tooltip:
|
||||
'If set to True then checks role grants at the beginning of the ingestion run. To be used for debugging purposes. If you think everything is working fine then set it to False. In some cases this can take long depending on how many roles you might have.',
|
||||
type: FieldType.BOOLEAN,
|
||||
fieldPath: 'source.config.check_role_grants',
|
||||
rules: null,
|
||||
};
|
||||
|
||||
export const PROFILING_ENABLED: RecipeField = {
|
||||
name: 'profiling.enabled',
|
||||
label: 'Enable Profiling',
|
||||
tooltip: 'Whether profiling should be done.',
|
||||
type: FieldType.BOOLEAN,
|
||||
fieldPath: 'source.config.profiling.enabled',
|
||||
rules: null,
|
||||
};
|
||||
|
||||
export const STATEFUL_INGESTION_ENABLED: RecipeField = {
|
||||
name: 'stateful_ingestion.enabled',
|
||||
label: 'Enable Stateful Ingestion',
|
||||
tooltip: 'Enable the type of the ingestion state provider registered with datahub.',
|
||||
type: FieldType.BOOLEAN,
|
||||
fieldPath: 'source.config.stateful_ingestion.enabled',
|
||||
rules: null,
|
||||
};
|
||||
|
||||
const databaseAllowFieldPath = 'source.config.database_pattern.allow';
|
||||
export const DATABASE_ALLOW: RecipeField = {
|
||||
name: 'database_pattern.allow',
|
||||
label: 'Allow Patterns',
|
||||
tooltip: 'Use regex here.',
|
||||
type: FieldType.LIST,
|
||||
fieldPath: 'source.config.database_pattern.allow',
|
||||
rules: null,
|
||||
section: 'Databases',
|
||||
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
|
||||
setListValuesOnRecipe(recipe, values, databaseAllowFieldPath),
|
||||
};
|
||||
|
||||
const databaseDenyFieldPath = 'source.config.database_pattern.deny';
|
||||
export const DATABASE_DENY: RecipeField = {
|
||||
name: 'database_pattern.deny',
|
||||
label: 'Deny Patterns',
|
||||
tooltip: 'Use regex here.',
|
||||
type: FieldType.LIST,
|
||||
fieldPath: 'source.config.database_pattern.deny',
|
||||
rules: null,
|
||||
section: 'Databases',
|
||||
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
|
||||
setListValuesOnRecipe(recipe, values, databaseDenyFieldPath),
|
||||
};
|
||||
|
||||
const schemaAllowFieldPath = 'source.config.schema_pattern.allow';
|
||||
export const SCHEMA_ALLOW: RecipeField = {
|
||||
name: 'schema_pattern.allow',
|
||||
label: 'Allow Patterns',
|
||||
tooltip: 'Use regex here.',
|
||||
type: FieldType.LIST,
|
||||
fieldPath: 'source.config.schema_pattern.allow',
|
||||
rules: null,
|
||||
section: 'Schemas',
|
||||
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
|
||||
setListValuesOnRecipe(recipe, values, schemaAllowFieldPath),
|
||||
};
|
||||
|
||||
const schemaDenyFieldPath = 'source.config.schema_pattern.deny';
|
||||
export const SCHEMA_DENY: RecipeField = {
|
||||
name: 'schema_pattern.deny',
|
||||
label: 'Deny Patterns',
|
||||
tooltip: 'Use regex here.',
|
||||
type: FieldType.LIST,
|
||||
fieldPath: 'source.config.schema_pattern.deny',
|
||||
rules: null,
|
||||
section: 'Schemas',
|
||||
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
|
||||
setListValuesOnRecipe(recipe, values, schemaDenyFieldPath),
|
||||
};
|
||||
|
||||
const viewAllowFieldPath = 'source.config.view_pattern.allow';
|
||||
export const VIEW_ALLOW: RecipeField = {
|
||||
name: 'view_pattern.allow',
|
||||
label: 'Allow Patterns',
|
||||
tooltip: 'Use regex here.',
|
||||
type: FieldType.LIST,
|
||||
fieldPath: 'source.config.view_pattern.allow',
|
||||
rules: null,
|
||||
section: 'Views',
|
||||
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
|
||||
setListValuesOnRecipe(recipe, values, viewAllowFieldPath),
|
||||
};
|
||||
|
||||
const viewDenyFieldPath = 'source.config.view_pattern.deny';
|
||||
export const VIEW_DENY: RecipeField = {
|
||||
name: 'view_pattern.deny',
|
||||
label: 'Deny Patterns',
|
||||
tooltip: 'Use regex here.',
|
||||
type: FieldType.LIST,
|
||||
fieldPath: 'source.config.view_pattern.deny',
|
||||
rules: null,
|
||||
section: 'Views',
|
||||
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
|
||||
setListValuesOnRecipe(recipe, values, viewDenyFieldPath),
|
||||
};
|
||||
|
||||
const tableAllowFieldPath = 'source.config.table_pattern.allow';
|
||||
export const TABLE_ALLOW: RecipeField = {
|
||||
name: 'table_pattern.allow',
|
||||
label: 'Allow Patterns',
|
||||
tooltip: 'Use regex here.',
|
||||
type: FieldType.LIST,
|
||||
fieldPath: 'source.config.table_pattern.allow',
|
||||
rules: null,
|
||||
section: 'Tables',
|
||||
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
|
||||
setListValuesOnRecipe(recipe, values, tableAllowFieldPath),
|
||||
};
|
||||
|
||||
const tableDenyFieldPath = 'source.config.table_pattern.deny';
|
||||
export const TABLE_DENY: RecipeField = {
|
||||
name: 'table_pattern.deny',
|
||||
label: 'Deny Patterns',
|
||||
tooltip: 'Use regex here.',
|
||||
type: FieldType.LIST,
|
||||
fieldPath: 'source.config.table_pattern.deny',
|
||||
rules: null,
|
||||
section: 'Tables',
|
||||
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
|
||||
setListValuesOnRecipe(recipe, values, tableDenyFieldPath),
|
||||
};
|
||||
|
||||
export const RECIPE_FIELDS = {
|
||||
[SNOWFLAKE]: {
|
||||
fields: [SNOWFLAKE_ACCOUNT_ID, SNOWFLAKE_WAREHOUSE, SNOWFLAKE_USERNAME, SNOWFLAKE_PASSWORD, SNOWFLAKE_ROLE],
|
||||
advancedFields: [
|
||||
INCLUDE_LINEAGE,
|
||||
IGNORE_START_TIME_LINEAGE,
|
||||
CHECK_ROLE_GRANTS,
|
||||
PROFILING_ENABLED,
|
||||
STATEFUL_INGESTION_ENABLED,
|
||||
],
|
||||
filterFields: [
|
||||
TABLE_ALLOW,
|
||||
TABLE_DENY,
|
||||
DATABASE_ALLOW,
|
||||
DATABASE_DENY,
|
||||
SCHEMA_ALLOW,
|
||||
SCHEMA_DENY,
|
||||
VIEW_ALLOW,
|
||||
VIEW_DENY,
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));
|
||||
@ -0,0 +1,35 @@
|
||||
import { render } from '@testing-library/react';
|
||||
import React from 'react';
|
||||
import { DefineRecipeStep } from '../DefineRecipeStep';
|
||||
|
||||
describe('DefineRecipeStep', () => {
|
||||
it('should render the RecipeBuilder if the type is in CONNECTORS_WITH_FORM', () => {
|
||||
const { getByText, queryByText } = render(
|
||||
<DefineRecipeStep
|
||||
state={{ type: 'snowflake' }}
|
||||
updateState={() => {}}
|
||||
goTo={() => {}}
|
||||
submit={() => {}}
|
||||
cancel={() => {}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(getByText('Connection')).toBeInTheDocument();
|
||||
expect(queryByText('For more information about how to configure a recipe')).toBeNull();
|
||||
});
|
||||
|
||||
it('should not render the RecipeBuilder if the type is not in CONNECTORS_WITH_FORM', () => {
|
||||
const { getByText, queryByText } = render(
|
||||
<DefineRecipeStep
|
||||
state={{ type: 'postgres' }}
|
||||
updateState={() => {}}
|
||||
goTo={() => {}}
|
||||
submit={() => {}}
|
||||
cancel={() => {}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(getByText('Configure Postgres Recipe')).toBeInTheDocument();
|
||||
expect(queryByText('Connection')).toBeNull();
|
||||
});
|
||||
});
|
||||
@ -5,52 +5,23 @@ const placeholderRecipe = `\
|
||||
source:
|
||||
type: snowflake
|
||||
config:
|
||||
# Uncomment this section to provision the role required for ingestion
|
||||
# provision_role:
|
||||
# enabled: true
|
||||
# dry_run: false
|
||||
# run_ingestion: true
|
||||
# admin_username: "\${SNOWFLAKE_ADMIN_USER}"
|
||||
# admin_password: "\${SNOWFLAKE_ADMIN_PASS}"
|
||||
|
||||
# Your Snowflake account name
|
||||
# e.g. if URL is example48144.us-west-2.snowflakecomputing.com then use "example48144"
|
||||
account_id: "example48144"
|
||||
warehouse: # Your Snowflake warehouse name, e.g. "PROD_WH"
|
||||
|
||||
# Credentials
|
||||
username: "\${SNOWFLAKE_USER}" # Create a secret SNOWFLAKE_USER in secrets Tab
|
||||
password: "\${SNOWFLAKE_PASS}" # Create a secret SNOWFLAKE_PASS in secrets Tab
|
||||
account_id: "example_id"
|
||||
warehouse: "example_warehouse"
|
||||
role: "datahub_role"
|
||||
|
||||
# Suggest to have this set to true initially to get all lineage
|
||||
ignore_start_time_lineage: true
|
||||
|
||||
# This is an alternative option to specify the start_time for lineage
|
||||
# if you don't want to look back since beginning
|
||||
# start_time: '2022-03-01T00:00:00Z'
|
||||
|
||||
# Uncomment and change to only allow some database metadata to be ingested
|
||||
# database_pattern:
|
||||
# allow:
|
||||
# - "^ACCOUNTING_DB$"
|
||||
# - "^MARKETING_DB$"
|
||||
|
||||
# Uncomment and change to deny some metadata from few schemas
|
||||
# schema_pattern:
|
||||
# deny:
|
||||
# - "information_schema.*"
|
||||
|
||||
# If you want to ingest only few tables with name revenue and sales
|
||||
# table_pattern:
|
||||
# allow:
|
||||
# - ".*revenue"
|
||||
# - ".*sales"
|
||||
|
||||
include_table_lineage: true
|
||||
include_view_lineage: true
|
||||
check_role_grants: true
|
||||
profiling:
|
||||
enabled: true
|
||||
stateful_ingestion:
|
||||
enabled: true
|
||||
`;
|
||||
|
||||
export const SNOWFLAKE = 'snowflake';
|
||||
|
||||
const snowflakeConfig: SourceConfig = {
|
||||
type: 'snowflake',
|
||||
type: SNOWFLAKE,
|
||||
placeholderRecipe,
|
||||
displayName: 'Snowflake',
|
||||
docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/snowflake/',
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user