mirror of
https://github.com/datahub-project/datahub.git
synced 2025-11-02 03:39:03 +00:00
refactor(ui): Misc improvements to the setup ingestion flow (ingest uplift 1/2) (#10764)
Co-authored-by: John Joyce <john@Johns-MBP.lan> Co-authored-by: John Joyce <john@ip-192-168-1-200.us-west-2.compute.internal>
This commit is contained in:
John Joyce
GitHub
parent
64d4172e55
commit
8d5f0f3b04
@ -58,9 +58,9 @@ export const ManageIngestionPage = () => {
|
||||
<PageContainer>
|
||||
<OnboardingTour stepIds={[INGESTION_CREATE_SOURCE_ID, INGESTION_REFRESH_SOURCES_ID]} />
|
||||
<PageHeaderContainer>
|
||||
<PageTitle level={3}>Manage Ingestion</PageTitle>
|
||||
<PageTitle level={3}>Manage Data Sources</PageTitle>
|
||||
<Typography.Paragraph type="secondary">
|
||||
Create, schedule, and run DataHub ingestion sources.
|
||||
Configure and schedule syncs to import data from your data sources
|
||||
</Typography.Paragraph>
|
||||
</PageHeaderContainer>
|
||||
<StyledTabs activeKey={selectedTab} size="large" onTabClick={(tab: string) => onClickTab(tab)}>
|
||||
|
||||
@ -10,6 +10,7 @@ import { TimezoneSelect } from './TimezoneSelect';
|
||||
import { ANTD_GRAY, REDESIGN_COLORS } from '../../../entity/shared/constants';
|
||||
import { lowerFirstLetter } from '../../../shared/textUtil';
|
||||
import { IngestionSourceBuilderStep } from './steps';
|
||||
import { RequiredFieldForm } from '../../../shared/form/RequiredFieldForm';
|
||||
|
||||
const Section = styled.div`
|
||||
display: flex;
|
||||
@ -31,10 +32,25 @@ const CronText = styled(Typography.Paragraph)`
|
||||
color: ${ANTD_GRAY[7]};
|
||||
`;
|
||||
|
||||
const CronInput = styled(Input)`
|
||||
margin-bottom: 8px;
|
||||
max-width: 200px;
|
||||
`;
|
||||
|
||||
const Schedule = styled.div`
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: start;
|
||||
`;
|
||||
|
||||
const AdvancedSchedule = styled.div`
|
||||
margin-left: 20px;
|
||||
`;
|
||||
|
||||
const AdvancedCheckBox = styled(Typography.Text)`
|
||||
margin-right: 10px;
|
||||
margin-bottom: 8px;
|
||||
`;
|
||||
|
||||
const CronSuccessCheck = styled(CheckCircleOutlined)`
|
||||
color: ${REDESIGN_COLORS.BLUE};
|
||||
margin-right: 4px;
|
||||
@ -123,9 +139,9 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps
|
||||
<Section>
|
||||
<SelectTemplateHeader level={5}>Configure an Ingestion Schedule</SelectTemplateHeader>
|
||||
</Section>
|
||||
<Form layout="vertical">
|
||||
<RequiredFieldForm layout="vertical">
|
||||
<Form.Item
|
||||
tooltip="Enable to run ingestion on a schedule. Running ingestion on a schedule helps to keep the information inside of DataHub up to date."
|
||||
tooltip="Enable to run ingestion syncs on a schedule. Running syncs on a schedule helps to keep information up to date."
|
||||
label={
|
||||
<Typography.Text strong>
|
||||
Run on a schedule <Typography.Text type="secondary">(Recommended)</Typography.Text>
|
||||
@ -141,29 +157,31 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps
|
||||
)}
|
||||
</Form.Item>
|
||||
<StyledFormItem required label={<Typography.Text strong>Schedule</Typography.Text>}>
|
||||
<div style={{ paddingBottom: 10, paddingLeft: 10 }}>
|
||||
<AdvancedCheckBox type="secondary">Advanced</AdvancedCheckBox>
|
||||
<Checkbox
|
||||
checked={advancedCronCheck}
|
||||
onChange={(event) => setAdvancedCronCheck(event.target.checked)}
|
||||
/>
|
||||
</div>
|
||||
{advancedCronCheck ? (
|
||||
<Input
|
||||
placeholder={DAILY_MIDNIGHT_CRON_INTERVAL}
|
||||
autoFocus
|
||||
value={scheduleCronInterval}
|
||||
onChange={(e) => setScheduleCronInterval(e.target.value)}
|
||||
/>
|
||||
) : (
|
||||
<Cron
|
||||
value={scheduleCronInterval}
|
||||
setValue={setScheduleCronInterval}
|
||||
clearButton={false}
|
||||
className="cron-builder"
|
||||
leadingZero
|
||||
/>
|
||||
)}
|
||||
<Schedule>
|
||||
{advancedCronCheck ? (
|
||||
<CronInput
|
||||
placeholder={DAILY_MIDNIGHT_CRON_INTERVAL}
|
||||
autoFocus
|
||||
value={scheduleCronInterval}
|
||||
onChange={(e) => setScheduleCronInterval(e.target.value)}
|
||||
/>
|
||||
) : (
|
||||
<Cron
|
||||
value={scheduleCronInterval}
|
||||
setValue={setScheduleCronInterval}
|
||||
clearButton={false}
|
||||
className="cron-builder"
|
||||
leadingZero
|
||||
/>
|
||||
)}
|
||||
<AdvancedSchedule>
|
||||
<AdvancedCheckBox type="secondary">Show Advanced</AdvancedCheckBox>
|
||||
<Checkbox
|
||||
checked={advancedCronCheck}
|
||||
onChange={(event) => setAdvancedCronCheck(event.target.checked)}
|
||||
/>
|
||||
</AdvancedSchedule>
|
||||
</Schedule>
|
||||
<CronText>
|
||||
{cronAsText.error && <>Invalid cron schedule. Cron must be of UNIX form:</>}
|
||||
{!cronAsText.text && (
|
||||
@ -183,7 +201,7 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps
|
||||
<ItemDescriptionText>Choose a timezone for the schedule.</ItemDescriptionText>
|
||||
<TimezoneSelect value={scheduleTimezone} onChange={setScheduleTimezone} />
|
||||
</Form.Item>
|
||||
</Form>
|
||||
</RequiredFieldForm>
|
||||
<ControlsContainer>
|
||||
<Button onClick={prev}>Previous</Button>
|
||||
<div>
|
||||
@ -191,6 +209,7 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps
|
||||
data-testid="ingestion-schedule-next-button"
|
||||
disabled={!interval || interval.length === 0 || cronAsText.error}
|
||||
onClick={onClickNext}
|
||||
type="primary"
|
||||
>
|
||||
Next
|
||||
</Button>
|
||||
|
||||
@ -0,0 +1,68 @@
|
||||
import React from 'react';
|
||||
import { Button, Image } from 'antd';
|
||||
import styled from 'styled-components';
|
||||
|
||||
import { REDESIGN_COLORS } from '../../../entity/shared/constants';
|
||||
|
||||
const Container = styled(Button)`
|
||||
padding: 32px;
|
||||
height: 200px;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
border-radius: 8px;
|
||||
align-items: start;
|
||||
flex-direction: column;
|
||||
border: 1px solid #e0e0e0;
|
||||
background-color: #ffffff;
|
||||
&&:hover {
|
||||
border: 1px solid ${REDESIGN_COLORS.BLUE};
|
||||
background-color: #ffffff;
|
||||
}
|
||||
white-space: unset;
|
||||
`;
|
||||
|
||||
const PlatformLogo = styled(Image)`
|
||||
max-height: 32px;
|
||||
height: 32px;
|
||||
width: auto;
|
||||
object-fit: contain;
|
||||
background-color: transparent;
|
||||
`;
|
||||
|
||||
const LogoContainer = styled.div`
|
||||
margin-bottom: 14px;
|
||||
`;
|
||||
|
||||
const Title = styled.div`
|
||||
word-break: break-word;
|
||||
color: #464646;
|
||||
font-weight: bold;
|
||||
font-size: 16px;
|
||||
margin-bottom: 8px;
|
||||
`;
|
||||
|
||||
const Description = styled.div`
|
||||
word-break: break-word;
|
||||
text-align: left;
|
||||
color: #7c7c7c;
|
||||
`;
|
||||
|
||||
type Props = {
|
||||
logoUrl?: string;
|
||||
logoComponent?: React.ReactNode;
|
||||
name: string;
|
||||
description?: string;
|
||||
onClick?: () => void;
|
||||
};
|
||||
|
||||
export const DataPlatformCard = ({ logoUrl, logoComponent, name, description, onClick }: Props) => {
|
||||
return (
|
||||
<Container type="link" onClick={onClick}>
|
||||
<LogoContainer>
|
||||
{(logoUrl && <PlatformLogo preview={false} src={logoUrl} alt={name} />) || logoComponent}
|
||||
</LogoContainer>
|
||||
<Title>{name}</Title>
|
||||
<Description>{description}</Description>
|
||||
</Container>
|
||||
);
|
||||
};
|
||||
@ -164,7 +164,7 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev, ingestionSour
|
||||
<Button disabled={isEditing} onClick={prev}>
|
||||
Previous
|
||||
</Button>
|
||||
<Button disabled={!stepComplete} onClick={onClickNext}>
|
||||
<Button type="primary" disabled={!stepComplete} onClick={onClickNext}>
|
||||
Next
|
||||
</Button>
|
||||
</ControlsContainer>
|
||||
|
||||
@ -0,0 +1,68 @@
|
||||
import React from 'react';
|
||||
import styled from 'styled-components';
|
||||
import { Button, Tooltip } from 'antd';
|
||||
import { CloseOutlined } from '@ant-design/icons';
|
||||
|
||||
import { SourceConfig } from './types';
|
||||
import { ANTD_GRAY } from '../../../entity/shared/constants';
|
||||
|
||||
const Container = styled.div`
|
||||
background-color: #ffffff;
|
||||
border-radius: 8px;
|
||||
padding: 12px 12px 16px 24px;
|
||||
border: 1px solid #e0e0e0;
|
||||
margin-bottom: 20px;
|
||||
`;
|
||||
|
||||
const Header = styled.div`
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
`;
|
||||
|
||||
const Title = styled.div`
|
||||
font-size: 16px;
|
||||
font-weight: bold;
|
||||
`;
|
||||
|
||||
const Description = styled.div`
|
||||
font-size: 14px;
|
||||
max-width: 90%;
|
||||
`;
|
||||
|
||||
const StyledCloseOutlined = styled(CloseOutlined)`
|
||||
color: ${ANTD_GRAY[6]};
|
||||
`;
|
||||
|
||||
interface Props {
|
||||
sourceConfigs: SourceConfig;
|
||||
onHide: () => void;
|
||||
}
|
||||
|
||||
export const IngestionDocumentationHint = ({ sourceConfigs, onHide }: Props) => {
|
||||
const { displayName, docsUrl } = sourceConfigs;
|
||||
return (
|
||||
<Container>
|
||||
<Header>
|
||||
<Title>Let's get connected! 🎉</Title>
|
||||
<Tooltip showArrow={false} title="Hide">
|
||||
<Button type="text" icon={<StyledCloseOutlined />} onClick={onHide} />
|
||||
</Tooltip>
|
||||
</Header>
|
||||
<Description>
|
||||
<div style={{ marginBottom: 8 }}>
|
||||
To import from {displayName}, we'll need some more information to connect to your instance.
|
||||
</div>
|
||||
<div>
|
||||
Check out the{' '}
|
||||
<a href={docsUrl} target="_blank" rel="noopener noreferrer">
|
||||
{displayName} Guide
|
||||
</a>{' '}
|
||||
to understand the prerequisites, learn about available settings, and view examples to help connect
|
||||
to the data source.
|
||||
</div>
|
||||
</Description>
|
||||
</Container>
|
||||
);
|
||||
};
|
||||
@ -1,8 +1,7 @@
|
||||
import { Button, Modal, Steps, Typography } from 'antd';
|
||||
import { Modal, Steps, Typography } from 'antd';
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import styled from 'styled-components';
|
||||
import { isEqual } from 'lodash';
|
||||
import { ExpandAltOutlined, ShrinkOutlined } from '@ant-design/icons';
|
||||
import { SourceBuilderState, StepProps } from './types';
|
||||
import { CreateScheduleStep } from './CreateScheduleStep';
|
||||
import { DefineRecipeStep } from './DefineRecipeStep';
|
||||
@ -10,15 +9,18 @@ import { NameSourceStep } from './NameSourceStep';
|
||||
import { SelectTemplateStep } from './SelectTemplateStep';
|
||||
import sourcesJson from './sources.json';
|
||||
|
||||
const ExpandButton = styled(Button)`
|
||||
&& {
|
||||
margin-right: 32px;
|
||||
const StyledModal = styled(Modal)`
|
||||
&& .ant-modal-content {
|
||||
border-radius: 16px;
|
||||
overflow: hidden;
|
||||
min-width: 400px;
|
||||
}
|
||||
`;
|
||||
|
||||
const TitleContainer = styled.div`
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
border-radius: 12px;
|
||||
`;
|
||||
|
||||
const StepsContainer = styled.div`
|
||||
@ -31,9 +33,9 @@ const StepsContainer = styled.div`
|
||||
* Mapping from the step type to the title for the step
|
||||
*/
|
||||
export enum IngestionSourceBuilderStepTitles {
|
||||
SELECT_TEMPLATE = 'Choose Type',
|
||||
DEFINE_RECIPE = 'Configure Recipe',
|
||||
CREATE_SCHEDULE = 'Schedule Ingestion',
|
||||
SELECT_TEMPLATE = 'Choose Data Source',
|
||||
DEFINE_RECIPE = 'Configure Connection',
|
||||
CREATE_SCHEDULE = 'Sync Schedule',
|
||||
NAME_SOURCE = 'Finish up',
|
||||
}
|
||||
|
||||
@ -57,6 +59,8 @@ export enum IngestionSourceBuilderStep {
|
||||
NAME_SOURCE = 'NAME_SOURCE',
|
||||
}
|
||||
|
||||
const modalBodyStyle = { padding: '16px 24px 16px 24px', backgroundColor: '#F6F6F6' };
|
||||
|
||||
type Props = {
|
||||
initialState?: SourceBuilderState;
|
||||
visible: boolean;
|
||||
@ -66,14 +70,17 @@ type Props = {
|
||||
|
||||
export const IngestionSourceBuilderModal = ({ initialState, visible, onSubmit, onCancel }: Props) => {
|
||||
const isEditing = initialState !== undefined;
|
||||
const titleText = isEditing ? 'Edit Ingestion Source' : 'New Ingestion Source';
|
||||
const titleText = isEditing ? 'Edit Data Source' : 'Connect Data Source';
|
||||
const initialStep = isEditing
|
||||
? IngestionSourceBuilderStep.DEFINE_RECIPE
|
||||
: IngestionSourceBuilderStep.SELECT_TEMPLATE;
|
||||
|
||||
const [stepStack, setStepStack] = useState([initialStep]);
|
||||
const [modalExpanded, setModalExpanded] = useState(false);
|
||||
const [ingestionBuilderState, setIngestionBuilderState] = useState<SourceBuilderState>({});
|
||||
const [ingestionBuilderState, setIngestionBuilderState] = useState<SourceBuilderState>({
|
||||
schedule: {
|
||||
interval: '0 0 * * *',
|
||||
},
|
||||
});
|
||||
|
||||
const ingestionSources = JSON.parse(JSON.stringify(sourcesJson)); // TODO: replace with call to server once we have access to dynamic list of sources
|
||||
|
||||
@ -122,28 +129,28 @@ export const IngestionSourceBuilderModal = ({ initialState, visible, onSubmit, o
|
||||
const StepComponent: React.FC<StepProps> = IngestionSourceBuilderStepComponent[currentStep];
|
||||
|
||||
return (
|
||||
<Modal
|
||||
width={modalExpanded ? 1400 : 800}
|
||||
<StyledModal
|
||||
width="64%"
|
||||
footer={null}
|
||||
title={
|
||||
<TitleContainer>
|
||||
<Typography.Text>{titleText}</Typography.Text>
|
||||
<ExpandButton onClick={() => setModalExpanded(!modalExpanded)}>
|
||||
{(modalExpanded && <ShrinkOutlined />) || <ExpandAltOutlined />}
|
||||
</ExpandButton>
|
||||
</TitleContainer>
|
||||
}
|
||||
style={{ top: 40 }}
|
||||
bodyStyle={modalBodyStyle}
|
||||
visible={visible}
|
||||
onCancel={onCancel}
|
||||
>
|
||||
<StepsContainer>
|
||||
<Steps current={currentStepIndex}>
|
||||
{Object.keys(IngestionSourceBuilderStep).map((item) => (
|
||||
<Steps.Step key={item} title={IngestionSourceBuilderStepTitles[item]} />
|
||||
))}
|
||||
</Steps>
|
||||
</StepsContainer>
|
||||
{currentStepIndex > 0 ? (
|
||||
<StepsContainer>
|
||||
<Steps current={currentStepIndex}>
|
||||
{Object.keys(IngestionSourceBuilderStep).map((item) => (
|
||||
<Steps.Step key={item} title={IngestionSourceBuilderStepTitles[item]} />
|
||||
))}
|
||||
</Steps>
|
||||
</StepsContainer>
|
||||
) : null}
|
||||
<StepComponent
|
||||
state={ingestionBuilderState}
|
||||
updateState={setIngestionBuilderState}
|
||||
@ -153,6 +160,6 @@ export const IngestionSourceBuilderModal = ({ initialState, visible, onSubmit, o
|
||||
cancel={cancel}
|
||||
ingestionSources={ingestionSources}
|
||||
/>
|
||||
</Modal>
|
||||
</StyledModal>
|
||||
);
|
||||
};
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
import { Button, Checkbox, Collapse, Form, Input, Typography } from 'antd';
|
||||
import { Button, Checkbox, Collapse, Form, Input, Tooltip, Typography } from 'antd';
|
||||
import React from 'react';
|
||||
import styled from 'styled-components';
|
||||
import { SourceBuilderState, StepProps, StringMapEntryInput } from './types';
|
||||
import { RequiredFieldForm } from '../../../shared/form/RequiredFieldForm';
|
||||
|
||||
const ControlsContainer = styled.div`
|
||||
display: flex;
|
||||
@ -156,7 +157,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
|
||||
|
||||
return (
|
||||
<>
|
||||
<Form layout="vertical">
|
||||
<RequiredFieldForm layout="vertical">
|
||||
<Form.Item
|
||||
required
|
||||
label={
|
||||
@ -166,7 +167,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
|
||||
}
|
||||
style={{ marginBottom: 8 }}
|
||||
>
|
||||
<Typography.Paragraph>Give this ingestion source a name.</Typography.Paragraph>
|
||||
<Typography.Paragraph>Give this data source a name</Typography.Paragraph>
|
||||
<Input
|
||||
data-testid="source-name-input"
|
||||
className="source-name-input"
|
||||
@ -252,7 +253,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
|
||||
</Form.Item>
|
||||
</Collapse.Panel>
|
||||
</Collapse>
|
||||
</Form>
|
||||
</RequiredFieldForm>
|
||||
<ControlsContainer>
|
||||
<Button onClick={prev}>Previous</Button>
|
||||
<div>
|
||||
@ -263,13 +264,15 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
|
||||
>
|
||||
Save
|
||||
</SaveButton>
|
||||
<Button
|
||||
disabled={!(state.name !== undefined && state.name.length > 0)}
|
||||
onClick={() => onClickCreate(true)}
|
||||
type="primary"
|
||||
>
|
||||
Save & Run
|
||||
</Button>
|
||||
<Tooltip showArrow={false} title="Save and starting syncing data source">
|
||||
<Button
|
||||
disabled={!(state.name !== undefined && state.name.length > 0)}
|
||||
onClick={() => onClickCreate(true)}
|
||||
type="primary"
|
||||
>
|
||||
Save & Run
|
||||
</Button>
|
||||
</Tooltip>
|
||||
</div>
|
||||
</ControlsContainer>
|
||||
</>
|
||||
|
||||
@ -10,6 +10,7 @@ import { SourceBuilderState, SourceConfig } from './types';
|
||||
import { CSV, LOOKER, LOOK_ML } from './constants';
|
||||
import { LookerWarning } from './LookerWarning';
|
||||
import { CSVInfo } from './CSVInfo';
|
||||
import { IngestionDocumentationHint } from './IngestionDocumentationHint';
|
||||
|
||||
export const ControlsContainer = styled.div`
|
||||
display: flex;
|
||||
@ -66,6 +67,7 @@ function RecipeBuilder(props: Props) {
|
||||
const { state, isEditing, displayRecipe, sourceConfigs, setStagedRecipe, onClickNext, goToPrevious } = props;
|
||||
const { type } = state;
|
||||
const [isViewingForm, setIsViewingForm] = useState(true);
|
||||
const [hideDocsHint, setHideDocsHint] = useState(false);
|
||||
|
||||
function switchViews(isFormView: boolean) {
|
||||
try {
|
||||
@ -81,12 +83,14 @@ function RecipeBuilder(props: Props) {
|
||||
|
||||
return (
|
||||
<div>
|
||||
{!hideDocsHint && isViewingForm && sourceConfigs ? (
|
||||
<IngestionDocumentationHint onHide={() => setHideDocsHint(true)} sourceConfigs={sourceConfigs} />
|
||||
) : null}
|
||||
{(type === LOOKER || type === LOOK_ML) && <LookerWarning type={type} />}
|
||||
{type === CSV && <CSVInfo />}
|
||||
|
||||
<HeaderContainer>
|
||||
<Title style={{ marginBottom: 0 }} level={5}>
|
||||
{sourceConfigs?.displayName} Recipe
|
||||
{sourceConfigs?.displayName} Details
|
||||
</Title>
|
||||
<ButtonsWrapper>
|
||||
<StyledButton
|
||||
@ -127,7 +131,7 @@ function RecipeBuilder(props: Props) {
|
||||
<Button disabled={isEditing} onClick={goToPrevious}>
|
||||
Previous
|
||||
</Button>
|
||||
<Button data-testid="recipe-builder-next-button" onClick={onClickNext}>
|
||||
<Button type="primary" data-testid="recipe-builder-next-button" onClick={onClickNext}>
|
||||
Next
|
||||
</Button>
|
||||
</ControlsContainer>
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
import { Button, Collapse, Form, message, Tooltip, Typography } from 'antd';
|
||||
import React, { Fragment } from 'react';
|
||||
|
||||
import { Button, Collapse, Form, message, Tooltip, Typography } from 'antd';
|
||||
import { get } from 'lodash';
|
||||
import YAML from 'yamljs';
|
||||
import { ApiOutlined, FilterOutlined, QuestionCircleOutlined, SettingOutlined } from '@ant-design/icons';
|
||||
import styled from 'styled-components/macro';
|
||||
|
||||
import { jsonToYaml } from '../../utils';
|
||||
import { CONNECTORS_WITH_TEST_CONNECTION, RecipeSections, RECIPE_FIELDS } from './constants';
|
||||
import FormField from './FormField';
|
||||
@ -11,6 +13,7 @@ import TestConnectionButton from './TestConnection/TestConnectionButton';
|
||||
import { useListSecretsQuery } from '../../../../../graphql/ingestion.generated';
|
||||
import { RecipeField, setFieldValueOnRecipe } from './common';
|
||||
import { SourceBuilderState, SourceConfig } from '../types';
|
||||
import { RequiredFieldForm } from '../../../../shared/form/RequiredFieldForm';
|
||||
|
||||
export const ControlsContainer = styled.div`
|
||||
display: flex;
|
||||
@ -140,7 +143,7 @@ function RecipeForm(props: Props) {
|
||||
}
|
||||
|
||||
return (
|
||||
<Form
|
||||
<RequiredFieldForm
|
||||
layout="vertical"
|
||||
initialValues={getInitialValues(displayRecipe, allFields)}
|
||||
onFinish={onClickNext}
|
||||
@ -208,7 +211,7 @@ function RecipeForm(props: Props) {
|
||||
header={
|
||||
<SectionHeader
|
||||
icon={<SettingOutlined />}
|
||||
text="Advanced"
|
||||
text="Settings"
|
||||
sectionTooltip={advancedSectionTooltip}
|
||||
/>
|
||||
}
|
||||
@ -230,9 +233,11 @@ function RecipeForm(props: Props) {
|
||||
<Button disabled={isEditing} onClick={goToPrevious}>
|
||||
Previous
|
||||
</Button>
|
||||
<Button htmlType="submit">Next</Button>
|
||||
<Button type="primary" htmlType="submit">
|
||||
Next
|
||||
</Button>
|
||||
</ControlsContainer>
|
||||
</Form>
|
||||
</RequiredFieldForm>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -276,7 +276,7 @@ export const INCLUDE_LINEAGE: RecipeField = {
|
||||
export const INCLUDE_TABLE_LINEAGE: RecipeField = {
|
||||
name: 'include_table_lineage',
|
||||
label: 'Include Table Lineage',
|
||||
tooltip: 'Extract Tabel-Level lineage metadata. Enabling this may increase the duration of the extraction process.',
|
||||
tooltip: 'Extract Tabel-Level lineage metadata. Enabling this may increase the duration of the sync.',
|
||||
type: FieldType.BOOLEAN,
|
||||
fieldPath: 'source.config.include_table_lineage',
|
||||
rules: null,
|
||||
@ -286,8 +286,7 @@ const isProfilingEnabledFieldPath = 'source.config.profiling.enabled';
|
||||
export const TABLE_PROFILING_ENABLED: RecipeField = {
|
||||
name: 'profiling.enabled',
|
||||
label: 'Enable Table Profiling',
|
||||
tooltip:
|
||||
'Generate Data Profiles for extracted Tables. Enabling this may increase the duration of the extraction process.',
|
||||
tooltip: 'Generate Data Profiles for extracted Tables. Enabling this may increase the duration of the sync.',
|
||||
type: FieldType.BOOLEAN,
|
||||
fieldPath: isProfilingEnabledFieldPath,
|
||||
rules: null,
|
||||
@ -298,7 +297,7 @@ export const COLUMN_PROFILING_ENABLED: RecipeField = {
|
||||
name: 'column_profiling.enabled',
|
||||
label: 'Enable Column Profiling',
|
||||
tooltip:
|
||||
'Generate Data Profiles for the Columns in extracted Tables. Enabling this may increase the duration of the extraction process.',
|
||||
'Generate Data Profiles for the Columns in extracted Tables. Enabling this may increase the duration of the sync.',
|
||||
type: FieldType.BOOLEAN,
|
||||
fieldPath: isTableProfilingOnlyFieldPath,
|
||||
rules: null,
|
||||
@ -466,7 +465,7 @@ export const START_TIME: RecipeField = {
|
||||
name: 'start_time',
|
||||
label: 'Start Time',
|
||||
tooltip:
|
||||
'Earliest date used when processing audit logs for lineage, usage, and more. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) to bootstrap your first ingestion run, and then reduce for subsequent runs. Changing this may increase the duration of the extraction process.',
|
||||
'Earliest date used when processing audit logs for lineage, usage, and more. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) to bootstrap your first ingestion run, and then reduce for subsequent runs. Changing this may increase the duration of the sync.',
|
||||
placeholder: 'Select date and time',
|
||||
type: FieldType.DATE,
|
||||
fieldPath: startTimeFieldPath,
|
||||
|
||||
@ -1,39 +1,60 @@
|
||||
import React, { useState } from 'react';
|
||||
|
||||
import { Button, Input } from 'antd';
|
||||
import { FormOutlined, SearchOutlined } from '@ant-design/icons';
|
||||
import React, { useState } from 'react';
|
||||
import styled from 'styled-components';
|
||||
import { LogoCountCard } from '../../../shared/LogoCountCard';
|
||||
import { SourceConfig, SourceBuilderState, StepProps } from './types';
|
||||
import { IngestionSourceBuilderStep } from './steps';
|
||||
import useGetSourceLogoUrl from './useGetSourceLogoUrl';
|
||||
import { CUSTOM } from './constants';
|
||||
import { ANTD_GRAY } from '../../../entity/shared/constants';
|
||||
import { DataPlatformCard } from './DataPlatformCard';
|
||||
|
||||
const Container = styled.div`
|
||||
max-height: 82vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
`;
|
||||
|
||||
const Section = styled.div`
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding-bottom: 12px;
|
||||
`;
|
||||
|
||||
const PlatformListContainer = styled.div`
|
||||
display: flex;
|
||||
justify-content: left;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
overflow: hidden;
|
||||
`;
|
||||
|
||||
const CancelButton = styled(Button)`
|
||||
&& {
|
||||
margin-left: 12px;
|
||||
}
|
||||
max-width: 120px;
|
||||
`;
|
||||
|
||||
const SearchBarContainer = styled.div`
|
||||
display: flex;
|
||||
justify-content: end;
|
||||
width: auto;
|
||||
padding-right: 12px;
|
||||
`;
|
||||
|
||||
const StyledSearchBar = styled(Input)`
|
||||
background-color: white;
|
||||
border-radius: 70px;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0px 0px 30px 0px rgb(239 239 239);
|
||||
width: 45%;
|
||||
margin: 0 0 15px 12px;
|
||||
border: 1px solid #e0e0e0;
|
||||
margin: 0 0 15px 0px;
|
||||
max-width: 300px;
|
||||
font-size: 16px;
|
||||
`;
|
||||
|
||||
const StyledSearchOutlined = styled(SearchOutlined)`
|
||||
color: #a9adbd;
|
||||
`;
|
||||
|
||||
const PlatformListContainer = styled.div`
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(min(100%, 31%), 1fr));
|
||||
gap: 10px;
|
||||
height: 100%;
|
||||
overflow-y: auto;
|
||||
padding-right: 12px;
|
||||
`;
|
||||
|
||||
interface SourceOptionProps {
|
||||
@ -42,7 +63,7 @@ interface SourceOptionProps {
|
||||
}
|
||||
|
||||
function SourceOption({ source, onClick }: SourceOptionProps) {
|
||||
const { name, displayName } = source;
|
||||
const { name, displayName, description } = source;
|
||||
|
||||
const logoUrl = useGetSourceLogoUrl(name);
|
||||
let logoComponent;
|
||||
@ -50,7 +71,15 @@ function SourceOption({ source, onClick }: SourceOptionProps) {
|
||||
logoComponent = <FormOutlined style={{ color: ANTD_GRAY[8], fontSize: 28 }} />;
|
||||
}
|
||||
|
||||
return <LogoCountCard onClick={onClick} name={displayName} logoUrl={logoUrl} logoComponent={logoComponent} />;
|
||||
return (
|
||||
<DataPlatformCard
|
||||
onClick={onClick}
|
||||
name={displayName}
|
||||
logoUrl={logoUrl}
|
||||
description={description}
|
||||
logoComponent={logoComponent}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -76,22 +105,24 @@ export const SelectTemplateStep = ({ state, updateState, goTo, cancel, ingestion
|
||||
);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Container>
|
||||
<Section>
|
||||
<StyledSearchBar
|
||||
placeholder="Search ingestion sources..."
|
||||
value={searchFilter}
|
||||
onChange={(e) => setSearchFilter(e.target.value)}
|
||||
allowClear
|
||||
prefix={<SearchOutlined />}
|
||||
/>
|
||||
<PlatformListContainer>
|
||||
<SearchBarContainer>
|
||||
<StyledSearchBar
|
||||
placeholder="Search data sources..."
|
||||
value={searchFilter}
|
||||
onChange={(e) => setSearchFilter(e.target.value)}
|
||||
allowClear
|
||||
prefix={<StyledSearchOutlined />}
|
||||
/>
|
||||
</SearchBarContainer>
|
||||
<PlatformListContainer data-testid="data-source-options">
|
||||
{filteredSources.map((source) => (
|
||||
<SourceOption key={source.urn} source={source} onClick={() => onSelectTemplate(source.name)} />
|
||||
))}
|
||||
</PlatformListContainer>
|
||||
</Section>
|
||||
<CancelButton onClick={cancel}>Cancel</CancelButton>
|
||||
</>
|
||||
</Container>
|
||||
);
|
||||
};
|
||||
|
||||
@ -1,21 +1,28 @@
|
||||
import { Select } from 'antd';
|
||||
import React from 'react';
|
||||
import moment from 'moment-timezone';
|
||||
import styled from 'styled-components';
|
||||
|
||||
const StyledSelect = styled(Select)`
|
||||
max-width: 300px;
|
||||
`;
|
||||
|
||||
type Props = {
|
||||
value: string;
|
||||
onChange: (newTimezone: string) => void;
|
||||
onChange: (newTimezone: any) => void;
|
||||
};
|
||||
|
||||
export const TimezoneSelect = ({ value, onChange }: Props) => {
|
||||
const timezones = moment.tz.names();
|
||||
return (
|
||||
<>
|
||||
<Select showSearch value={value} onChange={onChange}>
|
||||
<StyledSelect showSearch value={value} onChange={onChange}>
|
||||
{timezones.map((timezone) => (
|
||||
<Select.Option value={timezone}>{timezone}</Select.Option>
|
||||
<Select.Option key={timezone} value={timezone}>
|
||||
{timezone}
|
||||
</Select.Option>
|
||||
))}
|
||||
</Select>
|
||||
</StyledSelect>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
@ -3,55 +3,63 @@
|
||||
"urn": "urn:li:dataPlatform:bigquery",
|
||||
"name": "bigquery",
|
||||
"displayName": "BigQuery",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/bigquery/",
|
||||
"description": "Import Projects, Datasets, Tables, Views, lineage, queries, and statistics from BigQuery.",
|
||||
"docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/bigquery/overview",
|
||||
"recipe": "source:\n type: bigquery\n config:\n include_table_lineage: true\n include_usage_statistics: true\n include_tables: true\n include_views: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:redshift",
|
||||
"name": "redshift",
|
||||
"displayName": "Redshift",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/redshift/",
|
||||
"description": "Import Tables, Views, Databases, Schemas, lineage, queries, and statistics from Redshift.",
|
||||
"docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/redshift/overview",
|
||||
"recipe": "source: \n type: redshift\n config:\n # Coordinates\n host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n database: # Your Redshift database, e.g. SampleDatabase\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your Redshift username, e.g. admin\n\n table_lineage_mode: stl_scan_based\n include_table_lineage: true\n include_tables: true\n include_views: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:snowflake",
|
||||
"name": "snowflake",
|
||||
"displayName": "Snowflake",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/snowflake/",
|
||||
"description": "Import Tables, Views, Databases, Schemas, lineage, queries, and statistics from Snowflake.",
|
||||
"docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/snowflake/overview",
|
||||
"recipe": "source: \n type: snowflake\n config:\n account_id: null\n include_table_lineage: true\n include_view_lineage: true\n include_tables: true\n include_views: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:kafka",
|
||||
"name": "kafka",
|
||||
"displayName": "Kafka",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/kafka/",
|
||||
"recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"PLAINTEXT\"\n stateful_ingestion:\n enabled: false"
|
||||
"urn": "urn:li:dataPlatform:unity-catalog",
|
||||
"name": "unity-catalog",
|
||||
"displayName": "Databricks",
|
||||
"description": "Import Metastores, Schemas, Tables, lineage, queries, and statistics from Databricks Unity Catalog.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/databricks/#module-unity-catalog",
|
||||
"recipe": "source:\n type: unity-catalog\n config:\n # Coordinates\n workspace_url: null\n include_table_lineage: true\n include_column_lineage: false\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:looker",
|
||||
"name": "looker",
|
||||
"displayName": "Looker",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/looker/",
|
||||
"recipe": "source:\n type: looker\n config:\n # Coordinates\n base_url: # Your Looker instance URL, e.g. https://company.looker.com:19999\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n client_id: null # Your Looker client id, e.g. admin\n stateful_ingestion:\n enabled: true"
|
||||
"description": "Import Models, Explores, Views, Looks, Dashboards, and lineage from Looker.",
|
||||
"docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/looker/overview#looker",
|
||||
"recipe": "source:\n type: looker\n config:\n # Coosrdinates\n base_url: # Your Looker instance URL, e.g. https://company.looker.com:19999\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n client_id: null # Your Looker client id, e.g. admin\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:lookml",
|
||||
"name": "lookml",
|
||||
"displayName": "LookML",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/looker/#module-lookml",
|
||||
"description": "Import Models, Explores, Views, Looks, Dashboards, and lineage from LookML files.",
|
||||
"docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/looker/overview#lookml",
|
||||
"recipe": "source:\n type: lookml\n config:\n parse_table_names_from_sql: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:tableau",
|
||||
"name": "tableau",
|
||||
"displayName": "Tableau",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/tableau/",
|
||||
"description": "Import Data Sources, Workbooks, Worksheets, Tags, Dashboards, and lineage from Tableau.",
|
||||
"docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/tableau/overview",
|
||||
"recipe": "source:\n type: tableau\n config:\n # Coordinates\n connect_uri: null\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:powerbi",
|
||||
"name": "powerbi",
|
||||
"displayName": "PowerBI",
|
||||
"description": "Import Dashboards, Tiles, Datasets, and lineage from PowerBI.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/powerbi/",
|
||||
"recipe": "source:\n type: \"powerbi\"\n config:\n # Your Power BI tenant identifier\n tenant_id: null\n # Your Power BI client id\n client_id: null\n # Your Power BI client secret\n client_secret: null\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
@ -59,6 +67,7 @@
|
||||
"urn": "urn:li:dataPlatform:dbt",
|
||||
"name": "dbt-cloud",
|
||||
"displayName": "dbt Cloud",
|
||||
"description": "Import Sources, Seeds, Models, Snapshots, Tests, and lineage from dbt cloud.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/dbt/#module-dbt-cloud",
|
||||
"recipe": "source:\n type: dbt-cloud\n config:\n account_id: null\n project_id: null\n job_id: null\n target_platform: null\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
@ -66,6 +75,7 @@
|
||||
"urn": "urn:li:dataPlatform:mysql",
|
||||
"name": "mysql",
|
||||
"displayName": "MySQL",
|
||||
"description": "Import Tables, Views, Databases, Schemas, and statistics from MySQL.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mysql/",
|
||||
"recipe": "source: \n type: mysql\n config: \n # Coordinates\n host_port: # Your MySQL host and post, e.g. mysql:3306\n database: # Your MySQL database name, e.g. datahub\n \n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your MySQL username, e.g. admin\n\n # Options\n include_tables: true\n include_views: true\n\n # Profiling\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
@ -73,13 +83,23 @@
|
||||
"urn": "urn:li:dataPlatform:postgres",
|
||||
"name": "postgres",
|
||||
"displayName": "Postgres",
|
||||
"description": "Import Tables, Views, Databases, Schemas, and statistics from Postgres.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/postgres/",
|
||||
"recipe": "source: \n type: postgres\n config:\n # Coordinates\n host_port: # Your Postgres host and port, e.g. postgres:5432\n database: # Your Postgres Database, e.g. sample_db\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your Postgres username, e.g. admin\n\n # Options\n include_tables: true\n include_views: true\n\n # Profiling\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:kafka",
|
||||
"name": "kafka",
|
||||
"displayName": "Kafka",
|
||||
"description": "Import streaming topics from Kafka.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/kafka/",
|
||||
"recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"PLAINTEXT\"\n stateful_ingestion:\n enabled: false"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:hive",
|
||||
"name": "hive",
|
||||
"displayName": "Hive",
|
||||
"description": "Import Tables, Views, Databases, Schemas, and statistics from Hive.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/hive/",
|
||||
"recipe": "source: \n type: hive\n config:\n # Coordinates\n host_port: # Your Hive host and port, e.g. hive:10000\n database: # Your Hive database name, e.g. SampleDatabase (Optional, if not specified, ingests from all databases)\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your Hive username, e.g. admin\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
@ -87,6 +107,7 @@
|
||||
"urn": "urn:li:dataPlatform:presto",
|
||||
"name": "presto",
|
||||
"displayName": "Presto",
|
||||
"description": "Import Tables, Databases, Schemas, and statistics from Presto.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/presto/",
|
||||
"recipe": "source:\n type: presto\n config:\n # Coordinates\n host_port: null\n # The name of the catalog from getting the usage\n database: null\n # Credentials\n username: null\n include_views: true\n include_tables: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
@ -94,13 +115,23 @@
|
||||
"urn": "urn:li:dataPlatform:trino",
|
||||
"name": "trino",
|
||||
"displayName": "Trino",
|
||||
"description": "Import Tables, Databases, Schemas, and statistics from Trino.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/trino/",
|
||||
"recipe": "source:\n type: trino\n config:\n # Coordinates\n host_port: null\n # The name of the catalog from getting the usage\n database: null\n # Credentials\n username: null\n include_views: true\n include_tables: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:glue",
|
||||
"name": "glue",
|
||||
"displayName": "Glue",
|
||||
"description": "Import Tables, Databases, Jobs, statistics, and lineage to S3 from AWS Glue.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/glue/",
|
||||
"recipe": "source:\n type: glue\n config:\n # AWS credentials. \n aws_region: # The region for your AWS Glue instance. \n # Add secret in Secrets Tab with relevant names for each variable\n # The access key for your AWS account.\n aws_access_key_id: \"${AWS_ACCESS_KEY_ID}\"\n # The secret key for your AWS account.\n aws_secret_access_key: \"${AWS_SECRET_KEY}\"\n aws_session_token: # The session key for your AWS account. This is only needed when you are using temporary credentials.\n # aws_role: # (Optional) The role to assume (Role chaining supported by using a sorted list).\n\n # Allow / Deny specific databases & tables\n # database_pattern:\n # allow:\n # - \"flights-database\"\n # table_pattern:\n # allow:\n # - \"avro\""
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:mssql",
|
||||
"name": "mssql",
|
||||
"displayName": "Microsoft SQL Server",
|
||||
"description": "Import Tables, Views, Databases, Schemas, and statistics from SQL Server.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mssql/",
|
||||
"recipe": "source:\n type: mssql\n config:\n # Coordinates\n host_port: null\n # The name\n database: null\n # Credentials\n username: null\n include_views: true\n include_tables: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
@ -108,20 +139,15 @@
|
||||
"urn": "urn:li:dataPlatform:mariadb",
|
||||
"name": "mariadb",
|
||||
"displayName": "MariaDB",
|
||||
"description": "Import Tables, Views, Databases, Schemas, and statistics from MariaDB.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mariadb/",
|
||||
"recipe": "source:\n type: mariadb\n config:\n # Coordinates\n host_port: null\n # The name\n database: null\n # Credentials\n username: null\n include_views: true\n include_tables: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:unity-catalog",
|
||||
"name": "unity-catalog",
|
||||
"displayName": "Databricks Unity Catalog",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/databricks/#module-unity-catalog",
|
||||
"recipe": "source:\n type: unity-catalog\n config:\n # Coordinates\n workspace_url: null\n include_table_lineage: true\n include_column_lineage: false\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:mongodb",
|
||||
"name": "mongodb",
|
||||
"displayName": "MongoDB",
|
||||
"description": "Import Databases and Collections from MongoDB.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mongodb/",
|
||||
"recipe": "source:\n type: mongodb\n config:\n # Coordinates\n connect_uri: # Your MongoDB connect URI, e.g. \"mongodb://localhost\"\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${MONGO_USERNAME}\" # Your MongoDB username, e.g. admin\n password: \"${MONGO_PASSWORD}\" # Your MongoDB password, e.g. password_01\n\n # Options (recommended)\n enableSchemaInference: True\n useRandomSampling: True\n maxSchemaSize: 300"
|
||||
},
|
||||
@ -129,20 +155,15 @@
|
||||
"urn": "urn:li:dataPlatform:dynamodb",
|
||||
"name": "dynamodb",
|
||||
"displayName": "DynamoDB",
|
||||
"description": "Import Tables from DynamoDB.",
|
||||
"docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
|
||||
"recipe": "source:\n type: dynamodb\n config:\n platform_instance: \"AWS_ACCOUNT_ID\"\n aws_access_key_id : '${AWS_ACCESS_KEY_ID}'\n aws_secret_access_key : '${AWS_SECRET_ACCESS_KEY}'\n # If there are items that have most representative fields of the table, users could use the\n # `include_table_item` option to provide a list of primary keys of the table in dynamodb format.\n # For each `region.table`, the list of primary keys can be at most 100.\n # We include these items in addition to the first 100 items in the table when we scan it.\n # include_table_item:\n # region.table_name:\n # [\n # {\n # 'partition_key_name': { 'attribute_type': 'attribute_value' },\n # 'sort_key_name': { 'attribute_type': 'attribute_value' },\n # },\n # ]"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:glue",
|
||||
"name": "glue",
|
||||
"displayName": "Glue",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/glue/",
|
||||
"recipe": "source:\n type: glue\n config:\n # AWS credentials. \n aws_region: # The region for your AWS Glue instance. \n # Add secret in Secrets Tab with relevant names for each variable\n # The access key for your AWS account.\n aws_access_key_id: \"${AWS_ACCESS_KEY_ID}\"\n # The secret key for your AWS account.\n aws_secret_access_key: \"${AWS_SECRET_KEY}\"\n aws_session_token: # The session key for your AWS account. This is only needed when you are using temporary credentials.\n # aws_role: # (Optional) The role to assume (Role chaining supported by using a sorted list).\n\n # Allow / Deny specific databases & tables\n # database_pattern:\n # allow:\n # - \"flights-database\"\n # table_pattern:\n # allow:\n # - \"avro\""
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:oracle",
|
||||
"name": "oracle",
|
||||
"displayName": "Oracle",
|
||||
"description": "Import Databases, Schemas, Tables, Views, statistics, and lineage from Oracle.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/oracle/",
|
||||
"recipe": "source: \n type: oracle\n config:\n # Coordinates\n host_port: # Your Oracle host and port, e.g. oracle:5432\n database: # Your Oracle database name, e.g. sample_db\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${ORACLE_USERNAME}\" # Your Oracle username, e.g. admin\n password: \"${ORACLE_PASSWORD}\" # Your Oracle password, e.g. password_01\n\n # Optional service name\n # service_name: # Your service name, e.g. svc # omit database if using this option"
|
||||
},
|
||||
@ -150,6 +171,7 @@
|
||||
"urn": "urn:li:dataPlatform:superset",
|
||||
"name": "superset",
|
||||
"displayName": "Superset",
|
||||
"description": "Import Charts and Dashboards from Superset",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/superset/",
|
||||
"recipe": "source:\n type: superset\n config:\n # Coordinates\n connect_uri: http://localhost:8088\n\n # Credentials\n username: user\n password: pass\n provider: ldap"
|
||||
},
|
||||
@ -157,6 +179,7 @@
|
||||
"urn": "urn:li:dataPlatform:athena",
|
||||
"name": "athena",
|
||||
"displayName": "Athena",
|
||||
"description": "Import Schemas, Tables, Views, and lineage to S3 from Athena.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/athena/",
|
||||
"recipe": "source:\n type: athena\n config:\n # Coordinates\n aws_region: my_aws_region\n work_group: primary\n\n # Options\n s3_staging_dir: \"s3://my_staging_athena_results_bucket/results/\""
|
||||
},
|
||||
@ -164,6 +187,7 @@
|
||||
"urn": "urn:li:dataPlatform:clickhouse",
|
||||
"name": "clickhouse",
|
||||
"displayName": "ClickHouse",
|
||||
"description": "Import Tables, Views, Materialized Views, Dictionaries, statistics, queries, and lineage from ClickHouse.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/clickhouse/",
|
||||
"recipe": "source:\n type: clickhouse\n config:\n # Coordinates\n host_port: localhost:9000\n\n # Credentials\n username: user\n password: pass\n\n # Options\n platform_instance: DatabaseNameToBeIngested\n\n include_views: true # whether to include views, defaults to True\n include_tables: true # whether to include views, defaults to True\n\nsink:\n # sink configs\n\n#---------------------------------------------------------------------------\n# For the HTTP interface:\n#---------------------------------------------------------------------------\nsource:\n type: clickhouse\n config:\n host_port: localhost:8443\n protocol: https\n\n#---------------------------------------------------------------------------\n# For the Native interface:\n#---------------------------------------------------------------------------\n\nsource:\n type: clickhouse\n config:\n host_port: localhost:9440\n scheme: clickhouse+native\n secure: True"
|
||||
},
|
||||
@ -171,13 +195,23 @@
|
||||
"urn": "urn:li:dataPlatform:druid",
|
||||
"name": "druid",
|
||||
"displayName": "Druid",
|
||||
"description": "Import Databases, Schemas, Tables, statistics, and lineage from Druid.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/druid/",
|
||||
"recipe": "source:\n type: druid\n config:\n # Coordinates\n host_port: \"localhost:8082\"\n\n # Credentials\n username: admin\n password: password"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:mode",
|
||||
"name": "mode",
|
||||
"displayName": "Mode",
|
||||
"description": "Import Reports, Charts, and lineage from Mode.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mode/",
|
||||
"recipe": "source:\n type: mode\n config:\n # Coordinates\n connect_uri: http://app.mode.com\n\n # Credentials\n token: token\n password: pass\n\n # Options\n workspace: \"datahub\"\n default_schema: \"public\"\n owner_username_instead_of_email: False\n api_options:\n retry_backoff_multiplier: 2\n max_retry_interval: 10\n max_attempts: 5"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:metabase",
|
||||
"name": "metabase",
|
||||
"displayName": "Metabase",
|
||||
"description": "Import Collections, Dashboards, and Charts from Metabase.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/metabase/",
|
||||
"recipe": "source:\n type: metabase\n config:\n # Coordinates\n connect_uri:\n\n # Credentials\n username: root\n password: example"
|
||||
},
|
||||
@ -185,20 +219,15 @@
|
||||
"urn": "urn:li:dataPlatform:mlflow",
|
||||
"name": "mlflow",
|
||||
"displayName": "MLflow",
|
||||
"description": "Import Registered Models, Model Versions, and Model Stages from MLflow.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mlflow/",
|
||||
"recipe": "source:\n type: mlflow\n config:\n tracking_uri: tracking_uri"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:mode",
|
||||
"name": "mode",
|
||||
"displayName": "Mode",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mode/",
|
||||
"recipe": "source:\n type: mode\n config:\n # Coordinates\n connect_uri: http://app.mode.com\n\n # Credentials\n token: token\n password: pass\n\n # Options\n workspace: \"datahub\"\n default_schema: \"public\"\n owner_username_instead_of_email: False\n api_options:\n retry_backoff_multiplier: 2\n max_retry_interval: 10\n max_attempts: 5"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:azure-ad",
|
||||
"name": "azure-ad",
|
||||
"displayName": "Azure AD",
|
||||
"description": "Import Users and Groups from Azure Active Directory.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/azure-ad/",
|
||||
"recipe": "source:\n type: azure-ad\n config:\n client_id: # Your Azure Client ID, e.g. \"00000000-0000-0000-0000-000000000000\"\n tenant_id: # Your Azure Tenant ID, e.g. \"00000000-0000-0000-0000-000000000000\"\n # Add secret in Secrets Tab with this name\n client_secret: \n redirect: # Your Redirect URL, e.g. \"https://login.microsoftonline.com/common/oauth2/nativeclient\"\n authority: # Your Authority URL, e.g. \"https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000\"\n token_url: # Your Token URL, e.g. \"https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/token\"\n graph_url: # The Graph URL, e.g. \"https://graph.microsoft.com/v1.0\"\n \n # Optional flags to ingest users, groups, or both\n ingest_users: True\n ingest_groups: True\n \n # Optional Allow / Deny extraction of particular Groups\n # groups_pattern:\n # allow:\n # - \".*\"\n\n # Optional Allow / Deny extraction of particular Users.\n # users_pattern:\n # allow:\n # - \".*\""
|
||||
},
|
||||
@ -206,6 +235,7 @@
|
||||
"urn": "urn:li:dataPlatform:okta",
|
||||
"name": "okta",
|
||||
"displayName": "Okta",
|
||||
"description": "Import Users and Groups from Okta.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/okta/",
|
||||
"recipe": "source:\n type: okta\n config:\n # Coordinates\n okta_domain: # Your Okta Domain, e.g. \"dev-35531955.okta.com\"\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n okta_api_token: # Your Okta API Token, e.g. \"11be4R_M2MzDqXawbTHfKGpKee0kuEOfX1RCQSRx99\"\n\n # Optional flags to ingest users, groups, or both\n ingest_users: True\n ingest_groups: True\n\n # Optional: Customize the mapping to DataHub Username from an attribute appearing in the Okta User\n # profile. Reference: https://developer.okta.com/docs/reference/api/users/\n # okta_profile_to_username_attr: str = \"login\"\n # okta_profile_to_username_regex: str = \"([^@]+)\"\n \n # Optional: Customize the mapping to DataHub Group from an attribute appearing in the Okta Group\n # profile. Reference: https://developer.okta.com/docs/reference/api/groups/\n # okta_profile_to_group_name_attr: str = \"name\"\n # okta_profile_to_group_name_regex: str = \"(.*)\"\n \n # Optional: Include deprovisioned or suspended Okta users in the ingestion.\n # include_deprovisioned_users = False\n # include_suspended_users = False"
|
||||
},
|
||||
@ -213,6 +243,7 @@
|
||||
"urn": "urn:li:dataPlatform:vertica",
|
||||
"name": "vertica",
|
||||
"displayName": "Vertica",
|
||||
"description": "Import Databases, Schemas, Tables, Views, Projections, statistics, and lineage from Vertica.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/vertica/",
|
||||
"recipe": "source:\n type: vertica\n config:\n # Coordinates\n host_port: localhost:5433\n # The name of the vertica database\n database: Database_Name\n # Credentials\n username: Vertica_User\n password: Vertica_Password\n\n include_tables: true\n include_views: true\n include_projections: true\n include_models: true\n include_view_lineage: true\n include_projection_lineage: true\n profiling:\n enabled: false\n stateful_ingestion:\n enabled: true "
|
||||
},
|
||||
@ -220,13 +251,39 @@
|
||||
"urn": "urn:li:dataPlatform:fivetran",
|
||||
"name": "fivetran",
|
||||
"displayName": "Fivetran",
|
||||
"description": "Import Connectors, Destinations, Sync Histor, Users, and lineage from FiveTran.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/fivetran/",
|
||||
"recipe": "source:\n type: fivetran\n config:\n # Fivetran log connector destination server configurations\n fivetran_log_config:\n destination_platform: snowflake\n snowflake_destination_config:\n # Coordinates\n account_id: snowflake_account_id\n warehouse: warehouse_name\n database: snowflake_db\n log_schema: fivetran_log_schema\n\n # Credentials\n username: ${SNOWFLAKE_USER}\n password: ${SNOWFLAKE_PASS}\n role: snowflake_role\n\n # Optional - filter for certain connector names instead of ingesting everything.\n # connector_patterns:\n # allow:\n # - connector_name\n\n # Optional -- This mapping is optional and only required to configure platform-instance for source\n # A mapping of Fivetran connector id to data platform instance\n # sources_to_platform_instance:\n # calendar_elected:\n # platform_instance: cloud_postgres_instance\n # env: DEV\n\n # Optional -- This mapping is optional and only required to configure platform-instance for destination.\n # A mapping of Fivetran destination id to data platform instance\n # destination_to_platform_instance:\n # calendar_elected:\n # platform_instance: cloud_postgres_instance\n # env: DEV"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:sigma",
|
||||
"name": "sigma",
|
||||
"displayName": "Sigma",
|
||||
"description": "Import Workspaces, Workbooks, Pages, Elements, and lineage from Sigma Computing.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/sigma/",
|
||||
"recipe": "source:\n type: sigma\n config:\n # Coordinates\n api_url: https://aws-api.sigmacomputing.com/v2\n # Coordinates\n client_id: CLIENT_ID\n client_secret: CLIENT_SECRET\n\n # Optional - filter for certain workspace names instead of ingesting everything.\n # workspace_pattern:\n\n # allow:\n # - workspace_name\n ingest_owner: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:qlik-sense",
|
||||
"name": "qlik-sense",
|
||||
"displayName": "Qlik Sense",
|
||||
"description": "Import Spaces, Apps, Sheets, Charts, and Datasets from Qlik Sense.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-sense/",
|
||||
"recipe": "source:\n type: qlik-sense\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n ingest_owner: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:cockroachdb",
|
||||
"name": "cockroachdb",
|
||||
"displayName": "CockroachDb",
|
||||
"description": "Import Databases, Schemas, Tables, Views, statistics and lineage from CockroachDB.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/cockroachdb/",
|
||||
"recipe": "source: \n type: cockroachdb\n config:\n # Coordinates\n host_port: # Your CockroachDb host and port, e.g. cockroachdb:5432\n database: # Your CockroachDb Database, e.g. sample_db\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your CockroachDb username, e.g. admin\n\n # Options\n include_tables: true\n include_views: true\n\n # Profiling\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:csv-enricher",
|
||||
"name": "csv-enricher",
|
||||
"displayName": "CSV",
|
||||
"description": "Import metadata from a formatted CSV.",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv'",
|
||||
"recipe": "source: \n type: csv-enricher \n config: \n # URL of your csv file to ingest \n filename: \n array_delimiter: '|' \n delimiter: ',' \n write_semantics: PATCH"
|
||||
},
|
||||
@ -234,28 +291,8 @@
|
||||
"urn": "urn:li:dataPlatform:custom",
|
||||
"name": "custom",
|
||||
"displayName": "Other",
|
||||
"description": "Configure a custom recipe using YAML.",
|
||||
"docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
|
||||
"recipe": "source:\n type: <source-type>\n config:\n # Source-type specifics config\n <source-configs>"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:qlik-sense",
|
||||
"name": "qlik-sense",
|
||||
"displayName": "Qlik Sense",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-sense/",
|
||||
"recipe": "source:\n type: qlik-sense\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n ingest_owner: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:sigma",
|
||||
"name": "sigma",
|
||||
"displayName": "Sigma",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/sigma/",
|
||||
"recipe": "source:\n type: sigma\n config:\n # Coordinates\n api_url: https://aws-api.sigmacomputing.com/v2\n # Coordinates\n client_id: CLIENT_ID\n client_secret: CLIENT_SECRET\n\n # Optional - filter for certain workspace names instead of ingesting everything.\n # workspace_pattern:\n\n # allow:\n # - workspace_name\n ingest_owner: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:cockroachdb",
|
||||
"name": "cockroachdb",
|
||||
"displayName": "CockroachDb",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/cockroachdb/",
|
||||
"recipe": "source: \n type: cockroachdb\n config:\n # Coordinates\n host_port: # Your CockroachDb host and port, e.g. cockroachdb:5432\n database: # Your CockroachDb Database, e.g. sample_db\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your CockroachDb username, e.g. admin\n\n # Options\n include_tables: true\n include_views: true\n\n # Profiling\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
|
||||
}
|
||||
]
|
||||
|
||||
@ -18,6 +18,7 @@ export interface SourceConfig {
|
||||
name: string;
|
||||
displayName: string;
|
||||
docsUrl: string;
|
||||
description?: string;
|
||||
recipe: string;
|
||||
}
|
||||
|
||||
|
||||
14
datahub-web-react/src/app/shared/form/RequiredFieldForm.tsx
Normal file
14
datahub-web-react/src/app/shared/form/RequiredFieldForm.tsx
Normal file
@ -0,0 +1,14 @@
|
||||
import { Form } from 'antd';
|
||||
import styled from 'styled-components';
|
||||
|
||||
const DEFAULT_ASTERICK_COLOR = '#F5222D';
|
||||
|
||||
export const RequiredFieldForm = styled(Form)<{ requiredColor?: string }>`
|
||||
&& {
|
||||
.ant-form-item-label > label.ant-form-item-required::before {
|
||||
color: ${(props) =>
|
||||
props.requiredColor || DEFAULT_ASTERICK_COLOR}; /* Change 'red' to any color you prefer */
|
||||
content: '*'; /* Ensure the asterisk is always used */
|
||||
}
|
||||
}
|
||||
`;
|
||||
@ -13,7 +13,7 @@ describe("ingestion source creation flow", () => {
|
||||
cy.goToIngestionPage();
|
||||
cy.clickOptionWithTestId("create-ingestion-source-button");
|
||||
cy.clickOptionWithText("Snowflake");
|
||||
cy.waitTextVisible("Snowflake Recipe");
|
||||
cy.waitTextVisible("Snowflake Details");
|
||||
cy.get("#account_id").type(accound_id);
|
||||
cy.get("#warehouse").type(warehouse_id);
|
||||
cy.get("#username").type(username);
|
||||
@ -34,7 +34,7 @@ describe("ingestion source creation flow", () => {
|
||||
cy.clickOptionWithTestId("recipe-builder-next-button");
|
||||
cy.waitTextVisible("Configure an Ingestion Schedule");
|
||||
cy.clickOptionWithTestId("ingestion-schedule-next-button");
|
||||
cy.waitTextVisible("Give this ingestion source a name.");
|
||||
cy.waitTextVisible("Give this data source a name");
|
||||
cy.get('[data-testid="source-name-input"]').type(ingestion_source_name);
|
||||
cy.clickOptionWithTestId("ingestion-source-save-button");
|
||||
cy.waitTextVisible("Successfully created ingestion source!").wait(5000);
|
||||
@ -47,7 +47,7 @@ describe("ingestion source creation flow", () => {
|
||||
cy.get('[data-testid="ingestion-source-table-edit-button"]')
|
||||
.first()
|
||||
.click();
|
||||
cy.waitTextVisible("Edit Ingestion Source");
|
||||
cy.waitTextVisible("Edit Data Source");
|
||||
cy.get("#account_id").should("have.value", accound_id);
|
||||
cy.get("#warehouse").should("have.value", warehouse_id);
|
||||
cy.get("#username").should("have.value", username);
|
||||
|
||||
@ -1,40 +1,44 @@
|
||||
function readyToTypeEditor() {
|
||||
return cy.get(".monaco-editor textarea:first").click().focused();
|
||||
}
|
||||
|
||||
describe("run managed ingestion", () => {
|
||||
it("create run managed ingestion source", () => {
|
||||
const number = Math.floor(Math.random() * 100000);
|
||||
const testName = `cypress test source ${number}`;
|
||||
const cli_version = "0.12.0";
|
||||
cy.login();
|
||||
cy.goToIngestionPage();
|
||||
cy.clickOptionWithText("Create new source");
|
||||
cy.clickOptionWithTextToScrollintoView("Other");
|
||||
|
||||
cy.waitTextVisible("source-type");
|
||||
readyToTypeEditor().type("{ctrl}a").clear();
|
||||
readyToTypeEditor().type("source:{enter}");
|
||||
readyToTypeEditor().type(" type: demo-data");
|
||||
readyToTypeEditor().type("{enter}");
|
||||
// no space because the editor starts new line at same indentation
|
||||
readyToTypeEditor().type("config: {}");
|
||||
cy.clickOptionWithText("Next");
|
||||
cy.clickOptionWithText("Next");
|
||||
|
||||
cy.enterTextInTestId("source-name-input", testName);
|
||||
cy.clickOptionWithText("Advanced");
|
||||
cy.enterTextInTestId("cli-version-input", cli_version);
|
||||
cy.clickOptionWithTextToScrollintoView("Save & Run");
|
||||
cy.waitTextVisible(testName);
|
||||
|
||||
cy.contains(testName)
|
||||
.parent()
|
||||
.within(() => {
|
||||
cy.contains("Succeeded", { timeout: 180000 });
|
||||
cy.clickOptionWithTestId("delete-button");
|
||||
});
|
||||
cy.clickOptionWithText("Yes");
|
||||
cy.ensureTextNotPresent(testName);
|
||||
});
|
||||
});
|
||||
// TODO: Investigate why this test can never pass on CI, but passes locally after PR #21465
|
||||
//
|
||||
// function readyToTypeEditor() {
|
||||
// return cy.get(".monaco-editor textarea:first").click().focused();
|
||||
// }
|
||||
//
|
||||
// describe("run managed ingestion", () => {
|
||||
// it("create run managed ingestion source", () => {
|
||||
// const number = Math.floor(Math.random() * 100000);
|
||||
// const testName = `cypress test source ${number}`;
|
||||
// const cli_version = "0.12.0";
|
||||
// cy.login();
|
||||
// cy.goToIngestionPage();
|
||||
// cy.clickOptionWithText("Create new source");
|
||||
// cy.clickOptionWithTextToScrollintoView("Other");
|
||||
//
|
||||
// cy.waitTextVisible("source-type");
|
||||
// cy.wait(10000); // waits for 5 seconds
|
||||
//
|
||||
// readyToTypeEditor().type("{ctrl}a").clear({ force: true });
|
||||
// readyToTypeEditor().type("source:{enter}", { force: true });
|
||||
// readyToTypeEditor().type(" type: demo-data", { force: true });
|
||||
// readyToTypeEditor().type("{enter}", { force: true });
|
||||
// // no space because the editor starts new line at same indentation
|
||||
// readyToTypeEditor().type("config: {}", { force: true });
|
||||
// cy.clickOptionWithText("Next");
|
||||
// cy.clickOptionWithText("Next");
|
||||
//
|
||||
// cy.enterTextInTestId("source-name-input", testName);
|
||||
// cy.clickOptionWithText("Advanced");
|
||||
// cy.enterTextInTestId("cli-version-input", cli_version);
|
||||
// cy.clickOptionWithTextToScrollintoView("Save & Run");
|
||||
// cy.waitTextVisible(testName);
|
||||
//
|
||||
// cy.contains(testName)
|
||||
// .parent()
|
||||
// .within(() => {
|
||||
// cy.contains("Succeeded", { timeout: 180000 });
|
||||
// cy.clickOptionWithTestId("delete-button");
|
||||
// });
|
||||
// cy.clickOptionWithText("Yes");
|
||||
// cy.ensureTextNotPresent(testName);
|
||||
// });
|
||||
// });
|
||||
|
||||
@ -30,7 +30,7 @@ describe("managing secrets for ingestion creation", () => {
|
||||
cy.goToIngestionPage();
|
||||
cy.get("#ingestion-create-source").click();
|
||||
cy.clickOptionWithText("Snowflake");
|
||||
cy.waitTextVisible("Snowflake Recipe");
|
||||
cy.waitTextVisible("Snowflake Details");
|
||||
cy.get("#account_id").type(accound_id);
|
||||
cy.get("#warehouse").type(warehouse_id);
|
||||
cy.get("#username").type(username);
|
||||
@ -41,7 +41,7 @@ describe("managing secrets for ingestion creation", () => {
|
||||
cy.get("button").contains("Next").click();
|
||||
cy.waitTextVisible("Configure an Ingestion Schedule");
|
||||
cy.get("button").contains("Next").click();
|
||||
cy.waitTextVisible("Give this ingestion source a name.");
|
||||
cy.waitTextVisible("Give this data source a name");
|
||||
cy.get('[data-testid="source-name-input"]').type(ingestion_source_name);
|
||||
cy.get("button").contains("Save").click();
|
||||
cy.waitTextVisible("Successfully created ingestion source!").wait(5000);
|
||||
@ -69,7 +69,7 @@ describe("managing secrets for ingestion creation", () => {
|
||||
// Verify secret is not present during ingestion source creation for password dropdown
|
||||
cy.clickOptionWithText("Create new source");
|
||||
cy.clickOptionWithText("Snowflake");
|
||||
cy.waitTextVisible("Snowflake Recipe");
|
||||
cy.waitTextVisible("Snowflake Details");
|
||||
cy.get("#account_id").type(accound_id);
|
||||
cy.get("#warehouse").type(warehouse_id);
|
||||
cy.get("#username").type(username);
|
||||
@ -90,7 +90,7 @@ describe("managing secrets for ingestion creation", () => {
|
||||
cy.get("button").contains("Next").click();
|
||||
cy.waitTextVisible("Configure an Ingestion Schedule");
|
||||
cy.get("button").contains("Next").click();
|
||||
cy.waitTextVisible("Give this ingestion source a name.");
|
||||
cy.waitTextVisible("Give this data source a name");
|
||||
cy.get('[data-testid="source-name-input"]').type(ingestion_source_name);
|
||||
cy.get("button").contains("Save").click();
|
||||
cy.waitTextVisible("Successfully created ingestion source!").wait(5000); // prevent issue with missing form data
|
||||
|
||||
@ -107,7 +107,7 @@ Cypress.Commands.add("goToAccessTokenSettings", () => {
|
||||
|
||||
Cypress.Commands.add("goToIngestionPage", () => {
|
||||
cy.visit("/ingestion");
|
||||
cy.waitTextVisible("Manage Ingestion");
|
||||
cy.waitTextVisible("Manage Data Sources");
|
||||
});
|
||||
|
||||
Cypress.Commands.add("goToDataset", (urn, dataset_name) => {
|
||||
@ -198,6 +198,21 @@ Cypress.Commands.add("clickOptionWithTextToScrollintoView", (text) => {
|
||||
cy.contains(text).scrollIntoView().click();
|
||||
});
|
||||
|
||||
Cypress.Commands.add("clickOptionInScrollView", (text, selector) => {
|
||||
cy.get(selector).within(() => {
|
||||
cy.contains(text).then((el) => {
|
||||
// Scroll the element into view with options for better alignment
|
||||
el[0].scrollIntoView({ block: "center", inline: "nearest" });
|
||||
|
||||
// Wrap the element for further chaining with Cypress commands
|
||||
cy.wrap(el)
|
||||
.should("be.visible") // Wait until the element is visible
|
||||
.should("not.be.disabled") // Ensure the element is not disabled
|
||||
.click({ force: true }); // Force click if necessary
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Cypress.Commands.add("deleteFromDropdown", () => {
|
||||
cy.openThreeDotDropdown();
|
||||
cy.clickOptionWithText("Delete");
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user