mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-12 18:47:45 +00:00
feat(ingestion-ui) Display ingestion sources in UI more dynamically (#5789)
This commit is contained in:
Chris Collins
GitHub
parent
dfeced8eee
commit
0be5c39802
@ -49,6 +49,7 @@ import com.linkedin.datahub.graphql.generated.GetRootGlossaryTermsResult;
|
||||
import com.linkedin.datahub.graphql.generated.GlossaryNode;
|
||||
import com.linkedin.datahub.graphql.generated.GlossaryTerm;
|
||||
import com.linkedin.datahub.graphql.generated.GlossaryTermAssociation;
|
||||
import com.linkedin.datahub.graphql.generated.IngestionSource;
|
||||
import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata;
|
||||
import com.linkedin.datahub.graphql.generated.LineageRelationship;
|
||||
import com.linkedin.datahub.graphql.generated.ListAccessTokenResult;
|
||||
@ -1475,6 +1476,13 @@ public class GmsGraphQLEngine {
|
||||
}
|
||||
|
||||
private void configureIngestionSourceResolvers(final RuntimeWiring.Builder builder) {
|
||||
builder.type("IngestionSource", typeWiring -> typeWiring.dataFetcher("executions", new IngestionSourceExecutionRequestsResolver(entityClient)));
|
||||
builder.type("IngestionSource", typeWiring -> typeWiring
|
||||
.dataFetcher("executions", new IngestionSourceExecutionRequestsResolver(entityClient))
|
||||
.dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType,
|
||||
(env) -> {
|
||||
final IngestionSource ingestionSource = env.getSource();
|
||||
return ingestionSource.getPlatform() != null ? ingestionSource.getPlatform().getUrn() : null;
|
||||
})
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@ -389,6 +389,11 @@ type IngestionSource {
|
||||
"""
|
||||
schedule: IngestionSchedule
|
||||
|
||||
"""
|
||||
The data platform associated with this ingestion source
|
||||
"""
|
||||
platform: DataPlatform
|
||||
|
||||
"""
|
||||
An type-specific set of configurations for the ingestion source
|
||||
"""
|
||||
|
||||
@ -108,6 +108,7 @@ function IngestionSourceTable({
|
||||
urn: source.urn,
|
||||
type: source.type,
|
||||
name: source.name,
|
||||
platformUrn: source.platform?.urn,
|
||||
schedule: source.schedule?.interval,
|
||||
timezone: source.schedule?.timezone,
|
||||
execCount: source.executions?.total || 0,
|
||||
|
||||
@ -6,12 +6,12 @@ import React from 'react';
|
||||
import styled from 'styled-components/macro';
|
||||
import { ANTD_GRAY } from '../../entity/shared/constants';
|
||||
import { capitalizeFirstLetter } from '../../shared/textUtil';
|
||||
import useGetSourceLogoUrl from './builder/useGetSourceLogoUrl';
|
||||
import {
|
||||
getExecutionRequestStatusDisplayColor,
|
||||
getExecutionRequestStatusDisplayText,
|
||||
getExecutionRequestStatusIcon,
|
||||
RUNNING,
|
||||
sourceTypeToIconUrl,
|
||||
} from './utils';
|
||||
|
||||
const PreviewImage = styled(Image)`
|
||||
@ -63,7 +63,7 @@ const CliBadge = styled.span`
|
||||
`;
|
||||
|
||||
export function TypeColumn(type: string, record: any) {
|
||||
const iconUrl = sourceTypeToIconUrl(type);
|
||||
const iconUrl = useGetSourceLogoUrl(type);
|
||||
const typeDisplayName = capitalizeFirstLetter(type);
|
||||
|
||||
return (
|
||||
|
||||
@ -2,7 +2,7 @@ import { Alert, Button, Space, Typography } from 'antd';
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import styled from 'styled-components';
|
||||
import { StepProps } from './types';
|
||||
import { getSourceConfigs, jsonToYaml } from '../utils';
|
||||
import { getPlaceholderRecipe, getSourceConfigs, jsonToYaml } from '../utils';
|
||||
import { YamlEditor } from './YamlEditor';
|
||||
import { ANTD_GRAY } from '../../../entity/shared/constants';
|
||||
import { IngestionSourceBuilderStep } from './steps';
|
||||
@ -37,13 +37,14 @@ const ControlsContainer = styled.div`
|
||||
/**
|
||||
* The step for defining a recipe
|
||||
*/
|
||||
export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps) => {
|
||||
export const DefineRecipeStep = ({ state, updateState, goTo, prev, ingestionSources }: StepProps) => {
|
||||
const existingRecipeJson = state.config?.recipe;
|
||||
const existingRecipeYaml = existingRecipeJson && jsonToYaml(existingRecipeJson);
|
||||
const { type } = state;
|
||||
const sourceConfigs = getSourceConfigs(type as string);
|
||||
const sourceConfigs = getSourceConfigs(ingestionSources, type as string);
|
||||
const placeholderRecipe = getPlaceholderRecipe(ingestionSources, type);
|
||||
|
||||
const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || sourceConfigs.placeholderRecipe);
|
||||
const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || placeholderRecipe);
|
||||
|
||||
useEffect(() => {
|
||||
if (existingRecipeYaml) {
|
||||
@ -54,12 +55,12 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps)
|
||||
const [stepComplete, setStepComplete] = useState(false);
|
||||
|
||||
const isEditing: boolean = prev === undefined;
|
||||
const displayRecipe = stagedRecipeYml || sourceConfigs.placeholderRecipe;
|
||||
const sourceDisplayName = sourceConfigs.displayName;
|
||||
const sourceDocumentationUrl = sourceConfigs.docsUrl; // Maybe undefined (in case of "custom")
|
||||
const displayRecipe = stagedRecipeYml || placeholderRecipe;
|
||||
const sourceDisplayName = sourceConfigs?.displayName;
|
||||
const sourceDocumentationUrl = sourceConfigs?.docsUrl;
|
||||
|
||||
// TODO: Delete LookML banner specific code
|
||||
const isSourceLooker: boolean = sourceConfigs.type === 'looker';
|
||||
const isSourceLooker: boolean = sourceConfigs?.name === 'looker';
|
||||
const [showLookerBanner, setShowLookerBanner] = useState(isSourceLooker && !isEditing);
|
||||
|
||||
useEffect(() => {
|
||||
@ -90,6 +91,7 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps)
|
||||
type={type}
|
||||
isEditing={isEditing}
|
||||
displayRecipe={displayRecipe}
|
||||
sourceConfigs={sourceConfigs}
|
||||
setStagedRecipe={setStagedRecipeYml}
|
||||
onClickNext={onClickNext}
|
||||
goToPrevious={prev}
|
||||
|
||||
@ -8,6 +8,7 @@ import { CreateScheduleStep } from './CreateScheduleStep';
|
||||
import { DefineRecipeStep } from './DefineRecipeStep';
|
||||
import { NameSourceStep } from './NameSourceStep';
|
||||
import { SelectTemplateStep } from './SelectTemplateStep';
|
||||
import sourcesJson from './sources.json';
|
||||
|
||||
const ExpandButton = styled(Button)`
|
||||
&& {
|
||||
@ -74,6 +75,8 @@ export const IngestionSourceBuilderModal = ({ initialState, visible, onSubmit, o
|
||||
const [modalExpanded, setModalExpanded] = useState(false);
|
||||
const [ingestionBuilderState, setIngestionBuilderState] = useState<SourceBuilderState>({});
|
||||
|
||||
const ingestionSources = JSON.parse(JSON.stringify(sourcesJson)); // TODO: replace with call to server once we have access to dynamic list of sources
|
||||
|
||||
// Reset the ingestion builder modal state when the modal is re-opened.
|
||||
const prevInitialState = useRef(initialState);
|
||||
useEffect(() => {
|
||||
@ -148,6 +151,7 @@ export const IngestionSourceBuilderModal = ({ initialState, visible, onSubmit, o
|
||||
prev={stepStack.length > 1 ? prev : undefined}
|
||||
submit={submit}
|
||||
cancel={cancel}
|
||||
ingestionSources={ingestionSources}
|
||||
/>
|
||||
</Modal>
|
||||
);
|
||||
|
||||
@ -6,6 +6,7 @@ import styled from 'styled-components/macro';
|
||||
import { ANTD_GRAY } from '../../../entity/shared/constants';
|
||||
import { YamlEditor } from './YamlEditor';
|
||||
import RecipeForm from './RecipeForm/RecipeForm';
|
||||
import { SourceConfig } from './types';
|
||||
|
||||
export const ControlsContainer = styled.div`
|
||||
display: flex;
|
||||
@ -41,13 +42,14 @@ interface Props {
|
||||
type: string;
|
||||
isEditing: boolean;
|
||||
displayRecipe: string;
|
||||
sourceConfigs?: SourceConfig;
|
||||
setStagedRecipe: (recipe: string) => void;
|
||||
onClickNext: () => void;
|
||||
goToPrevious?: () => void;
|
||||
}
|
||||
|
||||
function RecipeBuilder(props: Props) {
|
||||
const { type, isEditing, displayRecipe, setStagedRecipe, onClickNext, goToPrevious } = props;
|
||||
const { type, isEditing, displayRecipe, sourceConfigs, setStagedRecipe, onClickNext, goToPrevious } = props;
|
||||
|
||||
const [isViewingForm, setIsViewingForm] = useState(true);
|
||||
|
||||
@ -78,6 +80,7 @@ function RecipeBuilder(props: Props) {
|
||||
type={type}
|
||||
isEditing={isEditing}
|
||||
displayRecipe={displayRecipe}
|
||||
sourceConfigs={sourceConfigs}
|
||||
setStagedRecipe={setStagedRecipe}
|
||||
onClickNext={onClickNext}
|
||||
goToPrevious={goToPrevious}
|
||||
|
||||
@ -10,6 +10,7 @@ import FormField from './FormField';
|
||||
import TestConnectionButton from './TestConnection/TestConnectionButton';
|
||||
import { useListSecretsQuery } from '../../../../../graphql/ingestion.generated';
|
||||
import { RecipeField, setFieldValueOnRecipe } from './common';
|
||||
import { SourceConfig } from '../types';
|
||||
|
||||
export const ControlsContainer = styled.div`
|
||||
display: flex;
|
||||
@ -91,13 +92,14 @@ interface Props {
|
||||
type: string;
|
||||
isEditing: boolean;
|
||||
displayRecipe: string;
|
||||
sourceConfigs?: SourceConfig;
|
||||
setStagedRecipe: (recipe: string) => void;
|
||||
onClickNext: () => void;
|
||||
goToPrevious?: () => void;
|
||||
}
|
||||
|
||||
function RecipeForm(props: Props) {
|
||||
const { type, isEditing, displayRecipe, setStagedRecipe, onClickNext, goToPrevious } = props;
|
||||
const { type, isEditing, displayRecipe, sourceConfigs, setStagedRecipe, onClickNext, goToPrevious } = props;
|
||||
const { fields, advancedFields, filterFields, filterSectionTooltip } = RECIPE_FIELDS[type];
|
||||
const allFields = [...fields, ...advancedFields, ...filterFields];
|
||||
const { data, refetch: refetchSecrets } = useListSecretsQuery({
|
||||
@ -146,7 +148,7 @@ function RecipeForm(props: Props) {
|
||||
))}
|
||||
{CONNECTORS_WITH_TEST_CONNECTION.has(type) && (
|
||||
<TestConnectionWrapper>
|
||||
<TestConnectionButton type={type} recipe={displayRecipe} />
|
||||
<TestConnectionButton recipe={displayRecipe} sourceConfigs={sourceConfigs} />
|
||||
</TestConnectionWrapper>
|
||||
)}
|
||||
</Collapse.Panel>
|
||||
|
||||
@ -6,9 +6,10 @@ import {
|
||||
useCreateTestConnectionRequestMutation,
|
||||
useGetIngestionExecutionRequestLazyQuery,
|
||||
} from '../../../../../../graphql/ingestion.generated';
|
||||
import { FAILURE, getSourceConfigs, RUNNING, yamlToJson } from '../../../utils';
|
||||
import { FAILURE, RUNNING, yamlToJson } from '../../../utils';
|
||||
import { TestConnectionResult } from './types';
|
||||
import TestConnectionModal from './TestConnectionModal';
|
||||
import { SourceConfig } from '../../types';
|
||||
|
||||
export function getRecipeJson(recipeYaml: string) {
|
||||
// Convert the recipe into it's json representation, and catch + report exceptions while we do it.
|
||||
@ -26,12 +27,12 @@ export function getRecipeJson(recipeYaml: string) {
|
||||
}
|
||||
|
||||
interface Props {
|
||||
type: string;
|
||||
recipe: string;
|
||||
sourceConfigs?: SourceConfig;
|
||||
}
|
||||
|
||||
function TestConnectionButton(props: Props) {
|
||||
const { type, recipe } = props;
|
||||
const { recipe, sourceConfigs } = props;
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [isModalVisible, setIsModalVisible] = useState(false);
|
||||
const [pollingInterval, setPollingInterval] = useState<null | NodeJS.Timeout>(null);
|
||||
@ -39,8 +40,6 @@ function TestConnectionButton(props: Props) {
|
||||
const [createTestConnectionRequest, { data: requestData }] = useCreateTestConnectionRequestMutation();
|
||||
const [getIngestionExecutionRequest, { data: resultData, loading }] = useGetIngestionExecutionRequestLazyQuery();
|
||||
|
||||
const sourceConfigs = getSourceConfigs(type);
|
||||
|
||||
useEffect(() => {
|
||||
if (requestData && requestData.createTestConnectionRequest) {
|
||||
const interval = setInterval(
|
||||
|
||||
@ -6,8 +6,9 @@ import styled from 'styled-components/macro';
|
||||
import { ReactComponent as LoadingSvg } from '../../../../../../images/datahub-logo-color-loading_pendulum.svg';
|
||||
import { ANTD_GRAY } from '../../../../../entity/shared/constants';
|
||||
import ConnectionCapabilityView from './ConnectionCapabilityView';
|
||||
import { SourceConfig } from '../../../conf/types';
|
||||
import { CapabilityReport, SourceCapability, TestConnectionResult } from './types';
|
||||
import { SourceConfig } from '../../types';
|
||||
import useGetSourceLogoUrl from '../../useGetSourceLogoUrl';
|
||||
|
||||
const LoadingWrapper = styled.div`
|
||||
display: flex;
|
||||
@ -84,7 +85,7 @@ const StyledClose = styled(CloseOutlined)`
|
||||
interface Props {
|
||||
isLoading: boolean;
|
||||
testConnectionFailed: boolean;
|
||||
sourceConfig: SourceConfig;
|
||||
sourceConfig?: SourceConfig;
|
||||
testConnectionResult: TestConnectionResult | null;
|
||||
hideModal: () => void;
|
||||
}
|
||||
@ -96,6 +97,8 @@ function TestConnectionModal({
|
||||
testConnectionResult,
|
||||
hideModal,
|
||||
}: Props) {
|
||||
const logoUrl = useGetSourceLogoUrl(sourceConfig?.name || '');
|
||||
|
||||
return (
|
||||
<Modal
|
||||
visible
|
||||
@ -103,8 +106,8 @@ function TestConnectionModal({
|
||||
footer={<Button onClick={hideModal}>Done</Button>}
|
||||
title={
|
||||
<ModalHeader style={{ margin: 0 }}>
|
||||
<SourceIcon alt="source logo" src={sourceConfig.logoUrl} />
|
||||
{sourceConfig.displayName} Connection Test
|
||||
<SourceIcon alt="source logo" src={logoUrl} />
|
||||
{sourceConfig?.displayName} Connection Test
|
||||
</ModalHeader>
|
||||
}
|
||||
width={750}
|
||||
@ -133,8 +136,8 @@ function TestConnectionModal({
|
||||
</ResultsHeader>
|
||||
<ResultsSubHeader>
|
||||
{testConnectionFailed
|
||||
? `A connection was not able to be established with ${sourceConfig.displayName}.`
|
||||
: `A connection was successfully established with ${sourceConfig.displayName}.`}
|
||||
? `A connection was not able to be established with ${sourceConfig?.displayName}.`
|
||||
: `A connection was successfully established with ${sourceConfig?.displayName}.`}
|
||||
</ResultsSubHeader>
|
||||
<Divider />
|
||||
{testConnectionResult?.internal_failure ? (
|
||||
|
||||
@ -1,10 +1,13 @@
|
||||
import { Button } from 'antd';
|
||||
import React from 'react';
|
||||
import { Button, Input } from 'antd';
|
||||
import { FormOutlined, SearchOutlined } from '@ant-design/icons';
|
||||
import React, { useState } from 'react';
|
||||
import styled from 'styled-components';
|
||||
import { LogoCountCard } from '../../../shared/LogoCountCard';
|
||||
import { SourceBuilderState, StepProps } from './types';
|
||||
import { SOURCE_TEMPLATE_CONFIGS } from '../conf/sources';
|
||||
import { SourceConfig, SourceBuilderState, StepProps } from './types';
|
||||
import { IngestionSourceBuilderStep } from './steps';
|
||||
import useGetSourceLogoUrl from './useGetSourceLogoUrl';
|
||||
import { CUSTOM } from './constants';
|
||||
import { ANTD_GRAY } from '../../../entity/shared/constants';
|
||||
|
||||
const Section = styled.div`
|
||||
display: flex;
|
||||
@ -25,11 +28,36 @@ const CancelButton = styled(Button)`
|
||||
}
|
||||
`;
|
||||
|
||||
const StyledSearchBar = styled(Input)`
|
||||
background-color: white;
|
||||
border-radius: 70px;
|
||||
box-shadow: 0px 0px 30px 0px rgb(239 239 239);
|
||||
width: 45%;
|
||||
margin: 0 0 15px 12px;
|
||||
`;
|
||||
|
||||
interface SourceOptionProps {
|
||||
source: SourceConfig;
|
||||
onClick: () => void;
|
||||
}
|
||||
|
||||
function SourceOption({ source, onClick }: SourceOptionProps) {
|
||||
const { name, displayName } = source;
|
||||
|
||||
const logoUrl = useGetSourceLogoUrl(name);
|
||||
let logoComponent;
|
||||
if (name === CUSTOM) {
|
||||
logoComponent = <FormOutlined style={{ color: ANTD_GRAY[8], fontSize: 28 }} />;
|
||||
}
|
||||
|
||||
return <LogoCountCard onClick={onClick} name={displayName} logoUrl={logoUrl} logoComponent={logoComponent} />;
|
||||
}
|
||||
|
||||
/**
|
||||
* Component responsible for selecting the mechanism for constructing a new Ingestion Source
|
||||
*/
|
||||
export const SelectTemplateStep = ({ state, updateState, goTo, cancel }: StepProps) => {
|
||||
// Reoslve the supported platform types to their logos and names.
|
||||
export const SelectTemplateStep = ({ state, updateState, goTo, cancel, ingestionSources }: StepProps) => {
|
||||
const [searchFilter, setSearchFilter] = useState('');
|
||||
|
||||
const onSelectTemplate = (type: string) => {
|
||||
const newState: SourceBuilderState = {
|
||||
@ -41,18 +69,23 @@ export const SelectTemplateStep = ({ state, updateState, goTo, cancel }: StepPro
|
||||
goTo(IngestionSourceBuilderStep.DEFINE_RECIPE);
|
||||
};
|
||||
|
||||
const filteredSources = ingestionSources.filter(
|
||||
(source) => source.displayName.includes(searchFilter) || source.name.includes(searchFilter),
|
||||
);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Section>
|
||||
<StyledSearchBar
|
||||
placeholder="Search ingestion sources..."
|
||||
value={searchFilter}
|
||||
onChange={(e) => setSearchFilter(e.target.value)}
|
||||
allowClear
|
||||
prefix={<SearchOutlined />}
|
||||
/>
|
||||
<PlatformListContainer>
|
||||
{SOURCE_TEMPLATE_CONFIGS.map((configs, _) => (
|
||||
<LogoCountCard
|
||||
key={configs.type}
|
||||
onClick={() => onSelectTemplate(configs.type)}
|
||||
name={configs.displayName}
|
||||
logoUrl={configs.logoUrl}
|
||||
logoComponent={configs.logoComponent}
|
||||
/>
|
||||
{filteredSources.map((source) => (
|
||||
<SourceOption key={source.urn} source={source} onClick={() => onSelectTemplate(source.name)} />
|
||||
))}
|
||||
</PlatformListContainer>
|
||||
</Section>
|
||||
|
||||
@ -2,6 +2,7 @@ import { MockedProvider } from '@apollo/client/testing';
|
||||
import { render } from '@testing-library/react';
|
||||
import React from 'react';
|
||||
import { DefineRecipeStep } from '../DefineRecipeStep';
|
||||
import { SourceConfig } from '../types';
|
||||
|
||||
describe('DefineRecipeStep', () => {
|
||||
it('should render the RecipeBuilder if the type is in CONNECTORS_WITH_FORM', () => {
|
||||
@ -13,6 +14,7 @@ describe('DefineRecipeStep', () => {
|
||||
goTo={() => {}}
|
||||
submit={() => {}}
|
||||
cancel={() => {}}
|
||||
ingestionSources={[{ name: 'snowflake', displayName: 'Snowflake' } as SourceConfig]}
|
||||
/>
|
||||
</MockedProvider>,
|
||||
);
|
||||
@ -30,6 +32,7 @@ describe('DefineRecipeStep', () => {
|
||||
goTo={() => {}}
|
||||
submit={() => {}}
|
||||
cancel={() => {}}
|
||||
ingestionSources={[{ name: 'glue', displayName: 'Glue' } as SourceConfig]}
|
||||
/>
|
||||
</MockedProvider>,
|
||||
);
|
||||
|
||||
135
datahub-web-react/src/app/ingest/source/builder/constants.ts
Normal file
135
datahub-web-react/src/app/ingest/source/builder/constants.ts
Normal file
@ -0,0 +1,135 @@
|
||||
import snowflakeLogo from '../../../../images/snowflakelogo.png';
|
||||
import bigqueryLogo from '../../../../images/bigquerylogo.png';
|
||||
import redshiftLogo from '../../../../images/redshiftlogo.png';
|
||||
import kafkaLogo from '../../../../images/kafkalogo.png';
|
||||
import lookerLogo from '../../../../images/lookerlogo.png';
|
||||
import tableauLogo from '../../../../images/tableaulogo.png';
|
||||
import mysqlLogo from '../../../../images/mysqllogo-2.png';
|
||||
import postgresLogo from '../../../../images/postgreslogo.png';
|
||||
import mongodbLogo from '../../../../images/mongodblogo.png';
|
||||
import azureLogo from '../../../../images/azure-ad.png';
|
||||
import oktaLogo from '../../../../images/oktalogo.png';
|
||||
import glueLogo from '../../../../images/gluelogo.png';
|
||||
import oracleLogo from '../../../../images/oraclelogo.png';
|
||||
import hiveLogo from '../../../../images/hivelogo.png';
|
||||
import supersetLogo from '../../../../images/supersetlogo.png';
|
||||
import athenaLogo from '../../../../images/awsathenalogo.png';
|
||||
import mssqlLogo from '../../../../images/mssqllogo.png';
|
||||
import clickhouseLogo from '../../../../images/clickhouselogo.png';
|
||||
import trinoLogo from '../../../../images/trinologo.png';
|
||||
import dbtLogo from '../../../../images/dbtlogo.png';
|
||||
import druidLogo from '../../../../images/druidlogo.png';
|
||||
import elasticsearchLogo from '../../../../images/elasticsearchlogo.png';
|
||||
import feastLogo from '../../../../images/feastlogo.png';
|
||||
import mariadbLogo from '../../../../images/mariadblogo.png';
|
||||
import metabaseLogo from '../../../../images/metabaselogo.png';
|
||||
import powerbiLogo from '../../../../images/powerbilogo.png';
|
||||
import modeLogo from '../../../../images/modelogo.png';
|
||||
|
||||
export const ATHENA = 'athena';
|
||||
export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`;
|
||||
export const AZURE = 'azure-ad';
|
||||
export const AZURE_URN = `urn:li:dataPlatform:${AZURE}`;
|
||||
export const BIGQUERY = 'bigquery';
|
||||
export const BIGQUERY_USAGE = 'bigquery-usage';
|
||||
export const BIGQUERY_BETA = 'bigquery-beta';
|
||||
export const BIGQUERY_URN = `urn:li:dataPlatform:${BIGQUERY}`;
|
||||
export const CLICKHOUSE = 'clickhouse';
|
||||
export const CLICKHOUSE_USAGE = 'clickhouse-usage';
|
||||
export const CLICKHOUSE_URN = `urn:li:dataPlatform:${CLICKHOUSE}`;
|
||||
export const DBT = 'dbt';
|
||||
export const DBT_URN = `urn:li:dataPlatform:${DBT}`;
|
||||
export const DRUID = 'druid';
|
||||
export const DRUID_URN = `urn:li:dataPlatform:${DRUID}`;
|
||||
export const ELASTICSEARCH = 'elasticsearch';
|
||||
export const ELASTICSEARCH_URN = `urn:li:dataPlatform:${ELASTICSEARCH}`;
|
||||
export const FEAST = 'feast';
|
||||
export const FEAST_LEGACY = 'feast-legacy';
|
||||
export const FEAST_URN = `urn:li:dataPlatform:${FEAST}`;
|
||||
export const GLUE = 'glue';
|
||||
export const GLUE_URN = `urn:li:dataPlatform:${GLUE}`;
|
||||
export const HIVE = 'hive';
|
||||
export const HIVE_URN = `urn:li:dataPlatform:${HIVE}`;
|
||||
export const KAFKA = 'kafka';
|
||||
export const KAFKA_URN = `urn:li:dataPlatform:${KAFKA}`;
|
||||
export const LOOKER = 'looker';
|
||||
export const LOOK_ML = 'lookml';
|
||||
export const LOOKER_URN = `urn:li:dataPlatform:${LOOKER}`;
|
||||
export const MARIA_DB = 'mariadb';
|
||||
export const MARIA_DB_URN = `urn:li:dataPlatform:${MARIA_DB}`;
|
||||
export const METABASE = 'metabase';
|
||||
export const METABASE_URN = `urn:li:dataPlatform:${METABASE}`;
|
||||
export const MODE = 'mode';
|
||||
export const MODE_URN = `urn:li:dataPlatform:${MODE}`;
|
||||
export const MONGO_DB = 'mongodb';
|
||||
export const MONGO_DB_URN = `urn:li:dataPlatform:${MONGO_DB}`;
|
||||
export const MSSQL = 'mssql';
|
||||
export const MSSQL_URN = `urn:li:dataPlatform:${MSSQL}`;
|
||||
export const MYSQL = 'mysql';
|
||||
export const MYSQL_URN = `urn:li:dataPlatform:${MYSQL}`;
|
||||
export const OKTA = 'okta';
|
||||
export const OKTA_URN = `urn:li:dataPlatform:${OKTA}`;
|
||||
export const ORACLE = 'oracle';
|
||||
export const ORACLE_URN = `urn:li:dataPlatform:${ORACLE}`;
|
||||
export const POSTGRES = 'postgres';
|
||||
export const POSTGRES_URN = `urn:li:dataPlatform:${POSTGRES}`;
|
||||
export const POWER_BI = 'powerbi';
|
||||
export const POWER_BI_URN = `urn:li:dataPlatform:${POWER_BI}`;
|
||||
export const REDSHIFT = 'redshift';
|
||||
export const REDSHIFT_USAGE = 'redshift-usage';
|
||||
export const REDSHIFT_URN = `urn:li:dataPlatform:${REDSHIFT}`;
|
||||
export const SNOWFLAKE = 'snowflake';
|
||||
export const SNOWFLAKE_BETA = 'snowflake-beta';
|
||||
export const SNOWFLAKE_USAGE = 'snowflake-usage';
|
||||
export const SNOWFLAKE_URN = `urn:li:dataPlatform:${SNOWFLAKE}`;
|
||||
export const STARBURST_TRINO_USAGE = 'starburst-trino-usage';
|
||||
export const SUPERSET = 'superset';
|
||||
export const SUPERSET_URN = `urn:li:dataPlatform:${SUPERSET}`;
|
||||
export const TABLEAU = 'tableau';
|
||||
export const TABLEAU_URN = `urn:li:dataPlatform:${TABLEAU}`;
|
||||
export const TRINO = 'trino';
|
||||
export const TRINO_URN = `urn:li:dataPlatform:${TRINO}`;
|
||||
export const CUSTOM = 'custom';
|
||||
export const CUSTOM_URN = `urn:li:dataPlatform:${CUSTOM}`;
|
||||
|
||||
export const PLATFORM_URN_TO_LOGO = {
|
||||
[ATHENA_URN]: athenaLogo,
|
||||
[AZURE_URN]: azureLogo,
|
||||
[BIGQUERY_URN]: bigqueryLogo,
|
||||
[CLICKHOUSE_URN]: clickhouseLogo,
|
||||
[DBT_URN]: dbtLogo,
|
||||
[DRUID_URN]: druidLogo,
|
||||
[ELASTICSEARCH_URN]: elasticsearchLogo,
|
||||
[FEAST_URN]: feastLogo,
|
||||
[GLUE_URN]: glueLogo,
|
||||
[HIVE_URN]: hiveLogo,
|
||||
[KAFKA_URN]: kafkaLogo,
|
||||
[LOOKER_URN]: lookerLogo,
|
||||
[MARIA_DB_URN]: mariadbLogo,
|
||||
[METABASE_URN]: metabaseLogo,
|
||||
[MODE_URN]: modeLogo,
|
||||
[MONGO_DB_URN]: mongodbLogo,
|
||||
[MSSQL_URN]: mssqlLogo,
|
||||
[MYSQL_URN]: mysqlLogo,
|
||||
[OKTA_URN]: oktaLogo,
|
||||
[ORACLE_URN]: oracleLogo,
|
||||
[POSTGRES_URN]: postgresLogo,
|
||||
[POWER_BI_URN]: powerbiLogo,
|
||||
[REDSHIFT_URN]: redshiftLogo,
|
||||
[SNOWFLAKE_URN]: snowflakeLogo,
|
||||
[TABLEAU_URN]: tableauLogo,
|
||||
[TRINO_URN]: trinoLogo,
|
||||
[SUPERSET_URN]: supersetLogo,
|
||||
};
|
||||
|
||||
export const SOURCE_TO_PLATFORM_URN = {
|
||||
[BIGQUERY_BETA]: BIGQUERY_URN,
|
||||
[BIGQUERY_USAGE]: BIGQUERY_URN,
|
||||
[CLICKHOUSE_USAGE]: CLICKHOUSE_URN,
|
||||
[FEAST_LEGACY]: FEAST_URN,
|
||||
[LOOK_ML]: LOOKER_URN,
|
||||
[REDSHIFT_USAGE]: REDSHIFT_URN,
|
||||
[SNOWFLAKE_BETA]: SNOWFLAKE_URN,
|
||||
[SNOWFLAKE_USAGE]: SNOWFLAKE_URN,
|
||||
[STARBURST_TRINO_USAGE]: TRINO_URN,
|
||||
};
|
||||
177
datahub-web-react/src/app/ingest/source/builder/sources.json
Normal file
177
datahub-web-react/src/app/ingest/source/builder/sources.json
Normal file
@ -0,0 +1,177 @@
|
||||
[
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:bigquery",
|
||||
"name": "bigquery",
|
||||
"displayName": "BigQuery",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/bigquery/",
|
||||
"recipe": "source:\n type: bigquery\n config:\n # Coordinates\n project_id: # Your BigQuery project id, e.g. sample_project_id\n # Credentials\n credential:\n project_id: # Your BQ project id, e.g. sample_project_id\n private_key_id: \"${BQ_PRIVATE_KEY_ID}\"\n private_key: \"${BQ_PRIVATE_KEY}\"\n client_email: # Your BQ client email, e.g. \"test@suppproject-id-1234567.iam.gserviceaccount.com\"\n client_id: # Your BQ client id, e.g.\"123456678890\"\n \n include_table_lineage: true\n include_view_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:redshift",
|
||||
"name": "redshift",
|
||||
"displayName": "Redshift",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/redshift/",
|
||||
"recipe": "source: \n type: redshift\n config:\n # Coordinates\n host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n database: # Your Redshift database, e.g. SampleDatabase\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${REDSHIFT_USERNAME}\" # Your Redshift username, e.g. admin\n password: \"${REDSHIFT_PASSWORD}\" # Your Redshift password, e.g. password_01\n\n table_lineage_mode: stl_scan_based\n include_table_lineage: true\n include_view_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:snowflake",
|
||||
"name": "snowflake",
|
||||
"displayName": "Snowflake",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/snowflake/",
|
||||
"recipe": "source: \n type: snowflake\n config:\n account_id: \"example_id\"\n warehouse: \"example_warehouse\"\n role: \"datahub_role\"\n include_table_lineage: true\n include_view_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:kafka",
|
||||
"name": "kafka",
|
||||
"displayName": "Kafka",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/kafka/",
|
||||
"recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"SASL_SSL\"\n sasl.mechanism: \"PLAIN\"\n stateful_ingestion:\n enabled: true'"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:looker",
|
||||
"name": "looker",
|
||||
"displayName": "Looker",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/looker/",
|
||||
"recipe": "source:\n type: looker\n config:\n # Coordinates\n base_url: # Your Looker instance URL, e.g. https://company.looker.com:19999\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n client_id: \"${LOOKER_CLIENT_ID}\" # Your Looker client id, e.g. admin\n client_secret: \"${LOOKER_CLIENT_SECRET}\" # Your Looker password, e.g. password_01"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:tableau",
|
||||
"name": "tableau",
|
||||
"displayName": "Tableau",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/tableau/",
|
||||
"recipe": "source:\n type: tableau\n config:\n # Coordinates\n connect_uri: https://prod-ca-a.online.tableau.com\n site: acryl\n projects: [\"default\", \"Project 2\"]\n\n # Credentials\n username: \"${TABLEAU_USER}\"\n password: \"${TABLEAU_PASSWORD}\"\n\n # Options\n ingest_tags: True\n ingest_owner: True\n default_schema_map:\n mydatabase: public\n anotherdatabase: anotherschema"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:mysql",
|
||||
"name": "mysql",
|
||||
"displayName": "MySQL",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mysql/",
|
||||
"recipe": "source: \n type: mysql\n config: \n # Coordinates\n host_port: # Your MySQL host and post, e.g. mysql:3306\n database: # Your MySQL database name, e.g. datahub\n \n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${MYSQL_USERNAME}\" # Your MySQL username, e.g. admin\n password: \"${MYSQL_PASSWORD}\" # Your MySQL password, e.g. password_01\n\n # Options\n include_tables: True\n include_views: True\n\n # Profiling\n profiling:\n enabled: false"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:postgres",
|
||||
"name": "postgres",
|
||||
"displayName": "Postgres",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/postgres/",
|
||||
"recipe": "source: \n type: postgres\n config:\n # Coordinates\n host_port: # Your Postgres host and port, e.g. postgres:5432\n database: # Your Postgres Database, e.g. sample_db\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${POSTGRES_USERNAME}\" # Your Postgres username, e.g. admin\n password: \"${POSTGRES_PASSWORD}\" # Your Postgres password, e.g. password_01\n\n # Options\n include_tables: True\n include_views: True\n\n # Profiling\n profiling:\n enabled: false\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:mongodb",
|
||||
"name": "mongodb",
|
||||
"displayName": "Postgres",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mongodb/",
|
||||
"recipe": "source:\n type: mongodb\n config:\n # Coordinates\n connect_uri: # Your MongoDB connect URI, e.g. \"mongodb://localhost\"\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${MONGO_USERNAME}\" # Your MongoDB username, e.g. admin\n password: \"${MONGO_PASSWORD}\" # Your MongoDB password, e.g. password_01\n\n # Options (recommended)\n enableSchemaInference: True\n useRandomSampling: True\n maxSchemaSize: 300"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:azure-ad",
|
||||
"name": "azure-ad",
|
||||
"displayName": "Azure AD",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/azure-ad/",
|
||||
"recipe": "source:\n type: azure-ad\n config:\n client_id: # Your Azure Client ID, e.g. \"00000000-0000-0000-0000-000000000000\"\n tenant_id: # Your Azure Tenant ID, e.g. \"00000000-0000-0000-0000-000000000000\"\n # Add secret in Secrets Tab with this name\n client_secret: \"${AZURE_AD_CLIENT_SECRET}\"\n redirect: # Your Redirect URL, e.g. \"https://login.microsoftonline.com/common/oauth2/nativeclient\"\n authority: # Your Authority URL, e.g. \"https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000\"\n token_url: # Your Token URL, e.g. \"https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/token\"\n graph_url: # The Graph URL, e.g. \"https://graph.microsoft.com/v1.0\"\n \n # Optional flags to ingest users, groups, or both\n ingest_users: True\n ingest_groups: True\n \n # Optional Allow / Deny extraction of particular Groups\n # groups_pattern:\n # allow:\n # - \".*\"\n\n # Optional Allow / Deny extraction of particular Users.\n # users_pattern:\n # allow:\n # - \".*\""
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:okta",
|
||||
"name": "okta",
|
||||
"displayName": "Okta",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/okta/",
|
||||
"recipe": "source:\n type: okta\n config:\n # Coordinates\n okta_domain: # Your Okta Domain, e.g. \"dev-35531955.okta.com\"\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n okta_api_token: \"${OKTA_API_TOKEN}\" # Your Okta API Token, e.g. \"11be4R_M2MzDqXawbTHfKGpKee0kuEOfX1RCQSRx99\"\n\n # Optional flags to ingest users, groups, or both\n ingest_users: True\n ingest_groups: True\n\n # Optional: Customize the mapping to DataHub Username from an attribute appearing in the Okta User\n # profile. Reference: https://developer.okta.com/docs/reference/api/users/\n # okta_profile_to_username_attr: str = \"login\"\n # okta_profile_to_username_regex: str = \"([^@]+)\"\n \n # Optional: Customize the mapping to DataHub Group from an attribute appearing in the Okta Group\n # profile. Reference: https://developer.okta.com/docs/reference/api/groups/\n # okta_profile_to_group_name_attr: str = \"name\"\n # okta_profile_to_group_name_regex: str = \"(.*)\"\n \n # Optional: Include deprovisioned or suspended Okta users in the ingestion.\n # include_deprovisioned_users = False\n # include_suspended_users = False"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:glue",
|
||||
"name": "glue",
|
||||
"displayName": "Glue",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/glue/",
|
||||
"recipe": "source:\n type: glue\n config:\n # AWS credentials. \n aws_region: # The region for your AWS Glue instance. \n # Add secret in Secrets Tab with relevant names for each variable\n # The access key for your AWS account.\n aws_access_key_id: \"${AWS_ACCESS_KEY_ID}\"\n # The secret key for your AWS account.\n aws_secret_access_key: \"${AWS_SECRET_KEY}\"\n aws_session_token: # The session key for your AWS account. This is only needed when you are using temporary credentials.\n # aws_role: # (Optional) The role to assume (Role chaining supported by using a sorted list).\n\n # Allow / Deny specific databases & tables\n # database_pattern:\n # allow:\n # - \"flights-database\"\n # table_pattern:\n # allow:\n # - \"avro\""
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:oracle",
|
||||
"name": "oracle",
|
||||
"displayName": "Oracle",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/oracle/",
|
||||
"recipe": "source: \n type: oracle\n config:\n # Coordinates\n host_port: # Your Oracle host and port, e.g. oracle:5432\n database: # Your Oracle database name, e.g. sample_db\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${ORACLE_USERNAME}\" # Your Oracle username, e.g. admin\n password: \"${ORACLE_PASSWORD}\" # Your Oracle password, e.g. password_01\n\n # Optional service name\n # service_name: # Your service name, e.g. svc # omit database if using this option"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:hive",
|
||||
"name": "hive",
|
||||
"displayName": "Hive",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/hive/",
|
||||
"recipe": "source: \n type: hive\n config:\n # Coordinates\n host_port: # Your Hive host and port, e.g. hive:10000\n database: # Your Hive database name, e.g. SampleDatabase (Optional, if not specified, ingests from all databases)\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${HIVE_USERNAME}\" # Your Hive username, e.g. admin\n password: \"${HIVE_PASSWORD}\"# Your Hive password, e.g. password_01\n stateful_ingestion:\n enabled: true"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:superset",
|
||||
"name": "superset",
|
||||
"displayName": "Superset",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/superset/",
|
||||
"recipe": "source:\n type: superset\n config:\n # Coordinates\n connect_uri: http://localhost:8088\n\n # Credentials\n username: user\n password: pass\n provider: ldap"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:athena",
|
||||
"name": "athena",
|
||||
"displayName": "Athena",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/athena/",
|
||||
"recipe": "source:\n type: athena\n config:\n # Coordinates\n aws_region: my_aws_region\n work_group: primary\n\n # Options\n s3_staging_dir: \"s3://my_staging_athena_results_bucket/results/\""
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:mssql",
|
||||
"name": "mssql",
|
||||
"displayName": "SQL Server",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mssql/",
|
||||
"recipe": "source:\n type: mssql\n config:\n # Coordinates\n host_port: localhost:1433\n database: DemoDatabase\n\n # Credentials\n username: user\n password: pass"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:clickhouse",
|
||||
"name": "clickhouse",
|
||||
"displayName": "ClickHouse",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/clickhouse/",
|
||||
"recipe": "source:\n type: clickhouse\n config:\n # Coordinates\n host_port: localhost:9000\n\n # Credentials\n username: user\n password: pass\n\n # Options\n platform_instance: DatabaseNameToBeIngested\n\n include_views: True # whether to include views, defaults to True\n include_tables: True # whether to include views, defaults to True\n\nsink:\n # sink configs\n\n#---------------------------------------------------------------------------\n# For the HTTP interface:\n#---------------------------------------------------------------------------\nsource:\n type: clickhouse\n config:\n host_port: localhost:8443\n protocol: https\n\n#---------------------------------------------------------------------------\n# For the Native interface:\n#---------------------------------------------------------------------------\n\nsource:\n type: clickhouse\n config:\n host_port: localhost:9440\n scheme: clickhouse+native\n secure: True"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:trino",
|
||||
"name": "trino",
|
||||
"displayName": "Trino",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/trino/",
|
||||
"recipe": "source:\n type: starburst-trino-usage\n config:\n # Coordinates\n host_port: yourtrinohost:port\n # The name of the catalog from getting the usage\n database: hive\n # Credentials\n username: trino_username\n password: trino_password\n email_domain: test.com\n audit_catalog: audit\n audit_schema: audit_schema"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:druid",
|
||||
"name": "druid",
|
||||
"displayName": "Druid",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/druid/",
|
||||
"recipe": "source:\n type: druid\n config:\n # Coordinates\n host_port: \"localhost:8082\"\n\n # Credentials\n username: admin\n password: password"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:metabase",
|
||||
"name": "metabase",
|
||||
"displayName": "Metabase",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/metabase/",
|
||||
"recipe": "source:\n type: metabase\n config:\n # Coordinates\n connect_uri:\n\n # Credentials\n username: root\n password: example"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:mariadb",
|
||||
"name": "mariadb",
|
||||
"displayName": "MariaDB",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mariadb/",
|
||||
"recipe": "source:\n type: mariadb\n config:\n # Coordinates\n host_port: localhost:3306\n database: dbname\n\n # Credentials\n username: root\n password: example"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:powerbi",
|
||||
"name": "powerbi",
|
||||
"displayName": "Power BI",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/powerbi/",
|
||||
"recipe": "source:\n type: \"powerbi\"\n config:\n # Your Power BI tenant identifier\n tenant_id: a949d688-67c0-4bf1-a344-e939411c6c0a\n # Ingest elements of below PowerBi Workspace into Datahub\n workspace_id: 4bd10256-e999-45dd-8e56-571c77153a5f\n # Workspaces dataset environments (PROD, DEV, QA, STAGE)\n env: DEV\n # Azure AD Application identifier\n client_id: foo\n # Azure AD App client secret\n client_secret: bar\n # Enable / Disable ingestion of user information for dashboards\n extract_ownership: true\n # dataset_type_mapping is fixed mapping of Power BI datasources type to equivalent Datahub \"data platform\" dataset\n dataset_type_mapping:\n PostgreSql: postgres\n Oracle: oracle"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:mode",
|
||||
"name": "mode",
|
||||
"displayName": "Mode",
|
||||
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mode/",
|
||||
"recipe": "source:\n type: mode\n config:\n # Coordinates\n connect_uri: http://app.mode.com\n\n # Credentials\n token: token\n password: pass\n\n # Options\n workspace: \"datahub\"\n default_schema: \"public\"\n owner_username_instead_of_email: False\n api_options:\n retry_backoff_multiplier: 2\n max_retry_interval: 10\n max_attempts: 5"
|
||||
},
|
||||
{
|
||||
"urn": "urn:li:dataPlatform:custom",
|
||||
"name": "custom",
|
||||
"displayName": "Custom",
|
||||
"docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
|
||||
"recipe": "source:\n type: <source-type>\n config:\n # Source-type specifics config\n <source-configs>"
|
||||
}
|
||||
]
|
||||
@ -13,6 +13,14 @@ export enum ModalSize {
|
||||
*/
|
||||
export const DEFAULT_EXECUTOR_ID = 'default';
|
||||
|
||||
export interface SourceConfig {
|
||||
urn: string;
|
||||
name: string;
|
||||
displayName: string;
|
||||
docsUrl: string;
|
||||
recipe: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Props provided to each step as input.
|
||||
*/
|
||||
@ -23,6 +31,7 @@ export type StepProps = {
|
||||
prev?: () => void;
|
||||
submit: (shouldRun?: boolean) => void;
|
||||
cancel: () => void;
|
||||
ingestionSources: SourceConfig[];
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@ -0,0 +1,25 @@
|
||||
import { useEffect } from 'react';
|
||||
import { useGetDataPlatformLazyQuery } from '../../../../graphql/dataPlatform.generated';
|
||||
import { CUSTOM, SOURCE_TO_PLATFORM_URN, PLATFORM_URN_TO_LOGO } from './constants';
|
||||
|
||||
function generatePlatformUrn(platformName: string) {
|
||||
return `urn:li:dataPlatform:${platformName}`;
|
||||
}
|
||||
|
||||
export default function useGetSourceLogoUrl(sourceName: string) {
|
||||
const [getDataPlatform, { data, loading }] = useGetDataPlatformLazyQuery();
|
||||
|
||||
let platformUrn = SOURCE_TO_PLATFORM_URN[sourceName];
|
||||
if (!platformUrn) {
|
||||
platformUrn = generatePlatformUrn(sourceName);
|
||||
}
|
||||
const logoInMemory = PLATFORM_URN_TO_LOGO[platformUrn];
|
||||
|
||||
useEffect(() => {
|
||||
if (!logoInMemory && sourceName !== CUSTOM && !data && !loading) {
|
||||
getDataPlatform({ variables: { urn: platformUrn } });
|
||||
}
|
||||
});
|
||||
|
||||
return logoInMemory || (data?.dataPlatform?.properties?.logoUrl as string);
|
||||
}
|
||||
@ -7,19 +7,15 @@ import {
|
||||
WarningOutlined,
|
||||
} from '@ant-design/icons';
|
||||
import { ANTD_GRAY, REDESIGN_COLORS } from '../../entity/shared/constants';
|
||||
import { SOURCE_TEMPLATE_CONFIGS } from './conf/sources';
|
||||
import { EntityType, FacetMetadata } from '../../../types.generated';
|
||||
import { capitalizeFirstLetterOnly, pluralize } from '../../shared/textUtil';
|
||||
import EntityRegistry from '../../entity/EntityRegistry';
|
||||
import { SourceConfig } from './builder/types';
|
||||
|
||||
export const sourceTypeToIconUrl = (type: string) => {
|
||||
return SOURCE_TEMPLATE_CONFIGS.find((config) => config.type === type)?.logoUrl;
|
||||
};
|
||||
|
||||
export const getSourceConfigs = (sourceType: string) => {
|
||||
const sourceConfigs = SOURCE_TEMPLATE_CONFIGS.find((configs) => configs.type === sourceType);
|
||||
export const getSourceConfigs = (ingestionSources: SourceConfig[], sourceType: string) => {
|
||||
const sourceConfigs = ingestionSources.find((source) => source.name === sourceType);
|
||||
if (!sourceConfigs) {
|
||||
throw new Error(`Failed to find source configs with source type ${sourceType}`);
|
||||
console.error(`Failed to find source configs with source type ${sourceType}`);
|
||||
}
|
||||
return sourceConfigs;
|
||||
};
|
||||
@ -36,6 +32,11 @@ export const jsonToYaml = (json: string): string => {
|
||||
return yamlStr;
|
||||
};
|
||||
|
||||
export function getPlaceholderRecipe(ingestionSources: SourceConfig[], type?: string) {
|
||||
const selectedSource = ingestionSources.find((source) => source.name === type);
|
||||
return selectedSource?.recipe || '';
|
||||
}
|
||||
|
||||
export const RUNNING = 'RUNNING';
|
||||
export const SUCCESS = 'SUCCESS';
|
||||
export const FAILURE = 'FAILURE';
|
||||
|
||||
@ -17,6 +17,9 @@ query listIngestionSources($input: ListIngestionSourcesInput!) {
|
||||
interval
|
||||
timezone
|
||||
}
|
||||
platform {
|
||||
urn
|
||||
}
|
||||
executions(start: 0, count: 1) {
|
||||
start
|
||||
count
|
||||
@ -53,6 +56,9 @@ query getIngestionSource($urn: String!, $runStart: Int, $runCount: Int) {
|
||||
interval
|
||||
timezone
|
||||
}
|
||||
platform {
|
||||
urn
|
||||
}
|
||||
executions(start: $runStart, count: $runCount) {
|
||||
start
|
||||
count
|
||||
|
||||
@ -233,3 +233,19 @@ in [sql_common.py](./src/datahub/ingestion/source/sql/sql_common.py) if the sour
|
||||
### 9. Add logo for the platform
|
||||
|
||||
Add the logo image in [images folder](../datahub-web-react/src/images) and add it to be ingested at [startup](../metadata-service/war/src/main/resources/boot/data_platforms.json)
|
||||
|
||||
### 10. Update Frontend for UI-based ingestion
|
||||
|
||||
We are currently transitioning to a more dynamic approach to display available sources for UI-based Managed Ingestion. For the time being, adhere to these next steps to get your source to display in the UI Ingestion tab.
|
||||
|
||||
#### 10.1 Add to sources.json
|
||||
|
||||
Add new source to the list in [sources.json](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/ingest/source/builder/sources.json) including a default quickstart recipe. This will render your source in the list of options when creating a new recipe in the UI.
|
||||
|
||||
#### 10.2 Add logo to the React app
|
||||
|
||||
Add your source logo to the React [images folder](https://github.com/datahub-project/datahub/tree/master/datahub-web-react/src/images) so your image is available in memory.
|
||||
|
||||
#### 10.3 Update constants.ts
|
||||
|
||||
Create new constants in [constants.ts](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/ingest/source/builder/constants.ts) for the source urn and source name. Update PLATFORM_URN_TO_LOGO to map your source urn to the newly added logo in the images folder.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user