mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-06 14:26:28 +00:00
darth-coder00
GitHub
parent
685cb4733f
commit
ce149c86be
@ -22,7 +22,7 @@
|
||||
"connectionType": {
|
||||
"description": "Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres...",
|
||||
"type": "string",
|
||||
"enum": ["Database", "Dashboard", "Messaging"],
|
||||
"enum": ["Database", "Dashboard", "Messaging", "Pipeline"],
|
||||
"javaEnums": [
|
||||
{
|
||||
"name": "Database"
|
||||
@ -32,6 +32,9 @@
|
||||
},
|
||||
{
|
||||
"name": "Messaging"
|
||||
},
|
||||
{
|
||||
"name": "Pipeline"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -13,7 +13,7 @@ const rootDir = 'connTemp';
|
||||
const srcDir = 'schema/entity/services/connections';
|
||||
const destDir = 'src/jsons/connectionSchemas/connections';
|
||||
|
||||
const playDir = `${rootDir}/${srcDir}/${rootDir}`;
|
||||
const playDir = `${rootDir}/${srcDir}`;
|
||||
|
||||
const globalParserOptions = {
|
||||
continueOnError: true,
|
||||
@ -24,8 +24,11 @@ const globalParserOptions = {
|
||||
|
||||
async function parseSchema(filePath, destPath) {
|
||||
try {
|
||||
const fileDir = `${cwd}/${path.dirname(filePath)}`;
|
||||
const fileName = path.basename(filePath);
|
||||
process.chdir(fileDir);
|
||||
const parser = new $RefParser(globalParserOptions);
|
||||
const schema = await parser.parse(filePath);
|
||||
const schema = await parser.parse(fileName);
|
||||
const api = await parser.bundle(schema);
|
||||
const dirname = `${cwd}/${path.dirname(destPath)}`;
|
||||
if (!fs.existsSync(dirname)) {
|
||||
@ -38,54 +41,44 @@ async function parseSchema(filePath, destPath) {
|
||||
fs.writeFileSync(`${cwd}/${destPath}`, JSON.stringify(api, null, 2));
|
||||
} catch (err) {
|
||||
console.log(err);
|
||||
} finally {
|
||||
process.chdir(cwd);
|
||||
}
|
||||
}
|
||||
|
||||
function traverseDirectory(Directory) {
|
||||
fs.readdirSync(Directory).forEach((File) => {
|
||||
async function traverseDirectory(Directory) {
|
||||
const Files = fs.readdirSync(Directory);
|
||||
for (const File of Files) {
|
||||
const Absolute = path.join(Directory, File);
|
||||
if (fs.statSync(Absolute).isDirectory()) {
|
||||
return traverseDirectory(Absolute);
|
||||
await traverseDirectory(Absolute);
|
||||
} else {
|
||||
const name = Absolute.replace(srcDir, destDir);
|
||||
return parseSchema(Absolute, name);
|
||||
const name = Absolute.replace(playDir, destDir);
|
||||
await parseSchema(Absolute, name);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function copySourceFiles() {
|
||||
try {
|
||||
fse.copySync(schemaDir, `${rootDir}/schema`);
|
||||
fse.copySync(schemaDir, `${playDir}/schema`);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
async function main() {
|
||||
try {
|
||||
if (fs.existsSync(destDir)) {
|
||||
fs.rmSync(destDir, { recursive: true });
|
||||
}
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
copySourceFiles();
|
||||
|
||||
await traverseDirectory(`${playDir}`);
|
||||
} catch (err) {
|
||||
console.log(err);
|
||||
}
|
||||
|
||||
process.chdir(`${cwd}/${playDir}`);
|
||||
|
||||
fs.readdir(srcDir, (err, Files) => {
|
||||
if (err) console.log(err);
|
||||
else {
|
||||
Files.forEach((File, index) => {
|
||||
const Absolute = path.join(srcDir, File);
|
||||
if (fs.statSync(Absolute).isDirectory()) {
|
||||
traverseDirectory(Absolute);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
main();
|
||||
|
||||
@ -122,6 +122,11 @@ const AddIngestion = ({
|
||||
const [showChartFilter, setShowChartFilter] = useState(
|
||||
!isUndefined((data?.sourceConfig.config as ConfigClass)?.chartFilterPattern)
|
||||
);
|
||||
const [showPipelineFilter, setShowPipelineFilter] = useState(
|
||||
!isUndefined(
|
||||
(data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern
|
||||
)
|
||||
);
|
||||
const [showFqnFilter, setShowFqnFilter] = useState(
|
||||
!isUndefined((data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern)
|
||||
);
|
||||
@ -155,6 +160,9 @@ const AddIngestion = ({
|
||||
const [includeView, setIncludeView] = useState(
|
||||
Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews)
|
||||
);
|
||||
const [includeLineage, setIncludeLineage] = useState(
|
||||
Boolean((data?.sourceConfig.config as ConfigClass)?.includeLineage ?? true)
|
||||
);
|
||||
const [enableDebugLog, setEnableDebugLog] = useState(
|
||||
data?.loggerLevel === LogLevels.Debug
|
||||
);
|
||||
@ -184,6 +192,11 @@ const AddIngestion = ({
|
||||
(data?.sourceConfig.config as ConfigClass)?.chartFilterPattern ??
|
||||
INITIAL_FILTER_PATTERN
|
||||
);
|
||||
const [pipelineFilterPattern, setPipelineFilterPattern] =
|
||||
useState<FilterPattern>(
|
||||
(data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern ??
|
||||
INITIAL_FILTER_PATTERN
|
||||
);
|
||||
const [fqnFilterPattern, setFqnFilterPattern] = useState<FilterPattern>(
|
||||
(data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern ??
|
||||
INITIAL_FILTER_PATTERN
|
||||
@ -237,12 +250,16 @@ const AddIngestion = ({
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.CHART:
|
||||
setChartFilterPattern({ ...topicFilterPattern, includes: value });
|
||||
setChartFilterPattern({ ...chartFilterPattern, includes: value });
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.FQN:
|
||||
setFqnFilterPattern({ ...fqnFilterPattern, includes: value });
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.PIPELINE:
|
||||
setPipelineFilterPattern({ ...pipelineFilterPattern, includes: value });
|
||||
|
||||
break;
|
||||
}
|
||||
};
|
||||
@ -272,12 +289,16 @@ const AddIngestion = ({
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.CHART:
|
||||
setChartFilterPattern({ ...topicFilterPattern, excludes: value });
|
||||
setChartFilterPattern({ ...chartFilterPattern, excludes: value });
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.FQN:
|
||||
setFqnFilterPattern({ ...fqnFilterPattern, excludes: value });
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.PIPELINE:
|
||||
setPipelineFilterPattern({ ...pipelineFilterPattern, excludes: value });
|
||||
|
||||
break;
|
||||
}
|
||||
};
|
||||
@ -311,6 +332,10 @@ const AddIngestion = ({
|
||||
case FilterPatternEnum.FQN:
|
||||
setShowFqnFilter(value);
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.PIPELINE:
|
||||
setShowPipelineFilter(value);
|
||||
|
||||
break;
|
||||
}
|
||||
};
|
||||
@ -404,6 +429,16 @@ const AddIngestion = ({
|
||||
type: ConfigType.DashboardMetadata,
|
||||
};
|
||||
}
|
||||
case ServiceCategory.PIPELINE_SERVICES: {
|
||||
return {
|
||||
includeLineage: includeLineage,
|
||||
pipelineFilterPattern: getFilterPatternData(
|
||||
pipelineFilterPattern,
|
||||
showPipelineFilter
|
||||
),
|
||||
type: ConfigType.PipelineMetadata,
|
||||
};
|
||||
}
|
||||
default: {
|
||||
return {};
|
||||
}
|
||||
@ -580,6 +615,7 @@ const AddIngestion = ({
|
||||
getIncludeValue={getIncludeValue}
|
||||
handleDescription={(val) => setDescription(val)}
|
||||
handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)}
|
||||
handleIncludeLineage={() => setIncludeLineage((pre) => !pre)}
|
||||
handleIncludeView={() => setIncludeView((pre) => !pre)}
|
||||
handleIngestionName={(val) => setIngestionName(val)}
|
||||
handleMarkDeletedTables={() => setMarkDeletedTables((pre) => !pre)}
|
||||
@ -587,9 +623,11 @@ const AddIngestion = ({
|
||||
handleResultLimit={(val) => setResultLimit(val)}
|
||||
handleShowFilter={handleShowFilter}
|
||||
handleStageFileLocation={(val) => setStageFileLocation(val)}
|
||||
includeLineage={includeLineage}
|
||||
includeView={includeView}
|
||||
ingestionName={ingestionName}
|
||||
markDeletedTables={markDeletedTables}
|
||||
pipelineFilterPattern={pipelineFilterPattern}
|
||||
pipelineType={pipelineType}
|
||||
queryLogDuration={queryLogDuration}
|
||||
resultLimit={resultLimit}
|
||||
@ -599,6 +637,7 @@ const AddIngestion = ({
|
||||
showDashboardFilter={showDashboardFilter}
|
||||
showDatabaseFilter={showDatabaseFilter}
|
||||
showFqnFilter={showFqnFilter}
|
||||
showPipelineFilter={showPipelineFilter}
|
||||
showSchemaFilter={showSchemaFilter}
|
||||
showTableFilter={showTableFilter}
|
||||
showTopicFilter={showTopicFilter}
|
||||
|
||||
@ -54,10 +54,15 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
|
||||
includes: [],
|
||||
excludes: [],
|
||||
},
|
||||
pipelineFilterPattern: {
|
||||
includes: [],
|
||||
excludes: [],
|
||||
},
|
||||
fqnFilterPattern: {
|
||||
includes: [],
|
||||
excludes: [],
|
||||
},
|
||||
includeLineage: false,
|
||||
includeView: false,
|
||||
pipelineType: PipelineType.Metadata,
|
||||
queryLogDuration: 1,
|
||||
@ -70,7 +75,9 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
|
||||
showTableFilter: false,
|
||||
showTopicFilter: false,
|
||||
showChartFilter: false,
|
||||
showPipelineFilter: false,
|
||||
showFqnFilter: false,
|
||||
handleIncludeLineage: jest.fn(),
|
||||
handleIncludeView: jest.fn(),
|
||||
handleIngestionName: jest.fn(),
|
||||
handleMarkDeletedTables: jest.fn(),
|
||||
|
||||
@ -34,7 +34,9 @@ const ConfigureIngestion = ({
|
||||
tableFilterPattern,
|
||||
topicFilterPattern,
|
||||
chartFilterPattern,
|
||||
pipelineFilterPattern,
|
||||
fqnFilterPattern,
|
||||
includeLineage,
|
||||
includeView,
|
||||
markDeletedTables,
|
||||
serviceCategory,
|
||||
@ -45,6 +47,7 @@ const ConfigureIngestion = ({
|
||||
showTableFilter,
|
||||
showTopicFilter,
|
||||
showChartFilter,
|
||||
showPipelineFilter,
|
||||
showFqnFilter,
|
||||
queryLogDuration,
|
||||
stageFileLocation,
|
||||
@ -56,6 +59,7 @@ const ConfigureIngestion = ({
|
||||
handleIngestionName,
|
||||
handleDescription,
|
||||
handleShowFilter,
|
||||
handleIncludeLineage,
|
||||
handleIncludeView,
|
||||
handleMarkDeletedTables,
|
||||
handleQueryLogDuration,
|
||||
@ -128,6 +132,28 @@ const ConfigureIngestion = ({
|
||||
);
|
||||
};
|
||||
|
||||
const getPipelineFieldToggles = () => {
|
||||
return (
|
||||
<div>
|
||||
<Field>
|
||||
<div className="tw-flex tw-gap-1">
|
||||
<label>Include lineage</label>
|
||||
<ToggleSwitchV1
|
||||
checked={includeLineage}
|
||||
handleCheck={handleIncludeLineage}
|
||||
testId="include-lineage"
|
||||
/>
|
||||
</div>
|
||||
<p className="tw-text-grey-muted tw-mt-3">
|
||||
Configuration to turn off fetching lineage from pipelines.
|
||||
</p>
|
||||
{getSeparator('')}
|
||||
</Field>
|
||||
{getDebugLogToggle()}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const getMetadataFilterPatternField = () => {
|
||||
switch (serviceCategory) {
|
||||
case ServiceCategory.DATABASE_SERVICES:
|
||||
@ -221,6 +247,25 @@ const ConfigureIngestion = ({
|
||||
{getDebugLogToggle()}
|
||||
</Fragment>
|
||||
);
|
||||
case ServiceCategory.PIPELINE_SERVICES:
|
||||
return (
|
||||
<Fragment>
|
||||
<FilterPattern
|
||||
checked={showPipelineFilter}
|
||||
excludePattern={pipelineFilterPattern.excludes ?? []}
|
||||
getExcludeValue={getExcludeValue}
|
||||
getIncludeValue={getIncludeValue}
|
||||
handleChecked={(value) =>
|
||||
handleShowFilter(value, FilterPatternEnum.PIPELINE)
|
||||
}
|
||||
includePattern={pipelineFilterPattern.includes ?? []}
|
||||
showSeparator={false}
|
||||
type={FilterPatternEnum.PIPELINE}
|
||||
/>
|
||||
{getSeparator('')}
|
||||
{getPipelineFieldToggles()}
|
||||
</Fragment>
|
||||
);
|
||||
default:
|
||||
return <></>;
|
||||
}
|
||||
|
||||
@ -64,7 +64,9 @@ export interface ConfigureIngestionProps {
|
||||
tableFilterPattern: FilterPattern;
|
||||
topicFilterPattern: FilterPattern;
|
||||
chartFilterPattern: FilterPattern;
|
||||
pipelineFilterPattern: FilterPattern;
|
||||
fqnFilterPattern: FilterPattern;
|
||||
includeLineage: boolean;
|
||||
includeView: boolean;
|
||||
markDeletedTables?: boolean;
|
||||
enableDebugLog: boolean;
|
||||
@ -75,12 +77,14 @@ export interface ConfigureIngestionProps {
|
||||
showTableFilter: boolean;
|
||||
showTopicFilter: boolean;
|
||||
showChartFilter: boolean;
|
||||
showPipelineFilter: boolean;
|
||||
showFqnFilter: boolean;
|
||||
queryLogDuration: number;
|
||||
stageFileLocation: string;
|
||||
resultLimit: number;
|
||||
handleIngestionName: (value: string) => void;
|
||||
handleDescription?: (value: string) => void;
|
||||
handleIncludeLineage: () => void;
|
||||
handleIncludeView: () => void;
|
||||
handleMarkDeletedTables?: () => void;
|
||||
handleEnableDebugLog: () => void;
|
||||
|
||||
@ -21,17 +21,12 @@ import { FormSubmitType } from '../../enums/form.enum';
|
||||
import { PageLayoutType } from '../../enums/layout.enum';
|
||||
import { ServiceCategory } from '../../enums/service.enum';
|
||||
import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
|
||||
import {
|
||||
ConfigData,
|
||||
DataObj,
|
||||
DataService,
|
||||
} from '../../interface/service.interface';
|
||||
import { ConfigData, DataObj } from '../../interface/service.interface';
|
||||
import { getCurrentUserId } from '../../utils/CommonUtils';
|
||||
import { getAddServicePath } from '../../utils/RouterUtils';
|
||||
import {
|
||||
getServiceCreatedLabel,
|
||||
getServiceIngestionStepGuide,
|
||||
isIngestionSupported,
|
||||
} from '../../utils/ServiceUtils';
|
||||
import AddIngestion from '../AddIngestion/AddIngestion.component';
|
||||
import SuccessScreen from '../common/success-screen/SuccessScreen';
|
||||
@ -129,10 +124,7 @@ const AddService = ({
|
||||
});
|
||||
};
|
||||
|
||||
const handleConfigUpdate = (
|
||||
oData: ConfigData,
|
||||
serviceCat: ServiceCategory
|
||||
) => {
|
||||
const handleConfigUpdate = (oData: ConfigData) => {
|
||||
const data = {
|
||||
name: serviceName,
|
||||
serviceType: selectServiceType,
|
||||
@ -142,15 +134,12 @@ const AddService = ({
|
||||
type: 'user',
|
||||
},
|
||||
};
|
||||
const configData =
|
||||
serviceCat === ServiceCategory.PIPELINE_SERVICES
|
||||
? { ...data, pipelineUrl: oData.pipelineUrl }
|
||||
: {
|
||||
...data,
|
||||
connection: {
|
||||
config: oData,
|
||||
},
|
||||
};
|
||||
const configData = {
|
||||
...data,
|
||||
connection: {
|
||||
config: oData,
|
||||
},
|
||||
};
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
setSaveServiceState('waiting');
|
||||
@ -229,30 +218,23 @@ const AddService = ({
|
||||
{activeServiceStep === 3 && (
|
||||
<ConnectionConfigForm
|
||||
cancelText="Back"
|
||||
data={
|
||||
(serviceCategory !== ServiceCategory.PIPELINE_SERVICES
|
||||
? {
|
||||
connection: { config: { type: selectServiceType } },
|
||||
}
|
||||
: {}) as DataService
|
||||
}
|
||||
serviceCategory={serviceCategory}
|
||||
serviceType={selectServiceType}
|
||||
status={saveServiceState}
|
||||
onCancel={handleConnectionDetailsBackClick}
|
||||
onSave={(e) => {
|
||||
handleConfigUpdate(e.formData, serviceCategory);
|
||||
handleConfigUpdate(e.formData);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{activeServiceStep > 3 && (
|
||||
<SuccessScreen
|
||||
showIngestionButton
|
||||
handleIngestionClick={() => handleAddIngestion(true)}
|
||||
handleViewServiceClick={handleViewServiceClick}
|
||||
isAirflowSetup={isAirflowRunning}
|
||||
name={serviceName}
|
||||
showIngestionButton={isIngestionSupported(serviceCategory)}
|
||||
state={FormSubmitType.ADD}
|
||||
suffix={getServiceCreatedLabel(serviceCategory)}
|
||||
onCheckAirflowStatus={onAirflowStatusCheck}
|
||||
|
||||
@ -18,19 +18,22 @@ import React, { Fragment, FunctionComponent, useMemo } from 'react';
|
||||
import { TestConnection } from '../../axiosAPIs/serviceAPI';
|
||||
import { ServiceCategory } from '../../enums/service.enum';
|
||||
import {
|
||||
DashboardConnection,
|
||||
DashboardService,
|
||||
DashboardServiceType,
|
||||
} from '../../generated/entity/services/dashboardService';
|
||||
import {
|
||||
DatabaseConnection,
|
||||
DatabaseService,
|
||||
DatabaseServiceType,
|
||||
} from '../../generated/entity/services/databaseService';
|
||||
import {
|
||||
MessagingConnection,
|
||||
MessagingService,
|
||||
MessagingServiceType,
|
||||
} from '../../generated/entity/services/messagingService';
|
||||
import { PipelineService } from '../../generated/entity/services/pipelineService';
|
||||
import { ConfigData } from '../../interface/service.interface';
|
||||
import {
|
||||
PipelineService,
|
||||
PipelineServiceType,
|
||||
} from '../../generated/entity/services/pipelineService';
|
||||
import { ConfigData, DataService } from '../../interface/service.interface';
|
||||
import jsonData from '../../jsons/en';
|
||||
import { getDashboardConfig } from '../../utils/DashboardServiceUtils';
|
||||
import { getDatabaseConfig } from '../../utils/DatabaseServiceUtils';
|
||||
@ -45,7 +48,11 @@ import { showErrorToast } from '../../utils/ToastUtils';
|
||||
import FormBuilder from '../common/FormBuilder/FormBuilder';
|
||||
|
||||
interface Props {
|
||||
data: DatabaseService | MessagingService | DashboardService | PipelineService;
|
||||
data?:
|
||||
| DatabaseService
|
||||
| MessagingService
|
||||
| DashboardService
|
||||
| PipelineService;
|
||||
okText?: string;
|
||||
cancelText?: string;
|
||||
serviceType: string;
|
||||
@ -66,17 +73,11 @@ const ConnectionConfigForm: FunctionComponent<Props> = ({
|
||||
onSave,
|
||||
}: Props) => {
|
||||
const allowTestConn = useMemo(() => {
|
||||
return shouldTestConnection(serviceType, serviceCategory);
|
||||
}, [serviceType, serviceCategory]);
|
||||
return shouldTestConnection(serviceType);
|
||||
}, [serviceType]);
|
||||
|
||||
const config = !isNil(data)
|
||||
? /* eslint-disable-next-line no-prototype-builtins */
|
||||
data.hasOwnProperty('connection')
|
||||
? ((data as DatabaseService | MessagingService | DashboardService)
|
||||
.connection.config as ConfigData)
|
||||
: ({
|
||||
pipelineUrl: (data as PipelineService).connection.config?.hostPort,
|
||||
} as ConfigData)
|
||||
? ((data as DataService).connection.config as ConfigData)
|
||||
: ({} as ConfigData);
|
||||
|
||||
const handleSave = (data: ISubmitEvent<ConfigData>) => {
|
||||
@ -124,28 +125,22 @@ const ConnectionConfigForm: FunctionComponent<Props> = ({
|
||||
|
||||
switch (serviceCategory) {
|
||||
case ServiceCategory.DATABASE_SERVICES: {
|
||||
connSch = getDatabaseConfig(
|
||||
validConfig as DatabaseConnection['config']
|
||||
);
|
||||
connSch = getDatabaseConfig(serviceType as DatabaseServiceType);
|
||||
|
||||
break;
|
||||
}
|
||||
case ServiceCategory.MESSAGING_SERVICES: {
|
||||
connSch = getMessagingConfig(
|
||||
validConfig as MessagingConnection['config']
|
||||
);
|
||||
connSch = getMessagingConfig(serviceType as MessagingServiceType);
|
||||
|
||||
break;
|
||||
}
|
||||
case ServiceCategory.DASHBOARD_SERVICES: {
|
||||
connSch = getDashboardConfig(
|
||||
validConfig as DashboardConnection['config']
|
||||
);
|
||||
connSch = getDashboardConfig(serviceType as DashboardServiceType);
|
||||
|
||||
break;
|
||||
}
|
||||
case ServiceCategory.PIPELINE_SERVICES: {
|
||||
connSch = getPipelineConfig();
|
||||
connSch = getPipelineConfig(serviceType as PipelineServiceType);
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
@ -131,10 +131,17 @@ export const STEPS_FOR_ADD_SERVICE: Array<StepperStepType> = [
|
||||
{ name: 'Connection Details', step: 3 },
|
||||
];
|
||||
|
||||
export const COMMON_UI_SCHEMA = {
|
||||
const DEF_UI_SCHEMA = {
|
||||
supportsMetadataExtraction: { 'ui:widget': 'hidden', 'ui:hideError': true },
|
||||
supportsUsageExtraction: { 'ui:widget': 'hidden', 'ui:hideError': true },
|
||||
supportsProfiler: { 'ui:widget': 'hidden', 'ui:hideError': true },
|
||||
supportsDatabase: { 'ui:widget': 'hidden', 'ui:hideError': true },
|
||||
type: { 'ui:widget': 'hidden' },
|
||||
};
|
||||
|
||||
export const COMMON_UI_SCHEMA = {
|
||||
...DEF_UI_SCHEMA,
|
||||
connection: {
|
||||
...DEF_UI_SCHEMA,
|
||||
},
|
||||
};
|
||||
|
||||
@ -19,4 +19,5 @@ export enum FilterPatternEnum {
|
||||
DASHBOARD = 'dashboard',
|
||||
TOPIC = 'topic',
|
||||
FQN = 'fqn',
|
||||
PIPELINE = 'pipeline',
|
||||
}
|
||||
|
||||
@ -75,6 +75,5 @@ export interface ServiceResponse {
|
||||
|
||||
export type ConfigData = Partial<DatabaseService['connection']> &
|
||||
Partial<MessagingService['connection']> &
|
||||
Partial<DashboardService['connection']> & {
|
||||
pipelineUrl: string;
|
||||
};
|
||||
Partial<DashboardService['connection']> &
|
||||
Partial<PipelineService['connection']>;
|
||||
|
||||
@ -63,7 +63,7 @@ import { IngestionPipeline } from '../../generated/entity/services/ingestionPipe
|
||||
import { EntityReference } from '../../generated/type/entityReference';
|
||||
import { Paging } from '../../generated/type/paging';
|
||||
import { useAuth } from '../../hooks/authHooks';
|
||||
import { ServiceDataObj } from '../../interface/service.interface';
|
||||
import { ConfigData, ServiceDataObj } from '../../interface/service.interface';
|
||||
import jsonData from '../../jsons/en';
|
||||
import {
|
||||
getEntityDeleteMessage,
|
||||
@ -77,7 +77,6 @@ import { getInfoElements } from '../../utils/EntityUtils';
|
||||
import { getServicesWithTabPath } from '../../utils/RouterUtils';
|
||||
import {
|
||||
getCurrentServiceTab,
|
||||
getIsIngestionEnable,
|
||||
getServiceCategoryFromType,
|
||||
servicePageTabs,
|
||||
serviceTypeLogo,
|
||||
@ -96,9 +95,6 @@ const ServicePage: FunctionComponent = () => {
|
||||
const [serviceName, setServiceName] = useState(
|
||||
serviceCategory || getServiceCategoryFromType(serviceType)
|
||||
);
|
||||
const [isIngestionEnable] = useState(
|
||||
getIsIngestionEnable(serviceName as ServiceCategory)
|
||||
);
|
||||
const [slashedTableName, setSlashedTableName] = useState<
|
||||
TitleBreadcrumbProps['titleLinks']
|
||||
>([]);
|
||||
@ -155,7 +151,6 @@ const ServicePage: FunctionComponent = () => {
|
||||
title: 'Sample Data',
|
||||
selectedName: 'sample-data-color',
|
||||
},
|
||||
isHidden: !isIngestionEnable,
|
||||
isProtected: false,
|
||||
position: 2,
|
||||
count: ingestions.length,
|
||||
@ -340,32 +335,16 @@ const ServicePage: FunctionComponent = () => {
|
||||
}).finally(() => setIsloading(false));
|
||||
};
|
||||
|
||||
const handleConfigUpdate = (
|
||||
updatedData: ServicesData,
|
||||
serviceCategory: ServiceCategory
|
||||
) => {
|
||||
const configData =
|
||||
serviceCategory === ServiceCategory.PIPELINE_SERVICES
|
||||
? {
|
||||
databaseConnection: updatedData.databaseConnection,
|
||||
name: updatedData.name,
|
||||
serviceType: updatedData.serviceType,
|
||||
brokers: updatedData.brokers,
|
||||
schemaRegistry: updatedData.schemaRegistry,
|
||||
dashboardUrl: updatedData.dashboardUrl,
|
||||
username: updatedData.username,
|
||||
password: updatedData.password,
|
||||
pipelineUrl: updatedData.pipelineUrl,
|
||||
}
|
||||
: {
|
||||
name: serviceDetails?.name,
|
||||
serviceType: serviceDetails?.serviceType,
|
||||
description: serviceDetails?.description,
|
||||
owner: serviceDetails?.owner,
|
||||
connection: {
|
||||
config: updatedData,
|
||||
},
|
||||
};
|
||||
const handleConfigUpdate = (updatedData: ConfigData) => {
|
||||
const configData = {
|
||||
name: serviceDetails?.name,
|
||||
serviceType: serviceDetails?.serviceType,
|
||||
description: serviceDetails?.description,
|
||||
owner: serviceDetails?.owner,
|
||||
connection: {
|
||||
config: updatedData,
|
||||
},
|
||||
};
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
updateService(serviceName, serviceDetails?.id, configData)
|
||||
@ -735,16 +714,14 @@ const ServicePage: FunctionComponent = () => {
|
||||
activeTabHandler(1);
|
||||
}
|
||||
|
||||
if (isIngestionEnable) {
|
||||
getAirflowStatus()
|
||||
.then(() => {
|
||||
setIsAirflowRunning(true);
|
||||
getAllIngestionWorkflows();
|
||||
})
|
||||
.catch(() => {
|
||||
setIsAirflowRunning(false);
|
||||
});
|
||||
}
|
||||
getAirflowStatus()
|
||||
.then(() => {
|
||||
setIsAirflowRunning(true);
|
||||
getAllIngestionWorkflows();
|
||||
})
|
||||
.catch(() => {
|
||||
setIsAirflowRunning(false);
|
||||
});
|
||||
}, []);
|
||||
|
||||
const onCancel = () => {
|
||||
|
||||
@ -30,10 +30,10 @@ export const getDashboardURL = (config: DashboardConnection['config']) => {
|
||||
: '--';
|
||||
};
|
||||
|
||||
export const getDashboardConfig = (config?: DashboardConnection['config']) => {
|
||||
export const getDashboardConfig = (type: DashboardServiceType) => {
|
||||
let schema = {};
|
||||
const uiSchema = { ...COMMON_UI_SCHEMA };
|
||||
switch (config?.type) {
|
||||
switch (type) {
|
||||
case DashboardServiceType.Looker: {
|
||||
schema = lookerConnection;
|
||||
|
||||
|
||||
@ -13,10 +13,7 @@
|
||||
|
||||
import { cloneDeep } from 'lodash';
|
||||
import { COMMON_UI_SCHEMA } from '../constants/services.const';
|
||||
import {
|
||||
DatabaseConnection,
|
||||
DatabaseServiceType,
|
||||
} from '../generated/entity/services/databaseService';
|
||||
import { DatabaseServiceType } from '../generated/entity/services/databaseService';
|
||||
import athenaConnection from '../jsons/connectionSchemas/connections/database/athenaConnection.json';
|
||||
import azureSQLConnection from '../jsons/connectionSchemas/connections/database/azureSQLConnection.json';
|
||||
import bigQueryConnection from '../jsons/connectionSchemas/connections/database/bigQueryConnection.json';
|
||||
@ -43,10 +40,10 @@ import sqliteConnection from '../jsons/connectionSchemas/connections/database/sq
|
||||
import trinoConnection from '../jsons/connectionSchemas/connections/database/trinoConnection.json';
|
||||
import verticaConnection from '../jsons/connectionSchemas/connections/database/verticaConnection.json';
|
||||
|
||||
export const getDatabaseConfig = (config?: DatabaseConnection['config']) => {
|
||||
export const getDatabaseConfig = (type: DatabaseServiceType) => {
|
||||
let schema = {};
|
||||
const uiSchema = { ...COMMON_UI_SCHEMA };
|
||||
switch (config?.type as unknown as DatabaseServiceType) {
|
||||
switch (type as unknown as DatabaseServiceType) {
|
||||
case DatabaseServiceType.Athena: {
|
||||
schema = athenaConnection;
|
||||
|
||||
|
||||
@ -30,10 +30,10 @@ export const getBrokers = (config: MessagingConnection['config']) => {
|
||||
return !isUndefined(retVal) ? retVal : '--';
|
||||
};
|
||||
|
||||
export const getMessagingConfig = (config: MessagingConnection['config']) => {
|
||||
export const getMessagingConfig = (type: MessagingServiceType) => {
|
||||
let schema = {};
|
||||
const uiSchema = { ...COMMON_UI_SCHEMA };
|
||||
if (config?.type === MessagingServiceType.Kafka) {
|
||||
if (type === MessagingServiceType.Kafka) {
|
||||
schema = kafkaConnection;
|
||||
}
|
||||
|
||||
|
||||
@ -11,25 +11,27 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { JSONSchema7 } from 'json-schema';
|
||||
import { cloneDeep } from 'lodash';
|
||||
import { COMMON_UI_SCHEMA } from '../constants/services.const';
|
||||
import { PipelineServiceType } from '../generated/entity/services/pipelineService';
|
||||
import airflowConnection from '../jsons/connectionSchemas/connections/pipeline/airflowConnection.json';
|
||||
import glueConnection from '../jsons/connectionSchemas/connections/pipeline/glueConnection.json';
|
||||
|
||||
export const getPipelineConfig = () => {
|
||||
const schema = {
|
||||
$schema: 'http://json-schema.org/draft-07/schema#',
|
||||
title: 'PipelineConnection',
|
||||
description: 'Pipeline Connection Config',
|
||||
type: 'object',
|
||||
properties: {
|
||||
pipelineUrl: {
|
||||
description: 'Pipeline Service Management/UI URL.',
|
||||
type: 'string',
|
||||
format: 'uri',
|
||||
},
|
||||
},
|
||||
additionalProperties: false,
|
||||
} as JSONSchema7;
|
||||
const uiSchema = {};
|
||||
export const getPipelineConfig = (type: PipelineServiceType) => {
|
||||
let schema = {};
|
||||
const uiSchema = { ...COMMON_UI_SCHEMA };
|
||||
switch (type) {
|
||||
case PipelineServiceType.Airflow: {
|
||||
schema = airflowConnection;
|
||||
|
||||
break;
|
||||
}
|
||||
case PipelineServiceType.Glue: {
|
||||
schema = glueConnection;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return cloneDeep({ schema, uiSchema });
|
||||
};
|
||||
|
||||
@ -339,23 +339,6 @@ export const getTotalEntityCountByService = (buckets: Array<Bucket> = []) => {
|
||||
return entityCounts;
|
||||
};
|
||||
|
||||
export const getIsIngestionEnable = (serviceCategory: ServiceCategory) => {
|
||||
switch (serviceCategory) {
|
||||
case ServiceCategory.DATABASE_SERVICES:
|
||||
case ServiceCategory.MESSAGING_SERVICES:
|
||||
case ServiceCategory.DASHBOARD_SERVICES:
|
||||
return true;
|
||||
|
||||
case ServiceCategory.PIPELINE_SERVICES:
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
export const isIngestionSupported = (serviceCategory: ServiceCategory) => {
|
||||
return getIsIngestionEnable(serviceCategory);
|
||||
};
|
||||
|
||||
export const getKeyValuePair = (obj: DynamicObj) => {
|
||||
return Object.entries(obj).map((v) => {
|
||||
return {
|
||||
@ -558,14 +541,8 @@ export const getIngestionName = (
|
||||
}
|
||||
};
|
||||
|
||||
export const shouldTestConnection = (
|
||||
serviceType: string,
|
||||
serviceCategory: ServiceCategory
|
||||
) => {
|
||||
return (
|
||||
serviceType !== DatabaseServiceType.SampleData &&
|
||||
serviceCategory !== ServiceCategory.PIPELINE_SERVICES
|
||||
);
|
||||
export const shouldTestConnection = (serviceType: string) => {
|
||||
return serviceType !== DatabaseServiceType.SampleData;
|
||||
};
|
||||
|
||||
export const getTestConnectionType = (serviceCat: ServiceCategory) => {
|
||||
@ -574,6 +551,8 @@ export const getTestConnectionType = (serviceCat: ServiceCategory) => {
|
||||
return ConnectionType.Messaging;
|
||||
case ServiceCategory.DASHBOARD_SERVICES:
|
||||
return ConnectionType.Dashboard;
|
||||
case ServiceCategory.PIPELINE_SERVICES:
|
||||
return ConnectionType.Pipeline;
|
||||
case ServiceCategory.DATABASE_SERVICES:
|
||||
default:
|
||||
return ConnectionType.Database;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user