yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-25 17:08:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingestion-ui) Add ability to set debug_mode on UI ingestion sources (#5762)

Browse Source
This commit is contained in:
Chris Collins 2022-08-29 16:39:14 -04:00 committed by GitHub
parent 33754d41f5
commit 0d35b53b41
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 59 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java
View File

@ -142,6 +142,7 @@ public class IngestionResolverUtils {
result.setRecipe(config.getRecipe());
result.setVersion(config.getVersion());
result.setExecutorId(config.getExecutorId());
result.setDebugMode(config.isDebugMode());
return result;
}

6
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java
View File

@ -44,6 +44,7 @@ public class CreateIngestionExecutionRequestResolver implements DataFetcher<Comp
private static final String MANUAL_EXECUTION_SOURCE_NAME = "MANUAL_INGESTION_SOURCE";
private static final String RECIPE_ARG_NAME = "recipe";
private static final String VERSION_ARG_NAME = "version";
private static final String DEBUG_MODE_ARG_NAME = "debug_mode";
private final EntityClient _entityClient;
private final IngestionConfiguration _ingestionConfiguration;
@ -112,6 +113,11 @@ public class CreateIngestionExecutionRequestResolver implements DataFetcher<Comp
if (ingestionSourceInfo.getConfig().hasVersion()) {
arguments.put(VERSION_ARG_NAME, ingestionSourceInfo.getConfig().getVersion());
}
String debugMode = "false";
if (ingestionSourceInfo.getConfig().hasDebugMode()) {
debugMode = ingestionSourceInfo.getConfig().isDebugMode() ? "true" : "false";
}
arguments.put(DEBUG_MODE_ARG_NAME, debugMode);
execInput.setArgs(new StringMap(arguments));
proposal.setEntityType(Constants.EXECUTION_REQUEST_ENTITY_NAME);

1
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java
View File

@ -122,6 +122,7 @@ public class UpsertIngestionSourceResolver implements DataFetcher<CompletableFut
if (input.getExecutorId() != null) {
result.setExecutorId(input.getExecutorId());
}
result.setDebugMode(input.getDebugMode());
return result;
}

10
datahub-graphql-core/src/main/resources/ingestion.graphql
View File

@ -322,6 +322,11 @@ type IngestionConfig {
Advanced: The version of the ingestion framework to use
"""
version: String
"""
Advanced: Whether or not to run ingestion in debug mode
"""
debugMode: Boolean
}
"""
@ -458,6 +463,11 @@ input UpdateIngestionSourceConfigInput {
The id of the executor to use for executing the recipe
"""
executorId: String!
"""
Whether or not to run ingestion in debug mode
"""
debugMode: Boolean!
}
"""

2
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java
View File

@ -28,7 +28,7 @@ public class UpsertIngestionSourceResolverTest {
"Test source",
"mysql", "Test source description",
new UpdateIngestionSourceScheduleInput("* * * * *", "UTC"),
new UpdateIngestionSourceConfigInput("my test recipe", "0.8.18", "executor id")
new UpdateIngestionSourceConfigInput("my test recipe", "0.8.18", "executor id", false)
);
@Test

1
datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
View File

@ -263,6 +263,7 @@ export const IngestionSourceList = () => {
(recipeBuilderState.config?.executorId?.length &&
(recipeBuilderState.config?.executorId as string)) ||
DEFAULT_EXECUTOR_ID,
debugMode: recipeBuilderState.config?.debugMode || false,
},
schedule: recipeBuilderState.schedule && {
interval: recipeBuilderState.schedule?.interval as string,

22
datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
View File

@ -1,4 +1,4 @@
import { Button, Collapse, Form, Input, Typography } from 'antd';
import { Button, Checkbox, Collapse, Form, Input, Typography } from 'antd';
import React from 'react';
import styled from 'styled-components';
import { SourceBuilderState, StepProps } from './types';
@ -44,6 +44,17 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
updateState(newState);
};
const setDebugMode = (debugMode: boolean) => {
const newState: SourceBuilderState = {
...state,
config: {
...state.config,
debugMode,
},
};
updateState(newState);
};
const onClickCreate = (shouldRun?: boolean) => {
if (state.name !== undefined && state.name.length > 0) {
submit(shouldRun);
@ -92,6 +103,15 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
onChange={(event) => setVersion(event.target.value)}
/>
</Form.Item>
<Form.Item label={<Typography.Text strong>Debug Mode</Typography.Text>}>
<Typography.Paragraph>
Advanced: Turn on debug mode in order to get more verbose logs.
</Typography.Paragraph>
<Checkbox
checked={state.config?.debugMode || false}
onChange={(event) => setDebugMode(event.target.checked)}
/>
</Form.Item>
</Collapse.Panel>
</Collapse>
</Form>

5
datahub-web-react/src/app/ingest/source/builder/types.ts
View File

@ -77,5 +77,10 @@ export interface SourceBuilderState {
* Advanced: The version of the DataHub Ingestion Framework to use to perform ingestion
*/
version?: string | null;
/**
* Advanced: Whether or not to run this ingestion source in debug mode
*/
debugMode?: boolean | null;
};
}

2
datahub-web-react/src/graphql/ingestion.graphql
View File

@ -11,6 +11,7 @@ query listIngestionSources($input: ListIngestionSourcesInput!) {
recipe
version
executorId
debugMode
}
schedule {
interval
@ -46,6 +47,7 @@ query getIngestionSource($urn: String!, $runStart: Int, $runCount: Int) {
recipe
version
executorId
debugMode
}
schedule {
interval

6
ingestion-scheduler/src/main/java/com/datahub/metadata/ingestion/IngestionScheduler.java
View File

@ -283,6 +283,7 @@ public class IngestionScheduler {
private static final String EXECUTION_REQUEST_SOURCE_NAME = "SCHEDULED_INGESTION_SOURCE";
private static final String RECIPE_ARGUMENT_NAME = "recipe";
private static final String VERSION_ARGUMENT_NAME = "version";
private static final String DEBUG_MODE_ARG_NAME = "debug_mode";
private final Authentication _systemAuthentication;
private final EntityClient _entityClient;
@ -350,6 +351,11 @@ public class IngestionScheduler {
arguments.put(VERSION_ARGUMENT_NAME, _ingestionSourceInfo.getConfig().hasVersion()
? _ingestionSourceInfo.getConfig().getVersion()
: _ingestionConfiguration.getDefaultCliVersion());
String debugMode = "false";
if (_ingestionSourceInfo.getConfig().hasDebugMode()) {
debugMode = _ingestionSourceInfo.getConfig().isDebugMode() ? "true" : "false";
}
arguments.put(DEBUG_MODE_ARG_NAME, debugMode);
input.setArgs(new StringMap(arguments));
proposal.setEntityType(Constants.EXECUTION_REQUEST_ENTITY_NAME);

5
metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl
View File

@ -51,5 +51,10 @@ record DataHubIngestionSourceInfo {
* The id of the executor to use to execute the ingestion run
*/
executorId: optional string
/**
* Whether or not to run this ingestion source in debug mode
*/
debugMode: optional boolean
}
}