yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-06-27 05:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingestion) cleaning up ingestion page UI (#12710)

Browse Source
This commit is contained in:
Jay 2025-04-03 17:55:18 -04:00 committed by GitHub
parent 046c59bdb5
commit 372feeeade
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 549 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
View File

@ -110,6 +110,7 @@ import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateIngestionEx
import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateTestConnectionRequestResolver;
import com.linkedin.datahub.graphql.resolvers.ingest.execution.GetIngestionExecutionRequestResolver;
import com.linkedin.datahub.graphql.resolvers.ingest.execution.IngestionSourceExecutionRequestsResolver;
import com.linkedin.datahub.graphql.resolvers.ingest.execution.ListExecutionRequestsResolver;
import com.linkedin.datahub.graphql.resolvers.ingest.execution.RollbackIngestionResolver;
import com.linkedin.datahub.graphql.resolvers.ingest.secret.CreateSecretResolver;
import com.linkedin.datahub.graphql.resolvers.ingest.secret.DeleteSecretResolver;
@ -999,6 +1000,8 @@ public class GmsGraphQLEngine {
.dataFetcher(
"listIngestionSources", new ListIngestionSourcesResolver(this.entityClient))
.dataFetcher("ingestionSource", new GetIngestionSourceResolver(this.entityClient))
.dataFetcher(
"listExecutionRequests", new ListExecutionRequestsResolver(this.entityClient))
.dataFetcher(
"executionRequest", new GetIngestionExecutionRequestResolver(this.entityClient))
.dataFetcher("getSchemaBlame", new GetSchemaBlameResolver(this.timelineService))

6
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java
View File

@ -68,6 +68,9 @@ public class IngestionResolverUtils {
if (executionRequestInput.getActorUrn() != null) {
inputResult.setActorUrn(executionRequestInput.getActorUrn().toString());
}
if (executionRequestInput.hasExecutorId()) {
inputResult.setExecutorId(executionRequestInput.getExecutorId());
}
result.setInput(inputResult);
}
@ -88,6 +91,9 @@ public class IngestionResolverUtils {
final com.linkedin.datahub.graphql.generated.ExecutionRequestSource result =
new com.linkedin.datahub.graphql.generated.ExecutionRequestSource();
result.setType(execRequestSource.getType());
if (execRequestSource.hasIngestionSource()) {
result.setIngestionSource(execRequestSource.getIngestionSource().toString());
}
return result;
}

115
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/ListExecutionRequestsResolver.java Normal file
View File

@ -0,0 +1,115 @@
package com.linkedin.datahub.graphql.resolvers.ingest.execution;
import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.buildFilter;
import com.google.common.collect.ImmutableSet;
import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils;
import com.linkedin.datahub.graphql.generated.FacetFilterInput;
import com.linkedin.datahub.graphql.generated.IngestionSourceExecutionRequests;
import com.linkedin.datahub.graphql.generated.ListExecutionRequestsInput;
import com.linkedin.datahub.graphql.resolvers.ingest.IngestionResolverUtils;
import com.linkedin.entity.EntityResponse;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.query.filter.SortCriterion;
import com.linkedin.metadata.query.filter.SortOrder;
import com.linkedin.metadata.search.SearchEntity;
import com.linkedin.metadata.search.SearchResult;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
public class ListExecutionRequestsResolver
implements DataFetcher<CompletableFuture<IngestionSourceExecutionRequests>> {
private static final String DEFAULT_QUERY = "*";
private static final Integer DEFAULT_START = 0;
private static final Integer DEFAULT_COUNT = 20;
private final EntityClient _entityClient;
@Override
public CompletableFuture<IngestionSourceExecutionRequests> get(
final DataFetchingEnvironment environment) throws Exception {
final QueryContext context = environment.getContext();
final ListExecutionRequestsInput input =
bindArgument(environment.getArgument("input"), ListExecutionRequestsInput.class);
final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart();
final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount();
final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery();
final List<FacetFilterInput> filters =
input.getFilters() == null ? Collections.emptyList() : input.getFilters();
// construct sort criteria, defaulting to systemCreated
final SortCriterion sortCriterion;
// if query is expecting to sort by something, use that
final com.linkedin.datahub.graphql.generated.SortCriterion sortCriterionInput = input.getSort();
if (sortCriterionInput != null) {
sortCriterion =
new SortCriterion()
.setField(sortCriterionInput.getField())
.setOrder(SortOrder.valueOf(sortCriterionInput.getSortOrder().name()));
} else {
sortCriterion = new SortCriterion().setField("requestTimeMs").setOrder(SortOrder.DESCENDING);
}
return GraphQLConcurrencyUtils.supplyAsync(
() -> {
try {
// First, get all execution request Urns.
final SearchResult gmsResult =
_entityClient.search(
context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)),
Constants.EXECUTION_REQUEST_ENTITY_NAME,
query,
buildFilter(filters, Collections.emptyList()),
sortCriterion != null ? List.of(sortCriterion) : null,
start,
count);
log.info(String.format("Found %d execution requests", gmsResult.getNumEntities()));
final List<Urn> entitiesUrnList =
gmsResult.getEntities().stream().map(SearchEntity::getEntity).toList();
// Then, resolve all execution requests
final Map<Urn, EntityResponse> entities =
_entityClient.batchGetV2(
context.getOperationContext(),
Constants.EXECUTION_REQUEST_ENTITY_NAME,
new HashSet<>(entitiesUrnList),
ImmutableSet.of(
Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME,
Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME));
final List<EntityResponse> entitiesOrdered =
entitiesUrnList.stream().map(entities::get).filter(Objects::nonNull).toList();
// Now that we have entities we can bind this to a result.
final IngestionSourceExecutionRequests result = new IngestionSourceExecutionRequests();
result.setStart(gmsResult.getFrom());
result.setCount(gmsResult.getPageSize());
result.setTotal(gmsResult.getNumEntities());
result.setExecutionRequests(
IngestionResolverUtils.mapExecutionRequests(context, entitiesOrdered));
return result;
} catch (Exception e) {
throw new RuntimeException("Failed to list executions", e);
}
},
this.getClass().getSimpleName(),
"get");
}
}

42
datahub-graphql-core/src/main/resources/ingestion.graphql
View File

@ -22,6 +22,11 @@ extend type Query {
urn: The primary key associated with the ingestion source.
"""
ingestionSource(urn: String!): IngestionSource
"""
List all execution requests
"""
listExecutionRequests(input: ListExecutionRequestsInput!): IngestionSourceExecutionRequests
"""
Get an execution request
@ -92,6 +97,11 @@ type ExecutionRequestSource {
The type of the source, e.g. SCHEDULED_INGESTION_SOURCE
"""
type: String
"""
The urn of the ingestion source, if applicable
"""
ingestionSource: String
}
"""
@ -122,6 +132,11 @@ type ExecutionRequestInput {
Urn of the actor who created this execution request
"""
actorUrn: String
"""
The specific executor to route the request to. If none is provided, a "default" executor is used.
"""
executorId: String
}
"""
@ -360,6 +375,33 @@ type IngestionRun {
executionRequestUrn: String
}
input ListExecutionRequestsInput {
"""
The starting offset of the result set
"""
start: Int
"""
The number of results to be returned
"""
count: Int
"""
An optional search query
"""
query: String
"""
Optional Facet filters to apply to the result set
"""
filters: [FacetFilterInput!]
"""
Optional sort order. Defaults to use systemCreated.
"""
sort: SortCriterion
}
"""
Requests for execution associated with an ingestion source
"""

251
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/ListExecutionRequestsResolverTest.java Normal file
View File

@ -0,0 +1,251 @@
package com.linkedin.datahub.graphql.resolvers.ingest.execution;
import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.*;
import static org.mockito.ArgumentMatchers.any;
import static org.testng.Assert.*;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.ListExecutionRequestsInput;
import com.linkedin.entity.Aspect;
import com.linkedin.entity.EntityResponse;
import com.linkedin.entity.EnvelopedAspect;
import com.linkedin.entity.EnvelopedAspectMap;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.execution.ExecutionRequestInput;
import com.linkedin.execution.ExecutionRequestResult;
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.search.SearchEntity;
import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchResult;
import com.linkedin.r2.RemoteInvocationException;
import graphql.schema.DataFetchingEnvironment;
import java.util.HashSet;
import org.mockito.Mockito;
import org.testng.annotations.Test;
public class ListExecutionRequestsResolverTest {
private static final String TEST_EXECUTION_REQUEST_URN = "urn:li:executionRequest:test-execution";
private static final ListExecutionRequestsInput TEST_INPUT =
new ListExecutionRequestsInput(0, 20, null, null, null);
private ExecutionRequestInput getTestExecutionRequestInput() {
ExecutionRequestInput input = new ExecutionRequestInput();
input.setTask("test-task");
input.setRequestedAt(System.currentTimeMillis());
return input;
}
private ExecutionRequestResult getTestExecutionRequestResult() {
ExecutionRequestResult result = new ExecutionRequestResult();
result.setStatus("COMPLETED");
result.setStartTimeMs(System.currentTimeMillis());
result.setDurationMs(1000L);
return result;
}
@Test
public void testGetSuccess() throws Exception {
// Create resolver
EntityClient mockClient = Mockito.mock(EntityClient.class);
ExecutionRequestInput returnedInput = getTestExecutionRequestInput();
ExecutionRequestResult returnedResult = getTestExecutionRequestResult();
// Mock search response
Mockito.when(
mockClient.search(
any(),
Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME),
Mockito.eq("*"),
Mockito.any(),
Mockito.any(),
Mockito.eq(0),
Mockito.eq(20)))
.thenReturn(
new SearchResult()
.setFrom(0)
.setPageSize(1)
.setNumEntities(1)
.setEntities(
new SearchEntityArray(
ImmutableSet.of(
new SearchEntity()
.setEntity(Urn.createFromString(TEST_EXECUTION_REQUEST_URN))))));
// Mock batch get response
Mockito.when(
mockClient.batchGetV2(
any(),
Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME),
Mockito.eq(
new HashSet<>(
ImmutableSet.of(Urn.createFromString(TEST_EXECUTION_REQUEST_URN)))),
Mockito.eq(
ImmutableSet.of(
Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME,
Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME))))
.thenReturn(
ImmutableMap.of(
Urn.createFromString(TEST_EXECUTION_REQUEST_URN),
new EntityResponse()
.setEntityName(Constants.EXECUTION_REQUEST_ENTITY_NAME)
.setUrn(Urn.createFromString(TEST_EXECUTION_REQUEST_URN))
.setAspects(
new EnvelopedAspectMap(
ImmutableMap.of(
Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME,
new EnvelopedAspect().setValue(new Aspect(returnedInput.data())),
Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME,
new EnvelopedAspect()
.setValue(new Aspect(returnedResult.data())))))));
ListExecutionRequestsResolver resolver = new ListExecutionRequestsResolver(mockClient);
// Execute resolver
QueryContext mockContext = getMockAllowContext();
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
// Data Assertions
var result = resolver.get(mockEnv).get();
assertEquals(result.getStart(), 0);
assertEquals(result.getCount(), 1);
assertEquals(result.getTotal(), 1);
assertEquals(result.getExecutionRequests().size(), 1);
var executionRequest = result.getExecutionRequests().get(0);
assertEquals(executionRequest.getUrn(), TEST_EXECUTION_REQUEST_URN);
assertEquals(executionRequest.getInput().getTask(), returnedInput.getTask());
assertEquals(executionRequest.getResult().getStatus(), returnedResult.getStatus());
assertEquals(executionRequest.getResult().getDurationMs(), returnedResult.getDurationMs());
}
@Test
public void testGetWithNoResult() throws Exception {
// Create resolver
EntityClient mockClient = Mockito.mock(EntityClient.class);
ExecutionRequestInput returnedInput = getTestExecutionRequestInput();
// Mock search and batch get with only input aspect
Mockito.when(
mockClient.search(
any(),
Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME),
Mockito.eq("*"),
Mockito.any(),
Mockito.any(),
Mockito.eq(0),
Mockito.eq(20)))
.thenReturn(
new SearchResult()
.setFrom(0)
.setPageSize(1)
.setNumEntities(1)
.setEntities(
new SearchEntityArray(
ImmutableSet.of(
new SearchEntity()
.setEntity(Urn.createFromString(TEST_EXECUTION_REQUEST_URN))))));
Mockito.when(
mockClient.batchGetV2(
any(),
Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME),
Mockito.eq(
new HashSet<>(
ImmutableSet.of(Urn.createFromString(TEST_EXECUTION_REQUEST_URN)))),
Mockito.eq(
ImmutableSet.of(
Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME,
Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME))))
.thenReturn(
ImmutableMap.of(
Urn.createFromString(TEST_EXECUTION_REQUEST_URN),
new EntityResponse()
.setEntityName(Constants.EXECUTION_REQUEST_ENTITY_NAME)
.setUrn(Urn.createFromString(TEST_EXECUTION_REQUEST_URN))
.setAspects(
new EnvelopedAspectMap(
ImmutableMap.of(
Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME,
new EnvelopedAspect()
.setValue(new Aspect(returnedInput.data())))))));
ListExecutionRequestsResolver resolver = new ListExecutionRequestsResolver(mockClient);
// Execute resolver
QueryContext mockContext = getMockAllowContext();
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
// Verify execution request without result
var result = resolver.get(mockEnv).get();
assertEquals(result.getExecutionRequests().size(), 1);
var executionRequest = result.getExecutionRequests().get(0);
assertEquals(executionRequest.getInput().getTask(), returnedInput.getTask());
assertNull(executionRequest.getResult());
}
@Test
public void testGetEntityClientException() throws Exception {
// Create resolver
EntityClient mockClient = Mockito.mock(EntityClient.class);
Mockito.doThrow(RemoteInvocationException.class)
.when(mockClient)
.batchGetV2(any(), Mockito.any(), Mockito.anySet(), Mockito.anySet());
ListExecutionRequestsResolver resolver = new ListExecutionRequestsResolver(mockClient);
// Execute resolver
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
QueryContext mockContext = getMockAllowContext();
Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
// Verify exception is thrown
assertThrows(RuntimeException.class, () -> resolver.get(mockEnv).join());
}
@Test
public void testGetWithCustomQuery() throws Exception {
// Create resolver
EntityClient mockClient = Mockito.mock(EntityClient.class);
ListExecutionRequestsInput customInput =
new ListExecutionRequestsInput(0, 20, "custom-query", null, null);
// Verify custom query is passed to search
Mockito.when(
mockClient.search(
any(),
Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME),
Mockito.eq("custom-query"),
Mockito.any(),
Mockito.any(),
Mockito.eq(0),
Mockito.eq(20)))
.thenReturn(
new SearchResult()
.setFrom(0)
.setPageSize(0)
.setNumEntities(0)
.setEntities(new SearchEntityArray()));
ListExecutionRequestsResolver resolver = new ListExecutionRequestsResolver(mockClient);
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
QueryContext mockContext = getMockAllowContext();
Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(customInput);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
var result = resolver.get(mockEnv).get();
assertEquals(result.getExecutionRequests().size(), 0);
}
}

30
datahub-web-react/src/app/ingest/ManageIngestionPage.tsx
View File

@ -1,6 +1,7 @@
import { Tabs, Typography } from 'antd';
import React, { useEffect, useState } from 'react';
import styled from 'styled-components';
import { useHistory } from 'react-router';
import { IngestionSourceList } from './source/IngestionSourceList';
import { useAppConfig } from '../useAppConfig';
import { useUserContext } from '../context/useUserContext';
@ -11,6 +12,7 @@ import {
INGESTION_REFRESH_SOURCES_ID,
} from '../onboarding/config/IngestionOnboardingConfig';
import { useShowNavBarRedesign } from '../useShowNavBarRedesign';
import { TabType } from './types';
const PageContainer = styled.div<{ $isShowNavBarRedesign?: boolean }>`
padding-top: 20px;
@ -53,17 +55,6 @@ const Tab = styled(Tabs.TabPane)`
const ListContainer = styled.div``;
enum TabType {
Sources = 'Sources',
Secrets = 'Secrets',
RemoteExecutors = 'Executors',
}
const TabTypeToListComponent = {
[TabType.Sources]: <IngestionSourceList />,
[TabType.Secrets]: <SecretsList />,
};
export const ManageIngestionPage = () => {
/**
* Determines which view should be visible: ingestion sources or secrets.
@ -83,9 +74,18 @@ export const ManageIngestionPage = () => {
}
}, [loaded, me.loaded, showIngestionTab, selectedTab]);
const onClickTab = (newTab: string) => {
const history = useHistory();
const onSwitchTab = (newTab: string, options?: { clearQueryParams: boolean }) => {
const matchingTab = Object.values(TabType).find((tab) => tab === newTab);
setSelectedTab(matchingTab || selectedTab);
if (options?.clearQueryParams) {
history.push({ search: '' });
}
};
const TabTypeToListComponent = {
[TabType.Sources]: <IngestionSourceList />,
[TabType.Secrets]: <SecretsList />,
};
return (
@ -97,7 +97,11 @@ export const ManageIngestionPage = () => {
Configure and schedule syncs to import data from your data sources
</Typography.Paragraph>
</PageHeaderContainer>
<StyledTabs activeKey={selectedTab} size="large" onTabClick={(tab: string) => onClickTab(tab)}>
<StyledTabs
activeKey={selectedTab}
size="large"
onTabClick={(tab) => onSwitchTab(tab, { clearQueryParams: true })}
>
{showIngestionTab && <Tab key={TabType.Sources} tab={TabType.Sources} />}
{showSecretsTab && <Tab key={TabType.Secrets} tab={TabType.Secrets} />}
</StyledTabs>

3
datahub-web-react/src/app/ingest/constants.ts Normal file
View File

@ -0,0 +1,3 @@
export const INGESTION_TAB_QUERY_PARAMS = {
searchQuery: 'query',
};

13
datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
View File

@ -1,5 +1,5 @@
import { PlusOutlined, RedoOutlined } from '@ant-design/icons';
import React, { useCallback, useEffect, useState } from 'react';
import React, { useCallback, useEffect, useRef, useState } from 'react';
import { debounce } from 'lodash';
import * as QueryString from 'query-string';
import { useLocation } from 'react-router';
@ -80,7 +80,14 @@ export const IngestionSourceList = () => {
const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
const paramsQuery = (params?.query as string) || undefined;
const [query, setQuery] = useState<undefined | string>(undefined);
useEffect(() => setQuery(paramsQuery), [paramsQuery]);
const searchInputRef = useRef<HTMLInputElement | null>(null);
// highlight search input if user arrives with a query preset for salience
useEffect(() => {
if (paramsQuery?.length) {
setQuery(paramsQuery);
searchInputRef.current?.focus();
}
}, [paramsQuery]);
const [page, setPage] = useState(1);
@ -369,6 +376,7 @@ export const IngestionSourceList = () => {
},
onCancel() {},
okText: 'Yes',
okButtonProps: { ['data-testid' as any]: 'confirm-delete-ingestion-source' },
maskClosable: true,
closable: true,
});
@ -423,6 +431,7 @@ export const IngestionSourceList = () => {
</StyledSelect>
<SearchBar
searchInputRef={searchInputRef}
initialQuery={query || ''}
placeholderText="Search sources..."
suggestions={[]}

116
datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx
View File

@ -1,4 +1,4 @@
import { Empty, Typography } from 'antd';
import { Empty } from 'antd';
import React from 'react';
import styled from 'styled-components/macro';
import { SorterResult } from 'antd/lib/table/interface';
@ -6,17 +6,10 @@ import { useShowNavBarRedesign } from '@src/app/useShowNavBarRedesign';
import { StyledTable } from '../../entity/shared/components/styled/StyledTable';
import { ANTD_GRAY } from '../../entity/shared/constants';
import { CLI_EXECUTOR_ID, getIngestionSourceStatus } from './utils';
import {
LastStatusColumn,
TypeColumn,
ActionsColumn,
ScheduleColumn,
LastExecutionColumn,
} from './IngestionSourceTableColumns';
import { LastStatusColumn, TypeColumn, ActionsColumn, ScheduleColumn } from './IngestionSourceTableColumns';
import { IngestionSource } from '../../../types.generated';
import { IngestionSourceExecutionList } from './executions/IngestionSourceExecutionList';
const MIN_EXECUTION_COLUMN_WIDTH = 125;
const PAGE_HEADER_HEIGHT = 395;
const StyledSourceTable = styled(StyledTable)`
@ -61,65 +54,6 @@ function IngestionSourceTable({
}: Props) {
const isShowNavBarRedesign = useShowNavBarRedesign();
const tableColumns = [
{
title: 'Type',
dataIndex: 'type',
key: 'type',
render: (type: string, record: any) => <TypeColumn type={type} record={record} />,
sorter: true,
},
{
title: 'Name',
dataIndex: 'name',
key: 'name',
render: (name: string) => name || '',
sorter: true,
},
{
title: 'Schedule',
dataIndex: 'schedule',
key: 'schedule',
render: ScheduleColumn,
},
{
title: 'Execution Count',
dataIndex: 'execCount',
key: 'execCount',
width: isShowNavBarRedesign ? MIN_EXECUTION_COLUMN_WIDTH : undefined,
render: (execCount: any) => <Typography.Text>{execCount || '0'}</Typography.Text>,
},
{
title: 'Last Execution',
dataIndex: 'lastExecTime',
key: 'lastExecTime',
render: LastExecutionColumn,
},
{
title: 'Last Status',
dataIndex: 'lastExecStatus',
key: 'lastExecStatus',
render: (status: any, record) => (
<LastStatusColumn status={status} record={record} setFocusExecutionUrn={setFocusExecutionUrn} />
),
},
{
title: '',
dataIndex: '',
key: 'x',
render: (_, record: any) => (
<ActionsColumn
record={record}
setFocusExecutionUrn={setFocusExecutionUrn}
onExecute={onExecute}
onDelete={onDelete}
onView={onView}
onEdit={onEdit}
/>
),
},
];
const tableData = sources.map((source) => ({
urn: source.urn,
type: source.type,
@ -143,6 +77,52 @@ function IngestionSourceTable({
cliIngestion: source.config?.executorId === CLI_EXECUTOR_ID,
}));
const tableColumns = [
{
title: 'Type',
dataIndex: 'type',
key: 'type',
render: (type: string, record: any) => <TypeColumn type={type} record={record} />,
sorter: true,
},
{
title: 'Name',
dataIndex: 'name',
key: 'name',
render: (name: string) => name || '',
sorter: true,
},
{
title: 'Schedule',
dataIndex: 'schedule',
key: 'schedule',
render: ScheduleColumn,
},
{
title: 'Status',
dataIndex: 'lastExecStatus',
key: 'lastExecStatus',
render: (status: any, record) => (
<LastStatusColumn status={status} record={record} setFocusExecutionUrn={setFocusExecutionUrn} />
),
},
{
title: '',
dataIndex: '',
key: 'x',
render: (_, record: any) => (
<ActionsColumn
record={record}
setFocusExecutionUrn={setFocusExecutionUrn}
onExecute={onExecute}
onDelete={onDelete}
onView={onView}
onEdit={onEdit}
/>
),
},
];
const handleTableChange = (_: any, __: any, sorter: any) => {
const sorterTyped: SorterResult<any> = sorter;
const field = sorterTyped.field as string;

33
datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx
View File

@ -29,6 +29,11 @@ const StatusContainer = styled.div`
align-items: center;
`;
const AllStatusWrapper = styled.div`
display: flex;
flex-direction: column;
`;
const StatusButton = styled(Button)`
padding: 0px;
margin: 0px;
@ -100,10 +105,10 @@ export function TypeColumn({ type, record }: TypeColumnProps) {
);
}
export function LastExecutionColumn(time: any) {
export function LastExecutionColumn({ time }: { time: number }) {
const executionDate = time && new Date(time);
const localTime = executionDate && `${executionDate.toLocaleDateString()} at ${executionDate.toLocaleTimeString()}`;
return <Typography.Text>{localTime || 'None'}</Typography.Text>;
return <Typography.Text type="secondary">{localTime ? `Last run ${localTime}` : 'Never run'}</Typography.Text>;
}
export function ScheduleColumn(schedule: any, record: any) {
@ -131,17 +136,21 @@ export function LastStatusColumn({ status, record, setFocusExecutionUrn }: LastS
const Icon = getExecutionRequestStatusIcon(status);
const text = getExecutionRequestStatusDisplayText(status);
const color = getExecutionRequestStatusDisplayColor(status);
const { lastExecTime, lastExecUrn } = record;
return (
<StatusContainer>
{Icon && <Icon style={{ color, fontSize: 14 }} />}
<StatusButton
data-testid="ingestion-source-table-status"
type="link"
onClick={() => setFocusExecutionUrn(record.lastExecUrn)}
>
<StatusText color={color}>{text || 'Pending...'}</StatusText>
</StatusButton>
</StatusContainer>
<AllStatusWrapper>
<StatusContainer>
{Icon && <Icon style={{ color, fontSize: 14 }} />}
<StatusButton
data-testid="ingestion-source-table-status"
type="link"
onClick={() => setFocusExecutionUrn(lastExecUrn)}
>
<StatusText color={color}>{text || 'Pending...'}</StatusText>
</StatusButton>
</StatusContainer>
<LastExecutionColumn time={lastExecTime} />
</AllStatusWrapper>
);
}

1
datahub-web-react/src/app/ingest/source/builder/IngestionSourceBuilderModal.tsx
View File

@ -154,6 +154,7 @@ export const IngestionSourceBuilderModal = ({ initialState, open, onSubmit, onCa
<StepComponent
state={ingestionBuilderState}
updateState={setIngestionBuilderState}
isEditing={isEditing}
goTo={goTo}
prev={stepStack.length > 1 ? prev : undefined}
submit={submit}

3
datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
View File

@ -177,12 +177,13 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
</Form.Item>
<Collapse ghost>
<Collapse.Panel header={<Typography.Text type="secondary">Advanced</Typography.Text>} key="1">
{/* NOTE: Executor ID is OSS-only, used by actions pod */}
<Form.Item label={<Typography.Text strong>Executor ID</Typography.Text>}>
<Typography.Paragraph>
Provide the ID of the executor that should execute this ingestion recipe. This ID is
used to route execution requests of the recipe to the executor of the same ID. The
built-in DataHub executor ID is &apos;default&apos;. Do not change this unless you have
configured a remote or custom executor.
configured a custom executor via actions framework.
</Typography.Paragraph>
<Input
placeholder="default"

2
datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx
View File

@ -18,6 +18,7 @@ describe('DefineRecipeStep', () => {
submit={() => {}}
cancel={() => {}}
ingestionSources={[{ name: 'snowflake', displayName: 'Snowflake' } as SourceConfig]}
isEditing={false}
/>
</MockedProvider>
</ThemeProvider>,
@ -38,6 +39,7 @@ describe('DefineRecipeStep', () => {
submit={() => {}}
cancel={() => {}}
ingestionSources={[{ name: 'glue', displayName: 'Glue' } as SourceConfig]}
isEditing={false}
/>
</MockedProvider>
</ThemeProvider>,

1
datahub-web-react/src/app/ingest/source/builder/__tests__/NameSourceStep.test.tsx
View File

@ -18,6 +18,7 @@ describe('NameSourceStep', () => {
goTo={() => {}}
cancel={() => {}}
ingestionSources={[]}
isEditing
/>,
);
const nameInput = getByTestId('source-name-input') as HTMLInputElement;

1
datahub-web-react/src/app/ingest/source/builder/types.ts
View File

@ -33,6 +33,7 @@ export type StepProps = {
submit: (shouldRun?: boolean) => void;
cancel: () => void;
ingestionSources: SourceConfig[];
isEditing: boolean;
};
export type StringMapEntryInput = {

5
datahub-web-react/src/app/ingest/types.ts Normal file
View File

@ -0,0 +1,5 @@
export enum TabType {
Sources = 'Sources',
Secrets = 'Secrets',
RemoteExecutors = 'Executors',
}

22
datahub-web-react/src/app/search/SearchBar.tsx
View File

@ -1,4 +1,13 @@
import React, { useEffect, useMemo, useState, useRef, useCallback, EventHandler, SyntheticEvent } from 'react';
import React, {
useEffect,
useMemo,
useState,
useRef,
useCallback,
EventHandler,
SyntheticEvent,
MutableRefObject,
} from 'react';
import { Input, AutoComplete, Button } from 'antd';
import { CloseCircleFilled, SearchOutlined } from '@ant-design/icons';
import styled from 'styled-components/macro';
@ -123,6 +132,7 @@ interface Props {
onFocus?: () => void;
onBlur?: () => void;
showViewAllResults?: boolean;
searchInputRef?: MutableRefObject<any>;
}
const defaultProps = {
@ -152,6 +162,7 @@ export const SearchBar = ({
onFocus,
onBlur,
showViewAllResults = false,
...props
}: Props) => {
const history = useHistory();
const [searchQuery, setSearchQuery] = useState<string | undefined>(initialQuery);
@ -303,7 +314,8 @@ export const SearchBar = ({
}
}
const searchInputRef = useRef(null);
const searchInputFallbackRef: MutableRefObject<any> = useRef(null);
const searchInputRef: MutableRefObject<any> = props.searchInputRef || searchInputFallbackRef;
useEffect(() => {
if (showCommandK) {
@ -311,7 +323,7 @@ export const SearchBar = ({
// Support command-k to select the search bar.
// 75 is the keyCode for 'k'
if ((event.metaKey || event.ctrlKey) && event.keyCode === 75) {
(searchInputRef?.current as any)?.focus();
searchInputRef.current?.focus();
}
};
document.addEventListener('keydown', handleKeyDown);
@ -320,7 +332,7 @@ export const SearchBar = ({
};
}
return () => null;
}, [showCommandK]);
}, [showCommandK, searchInputRef]);
return (
<AutoCompleteContainer style={style} ref={searchBarWrapperRef}>
@ -377,6 +389,7 @@ export const SearchBar = ({
listHeight={480}
>
<StyledSearchBar
ref={searchInputRef}
bordered={false}
placeholder={placeholderText}
onPressEnter={() => {
@ -425,7 +438,6 @@ export const SearchBar = ({
/>
</>
}
ref={searchInputRef}
suffix={(showCommandK && !isFocused && <CommandK />) || null}
/>
</StyledAutoComplete>

2
smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js
View File

@ -67,7 +67,7 @@ describe("managing secrets for ingestion creation", () => {
`[data-testid="delete-ingestion-source-${ingestion_source_name}"]`,
).click();
cy.waitTextVisible("Confirm Ingestion Source Removal");
cy.get("button").contains("Yes").click();
cy.get(`[data-testid="confirm-delete-ingestion-source"]`).click();
cy.ensureTextNotPresent(ingestion_source_name);
// Verify secret is not present during ingestion source creation for password dropdown

2
smoke-test/tests/cypress/cypress/e2e/mutationsV2/v2_managed_ingestion.js
View File

@ -40,7 +40,7 @@ describe("run managed ingestion", () => {
cy.contains("Succeeded", { timeout: 180000 });
cy.clickOptionWithTestId(`delete-ingestion-source-${testName}`);
});
cy.clickOptionWithText("Yes");
cy.get(`[data-testid="confirm-delete-ingestion-source"]`).click();
cy.ensureTextNotPresent(testName);
});
});

2
smoke-test/tests/cypress/cypress/e2e/mutationsV2/v2_managing_secrets.js
View File

@ -72,7 +72,7 @@ describe("managing secrets for ingestion creation", () => {
`[data-testid="delete-ingestion-source-${ingestion_source_name}"]`,
).click();
cy.waitTextVisible("Confirm Ingestion Source Removal");
cy.get("button").contains("Yes").click();
cy.get(`[data-testid="confirm-delete-ingestion-source"]`).click();
cy.waitTextVisible("Removed ingestion source.");
cy.ensureTextNotPresent(ingestion_source_name);