diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index c69f8c748b..1572db17d5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -110,6 +110,7 @@ import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateIngestionEx import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateTestConnectionRequestResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.GetIngestionExecutionRequestResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.IngestionSourceExecutionRequestsResolver; +import com.linkedin.datahub.graphql.resolvers.ingest.execution.ListExecutionRequestsResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.RollbackIngestionResolver; import com.linkedin.datahub.graphql.resolvers.ingest.secret.CreateSecretResolver; import com.linkedin.datahub.graphql.resolvers.ingest.secret.DeleteSecretResolver; @@ -999,6 +1000,8 @@ public class GmsGraphQLEngine { .dataFetcher( "listIngestionSources", new ListIngestionSourcesResolver(this.entityClient)) .dataFetcher("ingestionSource", new GetIngestionSourceResolver(this.entityClient)) + .dataFetcher( + "listExecutionRequests", new ListExecutionRequestsResolver(this.entityClient)) .dataFetcher( "executionRequest", new GetIngestionExecutionRequestResolver(this.entityClient)) .dataFetcher("getSchemaBlame", new GetSchemaBlameResolver(this.timelineService)) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java index 3c3fed846e..e3e5c0f5c7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java @@ -68,6 +68,9 @@ public class IngestionResolverUtils { if (executionRequestInput.getActorUrn() != null) { inputResult.setActorUrn(executionRequestInput.getActorUrn().toString()); } + if (executionRequestInput.hasExecutorId()) { + inputResult.setExecutorId(executionRequestInput.getExecutorId()); + } result.setInput(inputResult); } @@ -88,6 +91,9 @@ public class IngestionResolverUtils { final com.linkedin.datahub.graphql.generated.ExecutionRequestSource result = new com.linkedin.datahub.graphql.generated.ExecutionRequestSource(); result.setType(execRequestSource.getType()); + if (execRequestSource.hasIngestionSource()) { + result.setIngestionSource(execRequestSource.getIngestionSource().toString()); + } return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/ListExecutionRequestsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/ListExecutionRequestsResolver.java new file mode 100644 index 0000000000..c5a1045656 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/ListExecutionRequestsResolver.java @@ -0,0 +1,115 @@ +package com.linkedin.datahub.graphql.resolvers.ingest.execution; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.buildFilter; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.IngestionSourceExecutionRequests; +import com.linkedin.datahub.graphql.generated.ListExecutionRequestsInput; +import com.linkedin.datahub.graphql.resolvers.ingest.IngestionResolverUtils; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchResult; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class ListExecutionRequestsResolver + implements DataFetcher> { + + private static final String DEFAULT_QUERY = "*"; + private static final Integer DEFAULT_START = 0; + private static final Integer DEFAULT_COUNT = 20; + + private final EntityClient _entityClient; + + @Override + public CompletableFuture get( + final DataFetchingEnvironment environment) throws Exception { + + final QueryContext context = environment.getContext(); + + final ListExecutionRequestsInput input = + bindArgument(environment.getArgument("input"), ListExecutionRequestsInput.class); + final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart(); + final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); + final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery(); + final List filters = + input.getFilters() == null ? Collections.emptyList() : input.getFilters(); + + // construct sort criteria, defaulting to systemCreated + final SortCriterion sortCriterion; + + // if query is expecting to sort by something, use that + final com.linkedin.datahub.graphql.generated.SortCriterion sortCriterionInput = input.getSort(); + if (sortCriterionInput != null) { + sortCriterion = + new SortCriterion() + .setField(sortCriterionInput.getField()) + .setOrder(SortOrder.valueOf(sortCriterionInput.getSortOrder().name())); + } else { + sortCriterion = new SortCriterion().setField("requestTimeMs").setOrder(SortOrder.DESCENDING); + } + + return GraphQLConcurrencyUtils.supplyAsync( + () -> { + try { + // First, get all execution request Urns. + final SearchResult gmsResult = + _entityClient.search( + context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)), + Constants.EXECUTION_REQUEST_ENTITY_NAME, + query, + buildFilter(filters, Collections.emptyList()), + sortCriterion != null ? List.of(sortCriterion) : null, + start, + count); + + log.info(String.format("Found %d execution requests", gmsResult.getNumEntities())); + + final List entitiesUrnList = + gmsResult.getEntities().stream().map(SearchEntity::getEntity).toList(); + // Then, resolve all execution requests + final Map entities = + _entityClient.batchGetV2( + context.getOperationContext(), + Constants.EXECUTION_REQUEST_ENTITY_NAME, + new HashSet<>(entitiesUrnList), + ImmutableSet.of( + Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME, + Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME)); + final List entitiesOrdered = + entitiesUrnList.stream().map(entities::get).filter(Objects::nonNull).toList(); + // Now that we have entities we can bind this to a result. + final IngestionSourceExecutionRequests result = new IngestionSourceExecutionRequests(); + result.setStart(gmsResult.getFrom()); + result.setCount(gmsResult.getPageSize()); + result.setTotal(gmsResult.getNumEntities()); + result.setExecutionRequests( + IngestionResolverUtils.mapExecutionRequests(context, entitiesOrdered)); + return result; + } catch (Exception e) { + throw new RuntimeException("Failed to list executions", e); + } + }, + this.getClass().getSimpleName(), + "get"); + } +} diff --git a/datahub-graphql-core/src/main/resources/ingestion.graphql b/datahub-graphql-core/src/main/resources/ingestion.graphql index 719ffea30c..9c46f66fb3 100644 --- a/datahub-graphql-core/src/main/resources/ingestion.graphql +++ b/datahub-graphql-core/src/main/resources/ingestion.graphql @@ -22,6 +22,11 @@ extend type Query { urn: The primary key associated with the ingestion source. """ ingestionSource(urn: String!): IngestionSource + + """ + List all execution requests + """ + listExecutionRequests(input: ListExecutionRequestsInput!): IngestionSourceExecutionRequests """ Get an execution request @@ -92,6 +97,11 @@ type ExecutionRequestSource { The type of the source, e.g. SCHEDULED_INGESTION_SOURCE """ type: String + + """ + The urn of the ingestion source, if applicable + """ + ingestionSource: String } """ @@ -122,6 +132,11 @@ type ExecutionRequestInput { Urn of the actor who created this execution request """ actorUrn: String + + """ + The specific executor to route the request to. If none is provided, a "default" executor is used. + """ + executorId: String } """ @@ -360,6 +375,33 @@ type IngestionRun { executionRequestUrn: String } +input ListExecutionRequestsInput { + """ + The starting offset of the result set + """ + start: Int + + """ + The number of results to be returned + """ + count: Int + + """ + An optional search query + """ + query: String + + """ + Optional Facet filters to apply to the result set + """ + filters: [FacetFilterInput!] + + """ + Optional sort order. Defaults to use systemCreated. + """ + sort: SortCriterion +} + """ Requests for execution associated with an ingestion source """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/ListExecutionRequestsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/ListExecutionRequestsResolverTest.java new file mode 100644 index 0000000000..089443935d --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/ListExecutionRequestsResolverTest.java @@ -0,0 +1,251 @@ +package com.linkedin.datahub.graphql.resolvers.ingest.execution; + +import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.*; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.*; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.ListExecutionRequestsInput; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.execution.ExecutionRequestInput; +import com.linkedin.execution.ExecutionRequestResult; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.HashSet; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class ListExecutionRequestsResolverTest { + + private static final String TEST_EXECUTION_REQUEST_URN = "urn:li:executionRequest:test-execution"; + + private static final ListExecutionRequestsInput TEST_INPUT = + new ListExecutionRequestsInput(0, 20, null, null, null); + + private ExecutionRequestInput getTestExecutionRequestInput() { + ExecutionRequestInput input = new ExecutionRequestInput(); + input.setTask("test-task"); + input.setRequestedAt(System.currentTimeMillis()); + return input; + } + + private ExecutionRequestResult getTestExecutionRequestResult() { + ExecutionRequestResult result = new ExecutionRequestResult(); + result.setStatus("COMPLETED"); + result.setStartTimeMs(System.currentTimeMillis()); + result.setDurationMs(1000L); + return result; + } + + @Test + public void testGetSuccess() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + + ExecutionRequestInput returnedInput = getTestExecutionRequestInput(); + ExecutionRequestResult returnedResult = getTestExecutionRequestResult(); + + // Mock search response + Mockito.when( + mockClient.search( + any(), + Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME), + Mockito.eq("*"), + Mockito.any(), + Mockito.any(), + Mockito.eq(0), + Mockito.eq(20))) + .thenReturn( + new SearchResult() + .setFrom(0) + .setPageSize(1) + .setNumEntities(1) + .setEntities( + new SearchEntityArray( + ImmutableSet.of( + new SearchEntity() + .setEntity(Urn.createFromString(TEST_EXECUTION_REQUEST_URN)))))); + + // Mock batch get response + Mockito.when( + mockClient.batchGetV2( + any(), + Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME), + Mockito.eq( + new HashSet<>( + ImmutableSet.of(Urn.createFromString(TEST_EXECUTION_REQUEST_URN)))), + Mockito.eq( + ImmutableSet.of( + Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME, + Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME)))) + .thenReturn( + ImmutableMap.of( + Urn.createFromString(TEST_EXECUTION_REQUEST_URN), + new EntityResponse() + .setEntityName(Constants.EXECUTION_REQUEST_ENTITY_NAME) + .setUrn(Urn.createFromString(TEST_EXECUTION_REQUEST_URN)) + .setAspects( + new EnvelopedAspectMap( + ImmutableMap.of( + Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(returnedInput.data())), + Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME, + new EnvelopedAspect() + .setValue(new Aspect(returnedResult.data()))))))); + + ListExecutionRequestsResolver resolver = new ListExecutionRequestsResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // Data Assertions + var result = resolver.get(mockEnv).get(); + assertEquals(result.getStart(), 0); + assertEquals(result.getCount(), 1); + assertEquals(result.getTotal(), 1); + assertEquals(result.getExecutionRequests().size(), 1); + + var executionRequest = result.getExecutionRequests().get(0); + assertEquals(executionRequest.getUrn(), TEST_EXECUTION_REQUEST_URN); + assertEquals(executionRequest.getInput().getTask(), returnedInput.getTask()); + assertEquals(executionRequest.getResult().getStatus(), returnedResult.getStatus()); + assertEquals(executionRequest.getResult().getDurationMs(), returnedResult.getDurationMs()); + } + + @Test + public void testGetWithNoResult() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + + ExecutionRequestInput returnedInput = getTestExecutionRequestInput(); + + // Mock search and batch get with only input aspect + Mockito.when( + mockClient.search( + any(), + Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME), + Mockito.eq("*"), + Mockito.any(), + Mockito.any(), + Mockito.eq(0), + Mockito.eq(20))) + .thenReturn( + new SearchResult() + .setFrom(0) + .setPageSize(1) + .setNumEntities(1) + .setEntities( + new SearchEntityArray( + ImmutableSet.of( + new SearchEntity() + .setEntity(Urn.createFromString(TEST_EXECUTION_REQUEST_URN)))))); + + Mockito.when( + mockClient.batchGetV2( + any(), + Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME), + Mockito.eq( + new HashSet<>( + ImmutableSet.of(Urn.createFromString(TEST_EXECUTION_REQUEST_URN)))), + Mockito.eq( + ImmutableSet.of( + Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME, + Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME)))) + .thenReturn( + ImmutableMap.of( + Urn.createFromString(TEST_EXECUTION_REQUEST_URN), + new EntityResponse() + .setEntityName(Constants.EXECUTION_REQUEST_ENTITY_NAME) + .setUrn(Urn.createFromString(TEST_EXECUTION_REQUEST_URN)) + .setAspects( + new EnvelopedAspectMap( + ImmutableMap.of( + Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME, + new EnvelopedAspect() + .setValue(new Aspect(returnedInput.data()))))))); + + ListExecutionRequestsResolver resolver = new ListExecutionRequestsResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // Verify execution request without result + var result = resolver.get(mockEnv).get(); + assertEquals(result.getExecutionRequests().size(), 1); + var executionRequest = result.getExecutionRequests().get(0); + assertEquals(executionRequest.getInput().getTask(), returnedInput.getTask()); + assertNull(executionRequest.getResult()); + } + + @Test + public void testGetEntityClientException() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.doThrow(RemoteInvocationException.class) + .when(mockClient) + .batchGetV2(any(), Mockito.any(), Mockito.anySet(), Mockito.anySet()); + ListExecutionRequestsResolver resolver = new ListExecutionRequestsResolver(mockClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // Verify exception is thrown + assertThrows(RuntimeException.class, () -> resolver.get(mockEnv).join()); + } + + @Test + public void testGetWithCustomQuery() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + + ListExecutionRequestsInput customInput = + new ListExecutionRequestsInput(0, 20, "custom-query", null, null); + + // Verify custom query is passed to search + Mockito.when( + mockClient.search( + any(), + Mockito.eq(Constants.EXECUTION_REQUEST_ENTITY_NAME), + Mockito.eq("custom-query"), + Mockito.any(), + Mockito.any(), + Mockito.eq(0), + Mockito.eq(20))) + .thenReturn( + new SearchResult() + .setFrom(0) + .setPageSize(0) + .setNumEntities(0) + .setEntities(new SearchEntityArray())); + + ListExecutionRequestsResolver resolver = new ListExecutionRequestsResolver(mockClient); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(customInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + var result = resolver.get(mockEnv).get(); + assertEquals(result.getExecutionRequests().size(), 0); + } +} diff --git a/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx b/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx index 5a9a5de896..a0d825693b 100644 --- a/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx +++ b/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx @@ -1,6 +1,7 @@ import { Tabs, Typography } from 'antd'; import React, { useEffect, useState } from 'react'; import styled from 'styled-components'; +import { useHistory } from 'react-router'; import { IngestionSourceList } from './source/IngestionSourceList'; import { useAppConfig } from '../useAppConfig'; import { useUserContext } from '../context/useUserContext'; @@ -11,6 +12,7 @@ import { INGESTION_REFRESH_SOURCES_ID, } from '../onboarding/config/IngestionOnboardingConfig'; import { useShowNavBarRedesign } from '../useShowNavBarRedesign'; +import { TabType } from './types'; const PageContainer = styled.div<{ $isShowNavBarRedesign?: boolean }>` padding-top: 20px; @@ -53,17 +55,6 @@ const Tab = styled(Tabs.TabPane)` const ListContainer = styled.div``; -enum TabType { - Sources = 'Sources', - Secrets = 'Secrets', - RemoteExecutors = 'Executors', -} - -const TabTypeToListComponent = { - [TabType.Sources]: , - [TabType.Secrets]: , -}; - export const ManageIngestionPage = () => { /** * Determines which view should be visible: ingestion sources or secrets. @@ -83,9 +74,18 @@ export const ManageIngestionPage = () => { } }, [loaded, me.loaded, showIngestionTab, selectedTab]); - const onClickTab = (newTab: string) => { + const history = useHistory(); + const onSwitchTab = (newTab: string, options?: { clearQueryParams: boolean }) => { const matchingTab = Object.values(TabType).find((tab) => tab === newTab); setSelectedTab(matchingTab || selectedTab); + if (options?.clearQueryParams) { + history.push({ search: '' }); + } + }; + + const TabTypeToListComponent = { + [TabType.Sources]: , + [TabType.Secrets]: , }; return ( @@ -97,7 +97,11 @@ export const ManageIngestionPage = () => { Configure and schedule syncs to import data from your data sources - onClickTab(tab)}> + onSwitchTab(tab, { clearQueryParams: true })} + > {showIngestionTab && } {showSecretsTab && } diff --git a/datahub-web-react/src/app/ingest/constants.ts b/datahub-web-react/src/app/ingest/constants.ts new file mode 100644 index 0000000000..0c3f1658e4 --- /dev/null +++ b/datahub-web-react/src/app/ingest/constants.ts @@ -0,0 +1,3 @@ +export const INGESTION_TAB_QUERY_PARAMS = { + searchQuery: 'query', +}; diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx index 8cc3845b1b..827d463e92 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx @@ -1,5 +1,5 @@ import { PlusOutlined, RedoOutlined } from '@ant-design/icons'; -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useRef, useState } from 'react'; import { debounce } from 'lodash'; import * as QueryString from 'query-string'; import { useLocation } from 'react-router'; @@ -80,7 +80,14 @@ export const IngestionSourceList = () => { const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); const paramsQuery = (params?.query as string) || undefined; const [query, setQuery] = useState(undefined); - useEffect(() => setQuery(paramsQuery), [paramsQuery]); + const searchInputRef = useRef(null); + // highlight search input if user arrives with a query preset for salience + useEffect(() => { + if (paramsQuery?.length) { + setQuery(paramsQuery); + searchInputRef.current?.focus(); + } + }, [paramsQuery]); const [page, setPage] = useState(1); @@ -369,6 +376,7 @@ export const IngestionSourceList = () => { }, onCancel() {}, okText: 'Yes', + okButtonProps: { ['data-testid' as any]: 'confirm-delete-ingestion-source' }, maskClosable: true, closable: true, }); @@ -423,6 +431,7 @@ export const IngestionSourceList = () => { , - sorter: true, - }, - { - title: 'Name', - dataIndex: 'name', - key: 'name', - render: (name: string) => name || '', - sorter: true, - }, - { - title: 'Schedule', - dataIndex: 'schedule', - key: 'schedule', - render: ScheduleColumn, - }, - { - title: 'Execution Count', - dataIndex: 'execCount', - key: 'execCount', - width: isShowNavBarRedesign ? MIN_EXECUTION_COLUMN_WIDTH : undefined, - render: (execCount: any) => {execCount || '0'}, - }, - { - title: 'Last Execution', - dataIndex: 'lastExecTime', - key: 'lastExecTime', - render: LastExecutionColumn, - }, - { - title: 'Last Status', - dataIndex: 'lastExecStatus', - key: 'lastExecStatus', - render: (status: any, record) => ( - - ), - }, - { - title: '', - dataIndex: '', - key: 'x', - render: (_, record: any) => ( - - ), - }, - ]; - const tableData = sources.map((source) => ({ urn: source.urn, type: source.type, @@ -143,6 +77,52 @@ function IngestionSourceTable({ cliIngestion: source.config?.executorId === CLI_EXECUTOR_ID, })); + const tableColumns = [ + { + title: 'Type', + dataIndex: 'type', + key: 'type', + render: (type: string, record: any) => , + sorter: true, + }, + { + title: 'Name', + dataIndex: 'name', + key: 'name', + render: (name: string) => name || '', + sorter: true, + }, + { + title: 'Schedule', + dataIndex: 'schedule', + key: 'schedule', + render: ScheduleColumn, + }, + { + title: 'Status', + dataIndex: 'lastExecStatus', + key: 'lastExecStatus', + render: (status: any, record) => ( + + ), + }, + { + title: '', + dataIndex: '', + key: 'x', + render: (_, record: any) => ( + + ), + }, + ]; + const handleTableChange = (_: any, __: any, sorter: any) => { const sorterTyped: SorterResult = sorter; const field = sorterTyped.field as string; diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx index bb89588293..aa183b2fa9 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx @@ -29,6 +29,11 @@ const StatusContainer = styled.div` align-items: center; `; +const AllStatusWrapper = styled.div` + display: flex; + flex-direction: column; +`; + const StatusButton = styled(Button)` padding: 0px; margin: 0px; @@ -100,10 +105,10 @@ export function TypeColumn({ type, record }: TypeColumnProps) { ); } -export function LastExecutionColumn(time: any) { +export function LastExecutionColumn({ time }: { time: number }) { const executionDate = time && new Date(time); const localTime = executionDate && `${executionDate.toLocaleDateString()} at ${executionDate.toLocaleTimeString()}`; - return {localTime || 'None'}; + return {localTime ? `Last run ${localTime}` : 'Never run'}; } export function ScheduleColumn(schedule: any, record: any) { @@ -131,17 +136,21 @@ export function LastStatusColumn({ status, record, setFocusExecutionUrn }: LastS const Icon = getExecutionRequestStatusIcon(status); const text = getExecutionRequestStatusDisplayText(status); const color = getExecutionRequestStatusDisplayColor(status); + const { lastExecTime, lastExecUrn } = record; return ( - - {Icon && } - setFocusExecutionUrn(record.lastExecUrn)} - > - {text || 'Pending...'} - - + + + {Icon && } + setFocusExecutionUrn(lastExecUrn)} + > + {text || 'Pending...'} + + + + ); } diff --git a/datahub-web-react/src/app/ingest/source/builder/IngestionSourceBuilderModal.tsx b/datahub-web-react/src/app/ingest/source/builder/IngestionSourceBuilderModal.tsx index be77e30afc..db854e9694 100644 --- a/datahub-web-react/src/app/ingest/source/builder/IngestionSourceBuilderModal.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/IngestionSourceBuilderModal.tsx @@ -154,6 +154,7 @@ export const IngestionSourceBuilderModal = ({ initialState, open, onSubmit, onCa 1 ? prev : undefined} submit={submit} diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx index 0290b5595e..9e63e91fbd 100644 --- a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx @@ -177,12 +177,13 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps) Advanced} key="1"> + {/* NOTE: Executor ID is OSS-only, used by actions pod */} Executor ID}> Provide the ID of the executor that should execute this ingestion recipe. This ID is used to route execution requests of the recipe to the executor of the same ID. The built-in DataHub executor ID is 'default'. Do not change this unless you have - configured a remote or custom executor. + configured a custom executor via actions framework. { submit={() => {}} cancel={() => {}} ingestionSources={[{ name: 'snowflake', displayName: 'Snowflake' } as SourceConfig]} + isEditing={false} /> , @@ -38,6 +39,7 @@ describe('DefineRecipeStep', () => { submit={() => {}} cancel={() => {}} ingestionSources={[{ name: 'glue', displayName: 'Glue' } as SourceConfig]} + isEditing={false} /> , diff --git a/datahub-web-react/src/app/ingest/source/builder/__tests__/NameSourceStep.test.tsx b/datahub-web-react/src/app/ingest/source/builder/__tests__/NameSourceStep.test.tsx index bd919999c9..bfcc2fa895 100644 --- a/datahub-web-react/src/app/ingest/source/builder/__tests__/NameSourceStep.test.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/__tests__/NameSourceStep.test.tsx @@ -18,6 +18,7 @@ describe('NameSourceStep', () => { goTo={() => {}} cancel={() => {}} ingestionSources={[]} + isEditing />, ); const nameInput = getByTestId('source-name-input') as HTMLInputElement; diff --git a/datahub-web-react/src/app/ingest/source/builder/types.ts b/datahub-web-react/src/app/ingest/source/builder/types.ts index e42bd0b790..2c7c19006e 100644 --- a/datahub-web-react/src/app/ingest/source/builder/types.ts +++ b/datahub-web-react/src/app/ingest/source/builder/types.ts @@ -33,6 +33,7 @@ export type StepProps = { submit: (shouldRun?: boolean) => void; cancel: () => void; ingestionSources: SourceConfig[]; + isEditing: boolean; }; export type StringMapEntryInput = { diff --git a/datahub-web-react/src/app/ingest/types.ts b/datahub-web-react/src/app/ingest/types.ts new file mode 100644 index 0000000000..b4fd16ebba --- /dev/null +++ b/datahub-web-react/src/app/ingest/types.ts @@ -0,0 +1,5 @@ +export enum TabType { + Sources = 'Sources', + Secrets = 'Secrets', + RemoteExecutors = 'Executors', +} diff --git a/datahub-web-react/src/app/search/SearchBar.tsx b/datahub-web-react/src/app/search/SearchBar.tsx index 15bdf62ead..99bdff894e 100644 --- a/datahub-web-react/src/app/search/SearchBar.tsx +++ b/datahub-web-react/src/app/search/SearchBar.tsx @@ -1,4 +1,13 @@ -import React, { useEffect, useMemo, useState, useRef, useCallback, EventHandler, SyntheticEvent } from 'react'; +import React, { + useEffect, + useMemo, + useState, + useRef, + useCallback, + EventHandler, + SyntheticEvent, + MutableRefObject, +} from 'react'; import { Input, AutoComplete, Button } from 'antd'; import { CloseCircleFilled, SearchOutlined } from '@ant-design/icons'; import styled from 'styled-components/macro'; @@ -123,6 +132,7 @@ interface Props { onFocus?: () => void; onBlur?: () => void; showViewAllResults?: boolean; + searchInputRef?: MutableRefObject; } const defaultProps = { @@ -152,6 +162,7 @@ export const SearchBar = ({ onFocus, onBlur, showViewAllResults = false, + ...props }: Props) => { const history = useHistory(); const [searchQuery, setSearchQuery] = useState(initialQuery); @@ -303,7 +314,8 @@ export const SearchBar = ({ } } - const searchInputRef = useRef(null); + const searchInputFallbackRef: MutableRefObject = useRef(null); + const searchInputRef: MutableRefObject = props.searchInputRef || searchInputFallbackRef; useEffect(() => { if (showCommandK) { @@ -311,7 +323,7 @@ export const SearchBar = ({ // Support command-k to select the search bar. // 75 is the keyCode for 'k' if ((event.metaKey || event.ctrlKey) && event.keyCode === 75) { - (searchInputRef?.current as any)?.focus(); + searchInputRef.current?.focus(); } }; document.addEventListener('keydown', handleKeyDown); @@ -320,7 +332,7 @@ export const SearchBar = ({ }; } return () => null; - }, [showCommandK]); + }, [showCommandK, searchInputRef]); return ( @@ -377,6 +389,7 @@ export const SearchBar = ({ listHeight={480} > { @@ -425,7 +438,6 @@ export const SearchBar = ({ /> } - ref={searchInputRef} suffix={(showCommandK && !isFocused && ) || null} /> diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js index 7e159b46e3..3245388360 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js @@ -67,7 +67,7 @@ describe("managing secrets for ingestion creation", () => { `[data-testid="delete-ingestion-source-${ingestion_source_name}"]`, ).click(); cy.waitTextVisible("Confirm Ingestion Source Removal"); - cy.get("button").contains("Yes").click(); + cy.get(`[data-testid="confirm-delete-ingestion-source"]`).click(); cy.ensureTextNotPresent(ingestion_source_name); // Verify secret is not present during ingestion source creation for password dropdown diff --git a/smoke-test/tests/cypress/cypress/e2e/mutationsV2/v2_managed_ingestion.js b/smoke-test/tests/cypress/cypress/e2e/mutationsV2/v2_managed_ingestion.js index 4c5b29c9f5..5ccf8874a9 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutationsV2/v2_managed_ingestion.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutationsV2/v2_managed_ingestion.js @@ -40,7 +40,7 @@ describe("run managed ingestion", () => { cy.contains("Succeeded", { timeout: 180000 }); cy.clickOptionWithTestId(`delete-ingestion-source-${testName}`); }); - cy.clickOptionWithText("Yes"); + cy.get(`[data-testid="confirm-delete-ingestion-source"]`).click(); cy.ensureTextNotPresent(testName); }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutationsV2/v2_managing_secrets.js b/smoke-test/tests/cypress/cypress/e2e/mutationsV2/v2_managing_secrets.js index e050993b09..3a74f9b0fc 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutationsV2/v2_managing_secrets.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutationsV2/v2_managing_secrets.js @@ -72,7 +72,7 @@ describe("managing secrets for ingestion creation", () => { `[data-testid="delete-ingestion-source-${ingestion_source_name}"]`, ).click(); cy.waitTextVisible("Confirm Ingestion Source Removal"); - cy.get("button").contains("Yes").click(); + cy.get(`[data-testid="confirm-delete-ingestion-source"]`).click(); cy.waitTextVisible("Removed ingestion source."); cy.ensureTextNotPresent(ingestion_source_name);