yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-04 19:44:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: revert param logic in search/list and introduce raw query param (#20244)

Browse Source 
* fix: revert  param logic in search/list and introduce raw query param

* unskip the playwright test related to DQ

---------

Co-authored-by: Shailesh Parmar <shailesh.parmar.webdev@gmail.com>
This commit is contained in:
Teddy 2025-03-14 12:11:56 -07:00 committed by GitHub
parent 2c90a2fe6f
commit dd3382aad8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 482 additions and 431 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
View File

@ -1355,9 +1355,11 @@ public abstract class EntityRepository<T extends EntityInterface> {
SearchListFilter searchListFilter,
int limit,
int offset,
String q)
String q,
String queryString)
throws IOException {
return listFromSearchWithOffset(uriInfo, fields, searchListFilter, limit, offset, null, q);
return listFromSearchWithOffset(
uriInfo, fields, searchListFilter, limit, offset, null, q, queryString);
}
public ResultList<T> listFromSearchWithOffset(
@ -1367,7 +1369,8 @@ public abstract class EntityRepository<T extends EntityInterface> {
int limit,
int offset,
SearchSortFilter searchSortFilter,
String q)
String q,
String queryString)
throws IOException {
List<T> entityList = new ArrayList<>();
Long total;
@ -1375,7 +1378,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
if (limit > 0) {
SearchResultListMapper results =
searchRepository.listWithOffset(
searchListFilter, limit, offset, entityType, searchSortFilter, q);
searchListFilter, limit, offset, entityType, searchSortFilter, q, queryString);
total = results.getTotal();
for (Map<String, Object> json : results.getResults()) {
T entity = JsonUtils.readOrConvertValueLenient(json, entityClass);
@ -1385,7 +1388,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
} else {
SearchResultListMapper results =
searchRepository.listWithOffset(
searchListFilter, limit, offset, entityType, searchSortFilter, q);
searchListFilter, limit, offset, entityType, searchSortFilter, q, queryString);
total = results.getTotal();
return new ResultList<>(entityList, null, limit, total.intValue());
}

7
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java
View File

@ -367,7 +367,8 @@ public abstract class EntityTimeSeriesRepository<T extends EntityTimeSeriesInter
int limit,
int offset,
SearchSortFilter searchSortFilter,
String q)
String q,
String queryString)
throws IOException {
List<T> entityList = new ArrayList<>();
long total;
@ -377,7 +378,7 @@ public abstract class EntityTimeSeriesRepository<T extends EntityTimeSeriesInter
if (limit > 0) {
SearchResultListMapper results =
searchRepository.listWithOffset(
searchListFilter, limit, offset, entityType, searchSortFilter, q);
searchListFilter, limit, offset, entityType, searchSortFilter, q, queryString);
total = results.getTotal();
for (Map<String, Object> json : results.getResults()) {
T entity = setFieldsInternal(JsonUtils.readOrConvertValue(json, entityClass), fields);
@ -389,7 +390,7 @@ public abstract class EntityTimeSeriesRepository<T extends EntityTimeSeriesInter
} else {
SearchResultListMapper results =
searchRepository.listWithOffset(
searchListFilter, limit, offset, entityType, searchSortFilter, q);
searchListFilter, limit, offset, entityType, searchSortFilter, q, queryString);
total = results.getTotal();
return new ResultList<>(entityList, null, limit, (int) total);
}

3
openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java
View File

@ -195,12 +195,13 @@ public abstract class EntityResource<T extends EntityInterface, K extends Entity
int offset,
SearchSortFilter searchSortFilter,
String q,
String queryString,
OperationContext operationContext,
ResourceContextInterface resourceContext)
throws IOException {
authorizer.authorize(securityContext, operationContext, resourceContext);
return repository.listFromSearchWithOffset(
uriInfo, fields, searchListFilter, limit, offset, searchSortFilter, q);
uriInfo, fields, searchListFilter, limit, offset, searchSortFilter, q, queryString);
}
public T getInternal(

6
openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityTimeSeriesResource.java
View File

@ -56,12 +56,13 @@ public abstract class EntityTimeSeriesResource<
int offset,
SearchSortFilter searchSortFilter,
String q,
String queryString,
OperationContext operationContext,
ResourceContextInterface resourceContext)
throws IOException {
authorizer.authorize(securityContext, operationContext, resourceContext);
return repository.listFromSearchWithOffset(
fields, searchListFilter, limit, offset, searchSortFilter, q);
fields, searchListFilter, limit, offset, searchSortFilter, q, queryString);
}
protected ResultList<T> listInternalFromSearch(
@ -72,12 +73,13 @@ public abstract class EntityTimeSeriesResource<
int offset,
SearchSortFilter searchSortFilter,
String q,
String queryString,
List<AuthRequest> authRequests,
AuthorizationLogic authorizationLogic)
throws IOException {
authorizer.authorizeRequests(securityContext, authRequests, authorizationLogic);
return repository.listFromSearchWithOffset(
fields, searchListFilter, limit, offset, searchSortFilter, q);
fields, searchListFilter, limit, offset, searchSortFilter, q, queryString);
}
public ResultList<T> listLatestFromSearch(

8
openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java
View File

@ -401,7 +401,12 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
description = "search query term to use in list",
schema = @Schema(type = "string"))
@QueryParam("q")
String q)
String q,
@Parameter(
description = "raw elasticsearch query to use in list",
schema = @Schema(type = "string"))
@QueryParam("queryString")
String queryString)
throws IOException {
if ((startTimestamp == null && endTimestamp != null)
|| (startTimestamp != null && endTimestamp == null)) {
@ -471,6 +476,7 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
offset,
searchSortFilter,
q,
queryString,
operationContext,
resourceContextInterface);
return PIIMasker.getTestCases(tests, authorizer, securityContext);

8
openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java
View File

@ -304,7 +304,12 @@ public class TestCaseResultResource
description = "search query term to use in list",
schema = @Schema(type = "string"))
@QueryParam("q")
String q)
String q,
@Parameter(
description = "raw elasticsearch query to use in list",
schema = @Schema(type = "string"))
@QueryParam("queryString")
String queryString)
throws IOException {
if (latest.equals("true") && (testSuiteId == null && entityFQN == null)) {
throw new IllegalArgumentException("latest=true requires testSuiteId");
@ -345,6 +350,7 @@ public class TestCaseResultResource
offset,
new SearchSortFilter("timestamp", "desc", null, null),
q,
queryString,
authRequests,
AuthorizationLogic.ANY);
}

9
openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java
View File

@ -274,7 +274,12 @@ public class TestSuiteResource extends EntityResource<TestSuite, TestSuiteReposi
description = "search query term to use in list",
schema = @Schema(type = "string"))
@QueryParam("q")
String q)
String q,
@Parameter(
description = "raw elasticsearch query to use in list",
schema = @Schema(type = "string"))
@QueryParam("queryString")
String queryString)
throws IOException {
SearchSortFilter searchSortFilter =
new SearchSortFilter(sortField, sortType, sortNestedPath, sortNestedMode);
@ -300,7 +305,7 @@ public class TestSuiteResource extends EntityResource<TestSuite, TestSuiteReposi
List<AuthRequest> authRequests = getAuthRequestsForListOps();
authorizer.authorizeRequests(securityContext, authRequests, AuthorizationLogic.ANY);
return repository.listFromSearchWithOffset(
uriInfo, fields, searchListFilter, limit, offset, searchSortFilter, q);
uriInfo, fields, searchListFilter, limit, offset, searchSortFilter, q, queryString);
}
@GET

3
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
View File

@ -187,7 +187,8 @@ public interface SearchClient {
int offset,
String index,
SearchSortFilter searchSortFilter,
String q)
String q,
String queryString)
throws IOException;
SearchResultListMapper listWithDeepPagination(

15
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
View File

@ -1019,6 +1019,18 @@ public class SearchRepository {
SearchSortFilter searchSortFilter,
String q)
throws IOException {
return listWithOffset(filter, limit, offset, entityType, searchSortFilter, q, null);
}
public SearchResultListMapper listWithOffset(
SearchListFilter filter,
int limit,
int offset,
String entityType,
SearchSortFilter searchSortFilter,
String q,
String queryString)
throws IOException {
IndexMapping index = entityIndexMap.get(entityType);
return searchClient.listWithOffset(
filter.getCondition(entityType),
@ -1026,7 +1038,8 @@ public class SearchRepository {
offset,
index.getIndexName(clusterAlias),
searchSortFilter,
q);
q,
queryString);
}
public SearchResultListMapper listWithDeepPagination(

10
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -733,16 +733,20 @@ public class ElasticSearchClient implements SearchClient {
int offset,
String index,
SearchSortFilter searchSortFilter,
String q)
String q,
String queryString)
throws IOException {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if (!nullOrEmpty(q)) {
XContentParser queryParser = createXContentParser(q);
searchSourceBuilder = getSearchSourceBuilder(index, q, offset, limit);
}
if (!nullOrEmpty(queryString)) {
XContentParser queryParser = createXContentParser(queryString);
searchSourceBuilder = SearchSourceBuilder.fromXContent(queryParser);
}
List<Map<String, Object>> results = new ArrayList<>();
getSearchFilter(filter, searchSourceBuilder, !nullOrEmpty(q));
getSearchFilter(filter, searchSourceBuilder, !nullOrEmpty(q) || !nullOrEmpty(queryString));
searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS));
searchSourceBuilder.from(offset);

10
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -717,16 +717,20 @@ public class OpenSearchClient implements SearchClient {
int offset,
String index,
SearchSortFilter searchSortFilter,
String q)
String q,
String queryString)
throws IOException {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if (!nullOrEmpty(q)) {
XContentParser queryParser = createXContentParser(q);
searchSourceBuilder = getSearchSourceBuilder(index, q, offset, limit);
}
if (!nullOrEmpty(queryString)) {
XContentParser queryParser = createXContentParser(queryString);
searchSourceBuilder = SearchSourceBuilder.fromXContent(queryParser);
}
List<Map<String, Object>> results = new ArrayList<>();
getSearchFilter(filter, searchSourceBuilder, !nullOrEmpty(q));
getSearchFilter(filter, searchSourceBuilder, !nullOrEmpty(q) || !nullOrEmpty(queryString));
searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS));
searchSourceBuilder.from(offset);

11
openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java
View File

@ -953,7 +953,7 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
listEntitiesFromSearch(queryParams, testCasesNum, 0, ADMIN_AUTH_HEADERS);
assertEquals(testCasesNum, allEntities.getData().size());
queryParams.put(
"q",
"queryString",
"%7B%22query%22%3A%20%7B%22term%22%3A%20%7B%22id%22%3A%20%22"
+ testCaseForEL.getId()
+ "%22%7D%7D%7D");
@ -965,6 +965,13 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
.allMatch(
ts -> ts.getFullyQualifiedName().equals(testCaseForEL.getFullyQualifiedName())));
queryParams.clear();
queryParams.put("q", "test_getSimpleListFromSearchb");
allEntities = listEntitiesFromSearch(queryParams, testCasesNum, 0, ADMIN_AUTH_HEADERS);
// Note: Since the "name" field and its ngram variant are prioritized in the search query
// and the test case names are very similar, the fuzzy matching returns all test cases.
assertEquals(testCasesNum, allEntities.getData().size());
queryParams.clear();
queryParams.put("entityLink", testCaseForEL.getEntityLink());
queryParams.put("includeAllTests", "true");
@ -3664,7 +3671,7 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
String id = testCaseResultResultList.getData().get(0).getId().toString();
queryParams.put(
"q",
"queryString",
"%7B%22query%22%3A%20%7B%22term%22%3A%20%7B%22id.keyword%22%3A%20%22"
+ id
+ "%22%7D%7D%7D");

11
openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java
View File

@ -439,7 +439,7 @@ public class TestSuiteResourceTest extends EntityResourceTest<TestSuite, CreateT
// 5. List test suite with a query
queryParams.clear();
queryParams.put(
"q",
"queryString",
"%7B%22query%22%3A%20%7B%22term%22%3A%20%7B%22id%22%3A%20%22"
+ logicalTestSuite.getId()
+ "%22%7D%7D%7D");
@ -449,6 +449,15 @@ public class TestSuiteResourceTest extends EntityResourceTest<TestSuite, CreateT
queryTestSuites.getData().stream()
.allMatch(
ts -> ts.getFullyQualifiedName().equals(logicalTestSuite.getFullyQualifiedName())));
queryParams.clear();
queryParams.put("q", logicalTestSuite.getFullyQualifiedName());
queryTestSuites = listEntitiesFromSearch(queryParams, 100, 0, ADMIN_AUTH_HEADERS);
Assertions.assertTrue(
queryTestSuites.getData().stream()
.allMatch(
ts -> ts.getFullyQualifiedName().equals(logicalTestSuite.getFullyQualifiedName())));
// 6. List test suites with a nested sort
queryParams.clear();
queryParams.put("fields", "tests");

4
openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TestSuiteMultiPipeline.spec.ts
View File

@ -18,7 +18,7 @@ import { getApiContext, redirectToHomePage, uuid } from '../../utils/common';
// use the admin user to login
test.use({ storageState: 'playwright/.auth/admin.json' });
test.skip(
test(
'TestSuite multi pipeline support',
PLAYWRIGHT_INGESTION_TAG_OBJ,
async ({ page }) => {
@ -185,7 +185,7 @@ test.skip(
}
);
test.skip(
test(
"Edit the pipeline's test case",
PLAYWRIGHT_INGESTION_TAG_OBJ,
async ({ page }) => {

25
openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts
View File

@ -322,7 +322,7 @@ test(
}
);
test.skip(
test(
'TestCase with Array params value',
PLAYWRIGHT_INGESTION_TAG_OBJ,
async ({ page }) => {
@ -639,10 +639,7 @@ test(
}
);
test.skip(
'TestCase filters',
PLAYWRIGHT_INGESTION_TAG_OBJ,
async ({ page }) => {
test('TestCase filters', PLAYWRIGHT_INGESTION_TAG_OBJ, async ({ page }) => {
test.setTimeout(360000);
const { apiContext, afterAction } = await getApiContext(page);
@ -705,8 +702,7 @@ test.skip(
});
const testCaseResult = {
result:
'Found min=10001, max=27809 vs. the expected min=90001, max=96162.',
result: 'Found min=10001, max=27809 vs. the expected min=90001, max=96162.',
testCaseStatus: 'Failed',
testResultValue: [
{
@ -793,9 +789,7 @@ test.skip(
);
await searchTestCaseResponse;
await expect(
page.locator(`[data-testid="${testCases[0]}"]`)
).toBeVisible();
await expect(page.locator(`[data-testid="${testCases[0]}"]`)).toBeVisible();
// clear the search filter
const getTestCaseResponse = page.waitForResponse(
@ -1001,9 +995,7 @@ test.skip(
await page.click('[value="testPlatforms"]');
await page.waitForTimeout(200);
await expect(
page.getByTestId('platform-select-filter')
).not.toBeVisible();
await expect(page.getByTestId('platform-select-filter')).not.toBeVisible();
await page.reload();
@ -1037,9 +1029,7 @@ test.skip(
.fill(testCases[0]);
await searchTestCase;
await expect(
page.locator(`[data-testid="${testCases[0]}"]`)
).toBeVisible();
await expect(page.locator(`[data-testid="${testCases[0]}"]`)).toBeVisible();
await expect(
page.locator(`[data-testid="${testCases[1]}"]`)
).not.toBeVisible();
@ -1051,5 +1041,4 @@ test.skip(
await domain.delete(apiContext);
await afterAction();
}
}
);
});

2
openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts
View File

@ -64,7 +64,7 @@ test.beforeEach(async ({ page }) => {
await redirectToHomePage(page);
});
test.skip('Logical TestSuite', async ({ page }) => {
test('Logical TestSuite', async ({ page }) => {
test.slow();
const NEW_TEST_SUITE = {