yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-18 14:06:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Appconfig fixes (#20960)

Browse Source 
* fix initial config for json logic query builder

* show alert for application

* hide hidden fields from doc
This commit is contained in:
Karan Hotchandani 2025-04-29 10:24:04 +05:30 committed by GitHub
parent b3d7f97590
commit dbebaff32b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 395 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
openmetadata-ui/src/main/resources/ui/generateApplicationDocs.js
View File

@ -17,6 +17,12 @@ const path = require('path');
const SCHEMA_DIR = path.join(__dirname, './src/utils/ApplicationSchemas');
const DOCS_DIR = path.join(__dirname, './public/locales/en-US/Applications');
const IGNORE_FIELDS = [
'moduleConfiguration.dataAssets.serviceFilter',
'entityLink',
'type',
];
const resolveRef = (schema, ref) => {
const path = ref.split('/').slice(1);
let current = schema;
@ -27,6 +33,12 @@ const resolveRef = (schema, ref) => {
};
const processProperty = (key, prop, schema) => {
// Skip if the key is in IGNORE_FIELDS
const currentKey = key.split('.').pop();
if (IGNORE_FIELDS.includes(currentKey)) {
return '';
}
let markdown = `$$section\n`;
markdown += `### ${prop.title || key} $(id="${key}")\n\n`;

14
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/AutoPilotApplication.md
View File

@ -2,23 +2,9 @@
Configuration for the AutoPilot Application.
$$section
### Application Type $(id="type")
Application Type
$$
$$section
### Active $(id="active")
Whether the AutoPilot Workflow should be active or not.
$$
$$section
### Service Entity Link $(id="entityLink")
Service Entity Link for which to trigger the application.
$$

7
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/DataInsightsApplication.md
View File

@ -2,13 +2,6 @@
This schema defines configuration for the Data Insights Application.
$$section
### Application Type $(id="type")
Application Type
$$
$$section
### batchSize $(id="batchSize")

8
openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx
View File

@ -30,13 +30,13 @@ import {
ValueSource,
} from 'react-awesome-query-builder';
import { useHistory, useParams } from 'react-router-dom';
import { emptyJsonTree } from '../../../constants/AdvancedSearch.constants';
import { SearchIndex } from '../../../enums/search.enum';
import useCustomLocation from '../../../hooks/useCustomLocation/useCustomLocation';
import { TabsInfoData } from '../../../pages/ExplorePage/ExplorePage.interface';
import { getAllCustomProperties } from '../../../rest/metadataTypeAPI';
import advancedSearchClassBase from '../../../utils/AdvancedSearchClassBase';
import {
getEmptyJsonTree,
getTierOptions,
getTreeConfig,
} from '../../../utils/AdvancedSearchUtils';
@ -114,7 +114,7 @@ export const AdvanceSearchProvider = ({
const [initialised, setInitialised] = useState(false);
const defaultTree = useMemo(
() => QbUtils.checkTree(QbUtils.loadTree(emptyJsonTree), config),
() => QbUtils.checkTree(QbUtils.loadTree(getEmptyJsonTree()), config),
[]
);
@ -197,7 +197,9 @@ export const AdvanceSearchProvider = ({
};
const handleReset = useCallback(() => {
setTreeInternal(QbUtils.checkTree(QbUtils.loadTree(emptyJsonTree), config));
setTreeInternal(
QbUtils.checkTree(QbUtils.loadTree(getEmptyJsonTree()), config)
);
setQueryFilter(undefined);
setSQLQuery('');
}, [config]);

96
openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget.tsx
View File

@ -35,19 +35,23 @@ import {
Query,
Utils as QbUtils,
} from 'react-awesome-query-builder';
import {
EntityFields,
EntityReferenceFields,
} from '../../../../../../enums/AdvancedSearch.enum';
import { EntityType } from '../../../../../../enums/entity.enum';
import { SearchIndex } from '../../../../../../enums/search.enum';
import {
EsBoolQuery,
QueryFieldInterface,
} from '../../../../../../pages/ExplorePage/ExplorePage.interface';
import { QueryFilterInterface } from '../../../../../../pages/ExplorePage/ExplorePage.interface';
import { searchQuery } from '../../../../../../rest/searchAPI';
import { getEmptyJsonTree } from '../../../../../../utils/AdvancedSearchUtils';
import {
elasticSearchFormat,
elasticSearchFormatForJSONLogic,
} from '../../../../../../utils/QueryBuilderElasticsearchFormatUtils';
import {
addEntityTypeFilter,
elasticsearchToJsonLogic,
getEntityTypeAggregationFilter,
getJsonTreeFromQueryFilter,
jsonLogicToElasticsearch,
READONLY_SETTINGS,
@ -82,29 +86,46 @@ const QueryBuilderWidget: FC<WidgetProps> = ({
const outputType = schema?.outputType ?? SearchOutputType.ElasticSearch;
const isSearchIndexUpdatedInContext = searchIndexFromContext === searchIndex;
const [initDone, setInitDone] = useState<boolean>(false);
const [queryURL, setQueryURL] = useState<string>('');
const fetchEntityCount = useCallback(
async (queryFilter: Record<string, unknown>) => {
const qFilter = getEntityTypeAggregationFilter(
queryFilter as unknown as QueryFilterInterface,
entityType
);
const tree = QbUtils.checkTree(
QbUtils.loadTree(getJsonTreeFromQueryFilter(qFilter) as JsonTree),
config
);
const queryFilterString = !isEmpty(tree)
? Qs.stringify({ queryFilter: JSON.stringify(tree) })
: '';
setQueryURL(`${getExplorePath({})}${queryFilterString}`);
try {
setIsCountLoading(true);
const res = await searchQuery({
query: '',
pageNumber: 0,
pageSize: 0,
queryFilter,
queryFilter: qFilter as unknown as Record<string, unknown>,
searchIndex: SearchIndex.ALL,
includeDeleted: false,
trackTotalHits: true,
fetchSource: false,
});
setSearchResults(res.hits.total.value ?? 0);
} catch (_) {
} catch {
// silent fail
} finally {
setIsCountLoading(false);
}
},
[]
[entityType]
);
const debouncedFetchEntityCount = useMemo(
@ -112,14 +133,6 @@ const QueryBuilderWidget: FC<WidgetProps> = ({
[fetchEntityCount]
);
const queryURL = useMemo(() => {
const queryFilterString = !isEmpty(treeInternal)
? Qs.stringify({ queryFilter: JSON.stringify(treeInternal) })
: '';
return `${getExplorePath({})}${queryFilterString}`;
}, [treeInternal]);
const showFilteredResourceCount = useMemo(
() =>
outputType === SearchOutputType.ElasticSearch &&
@ -138,31 +151,14 @@ const QueryBuilderWidget: FC<WidgetProps> = ({
query: data,
};
if (data) {
if (entityType !== EntityType.ALL) {
// Scope the search to the passed entity type
if (
Array.isArray(
((qFilter.query as QueryFieldInterface)?.bool as EsBoolQuery)
?.must
)
) {
(
(qFilter.query as QueryFieldInterface)?.bool
?.must as QueryFieldInterface[]
)?.push({
bool: {
must: [
{
term: {
entityType: entityType,
},
},
],
},
});
}
}
debouncedFetchEntityCount(qFilter);
const qFilterWithEntityType = addEntityTypeFilter(
qFilter as unknown as QueryFilterInterface,
entityType
);
debouncedFetchEntityCount(
qFilterWithEntityType as unknown as Record<string, unknown>
);
}
onChange(!isEmpty(data) ? JSON.stringify(qFilter) : '');
@ -187,22 +183,22 @@ const QueryBuilderWidget: FC<WidgetProps> = ({
const loadDefaultValueInTree = useCallback(() => {
if (!isEmpty(value)) {
const parsedValue = JSON.parse(value ?? '{}');
if (outputType === SearchOutputType.ElasticSearch) {
const parsedTree = getJsonTreeFromQueryFilter(
JSON.parse(value || ''),
parsedValue,
config.fields
) as JsonTree;
if (Object.keys(parsedTree).length > 0) {
const tree = QbUtils.checkTree(QbUtils.loadTree(parsedTree), config);
onTreeUpdate(tree, config);
// Fetch count for default value
debouncedFetchEntityCount(parsedValue);
}
} else {
try {
const query = jsonLogicToElasticsearch(
JSON.parse(value || ''),
config.fields
);
const query = jsonLogicToElasticsearch(parsedValue, config.fields);
const updatedQ = {
query: query,
};
@ -225,6 +221,16 @@ const QueryBuilderWidget: FC<WidgetProps> = ({
console.log(e);
}
}
} else {
const emptyJsonTree = getEmptyJsonTree(
outputType === SearchOutputType.JSONLogic
? EntityReferenceFields.OWNERS
: EntityFields.OWNERS
);
onTreeUpdate(
QbUtils.checkTree(QbUtils.loadTree(emptyJsonTree), config),
config
);
}
setInitDone(true);
}, [config, value, outputType]);

34
openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts
View File

@ -12,7 +12,6 @@
*/
import { t } from 'i18next';
import { JsonTree, Utils as QbUtils } from 'react-awesome-query-builder';
import { EntityFields } from '../enums/AdvancedSearch.enum';
import { SearchIndex } from '../enums/search.enum';
@ -304,39 +303,6 @@ export const RANGE_FIELD_OPERATORS = ['between', 'not_between'];
export const LIST_VALUE_OPERATORS = ['select_equals', 'select_not_equals'];
/**
* Generates a query builder tree with a group containing an empty rule
*/
export const emptyJsonTree: JsonTree = {
id: QbUtils.uuid(),
type: 'group',
properties: {
conjunction: 'AND',
not: false,
},
children1: {
[QbUtils.uuid()]: {
type: 'group',
properties: {
conjunction: 'AND',
not: false,
},
children1: {
[QbUtils.uuid()]: {
type: 'rule',
properties: {
// owner is common field , so setting owner as default field here
field: EntityFields.OWNERS,
operator: null,
value: [],
valueSrc: ['value'],
},
},
},
},
},
};
export const MISC_FIELDS = ['owner.displayName', 'tags.tagFQN'];
export const OWNER_QUICK_FILTER_DEFAULT_OPTIONS_KEY = 'displayName.keyword';

60
openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.test.tsx
View File

@ -12,10 +12,12 @@
*/
import { SearchDropdownOption } from '../components/SearchDropdown/SearchDropdown.interface';
import { EntityFields } from '../enums/AdvancedSearch.enum';
import { SearchIndex } from '../enums/search.enum';
import {
getChartsOptions,
getColumnsOptions,
getEmptyJsonTree,
getOptionsFromAggregationBucket,
getSchemaFieldOptions,
getSearchDropdownLabels,
@ -45,6 +47,13 @@ import {
mockShortOptionsArray,
} from './mocks/AdvancedSearchUtils.mock';
// Mock QbUtils
jest.mock('react-awesome-query-builder', () => ({
Utils: {
uuid: jest.fn().mockReturnValue('test-uuid'),
},
}));
describe('AdvancedSearchUtils tests', () => {
it('Function getSearchDropdownLabels should return menuItems for passed options', () => {
const resultMenuItems = getSearchDropdownLabels(mockOptionsArray, true);
@ -215,4 +224,55 @@ describe('AdvancedSearchUtils tests', () => {
{ count: 3, key: 'chart', label: 'chart' },
]);
});
describe('getEmptyJsonTree', () => {
it('should return a default JsonTree structure with OWNERS as the default field', () => {
const result = getEmptyJsonTree();
const expected = {
id: 'test-uuid',
type: 'group',
properties: {
conjunction: 'AND',
not: false,
},
children1: {
'test-uuid': {
type: 'group',
properties: {
conjunction: 'AND',
not: false,
},
children1: {
'test-uuid': {
type: 'rule',
properties: {
field: EntityFields.OWNERS,
operator: null,
value: [],
valueSrc: ['value'],
},
},
},
},
},
};
expect(result).toEqual(expected);
});
it('should use the provided field when passed as parameter', () => {
const customField = EntityFields.TAG;
const result = getEmptyJsonTree(customField);
const children1 = result.children1 as Record<
string,
{ children1: Record<string, { properties: { field: string } }> }
>;
expect(
children1['test-uuid'].children1['test-uuid'].properties.field
).toEqual(customField);
});
});
});

36
openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx
View File

@ -18,8 +18,10 @@ import { isArray, isEmpty, toLower } from 'lodash';
import React from 'react';
import {
AsyncFetchListValues,
JsonTree,
ListValues,
RenderSettings,
Utils as QbUtils,
} from 'react-awesome-query-builder';
import { ReactComponent as IconDeleteColored } from '../assets/svg/ic-delete-colored.svg';
import ProfilePicture from '../components/common/ProfilePicture/ProfilePicture';
@ -33,6 +35,7 @@ import {
LINEAGE_DROPDOWN_ITEMS,
} from '../constants/AdvancedSearch.constants';
import { NOT_INCLUDE_AGGREGATION_QUICK_FILTER } from '../constants/explore.constants';
import { EntityFields } from '../enums/AdvancedSearch.enum';
import { EntityType } from '../enums/entity.enum';
import { SearchIndex } from '../enums/search.enum';
import {
@ -410,3 +413,36 @@ export const getCustomPropertyAdvanceSearchEnumOptions = (
return acc;
}, {});
};
export const getEmptyJsonTree = (
defaultField: string = EntityFields.OWNERS
): JsonTree => {
return {
id: QbUtils.uuid(),
type: 'group',
properties: {
conjunction: 'AND',
not: false,
},
children1: {
[QbUtils.uuid()]: {
type: 'group',
properties: {
conjunction: 'AND',
not: false,
},
children1: {
[QbUtils.uuid()]: {
type: 'rule',
properties: {
field: defaultField,
operator: null,
value: [],
valueSrc: ['value'],
},
},
},
},
},
};
};

180
openmetadata-ui/src/main/resources/ui/src/utils/QueryBuilderUtils.test.ts
View File

@ -11,8 +11,14 @@
* limitations under the License.
*/
import { Fields } from 'react-awesome-query-builder';
import { QueryFilterInterface } from '../pages/ExplorePage/ExplorePage.interface';
import { EntityType } from '../enums/entity.enum';
import {
QueryFieldInterface,
QueryFilterInterface,
} from '../pages/ExplorePage/ExplorePage.interface';
import {
addEntityTypeFilter,
getEntityTypeAggregationFilter,
getJsonTreeFromQueryFilter,
resolveFieldType,
} from './QueryBuilderUtils';
@ -152,3 +158,175 @@ describe('resolveFieldType', () => {
expect(resolveFieldType(undefined, 'name')).toBe('');
});
});
describe('addEntityTypeFilter', () => {
const baseQueryFilter: QueryFilterInterface = {
query: {
bool: {
must: [
{
bool: {
must: [
{
term: {
field1: 'value1',
},
},
],
},
},
],
},
},
};
it('should return the original filter when entityType is ALL', () => {
const result = addEntityTypeFilter({ ...baseQueryFilter }, EntityType.ALL);
expect(result).toEqual(baseQueryFilter);
});
it('should add entity type filter for non-ALL entity types', () => {
const result = addEntityTypeFilter(
{ ...baseQueryFilter },
EntityType.TABLE
);
// Assert the must array exists and has correct length
expect(result.query?.bool?.must).toBeDefined();
const mustArray = result.query?.bool?.must as QueryFieldInterface[];
expect(Array.isArray(mustArray)).toBe(true);
expect(mustArray).toHaveLength(2);
// Assert the entity type filter is added correctly
expect(mustArray[1]).toEqual({
bool: {
must: [
{
term: {
entityType: EntityType.TABLE,
},
},
],
},
});
});
it('should handle undefined must array gracefully', () => {
const queryFilter: QueryFilterInterface = {
query: {
bool: {},
},
};
const result = addEntityTypeFilter(queryFilter, EntityType.TABLE);
expect(result).toEqual(queryFilter);
});
it('should handle empty query gracefully', () => {
const queryFilter = {} as QueryFilterInterface;
const result = addEntityTypeFilter(queryFilter, EntityType.TABLE);
expect(result).toEqual(queryFilter);
});
});
describe('getEntityTypeAggregationFilter', () => {
const baseQueryFilter: QueryFilterInterface = {
query: {
bool: {
must: [
{
bool: {
must: [
{
term: {
field1: 'value1',
},
},
],
},
},
],
},
},
};
it('should add entity type to the first must block', () => {
const result = getEntityTypeAggregationFilter(
{ ...baseQueryFilter },
EntityType.TABLE
);
// Assert the must array exists
expect(result.query?.bool?.must).toBeDefined();
const mustArray = result.query?.bool?.must as QueryFieldInterface[];
expect(Array.isArray(mustArray)).toBe(true);
expect(mustArray.length).toBeGreaterThan(0);
// Get the first must block and assert its structure
const firstMustBlock = mustArray[0];
const mustBlockArray = firstMustBlock.bool?.must as QueryFieldInterface[];
expect(mustBlockArray).toBeDefined();
expect(Array.isArray(mustBlockArray)).toBe(true);
expect(mustBlockArray).toHaveLength(2);
// Assert the entity type filter is added correctly
expect(mustBlockArray[1]).toEqual({
term: {
entityType: EntityType.TABLE,
},
});
});
it('should handle undefined must array in first block gracefully', () => {
const queryFilter: QueryFilterInterface = {
query: {
bool: {
must: [
{
bool: {},
},
],
},
},
};
const result = getEntityTypeAggregationFilter(
queryFilter,
EntityType.TABLE
);
expect(result).toEqual(queryFilter);
});
it('should handle empty must array gracefully', () => {
const queryFilter: QueryFilterInterface = {
query: {
bool: {
must: [],
},
},
};
const result = getEntityTypeAggregationFilter(
queryFilter,
EntityType.TABLE
);
expect(result).toEqual(queryFilter);
});
it('should handle empty query gracefully', () => {
const queryFilter = {} as QueryFilterInterface;
const result = getEntityTypeAggregationFilter(
queryFilter,
EntityType.TABLE
);
expect(result).toEqual(queryFilter);
});
});

52
openmetadata-ui/src/main/resources/ui/src/utils/QueryBuilderUtils.tsx
View File

@ -21,6 +21,7 @@ import {
RenderSettings,
} from 'react-awesome-query-builder';
import { EntityReferenceFields } from '../enums/AdvancedSearch.enum';
import { EntityType } from '../enums/entity.enum';
import {
EsBoolQuery,
EsExistsQuery,
@ -801,3 +802,54 @@ export const jsonLogicToElasticsearch = (
throw new Error('Unsupported JSON Logic format');
};
/**
* Adds entity type filter to the query filter if entity type is specified
* @param qFilter Query filter to add entity type to
* @param entityType Entity type to filter by
* @returns Updated query filter with entity type
*/
export const addEntityTypeFilter = (
qFilter: QueryFilterInterface,
entityType: string
): QueryFilterInterface => {
if (entityType === EntityType.ALL) {
return qFilter;
}
if (Array.isArray((qFilter.query?.bool as EsBoolQuery)?.must)) {
(qFilter.query?.bool?.must as QueryFieldInterface[])?.push({
bool: {
must: [
{
term: {
entityType: entityType,
},
},
],
},
});
}
return qFilter;
};
export const getEntityTypeAggregationFilter = (
qFilter: QueryFilterInterface,
entityType: string
): QueryFilterInterface => {
if (Array.isArray((qFilter.query?.bool as EsBoolQuery)?.must)) {
const firstMustBlock = (
qFilter.query?.bool?.must as QueryFieldInterface[]
)[0];
if (firstMustBlock?.bool?.must) {
(firstMustBlock.bool.must as QueryFieldInterface[]).push({
term: {
entityType: entityType,
},
});
}
}
return qFilter;
};