#11951 Column Profiling metrics based on data type (for UI and calculations) (#15987)

* #11951 Column Profiling metrics based on data type (for UI and calculations)

* fixed cypress

* fixed cypres test

* fixed cypress test
This commit is contained in:
Shailesh Parmar 2024-04-24 15:51:06 +05:30 committed by GitHub
parent 9f49b41acf
commit cd73328f87
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 831 additions and 0 deletions

View File

@ -0,0 +1,106 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const PROFILER_REQUEST_CONFIG = {
config_type: 'profilerConfiguration',
config_value: {
metricConfiguration: [
{
dataType: 'AGG_STATE',
metrics: [
'COLUMN_COUNT',
'COLUMN_NAMES',
'COUNT',
'COUNT_IN_SET',
'DISTINCT_COUNT',
'DISTINCT_RATIO',
'DUPLICATE_COUNT',
'FIRST_QUARTILE',
'HISTOGRAM',
'ILIKE_COUNT',
'ILIKE_RATIO',
'IQR',
'LIKE_COUNT',
'LIKE_RATIO',
'MAX',
'MAX_LENGTH',
'MEAN',
'MEDIAN',
'MIN',
'MIN_LENGTH',
'NON_PARAMETRIC_SKEW',
'NOT_LIKE_COUNT',
'NOT_REGEX_COUNT',
'NULL_COUNT',
'NULL_RATIO',
'REGEX_COUNT',
'ROW_COUNT',
'STDDEV',
'SUM',
'SYSTEM',
'THIRD_QUARTILE',
'UNIQUE_COUNT',
'UNIQUE_RATIO',
],
disabled: false,
},
{
dataType: 'AGGREGATEFUNCTION',
metrics: ['COLUMN_COUNT', 'COLUMN_NAMES'],
},
{
dataType: 'ARRAY',
metrics: [
'COLUMN_COUNT',
'COLUMN_NAMES',
'COUNT',
'COUNT_IN_SET',
'DISTINCT_COUNT',
'DISTINCT_RATIO',
'DUPLICATE_COUNT',
'FIRST_QUARTILE',
'HISTOGRAM',
'ILIKE_COUNT',
'ILIKE_RATIO',
'IQR',
'LIKE_COUNT',
'LIKE_RATIO',
'MAX',
'MAX_LENGTH',
'MEAN',
'MEDIAN',
'MIN',
'MIN_LENGTH',
'NON_PARAMETRIC_SKEW',
'NOT_LIKE_COUNT',
'NOT_REGEX_COUNT',
'NULL_COUNT',
'NULL_RATIO',
'REGEX_COUNT',
'ROW_COUNT',
'STDDEV',
'SUM',
'SYSTEM',
'THIRD_QUARTILE',
'UNIQUE_COUNT',
'UNIQUE_RATIO',
],
disabled: true,
},
],
},
};
export const PROFILER_EMPTY_RESPONSE_CONFIG = {
config_type: 'profilerConfiguration',
config_value: { metricConfiguration: [] },
};

View File

@ -0,0 +1,161 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { interceptURL } from '../../common/common';
import { getToken } from '../../common/Utils/LocalStorage';
import { generateRandomUser } from '../../common/Utils/Owner';
import { SidebarItem } from '../../constants/Entity.interface';
import {
PROFILER_EMPTY_RESPONSE_CONFIG,
PROFILER_REQUEST_CONFIG,
} from '../../constants/ProfilerConfiguration.constant';
const visitProfilerConfigurationPage = () => {
interceptURL(
'GET',
'/api/v1/system/settings/profilerConfiguration',
'getProfilerConfiguration'
);
cy.sidebarClick(SidebarItem.SETTINGS);
cy.get('[data-testid="preferences"]').scrollIntoView().click();
cy.get('[data-testid="preferences.profiler-configuration"]')
.scrollIntoView()
.click();
cy.wait('@getProfilerConfiguration');
};
const user = generateRandomUser();
let userId = '';
describe('ProfilerConfigurationPage', () => {
before(() => {
cy.login();
cy.getAllLocalStorage().then((data) => {
const token = getToken(data);
// Create a new user
cy.request({
method: 'POST',
url: `/api/v1/users/signup`,
headers: { Authorization: `Bearer ${token}` },
body: user,
}).then((response) => {
userId = response.body.id;
});
});
});
after(() => {
cy.login();
cy.getAllLocalStorage().then((data) => {
const token = getToken(data);
// Delete created user
cy.request({
method: 'DELETE',
url: `/api/v1/users/${userId}?hardDelete=true&recursive=false`,
headers: { Authorization: `Bearer ${token}` },
});
});
});
beforeEach(() => {
cy.login();
});
it('Verify validation', () => {
visitProfilerConfigurationPage();
cy.get('[data-testid="save-button"]').click();
cy.get('#metricConfiguration_0_dataType_help').should(
'contain',
'Data Type is required.'
);
cy.get('[data-testid="cancel-button"]').click();
cy.url().should('eq', `${Cypress.config().baseUrl}/settings/preferences`);
});
it('Update profiler configuration', () => {
visitProfilerConfigurationPage();
cy.get('#metricConfiguration_0_dataType').click();
cy.get(`[title="AGG_STATE"]`).filter(':visible').scrollIntoView().click();
cy.get('#metricConfiguration_0_metrics').click().type('All');
cy.get(`[role="tree"] [title="All"]`)
.filter(':visible')
.scrollIntoView()
.click();
cy.get('[data-testid="add-fields"]').click();
cy.get('#metricConfiguration_1_dataType').click();
cy.get(`.rc-virtual-list-holder-inner [title="AGG_STATE"]`)
.filter(':visible')
.should('have.class', 'ant-select-item-option-disabled');
cy.get(`.rc-virtual-list-holder-inner [title="AGGREGATEFUNCTION"]`)
.filter(':visible')
.scrollIntoView()
.click();
cy.clickOutside();
cy.get('#metricConfiguration_1_metrics').click().type('column');
cy.get(`[role="tree"] [title="COLUMN_COUNT"]`).filter(':visible').click();
cy.get(`[role="tree"] [title="COLUMN_NAMES"]`).filter(':visible').click();
cy.clickOutside();
cy.get('[data-testid="add-fields"]').click();
cy.get('#metricConfiguration_2_dataType').click();
cy.get(`.rc-virtual-list-holder-inner [title="ARRAY"]`)
.filter(':visible')
.scrollIntoView()
.click();
cy.get('#metricConfiguration_2_metrics').click().type('All');
cy.get(`[role="tree"] [title="All"]`)
.filter(':visible')
.scrollIntoView()
.click();
cy.clickOutside();
cy.get('#metricConfiguration_2_disabled').click();
interceptURL(
'PUT',
'/api/v1/system/settings',
'updateProfilerConfiguration'
);
cy.get('[data-testid="save-button"]').click();
cy.wait('@updateProfilerConfiguration').then((interception) => {
expect(interception.request.body).to.deep.eq(PROFILER_REQUEST_CONFIG);
});
});
it('Remove Configuration', () => {
visitProfilerConfigurationPage();
cy.get('[data-testid="remove-filter-2"]').click();
cy.get('[data-testid="remove-filter-1"]').click();
cy.get('[data-testid="remove-filter-0"]').click();
interceptURL(
'PUT',
'/api/v1/system/settings',
'updateProfilerConfiguration'
);
cy.get('[data-testid="save-button"]').click();
cy.wait('@updateProfilerConfiguration').then((interception) => {
expect(interception.request.body).to.deep.eq(
PROFILER_EMPTY_RESPONSE_CONFIG
);
});
});
it('permission check for non admin user', () => {
cy.logout();
cy.login(user.email, user.password);
cy.sidebarClick(SidebarItem.SETTINGS);
cy.get('[data-testid="preferences"]').should('not.exist');
cy.logout();
});
});

View File

@ -0,0 +1,54 @@
<svg width="50" height="51" viewBox="0 0 50 51" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M40.6898 3.55469H19.1594L11.6367 11.0774V44.2668C11.6367 45.9562 13.0079 47.3274 14.6973 47.3274H40.6973C42.3867 47.3274 43.7579 45.9562 43.7579 44.2668V6.61529C43.7504 4.9259 42.3791 3.55469 40.6898 3.55469Z" fill="url(#paint0_linear_4267_40961)"/>
<path opacity="0.1" d="M11.6367 25.6992V44.275C11.6367 45.9644 13.0079 47.3356 14.6973 47.3356H17.5458C20.9625 45.3053 23.2731 41.5856 23.2731 37.328C23.2655 30.9189 18.0458 25.6992 11.6367 25.6992Z" fill="black"/>
<path d="M16.0988 11.0774C17.7882 11.0774 19.1594 9.7062 19.1594 8.01681V3.55469L11.6367 11.0774H16.0988Z" fill="#FF8D61"/>
<path d="M30.993 26.3045C36.3109 26.3045 40.6218 21.9935 40.6218 16.6757C40.6218 11.3578 36.3109 7.04688 30.993 7.04688C25.6752 7.04688 21.3643 11.3578 21.3643 16.6757C21.3643 21.9935 25.6752 26.3045 30.993 26.3045Z" fill="url(#paint1_linear_4267_40961)"/>
<path d="M38.3945 26.7143L45.296 33.6158C46.0233 34.343 47.2127 34.343 47.94 33.6158C48.6673 32.8885 48.6673 31.6991 47.94 30.9718L41.0385 24.0703L38.3945 26.7143Z" fill="url(#paint2_linear_4267_40961)"/>
<path d="M37.8033 23.4781C36.826 24.4554 35.6897 25.1751 34.4775 25.6448L35.2654 26.4326C36.2048 27.372 37.7351 27.372 38.6745 26.4326L40.7578 24.3493C41.6972 23.4099 41.6972 21.8796 40.7578 20.9402L39.97 20.1523C39.5003 21.3645 38.7806 22.5008 37.8033 23.4781Z" fill="#8E76FF"/>
<path d="M25.6973 16.6602H28.0533V20.8192H25.6973V16.6602Z" fill="url(#paint3_linear_4267_40961)"/>
<path d="M29.8184 14.2969H32.1744V20.812H29.8184V14.2969Z" fill="url(#paint4_linear_4267_40961)"/>
<path d="M33.9395 12.5312H36.2955V20.8191H33.9395V12.5312Z" fill="url(#paint5_linear_4267_40961)"/>
<path d="M25.25 31.9805H29.8182V36.5487H25.25V31.9805Z" fill="#FF8D61"/>
<path d="M25.25 39.5781H29.8182V44.1463H25.25V39.5781Z" fill="#FF8D61"/>
<path d="M11.6368 27.2148C6.04593 27.2148 1.51562 31.7451 1.51562 37.3285C1.51562 40.1239 2.64441 42.6542 4.47775 44.48L11.6293 37.3285V27.2148H11.6368Z" fill="#653DC5"/>
<path d="M11.6367 27.2148V37.3361H21.7579C21.7504 31.7451 17.2201 27.2148 11.6367 27.2148Z" fill="url(#paint6_linear_4267_40961)"/>
<path d="M4.47754 44.4872C6.31087 46.3206 8.84118 47.4493 11.6291 47.4493C17.22 47.4493 21.7503 42.919 21.7503 37.3281H11.6366L4.47754 44.4872Z" fill="url(#paint7_linear_4267_40961)"/>
<path d="M32.9324 33.57H40.6218C40.9324 33.57 41.19 33.3124 41.19 33.0018C41.19 32.6912 40.9324 32.4336 40.6218 32.4336H32.9324C32.6218 32.4336 32.3643 32.6912 32.3643 33.0018C32.3643 33.3124 32.6218 33.57 32.9324 33.57Z" fill="#8E76FF"/>
<path d="M32.9324 36.0856H37.5915C37.9021 36.0856 38.1597 35.828 38.1597 35.5174C38.1597 35.2068 37.9021 34.9492 37.5915 34.9492H32.9324C32.6218 34.9492 32.3643 35.2068 32.3643 35.5174C32.3643 35.828 32.6218 36.0856 32.9324 36.0856Z" fill="#8E76FF"/>
<path d="M40.6218 40.0312H32.9324C32.6218 40.0312 32.3643 40.2888 32.3643 40.5994C32.3643 40.91 32.6218 41.1676 32.9324 41.1676H40.6218C40.9324 41.1676 41.19 40.91 41.19 40.5994C41.19 40.2888 40.94 40.0312 40.6218 40.0312Z" fill="#8E76FF"/>
<path d="M37.5915 42.5469H32.9324C32.6218 42.5469 32.3643 42.8045 32.3643 43.1151C32.3643 43.4257 32.6218 43.6832 32.9324 43.6832H37.5915C37.9021 43.6832 38.1597 43.4257 38.1597 43.1151C38.1597 42.8045 37.9021 42.5469 37.5915 42.5469Z" fill="#8E76FF"/>
<defs>
<linearGradient id="paint0_linear_4267_40961" x1="27.6928" y1="-3.38849" x2="27.6928" y2="36.8456" gradientUnits="userSpaceOnUse">
<stop stop-color="#FEFDFF"/>
<stop offset="1" stop-color="#DEDFFF"/>
</linearGradient>
<linearGradient id="paint1_linear_4267_40961" x1="30.9961" y1="7.0143" x2="30.9961" y2="26.0961" gradientUnits="userSpaceOnUse">
<stop stop-color="#8E76FF"/>
<stop offset="1" stop-color="#653DC5"/>
</linearGradient>
<linearGradient id="paint2_linear_4267_40961" x1="47.7885" y1="33.4658" x2="39.8423" y2="25.5196" gradientUnits="userSpaceOnUse">
<stop stop-color="#FF8D61"/>
<stop offset="1" stop-color="#FFCD0A"/>
</linearGradient>
<linearGradient id="paint3_linear_4267_40961" x1="26.8723" y1="12.6836" x2="26.8723" y2="20.4799" gradientUnits="userSpaceOnUse">
<stop stop-color="#FEFDFF"/>
<stop offset="1" stop-color="#DEDFFF"/>
</linearGradient>
<linearGradient id="paint4_linear_4267_40961" x1="30.9956" y1="12.684" x2="30.9956" y2="20.4802" gradientUnits="userSpaceOnUse">
<stop stop-color="#FEFDFF"/>
<stop offset="1" stop-color="#DEDFFF"/>
</linearGradient>
<linearGradient id="paint5_linear_4267_40961" x1="35.1198" y1="12.6835" x2="35.1198" y2="20.4797" gradientUnits="userSpaceOnUse">
<stop stop-color="#FEFDFF"/>
<stop offset="1" stop-color="#DEDFFF"/>
</linearGradient>
<linearGradient id="paint6_linear_4267_40961" x1="16.6928" y1="37.427" x2="16.6928" y2="27.9944" gradientUnits="userSpaceOnUse">
<stop stop-color="#FF8D61"/>
<stop offset="1" stop-color="#FFCD0A"/>
</linearGradient>
<linearGradient id="paint7_linear_4267_40961" x1="4.47905" y1="42.3895" x2="21.751" y2="42.3895" gradientUnits="userSpaceOnUse">
<stop stop-color="#FF8D61"/>
<stop offset="1" stop-color="#FFCD0A"/>
</linearGradient>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 5.0 KiB

View File

@ -44,6 +44,7 @@ import AddRulePage from '../../pages/PoliciesPage/PoliciesDetailPage/AddRulePage
import EditRulePage from '../../pages/PoliciesPage/PoliciesDetailPage/EditRulePage';
import PoliciesDetailPage from '../../pages/PoliciesPage/PoliciesDetailPage/PoliciesDetailPage';
import PoliciesListPage from '../../pages/PoliciesPage/PoliciesListPage/PoliciesListPage';
import ProfilerConfigurationPage from '../../pages/ProfilerConfigurationPage/ProfilerConfigurationPage';
import AddRolePage from '../../pages/RolesPage/AddRolePage/AddRolePage';
import RolesDetailPage from '../../pages/RolesPage/RolesDetailPage/RolesDetailPage';
import RolesListPage from '../../pages/RolesPage/RolesListPage/RolesListPage';
@ -288,6 +289,15 @@ const SettingsRouter = () => {
GlobalSettingOptions.APPEARANCE
)}
/>
<AdminProtectedRoute
exact
component={ProfilerConfigurationPage}
hasPermission={false}
path={getSettingPath(
GlobalSettingsMenuCategory.PREFERENCES,
GlobalSettingOptions.PROFILER_CONFIGURATION
)}
/>
<AdminProtectedRoute
exact
component={LoginConfigurationPage}

View File

@ -67,6 +67,7 @@ export enum GlobalSettingOptions {
CONTAINERS = 'containers',
APPLICATIONS = 'apps',
OM_HEALTH = 'om-health',
PROFILER_CONFIGURATION = 'profiler-configuration',
APPEARANCE = 'appearance',
}

View File

@ -23,6 +23,7 @@ import {
PartitionIntervalUnit,
ProfileSampleType,
} from '../generated/entity/data/table';
import { MetricType } from '../generated/settings/settings';
import { TestCaseStatus } from '../generated/tests/testCase';
import { TestPlatform } from '../generated/tests/testDefinition';
import { TestCaseType } from '../rest/testAPI';
@ -428,3 +429,26 @@ export const INITIAL_COLUMN_METRICS_VALUE = {
sumMetrics: INITIAL_SUM_METRIC_VALUE,
quartileMetrics: INITIAL_QUARTILE_METRIC_VALUE,
};
export const PROFILER_METRICS_TYPE_OPTIONS = [
{
label: 'All',
key: 'all',
value: 'all',
children: values(MetricType).map((value) => ({
label: value,
key: value,
value,
})),
},
];
export const DEFAULT_PROFILER_CONFIG_VALUE = {
metricConfiguration: [
{
dataType: undefined,
metrics: undefined,
disabled: false,
},
],
};

View File

@ -677,6 +677,7 @@
"metadata-to-es-config-optional": "Metadaten-zu-ES-Konfiguration (optional)",
"metapilot": "MetaPilot",
"metapilot-suggested-description": "Metapilot Suggested Description",
"metric-configuration": "Metric Configuration",
"metric-type": "Metriktyp",
"metric-value": "Metrikwert",
"metrics-summary": "Metrikzusammenfassung",
@ -845,6 +846,7 @@
"profile-sample-type": "Profil-Sample-Typ {{type}}",
"profiler": "Profiler",
"profiler-amp-data-quality": "Profiler & Datenqualität",
"profiler-configuration": "Profiler Configuration",
"profiler-ingestion": "Profiler-Erfassung",
"profiler-lowercase": "profiler",
"profiler-setting-plural": "Profiler Settings",
@ -1628,6 +1630,7 @@
"page-sub-header-for-persona": "Represent different persona that a user may have withing OpenMetadata.",
"page-sub-header-for-pipelines": "Ingestion von Metadaten aus den am häufigsten verwendeten Pipeline-Diensten.",
"page-sub-header-for-policies": "Definiere Richtlinien mit einer Reihe von Regeln für feinkörnige Zugriffssteuerung.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Weise Benutzern oder Teams umfassende rollenbasierte Zugriffsberechtigungen zu.",
"page-sub-header-for-search": "Ingestion von Metadaten aus den beliebtesten Suchdiensten.",
"page-sub-header-for-setting": "Ability to configure the OpenMetadata application to suit your needs.",

View File

@ -677,6 +677,7 @@
"metadata-to-es-config-optional": "Metadata To ES Config (Optional)",
"metapilot": "MetaPilot",
"metapilot-suggested-description": "Metapilot Suggested Description",
"metric-configuration": "Metric Configuration",
"metric-type": "Metric Type",
"metric-value": "Metric Value",
"metrics-summary": "Metrics Summary",
@ -845,6 +846,7 @@
"profile-sample-type": "Profile Sample {{type}}",
"profiler": "Profiler",
"profiler-amp-data-quality": "Profiler & Data Quality",
"profiler-configuration": "Profiler Configuration",
"profiler-ingestion": "Profiler Ingestion",
"profiler-lowercase": "profiler",
"profiler-setting-plural": "Profiler Settings",
@ -1628,6 +1630,7 @@
"page-sub-header-for-persona": "Enhance and customize the user experience with Personas.",
"page-sub-header-for-pipelines": "Ingest metadata from the most used pipeline services.",
"page-sub-header-for-policies": "Define policies with a set of rules for fine-grained access control.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Assign comprehensive role based access to Users or Teams.",
"page-sub-header-for-search": "Ingest metadata from the most popular search services.",
"page-sub-header-for-setting": "Ability to configure the OpenMetadata application to suit your needs.",

View File

@ -677,6 +677,7 @@
"metadata-to-es-config-optional": "Configuración de Metadatos a ES (Opcional)",
"metapilot": "MetaPilot",
"metapilot-suggested-description": "Sugerencia de Metapilot",
"metric-configuration": "Metric Configuration",
"metric-type": "Tipo de Métrica",
"metric-value": "Valor de Métrica",
"metrics-summary": "Resumen de Métricas",
@ -845,6 +846,7 @@
"profile-sample-type": "Muestra de perfil {{type}}",
"profiler": "Perfilador",
"profiler-amp-data-quality": "Perfilador y calidad de datos",
"profiler-configuration": "Profiler Configuration",
"profiler-ingestion": "Ingesta del Perfilador",
"profiler-lowercase": "perfilador",
"profiler-setting-plural": "Configuración del perfilador",
@ -1628,6 +1630,7 @@
"page-sub-header-for-persona": "Representa diferentes personas que un usuario puede tener dentro de OpenMetadata.",
"page-sub-header-for-pipelines": "Ingresa metadatos desde los servicios de pipeline más utilizados.",
"page-sub-header-for-policies": "Define políticas con un conjunto de reglas para el control de acceso detallado.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Asigna un acceso basado en roles integral a Usuarios o Equipos.",
"page-sub-header-for-search": "Ingresa metadatos desde los servicios de búsqueda más populares.",
"page-sub-header-for-setting": "Capacidad para configurar la aplicación OpenMetadata según tus necesidades.",

View File

@ -677,6 +677,7 @@
"metadata-to-es-config-optional": "Configuration de Métadonnées vers ES (Optionnelle)",
"metapilot": "MetaPilot",
"metapilot-suggested-description": "Metapilot Suggested Description",
"metric-configuration": "Metric Configuration",
"metric-type": "Type de Mesure",
"metric-value": "Valeur de la Mesure",
"metrics-summary": "Résumé des Mesures",
@ -845,6 +846,7 @@
"profile-sample-type": "Échantillon du Profil {{type}}",
"profiler": "Profilage",
"profiler-amp-data-quality": "Profilage & Contrôle Qualité",
"profiler-configuration": "Profiler Configuration",
"profiler-ingestion": "Ingestion de Profilage",
"profiler-lowercase": "profilage",
"profiler-setting-plural": "Profiler Settings",
@ -1628,6 +1630,7 @@
"page-sub-header-for-persona": "Represent different persona that a user may have withing OpenMetadata.",
"page-sub-header-for-pipelines": "Ingestion de métadonnées à partir des services de pipeline les plus utilisés.",
"page-sub-header-for-policies": "Définissez des politiques avec un ensemble de règles pour un contrôle d'accès précis.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Attribuez des autorisations basées sur les rôles aux utilisateurs ou aux équipes.",
"page-sub-header-for-search": "Ingestion de métadonnées à partir des services de recherche les plus populaires.",
"page-sub-header-for-setting": "Ability to configure the OpenMetadata application to suit your needs.",

View File

@ -677,6 +677,7 @@
"metadata-to-es-config-optional": "קונפיגורציית מטא-דאטה ל-Elasticsearch (אופציונלי)",
"metapilot": "MetaPilot",
"metapilot-suggested-description": "Metapilot Suggested Description",
"metric-configuration": "Metric Configuration",
"metric-type": "סוג מדד",
"metric-value": "ערך מדד",
"metrics-summary": "סיכום מדדים",
@ -845,6 +846,7 @@
"profile-sample-type": "דוגמת פרופיל {{type}}",
"profiler": "מדד ואיכות נתונים",
"profiler-amp-data-quality": "מדד ואיכות נתונים",
"profiler-configuration": "Profiler Configuration",
"profiler-ingestion": "הזנת מדד ואיכות נתונים",
"profiler-lowercase": "מדד ואיכות נתונים",
"profiler-setting-plural": "הגדרות מדד ואיכות נתונים",
@ -1628,6 +1630,7 @@
"page-sub-header-for-persona": "צור פרופיל משתמש (פרסונה) על מנת לשייך את המשתמש לפרופיל ולהתאים את הממשק לצרכים של המשתמשים כאשר הם נכנסים ל-UI.",
"page-sub-header-for-pipelines": "שלב מטה-דאטה ממוצרי טעינת נתונים (ETL, ELT) פופלריים (כגון Airflow, Glue וכד׳)",
"page-sub-header-for-policies": "הגדר מדיניות עם סט של כללים לבקרת גישה ברמת רזולוציה נמוכה.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "הקצאת גישה מבוססת תפקיד למשתמשים או קבוצות.",
"page-sub-header-for-search": "שלב מטה-דאטה משירותי החיפוש הפופולריים ביותר.",
"page-sub-header-for-setting": "Ability to configure the OpenMetadata application to suit your needs.",

View File

@ -677,6 +677,7 @@
"metadata-to-es-config-optional": "Metadata To ES Config (Optional)",
"metapilot": "MetaPilot",
"metapilot-suggested-description": "Metapilot Suggested Description",
"metric-configuration": "Metric Configuration",
"metric-type": "メトリクスのタイプ",
"metric-value": "メトリクスの値",
"metrics-summary": "メトリクスの要約",
@ -845,6 +846,7 @@
"profile-sample-type": "サンプル{{type}}のプロファイル",
"profiler": "プロファイラ",
"profiler-amp-data-quality": "プロファイラとデータ品質",
"profiler-configuration": "Profiler Configuration",
"profiler-ingestion": "プロファイラのインジェスチョン",
"profiler-lowercase": "プロファイラ",
"profiler-setting-plural": "Profiler Settings",
@ -1628,6 +1630,7 @@
"page-sub-header-for-persona": "Represent different persona that a user may have withing OpenMetadata.",
"page-sub-header-for-pipelines": "Ingest metadata from the most used pipeline services.",
"page-sub-header-for-policies": "Define policies with a set of rules for fine-grained access control.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Assign comprehensive role based access to Users or Teams.",
"page-sub-header-for-search": "Ingest metadata from the most popular search services.",
"page-sub-header-for-setting": "Ability to configure the OpenMetadata application to suit your needs.",

View File

@ -677,6 +677,7 @@
"metadata-to-es-config-optional": "Metadata naar ES-configuratie (Optioneel)",
"metapilot": "MetaPilot",
"metapilot-suggested-description": "Door Metapilot voorgestelde beschrijving",
"metric-configuration": "Metric Configuration",
"metric-type": "Type metriek",
"metric-value": "Metriekwaarde",
"metrics-summary": "Samenvatting van metingen",
@ -845,6 +846,7 @@
"profile-sample-type": "Voorbeeldprofiel {{type}}",
"profiler": "Profiler",
"profiler-amp-data-quality": "Profiler & datakwaliteit",
"profiler-configuration": "Profiler Configuration",
"profiler-ingestion": "Profileringestie",
"profiler-lowercase": "profiler",
"profiler-setting-plural": "Profilerinstellingen",
@ -1628,6 +1630,7 @@
"page-sub-header-for-persona": "De gebruikerservaring verbeteren en aanpassen met Persona's.",
"page-sub-header-for-pipelines": "Ingest metadata van de meestgebruikte pipelineservices.",
"page-sub-header-for-policies": "Definieer beleid met een reeks regels voor fijnmazige toegangscontrole.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Wijs uitgebreide rolgebaseerde toegang toe aan gebruikers of teams.",
"page-sub-header-for-search": "Ingest metadata van de meestgebruikte zoekservices.",
"page-sub-header-for-setting": "Mogelijkheid om de OpenMetadata-toepassing naar eigen behoefte te configureren.",

View File

@ -677,6 +677,7 @@
"metadata-to-es-config-optional": "Metadados para Configuração ES (Opcional)",
"metapilot": "MetaPilot",
"metapilot-suggested-description": "Metapilot Suggested Description",
"metric-configuration": "Metric Configuration",
"metric-type": "Tipo de Métrica",
"metric-value": "Valor da Métrica",
"metrics-summary": "Resumo de Métricas",
@ -845,6 +846,7 @@
"profile-sample-type": "Amostra de Perfil {{type}}",
"profiler": "Profiler",
"profiler-amp-data-quality": "Profiler & Qualidade de Dados",
"profiler-configuration": "Profiler Configuration",
"profiler-ingestion": "Ingestão de Profiler",
"profiler-lowercase": "profiler",
"profiler-setting-plural": "Configurações do Profiler",
@ -1628,6 +1630,7 @@
"page-sub-header-for-persona": "Crie Personas para associar a persona do usuário ao OpenMetadata",
"page-sub-header-for-pipelines": "Ingestão de metadados dos serviços de pipeline mais utilizados.",
"page-sub-header-for-policies": "Defina políticas com um conjunto de regras para controle de acesso detalhado.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Atribua acesso baseado em funções abrangentes a usuários ou equipes.",
"page-sub-header-for-search": "Ingestão de metadados dos serviços de pesquisa mais populares.",
"page-sub-header-for-setting": "Habilidade para configurar a aplicação OpenMetadata de acordo com suas necessidades.",

View File

@ -677,6 +677,7 @@
"metadata-to-es-config-optional": "Метаданные для конфигурации ES (необязательно)",
"metapilot": "MetaPilot",
"metapilot-suggested-description": "Metapilot Suggested Description",
"metric-configuration": "Metric Configuration",
"metric-type": "Тип метрики",
"metric-value": "Значение метрики",
"metrics-summary": "Сводка метрик",
@ -845,6 +846,7 @@
"profile-sample-type": "Образец профиля {{type}}",
"profiler": "Профайлер",
"profiler-amp-data-quality": "Профайлер & Качество данных",
"profiler-configuration": "Profiler Configuration",
"profiler-ingestion": "Получение профайлера",
"profiler-lowercase": "профайлер",
"profiler-setting-plural": "Profiler Settings",
@ -1628,6 +1630,7 @@
"page-sub-header-for-persona": "Represent different persona that a user may have withing OpenMetadata.",
"page-sub-header-for-pipelines": "Принимать метаданные из наиболее часто используемых конвейерных служб.",
"page-sub-header-for-policies": "Определите политики с набором правил для точного контроля доступа.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Назначьте полный доступ на основе ролей пользователям или командам.",
"page-sub-header-for-search": "Ingest metadata from the most popular search services.",
"page-sub-header-for-setting": "Ability to configure the OpenMetadata application to suit your needs.",

View File

@ -677,6 +677,7 @@
"metadata-to-es-config-optional": "元数据到 ES 配置(可选)",
"metapilot": "MetaPilot",
"metapilot-suggested-description": "Metapilot Suggested Description",
"metric-configuration": "Metric Configuration",
"metric-type": "指标类型",
"metric-value": "指标值",
"metrics-summary": "指标概要",
@ -845,6 +846,7 @@
"profile-sample-type": "分析样本{{type}}",
"profiler": "分析器",
"profiler-amp-data-quality": "数据分析质控",
"profiler-configuration": "Profiler Configuration",
"profiler-ingestion": "分析器提取",
"profiler-lowercase": "分析器",
"profiler-setting-plural": "Profiler Settings",
@ -1628,6 +1630,7 @@
"page-sub-header-for-persona": "Represent different persona that a user may have withing OpenMetadata.",
"page-sub-header-for-pipelines": "从最常用的工作流类型服务中提取元数据",
"page-sub-header-for-policies": "通过组合一系列规则定义权限策略以精细化控制访问权限",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "分配基于角色的访问权限给用户或团队",
"page-sub-header-for-search": "Ingest metadata from the most popular search services.",
"page-sub-header-for-setting": "Ability to configure the OpenMetadata application to suit your needs.",

View File

@ -0,0 +1,103 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { act, render, screen } from '@testing-library/react';
import React from 'react';
import { SettingType } from '../../generated/settings/settings';
import { getSettingsConfigFromConfigType } from '../../rest/settingConfigAPI';
import ProfilerConfigurationPage from './ProfilerConfigurationPage';
const mockHistory = {
goBack: jest.fn(),
};
jest.mock('../../components/common/Loader/Loader', () =>
jest.fn().mockReturnValue(<div>Loading...</div>)
);
jest.mock(
'../../components/common/TitleBreadcrumb/TitleBreadcrumb.component',
() => jest.fn().mockReturnValue(<div>TitleBreadcrumb.component</div>)
);
jest.mock('../../components/PageHeader/PageHeader.component', () =>
jest.fn().mockReturnValue(<div>PageHeader.component</div>)
);
jest.mock('react-router-dom', () => ({
useHistory: jest.fn().mockImplementation(() => mockHistory),
}));
jest.mock('../../rest/settingConfigAPI', () => ({
getSettingsConfigFromConfigType: jest.fn().mockResolvedValue({}),
updateSettingsConfig: jest.fn(),
}));
jest.mock('../../utils/GlobalSettingsUtils', () => ({
getSettingPageEntityBreadCrumb: jest.fn().mockReturnValue([]),
}));
jest.mock('../../components/PageLayoutV1/PageLayoutV1', () =>
jest.fn().mockImplementation(({ children }) => <div>{children}</div>)
);
jest.mock('../../constants/profiler.constant', () => ({
DEFAULT_PROFILER_CONFIG_VALUE: {
metricConfiguration: [
{
dataType: undefined,
metrics: undefined,
disabled: false,
},
],
},
PROFILER_METRICS_TYPE_OPTIONS: [],
}));
describe('ProfilerConfigurationPage', () => {
beforeEach(() => {
act(() => {
render(<ProfilerConfigurationPage />);
});
});
it('renders the page correctly', async () => {
expect(
await screen.findByText('TitleBreadcrumb.component')
).toBeInTheDocument();
expect(await screen.findByText('PageHeader.component')).toBeInTheDocument();
expect(await screen.findByText('label.data-type')).toBeInTheDocument();
expect(await screen.findByText('label.disable')).toBeInTheDocument();
expect(await screen.findByText('label.metric-type')).toBeInTheDocument();
expect(
await screen.findByTestId('profiler-config-form')
).toBeInTheDocument();
expect(await screen.findByTestId('data-type-select')).toBeInTheDocument();
expect(await screen.findByTestId('metric-type-select')).toBeInTheDocument();
expect(await screen.findByTestId('disabled-switch')).toBeInTheDocument();
expect(await screen.findByTestId('add-fields')).toBeInTheDocument();
expect(await screen.findByTestId('cancel-button')).toBeInTheDocument();
expect(await screen.findByTestId('save-button')).toBeInTheDocument();
});
it('should fetch the profiler config data on initial render', () => {
const mockGetSettingsConfigFromConfigType =
getSettingsConfigFromConfigType as jest.Mock;
expect(mockGetSettingsConfigFromConfigType).toHaveBeenCalledWith(
SettingType.ProfilerConfiguration
);
expect(mockGetSettingsConfigFromConfigType).toHaveBeenCalledTimes(1);
});
it("onCancel should call history's goBack method", () => {
const cancelButton = screen.getByTestId('cancel-button');
act(() => {
cancelButton.click();
});
expect(mockHistory.goBack).toHaveBeenCalledTimes(1);
});
});

View File

@ -0,0 +1,300 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { CloseOutlined } from '@ant-design/icons';
import {
Button,
Card,
Col,
Form,
Row,
Select,
Switch,
TreeSelect,
Typography,
} from 'antd';
import { AxiosError } from 'axios';
import { t } from 'i18next';
import { isEmpty, isEqual, values } from 'lodash';
import React, { Fragment, useEffect, useMemo, useState } from 'react';
import { useHistory } from 'react-router-dom';
import Loader from '../../components/common/Loader/Loader';
import TitleBreadcrumb from '../../components/common/TitleBreadcrumb/TitleBreadcrumb.component';
import { TitleBreadcrumbProps } from '../../components/common/TitleBreadcrumb/TitleBreadcrumb.interface';
import PageHeader from '../../components/PageHeader/PageHeader.component';
import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1';
import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.constants';
import {
DEFAULT_PROFILER_CONFIG_VALUE,
PROFILER_METRICS_TYPE_OPTIONS,
} from '../../constants/profiler.constant';
import {
DataType,
MetricConfigurationDefinition,
MetricType,
ProfilerConfiguration,
} from '../../generated/configuration/profilerConfiguration';
import { SettingType } from '../../generated/settings/settings';
import {
getSettingsConfigFromConfigType,
updateSettingsConfig,
} from '../../rest/settingConfigAPI';
import { getSettingPageEntityBreadCrumb } from '../../utils/GlobalSettingsUtils';
import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
import './profiler-configuration-page.style.less';
const ProfilerConfigurationPage = () => {
const [form] = Form.useForm();
const history = useHistory();
const [isLoading, setIsLoading] = useState(true);
const [isFormSubmitting, setIsFormSubmitting] = useState(false);
const breadcrumbs: TitleBreadcrumbProps['titleLinks'] = useMemo(
() =>
getSettingPageEntityBreadCrumb(
GlobalSettingsMenuCategory.PREFERENCES,
t('label.profiler-configuration')
),
[]
);
// Watchers
const selectedMetricConfiguration = Form.useWatch<
MetricConfigurationDefinition[]
>('metricConfiguration', form);
const dataTypeOptions = useMemo(() => {
return values(DataType).map((value) => ({
label: value,
key: value,
value,
// Disable the metric type selection if the data type is already selected
disabled: selectedMetricConfiguration?.some(
(data) => data?.dataType === value
),
}));
}, [selectedMetricConfiguration]);
const handleSubmit = async (data: ProfilerConfiguration) => {
setIsFormSubmitting(true);
const metricConfiguration = data.metricConfiguration?.map((item) => {
if (isEqual(item.metrics, ['all'])) {
return {
...item,
// If all metrics are selected, then we will send full array of metrics
metrics: values(MetricType),
};
}
return item;
});
try {
await updateSettingsConfig({
config_type: SettingType.ProfilerConfiguration,
config_value: {
metricConfiguration,
},
});
showSuccessToast(
t('server.update-entity-success', {
entity: t('label.profiler-configuration'),
})
);
} catch (error) {
showErrorToast(error as AxiosError);
} finally {
setIsFormSubmitting(false);
}
};
const fetchProfilerConfiguration = async () => {
setIsLoading(true);
try {
const { data } = await getSettingsConfigFromConfigType(
SettingType.ProfilerConfiguration
);
form.setFieldsValue(
isEmpty(data?.config_value?.metricConfiguration)
? DEFAULT_PROFILER_CONFIG_VALUE
: data?.config_value
);
} catch (error) {
// do nothing
} finally {
setIsLoading(false);
}
};
useEffect(() => {
fetchProfilerConfiguration();
}, []);
if (isLoading) {
return <Loader />;
}
return (
<PageLayoutV1 pageTitle={t('label.profiler-configuration')}>
<Row
align="middle"
className="profiler-configuration-page-container"
gutter={[0, 16]}>
<Col span={24}>
<TitleBreadcrumb titleLinks={breadcrumbs} />
</Col>
<Col span={24}>
<PageHeader
data={{
header: t('label.profiler-configuration'),
subHeader: t(
'message.page-sub-header-for-profiler-configuration'
),
}}
/>
</Col>
<Col span={24}>
<Card className="profiler-configuration-form-item-container">
<Form<ProfilerConfiguration>
data-testid="profiler-config-form"
form={form}
id="profiler-config"
layout="vertical"
onFinish={handleSubmit}>
<Form.List name="metricConfiguration">
{(fields, { add, remove }) => {
return (
<Row gutter={[16, 16]}>
<Col span={24}>
<Typography.Text className="text-grey-muted">
{t('label.add-entity', {
entity: t('label.metric-configuration'),
})}
</Typography.Text>
</Col>
<Col span={10}>
{t('label.data-type')}
<span className="text-failure">*</span>
</Col>
<Col span={11}>{t('label.metric-type')}</Col>
<Col span={3}>{t('label.disable')}</Col>
{fields.map(({ key, name }) => (
<Fragment key={key}>
<Col span={10}>
<Form.Item
name={[name, 'dataType']}
rules={[
{
required: true,
message: t('message.field-text-is-required', {
fieldText: t('label.data-type'),
}),
},
]}>
<Select
allowClear
data-testid="data-type-select"
options={dataTypeOptions}
placeholder={t('label.select-field', {
field: t('label.data-type'),
})}
/>
</Form.Item>
</Col>
<Col span={11}>
<Form.Item
noStyle
shouldUpdate={(prevValues, currentValues) => {
return !isEqual(
prevValues['metricConfiguration']?.[name]?.[
'disabled'
],
currentValues['metricConfiguration']?.[
name
]?.['disabled']
);
}}>
{() => (
<Form.Item name={[name, 'metrics']}>
<TreeSelect
allowClear
treeCheckable
data-testid="metric-type-select"
disabled={form.getFieldValue([
'metricConfiguration',
name,
'disabled',
])}
maxTagCount={5}
placeholder={t('label.select-field', {
field: t('label.metric-type'),
})}
showCheckedStrategy={TreeSelect.SHOW_PARENT}
treeData={PROFILER_METRICS_TYPE_OPTIONS}
/>
</Form.Item>
)}
</Form.Item>
</Col>
<Col className="d-flex justify-between" span={3}>
<Form.Item
name={[name, 'disabled']}
valuePropName="checked">
<Switch data-testid="disabled-switch" />
</Form.Item>
<Form.Item>
<Button
data-testid={`remove-filter-${name}`}
icon={<CloseOutlined />}
size="small"
onClick={() => remove(name)}
/>
</Form.Item>
</Col>
</Fragment>
))}
<Col span={24}>
<Button
data-testid="add-fields"
type="primary"
onClick={() => add()}>
{t('label.add-entity', {
entity: t('label.field'),
})}
</Button>
</Col>
</Row>
);
}}
</Form.List>
</Form>
</Card>
</Col>
<Col className="d-flex justify-end gap-2" span={24}>
<Button data-testid="cancel-button" onClick={() => history.goBack()}>
{t('label.cancel')}
</Button>
<Button
data-testid="save-button"
form="profiler-config"
htmlType="submit"
loading={isFormSubmitting}
type="primary">
{t('label.save')}
</Button>
</Col>
</Row>
</PageLayoutV1>
);
};
export default ProfilerConfigurationPage;

View File

@ -0,0 +1,32 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@import url('../../styles/variables.less');
.profiler-configuration-page-container {
width: 70%;
margin: 16px auto 0;
padding-bottom: 16px;
.ant-form-item {
margin: 0px;
}
.ant-card {
border-radius: 10px;
}
}
.profiler-configuration-form-item-container {
background-color: @grey-6;
.ant-card-body {
padding: 20px;
}
}

View File

@ -30,6 +30,7 @@ import { ReactComponent as OMHealthIcon } from '../assets/svg/om-health-colored.
import { ReactComponent as PersonasIcon } from '../assets/svg/persona-colored.svg';
import { ReactComponent as PipelineIcon } from '../assets/svg/pipeline-colored.svg';
import { ReactComponent as PoliciesIcon } from '../assets/svg/policies-colored.svg';
import { ReactComponent as ProfilerConfigIcon } from '../assets/svg/profiler-configuration-logo.svg';
import { ReactComponent as RolesIcon } from '../assets/svg/role-colored.svg';
import { ReactComponent as SearchIcon } from '../assets/svg/search-colored.svg';
import { ReactComponent as AccessControlIcon } from '../assets/svg/setting-access-control.svg';
@ -291,6 +292,15 @@ export const getGlobalSettingsMenuWithPermission = (
key: `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.OM_HEALTH}`,
icon: OMHealthIcon,
},
{
label: i18next.t('label.profiler-configuration'),
description: i18next.t(
'message.page-sub-header-for-profiler-configuration'
),
isProtected: Boolean(isAdminUser),
key: `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.PROFILER_CONFIGURATION}`,
icon: ProfilerConfigIcon,
},
],
},
{