Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

173 lines
5.7 KiB
MySQL
Raw Normal View History

-- Data quality failure status extension time series
CREATE TABLE test_case_resolution_status_time_series (
id varchar(36) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.id'))) VIRTUAL NOT NULL,
stateId varchar(36) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.stateId'))) VIRTUAL NOT NULL,
assignee varchar(256) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.testCaseResolutionStatusDetails.assignee.name'))) VIRTUAL NULL,
timestamp bigint unsigned GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.timestamp'))) VIRTUAL NOT NULL,
testCaseResolutionStatusType varchar(36) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.testCaseResolutionStatusType'))) VIRTUAL NOT NULL,
jsonSchema varchar(256) NOT NULL,
json json NOT NULL,
entityFQNHash varchar(768) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
CONSTRAINT test_case_resolution_status_unique_constraint UNIQUE (id,timestamp,entityFQNHash),
INDEX (id),
INDEX(testCaseResolutionStatusType),
INDEX(id, testCaseResolutionStatusType)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- DataInsightsApplication should not allow configuration
update apps_marketplace
set json = JSON_INSERT(
JSON_REMOVE(json, '$.allowConfiguration'),
'$.allowConfiguration',
false
)
where name = 'DataInsightsApplication';
update installed_apps
set json = JSON_INSERT(
JSON_REMOVE(json, '$.allowConfiguration'),
'$.allowConfiguration',
false
)
where name = 'DataInsightsApplication';
-- Remove mssql connection from airflow db
UPDATE pipeline_service_entity pse
SET json = JSON_REMOVE(json, '$.connection.config.connection')
WHERE serviceType = 'Airflow'
AND JSON_EXTRACT(json, '$.connection.config.connection.type') = 'Mssql';
-- Rename NOOP Secret Manager to DB
update metadata_service_entity
set json = JSON_REPLACE(json, '$.connection.config.secretsManagerProvider', 'db')
where name = 'OpenMetadata'
and JSON_EXTRACT(json, '$.connection.config.secretsManagerProvider') = 'noop';
-- Clean old test connections
TRUNCATE automations_workflow;
-- update service type to UnityCatalog - update database entity
UPDATE database_entity de
SET de.json = JSON_INSERT(
JSON_REMOVE(de.json, '$.serviceType'),
'$.serviceType',
'UnityCatalog'
)
where id in (
select toId from entity_relationship er
where
fromEntity = 'databaseService'
and toEntity = 'database'
and fromId in (
select id from dbservice_entity dbe
where
serviceType = 'Databricks'
and JSON_EXTRACT(
dbe.json, '$.connection.config.useUnityCatalog'
) = true
));
-- update service type to UnityCatalog - update database schema entity
UPDATE database_schema_entity dse
SET dse.json = JSON_INSERT(
JSON_REMOVE(dse.json, '$.serviceType'),
'$.serviceType',
'UnityCatalog'
)
where JSON_EXTRACT(dse.json, '$.database.id') in (
select toId from entity_relationship er
where
fromEntity = 'databaseService'
and toEntity = 'database'
and fromId in (
select id from dbservice_entity dbe
where
serviceType = 'Databricks'
and JSON_EXTRACT(
dbe.json, '$.connection.config.useUnityCatalog'
) = true
));
-- update service type to UnityCatalog - update table entity
UPDATE table_entity te
SET te.json = JSON_INSERT(
JSON_REMOVE(te.json, '$.serviceType'),
'$.serviceType',
'UnityCatalog'
)
where JSON_EXTRACT(te.json, '$.database.id') in (
select toId from entity_relationship er
where
fromEntity = 'databaseService'
and toEntity = 'database'
and fromId in (
select id from dbservice_entity dbe
where
serviceType = 'Databricks'
and JSON_EXTRACT(
dbe.json, '$.connection.config.useUnityCatalog'
) = true
));
-- update service type to UnityCatalog - update db service entity
UPDATE dbservice_entity de
SET de.json = JSON_INSERT(
JSON_REMOVE(de.json, '$.connection.config.type'),
'$.connection.config.type',
'UnityCatalog'
),de.json = JSON_INSERT(
JSON_REMOVE(de.json, '$.serviceType'),
'$.serviceType',
'UnityCatalog'
)
WHERE de.serviceType = 'Databricks'
AND JSON_EXTRACT(de.json, '$.connection.config.useUnityCatalog') = True
;
-- remove `useUnityCatalog` flag from service connection details of databricks
UPDATE dbservice_entity de
SET de.json = JSON_REMOVE(de.json, '$.connection.config.useUnityCatalog')
WHERE de.serviceType IN ('Databricks','UnityCatalog');
-- Add Incident ID for test case results
ALTER TABLE data_quality_data_time_series ADD COLUMN incidentId varchar(36);
ALTER TABLE data_quality_data_time_series ADD INDEX data_quality_data_time_series_incidentId(incidentId);
-- Add new table for event subscription extensions
CREATE TABLE IF NOT EXISTS change_event_consumers (
id VARCHAR(36) NOT NULL,
extension VARCHAR(256) NOT NULL,
jsonSchema VARCHAR(256) NOT NULL,
json JSON NOT NULL,
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
UNIQUE(id, extension)
);
DELETE FROM event_subscription_entity ese where name = 'DataInsightReport';
-- Update Change Event Table
feat(ui): Data observability and notifications (#14806) * Centralize EventType * Format better for readability * Add Handling for Thread as Change Event * Centralize logic for Entity Message Creation * Add Thread Parent Message in Conversation * Add Task and Announcement * Support for multiple destinations * Delete existing alerts for schema changes * Delete all tables data for event subscription * Centralize logic for Notifications and Observability * Change Field Name and Remove redundant code * Test Fixes * Add more fields * Add Optional * Logical Test Case Addition Chnage Event * Add Filter By Owner * Fix Email Entity Url * added data observability settings page * localization changes for other languages * Handler closure and cleanup * complete create observibility flow * fix searchIndex method error * Update Entity Observability Filter * working observability creation * complete creation flow * fix action with arguments * fix form validations and form data formulation for 'input.actions' field * localization changes for other languages * Fixes for Data Observability * Add mention notification * Only apply mention to conversation * fIx test case * bug fixes for add and edit observability alert form functionality * fIx test case * add feature to config internal along with external destinations in alert form * Fix Formatting * Fix Url Redirection * add observability alert details page * localization changes for other languages * remove filter and action required rules * fix styling for observability form and details page * add delete functionality on observability alert details page * update view page layout and add a warning in configure destination step * localization changes for other languages * feat(ui): support notification with new flow (#14848) * feat(ui): support notification with new flow * fix notifications * add missing filters for notification alert creation flow * update the filter hiding logic to show at the start always * add check to show Actions section in observability alert creation form * code cleanup and improvements * add breadcrumb to add notification alert page * Add General Metadata Filters * remove code smells and code cleanup for add notification page * add new filter * show activity feed alert on notifications list page * add notification alert details page changes * localization changes for other languages * show icons for all the entities in the alert trigger dropdown * redirect to notification alert listing page after deleting the notification alert * change add alert page title * add logic to disable all destinations except email for users * fixed unit tests * code improvements and cleanups --------- Co-authored-by: mohitdeuex <mohit.y@deuexsolutions.com> Co-authored-by: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com> Co-authored-by: Aniket Katkar <aniketkatkar97@gmail.com>
2024-01-26 13:56:18 +05:30
ALTER TABLE change_event ADD COLUMN offset INT AUTO_INCREMENT, ADD PRIMARY KEY (offset);
CREATE TABLE IF NOT EXISTS consumers_dlq (
id VARCHAR(36) NOT NULL,
extension VARCHAR(256) NOT NULL,
json JSON NOT NULL,
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
UNIQUE(id, extension)
);
-- Add supportsQueryComment to MSSQL
update dbservice_entity
set json = JSON_SET(json, '$.connection.config.supportsQueryComment', true)
where serviceType = 'Mssql';
DELETE FROM event_subscription_entity;
feat(ui): Data observability and notifications (#14806) * Centralize EventType * Format better for readability * Add Handling for Thread as Change Event * Centralize logic for Entity Message Creation * Add Thread Parent Message in Conversation * Add Task and Announcement * Support for multiple destinations * Delete existing alerts for schema changes * Delete all tables data for event subscription * Centralize logic for Notifications and Observability * Change Field Name and Remove redundant code * Test Fixes * Add more fields * Add Optional * Logical Test Case Addition Chnage Event * Add Filter By Owner * Fix Email Entity Url * added data observability settings page * localization changes for other languages * Handler closure and cleanup * complete create observibility flow * fix searchIndex method error * Update Entity Observability Filter * working observability creation * complete creation flow * fix action with arguments * fix form validations and form data formulation for 'input.actions' field * localization changes for other languages * Fixes for Data Observability * Add mention notification * Only apply mention to conversation * fIx test case * bug fixes for add and edit observability alert form functionality * fIx test case * add feature to config internal along with external destinations in alert form * Fix Formatting * Fix Url Redirection * add observability alert details page * localization changes for other languages * remove filter and action required rules * fix styling for observability form and details page * add delete functionality on observability alert details page * update view page layout and add a warning in configure destination step * localization changes for other languages * feat(ui): support notification with new flow (#14848) * feat(ui): support notification with new flow * fix notifications * add missing filters for notification alert creation flow * update the filter hiding logic to show at the start always * add check to show Actions section in observability alert creation form * code cleanup and improvements * add breadcrumb to add notification alert page * Add General Metadata Filters * remove code smells and code cleanup for add notification page * add new filter * show activity feed alert on notifications list page * add notification alert details page changes * localization changes for other languages * show icons for all the entities in the alert trigger dropdown * redirect to notification alert listing page after deleting the notification alert * change add alert page title * add logic to disable all destinations except email for users * fixed unit tests * code improvements and cleanups --------- Co-authored-by: mohitdeuex <mohit.y@deuexsolutions.com> Co-authored-by: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com> Co-authored-by: Aniket Katkar <aniketkatkar97@gmail.com>
2024-01-26 13:56:18 +05:30
DELETE FROM change_event_consumers;
DELETE FROM consumers_dlq;
UPDATE ingestion_pipeline_entity SET json = JSON_SET(json, '$.provider', 'user')
WHERE JSON_EXTRACT(json, '$.name') = 'OpenMetadata_dataInsight';