mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-07-12 11:39:39 +00:00
0a1018648c
* feat(nosql-profiler): row count 1. Implemented the NoSQLProfilerInterface as an entrypoint for the nosql profiler. 2. Added the NoSQLMetric as an abstract class. 3. Implemented the interface for the MongoDB database source. 4. Implemented an e2e test using testcontainers. * added profiler support for mongodb connection * doc * use int_admin_ometa in test setup * - fixed linting issue in gx - removed unused inheritance * moved the nosql function into the metric class * feat(profiler): add dynamodb row count * feat(profiler): add dynamodb row count * formatting * fixed import * format * dded dynamodb row count * format * removed unused factory file * removed "validate" * migrations * removed validations * format * linting * fixed: test_amundsen.py * Update schemaChanges.sql
276 lines
14 KiB
SQL
276 lines
14 KiB
SQL
-- Add the supportsProfiler field to the MongoDB connection configuration
|
|
UPDATE dbservice_entity
|
|
SET json = JSON_INSERT(json, '$.connection.config.supportsProfiler', TRUE)
|
|
WHERE serviceType = 'MongoDB';
|
|
|
|
ALTER TABLE query_entity ADD COLUMN checksum VARCHAR
|
|
(32) GENERATED ALWAYS AS
|
|
(json ->> '$.checksum') NOT NULL UNIQUE;
|
|
|
|
UPDATE query_entity SET json = JSON_INSERT(json, '$.checksum', MD5(JSON_UNQUOTE(JSON_EXTRACT(json, '$.checksum'))));
|
|
|
|
-- Restructure dbServiceNames in ingestion_pipeline_entity
|
|
update ingestion_pipeline_entity set json =
|
|
JSON_INSERT(
|
|
JSON_REMOVE(json, '$.sourceConfig.config.dbServiceNames'),
|
|
'$.sourceConfig.config.lineageInformation',
|
|
JSON_OBJECT(
|
|
'dbServiceNames',
|
|
JSON_EXTRACT(json, '$.sourceConfig.config.dbServiceNames')
|
|
)
|
|
)
|
|
where
|
|
JSON_EXTRACT(json, '$.sourceConfig.config.type') in ('DashboardMetadata', 'PipelineMetadata')
|
|
AND JSON_EXTRACT(json, '$.sourceConfig.config.dbServiceNames') is not null;
|
|
|
|
|
|
ALTER TABLE chart_entity ADD INDEX index_chart_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE dashboard_data_model_entity ADD INDEX index_dashboard_data_model_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE dashboard_entity ADD INDEX index_dashboard_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE data_insight_chart ADD INDEX index_data_insight_chart_deleted(fqnHash, deleted);
|
|
ALTER TABLE database_entity ADD INDEX index_database_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE database_schema_entity ADD INDEX index_database_schema_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE glossary_term_entity ADD INDEX index_glossary_term_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE ingestion_pipeline_entity ADD INDEX index_ingestion_pipeline_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE metric_entity ADD INDEX index_metric_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE ml_model_entity ADD INDEX index_ml_model_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE pipeline_entity ADD INDEX index_pipeline_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE policy_entity ADD INDEX index_policy_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE report_entity ADD INDEX index_report_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE search_index_entity ADD INDEX index_search_index_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE storage_container_entity ADD INDEX index_storage_container_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE stored_procedure_entity ADD INDEX index_stored_procedure_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE table_entity ADD INDEX index_table_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE tag ADD INDEX index_tag_deleted(fqnHash, deleted);
|
|
ALTER TABLE test_case ADD INDEX index_test_case_deleted(fqnHash, deleted);
|
|
ALTER TABLE test_suite ADD INDEX index_test_suite_deleted(fqnHash, deleted);
|
|
ALTER TABLE topic_entity ADD INDEX index_topic_entity_deleted(fqnHash, deleted);
|
|
ALTER TABLE web_analytic_event ADD INDEX index_web_analytic_event_deleted(fqnHash, deleted);
|
|
|
|
ALTER TABLE apps_marketplace ADD INDEX index_apps_marketplace_deleted(nameHash, deleted);
|
|
ALTER TABLE bot_entity ADD INDEX index_bot_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE classification ADD INDEX index_classification_deleted(nameHash, deleted);
|
|
ALTER TABLE dashboard_service_entity ADD INDEX index_dashboard_service_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE dbservice_entity ADD INDEX index_dbservice_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE glossary_entity ADD INDEX index_glossary_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE installed_apps ADD INDEX index_installed_apps_deleted(nameHash, deleted);
|
|
ALTER TABLE knowledge_center ADD INDEX index_knowledge_center_deleted(nameHash, deleted);
|
|
ALTER TABLE kpi_entity ADD INDEX index_kpi_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE messaging_service_entity ADD INDEX index_messaging_service_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE metadata_service_entity ADD INDEX index_metadata_service_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE mlmodel_service_entity ADD INDEX index_mlmodel_service_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE pipeline_service_entity ADD INDEX index_pipeline_service_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE role_entity ADD INDEX index_role_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE search_service_entity ADD INDEX index_search_service_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE storage_service_entity ADD INDEX index_storage_service_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE team_entity ADD INDEX index_team_entity_deleted(nameHash, deleted);
|
|
ALTER TABLE user_entity ADD INDEX index_user_entity_deleted(nameHash, deleted);
|
|
|
|
ALTER TABLE apps_extension_time_series ADD INDEX apps_extension_time_series_index(appId);
|
|
ALTER TABLE suggestions ADD INDEX index_suggestions_type(suggestionType);
|
|
ALTER TABLE suggestions ADD INDEX index_suggestions_status(status);
|
|
|
|
|
|
-- Add the supportsProfiler field to the DynamoDB connection configuration
|
|
UPDATE dbservice_entity
|
|
SET json = JSON_INSERT(json, '$.connection.config.supportsProfiler', TRUE)
|
|
WHERE serviceType = 'DynamoDB';
|
|
|
|
-- Migrate 'QlikSenseDataModel' & 'QlikCloudDataModel' into single entity 'QlikDataModel'
|
|
UPDATE dashboard_data_model_entity
|
|
SET json = JSON_SET(json, '$.dataModelType', 'QlikDataModel')
|
|
WHERE JSON_EXTRACT(json, '$.dataModelType') in ('QlikSenseDataModel', 'QlikCloudDataModel');
|
|
|
|
-- clean ES pipelines
|
|
DELETE FROM ingestion_pipeline_entity
|
|
WHERE LOWER(JSON_EXTRACT(json, '$.pipelineType')) = 'elasticsearchreindex';
|
|
|
|
|
|
UPDATE dbservice_entity
|
|
SET json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.sslCA'),
|
|
'$.connection.config.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.sslCA')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.sslKey'),
|
|
'$.connection.config.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.sslConfig.sslKey',
|
|
JSON_EXTRACT(json, '$.connection.config.sslKey')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.sslCert'),
|
|
'$.connection.config.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.sslConfig.sslCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.sslCert'))
|
|
where serviceType in ('Mysql', 'Doris') AND JSON_EXTRACT(json, '$.connection.config.sslCA') IS NOT NULL;
|
|
|
|
UPDATE dbservice_entity
|
|
SET json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.sslConfig.certificatePath'),
|
|
'$.connection.config.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.sslConfig.certificatePath')
|
|
) where serviceType in ('Redshift', 'Postgres', 'Greenplum') and JSON_EXTRACT(json, '$.connection.config.sslConfig.certificatePath') is not null;
|
|
|
|
|
|
UPDATE dbservice_entity
|
|
SET json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.metastoreConnection.sslCA'),
|
|
'$.connection.config.metastoreConnection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.metastoreConnection.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.metastoreConnection.sslCA')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.metastoreConnection.sslKey'),
|
|
'$.connection.config.metastoreConnection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.metastoreConnection.sslConfig.sslKey',
|
|
JSON_EXTRACT(json, '$.connection.config.metastoreConnection.sslKey')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.metastoreConnection.sslCert'),
|
|
'$.connection.config.metastoreConnection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.metastoreConnection.sslConfig.sslCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.metastoreConnection.sslCert'))
|
|
|
|
where serviceType = 'Hive'
|
|
AND JSON_EXTRACT(json, '$.connection.config.metastoreConnection.type') = 'Mysql'
|
|
AND JSON_EXTRACT(json, '$.connection.config.metastoreConnection.sslCA') IS NOT NULL;
|
|
|
|
UPDATE dbservice_entity
|
|
SET json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.metastoreConnection.sslConfig.certificatePath'),
|
|
'$.connection.config.metastoreConnection.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.metastoreConnection.sslConfig.certificatePath'))
|
|
where serviceType = 'Hive'
|
|
AND JSON_EXTRACT(json, '$.connection.config.metastoreConnection.type') = 'Postgres'
|
|
AND JSON_EXTRACT(json, '$.connection.config.metastoreConnection.sslConfig.certificatePath') IS NOT NULL;
|
|
|
|
UPDATE dashboard_service_entity
|
|
SET json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslCA'),
|
|
'$.connection.config.connection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.connection.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslCA')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslKey'),
|
|
'$.connection.config.connection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.connection.sslConfig.sslKey',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslKey')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslCert'),
|
|
'$.connection.config.connection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.connection.sslConfig.sslCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslCert'))
|
|
|
|
where serviceType = 'Superset'
|
|
AND JSON_EXTRACT(json, '$.connection.config.connection.type') = 'Mysql'
|
|
AND JSON_EXTRACT(json, '$.connection.config.connection.sslCA') IS NOT NULL;
|
|
|
|
UPDATE dashboard_service_entity
|
|
SET json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslConfig.certificatePath'),
|
|
'$.connection.config.connection.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslConfig.certificatePath'))
|
|
where serviceType = 'Superset'
|
|
AND JSON_EXTRACT(json, '$.connection.config.connection.type') = 'Postgres'
|
|
AND JSON_EXTRACT(json, '$.connection.config.connection.sslConfig.certificatePath') IS NOT NULL;
|
|
|
|
|
|
UPDATE metadata_service_entity
|
|
SET json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslCA'),
|
|
'$.connection.config.connection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.connection.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslCA')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslKey'),
|
|
'$.connection.config.connection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.connection.sslConfig.sslKey',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslKey')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslCert'),
|
|
'$.connection.config.connection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.connection.sslConfig.sslCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslCert'))
|
|
|
|
where serviceType = 'Alation'
|
|
AND JSON_EXTRACT(json, '$.connection.config.connection.type') = 'Mysql'
|
|
AND JSON_EXTRACT(json, '$.connection.config.connection.sslCA') IS NOT NULL;
|
|
|
|
|
|
UPDATE metadata_service_entity
|
|
SET json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslConfig.certificatePath'),
|
|
'$.connection.config.connection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.connection.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslConfig.certificatePath'))
|
|
where serviceType = 'Alation'
|
|
AND JSON_EXTRACT(json, '$.connection.config.connection.type') = 'Postgres'
|
|
AND JSON_EXTRACT(json, '$.connection.config.connection.sslConfig.certificatePath') IS NOT NULL;
|
|
|
|
|
|
UPDATE dashboard_service_entity
|
|
SET json = JSON_REMOVE(json,'$.connection.config.certificates.stagingDir'), json = JSON_INSERT(JSON_REMOVE(json,'$.connection.config.certificates.rootCertificateData'), '$.connection.config.certificates.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.certificates.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.certificates.rootCertificateData')), json = JSON_INSERT(JSON_REMOVE(json,'$.connection.config.certificates.clientCertificateData'), '$.connection.config.certificates.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.certificates.sslConfig.sslCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.certificates.clientCertificateData')),
|
|
json = JSON_INSERT(JSON_REMOVE(json,'$.connection.config.certificates.clientKeyCertificateData'), '$.connection.config.certificates.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.certificates.sslConfig.sslKey',
|
|
JSON_EXTRACT(json, '$.connection.config.certificates.clientKeyCertificateData'))
|
|
|
|
where lower(serviceType) = 'qliksense'
|
|
and JSON_EXTRACT(json, '$.connection.config.certificates.rootCertificateData') is not null;
|
|
|
|
|
|
|
|
UPDATE pipeline_service_entity
|
|
SET json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslCA'),
|
|
'$.connection.config.connection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.connection.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslCA')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslKey'),
|
|
'$.connection.config.connection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.connection.sslConfig.sslKey',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslKey')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.connection.sslCert'),
|
|
'$.connection.config.connection.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.connection.sslConfig.sslCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.connection.sslCert'))
|
|
|
|
where serviceType = 'Airflow'
|
|
AND JSON_EXTRACT(json, '$.connection.config.connection.type') = 'Mysql'
|
|
AND JSON_EXTRACT(json, '$.connection.config.connection.sslCA') IS NOT NULL;
|
|
|
|
|
|
|
|
UPDATE pipeline_service_entity
|
|
SET json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.SSLCALocation'),
|
|
'$.connection.config.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.sslConfig.caCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.SSLCALocation')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.SSLKeyLocation'),
|
|
'$.connection.config.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.sslConfig.sslKey',
|
|
JSON_EXTRACT(json, '$.connection.config.SSLKeyLocation')), json = JSON_INSERT(
|
|
JSON_REMOVE(json, '$.connection.config.SSLCertificateLocation'),
|
|
'$.connection.config.sslConfig',
|
|
JSON_OBJECT(),
|
|
'$.connection.config.sslConfig.sslCertificate',
|
|
JSON_EXTRACT(json, '$.connection.config.SSLCertificateLocation'))
|
|
|
|
where serviceType = 'OpenLineage'
|
|
AND JSON_EXTRACT(json, '$.connection.config.SSLCALocation') IS NOT NULL;
|