yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-04 03:15:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR - Renaming MetaPilot to CollateAI (#18393)

Browse Source 
* MINOR - Renaming MetaPilot to CollateAI

* MINOR - Renaming MetaPilot to CollateAI

* MINOR - Renaming MetaPilot to CollateAI

* MINOR - Renaming MetaPilot to CollateAI

* fix

* moving limits migrations for 1.5.9
This commit is contained in:
Pere Miquel Brull 2024-10-28 09:21:15 +01:00 committed by GitHub
parent ef77535f17
commit 7d01373261
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 158 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
bootstrap/sql/migrations/native/1.5.9/mysql/postDataMigrationSQLScript.sql Normal file
View File

5
bootstrap/sql/migrations/native/1.5.9/mysql/schemaChanges.sql Normal file
View File

@ -0,0 +1,5 @@
-- Extend app extension for limits
ALTER TABLE apps_extension_time_series ADD COLUMN extension VARCHAR(255);
UPDATE apps_extension_time_series SET extension = 'status' WHERE extension IS NULL;
ALTER TABLE apps_extension_time_series MODIFY COLUMN extension VARCHAR(255) NOT NULL;
CREATE INDEX apps_extension_time_series_extension ON apps_extension_time_series(extension);

0
bootstrap/sql/migrations/native/1.5.9/postgres/postDataMigrationSQLScript.sql Normal file
View File

5
bootstrap/sql/migrations/native/1.5.9/postgres/schemaChanges.sql Normal file
View File

@ -0,0 +1,5 @@
-- Extend app extension for limits
ALTER TABLE apps_extension_time_series ADD COLUMN extension VARCHAR(255);
UPDATE apps_extension_time_series SET extension = 'status' WHERE extension IS NULL;
ALTER TABLE apps_extension_time_series ALTER COLUMN extension SET NOT NULL;
CREATE INDEX IF NOT EXISTS apps_extension_time_series_extension ON apps_extension_time_series(extension);

6
bootstrap/sql/migrations/native/1.6.0/mysql/schemaChanges.sql
View File

@ -1,9 +1,3 @@
-- Extend app extension for limits
ALTER TABLE apps_extension_time_series ADD COLUMN extension VARCHAR(255);
UPDATE apps_extension_time_series SET extension = 'status' WHERE extension IS NULL;
ALTER TABLE apps_extension_time_series MODIFY COLUMN extension VARCHAR(255) NOT NULL;
CREATE INDEX apps_extension_time_series_extension ON apps_extension_time_series(extension);
-- Clean dangling workflows not removed after test connection
truncate automations_workflow;

6
bootstrap/sql/migrations/native/1.6.0/postgres/schemaChanges.sql
View File

@ -1,9 +1,3 @@
-- Extend app extension for limits
ALTER TABLE apps_extension_time_series ADD COLUMN extension VARCHAR(255);
UPDATE apps_extension_time_series SET extension = 'status' WHERE extension IS NULL;
ALTER TABLE apps_extension_time_series ALTER COLUMN extension SET NOT NULL;
CREATE INDEX IF NOT EXISTS apps_extension_time_series_extension ON apps_extension_time_series(extension);
-- Clean dangling workflows not removed after test connection
truncate automations_workflow;

20
openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v159/Migration.java Normal file
View File

@ -0,0 +1,20 @@
package org.openmetadata.service.migration.mysql.v159;
import static org.openmetadata.service.migration.utils.v159.MigrationUtil.addAppExtensionName;
import lombok.SneakyThrows;
import org.openmetadata.service.migration.api.MigrationProcessImpl;
import org.openmetadata.service.migration.utils.MigrationFile;
public class Migration extends MigrationProcessImpl {
public Migration(MigrationFile migrationFile) {
super(migrationFile);
}
@Override
@SneakyThrows
public void runDataMigration() {
addAppExtensionName(handle, collectionDAO, authenticationConfiguration, false);
}
}

2
openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java
View File

@ -1,6 +1,5 @@
package org.openmetadata.service.migration.mysql.v160;
import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addAppExtensionName;
import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addViewAllRuleToOrgPolicy;
import static org.openmetadata.service.migration.utils.v160.MigrationUtil.migrateServiceTypesAndConnections;
@ -17,7 +16,6 @@ public class Migration extends MigrationProcessImpl {
@Override
@SneakyThrows
public void runDataMigration() {
addAppExtensionName(handle, collectionDAO, authenticationConfiguration, false);
migrateServiceTypesAndConnections(handle, false);
addViewAllRuleToOrgPolicy(collectionDAO);
}

20
openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v159/Migration.java Normal file
View File

@ -0,0 +1,20 @@
package org.openmetadata.service.migration.postgres.v159;
import static org.openmetadata.service.migration.utils.v159.MigrationUtil.addAppExtensionName;
import lombok.SneakyThrows;
import org.openmetadata.service.migration.api.MigrationProcessImpl;
import org.openmetadata.service.migration.utils.MigrationFile;
public class Migration extends MigrationProcessImpl {
public Migration(MigrationFile migrationFile) {
super(migrationFile);
}
@Override
@SneakyThrows
public void runDataMigration() {
addAppExtensionName(handle, collectionDAO, authenticationConfiguration, true);
}
}

2
openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java
View File

@ -1,6 +1,5 @@
package org.openmetadata.service.migration.postgres.v160;
import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addAppExtensionName;
import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addViewAllRuleToOrgPolicy;
import static org.openmetadata.service.migration.utils.v160.MigrationUtil.migrateServiceTypesAndConnections;
@ -17,7 +16,6 @@ public class Migration extends MigrationProcessImpl {
@Override
@SneakyThrows
public void runDataMigration() {
addAppExtensionName(handle, collectionDAO, authenticationConfiguration, true);
migrateServiceTypesAndConnections(handle, true);
addViewAllRuleToOrgPolicy(collectionDAO);
}

83
openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v159/MigrationUtil.java Normal file
View File

@ -0,0 +1,83 @@
package org.openmetadata.service.migration.utils.v159;
import java.util.UUID;
import javax.json.JsonObject;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Handle;
import org.jdbi.v3.core.statement.Update;
import org.openmetadata.schema.api.security.AuthenticationConfiguration;
import org.openmetadata.schema.entity.app.App;
import org.openmetadata.schema.entity.app.AppExtension;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.AppRepository;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.util.JsonUtils;
@Slf4j
public class MigrationUtil {
// Just list status to make this ignore the new `limits` extension if it ever runs again
private static final String SELECT_ALL_APP_EXTENSION_TIME_SERIES =
"SELECT appId, json FROM apps_extension_time_series where extension = 'status'";
private static final String UPDATE_MYSQL_APP_EXTENSION =
"UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.appName', :appName) WHERE appId = :appId AND extension = 'status'";
private static final String UPDATE_PG_APP_EXTENSION =
"UPDATE apps_extension_time_series SET json = jsonb_set(json, '{appName}', to_jsonb(:appName)) WHERE appId = :appId AND extension = 'status'";
// We'll list the entries in app_extension_time_series, clean those whose appId
// is not installed, and for those that appId matches from installed Apps, we'll
// add the appName to the JSON data.
// Note that we only want to clean up old status data.
public static void addAppExtensionName(
Handle handle,
CollectionDAO daoCollection,
AuthenticationConfiguration config,
boolean postgres) {
LOG.info("Migrating app extension name...");
AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION);
try {
handle
.createQuery(SELECT_ALL_APP_EXTENSION_TIME_SERIES)
.mapToMap()
.forEach(
row -> {
try {
UUID appId = UUID.fromString(row.get("appid").toString());
// Ignore if this has already been migrated
JsonObject json = JsonUtils.readJson(row.get("json").toString()).asJsonObject();
if (json.containsKey("appName")) {
return;
}
// Else, update the name
App app = appRepository.find(appId, Include.ALL);
updateAppExtension(handle, app, postgres);
} catch (EntityNotFoundException ex) {
// Clean up the old status data
daoCollection
.appExtensionTimeSeriesDao()
.delete(
row.get("appid").toString(),
AppExtension.ExtensionType.STATUS.toString());
} catch (Exception ex) {
LOG.warn(
String.format("Error migrating app extension [%s] due to [%s]", row, ex));
}
});
} catch (Exception ex) {
LOG.warn("Error running app extension migration ", ex);
}
}
private static void updateAppExtension(Handle handle, App app, boolean postgres) {
Update update;
if (postgres) {
update = handle.createUpdate(UPDATE_PG_APP_EXTENSION);
} else {
update = handle.createUpdate(UPDATE_MYSQL_APP_EXTENSION);
}
update.bind("appId", app.getId().toString()).bind("appName", app.getName()).execute();
}
}

70
openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java
View File

@ -2,21 +2,14 @@ package org.openmetadata.service.migration.utils.v160;
import static org.openmetadata.common.utils.CommonUtil.listOf;
import java.util.UUID;
import javax.json.JsonObject;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Handle;
import org.jdbi.v3.core.statement.Update;
import org.openmetadata.schema.api.security.AuthenticationConfiguration;
import org.openmetadata.schema.entity.app.App;
import org.openmetadata.schema.entity.app.AppExtension;
import org.openmetadata.schema.entity.policies.Policy;
import org.openmetadata.schema.entity.policies.accessControl.Rule;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.AppRepository;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.PolicyRepository;
import org.openmetadata.service.util.JsonUtils;
@ -24,59 +17,6 @@ import org.openmetadata.service.util.JsonUtils;
@Slf4j
public class MigrationUtil {
// Just list status to make this ignore the new `limits` extension if it ever runs again
private static final String SELECT_ALL_APP_EXTENSION_TIME_SERIES =
"SELECT appId, json FROM apps_extension_time_series where extension = 'status'";
private static final String UPDATE_MYSQL_APP_EXTENSION =
"UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.appName', :appName) WHERE appId = :appId AND extension = 'status'";
private static final String UPDATE_PG_APP_EXTENSION =
"UPDATE apps_extension_time_series SET json = jsonb_set(json, '{appName}', to_jsonb(:appName)) WHERE appId = :appId AND extension = 'status'";
// We'll list the entries in app_extension_time_series, clean those whose appId
// is not installed, and for those that appId matches from installed Apps, we'll
// add the appName to the JSON data.
// Note that we only want to clean up old status data.
public static void addAppExtensionName(
Handle handle,
CollectionDAO daoCollection,
AuthenticationConfiguration config,
boolean postgres) {
LOG.info("Migrating app extension name...");
AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION);
try {
handle
.createQuery(SELECT_ALL_APP_EXTENSION_TIME_SERIES)
.mapToMap()
.forEach(
row -> {
try {
UUID appId = UUID.fromString(row.get("appid").toString());
// Ignore if this has already been migrated
JsonObject json = JsonUtils.readJson(row.get("json").toString()).asJsonObject();
if (json.containsKey("appName")) {
return;
}
// Else, update the name
App app = appRepository.find(appId, Include.ALL);
updateAppExtension(handle, app, postgres);
} catch (EntityNotFoundException ex) {
// Clean up the old status data
daoCollection
.appExtensionTimeSeriesDao()
.delete(
row.get("appid").toString(),
AppExtension.ExtensionType.STATUS.toString());
} catch (Exception ex) {
LOG.warn(
String.format("Error migrating app extension [%s] due to [%s]", row, ex));
}
});
} catch (Exception ex) {
LOG.warn("Error running app extension migration ", ex);
}
}
public static void addViewAllRuleToOrgPolicy(CollectionDAO collectionDAO) {
PolicyRepository repository = (PolicyRepository) Entity.getEntityRepository(Entity.POLICY);
try {
@ -109,16 +49,6 @@ public class MigrationUtil {
}
}
private static void updateAppExtension(Handle handle, App app, boolean postgres) {
Update update;
if (postgres) {
update = handle.createUpdate(UPDATE_PG_APP_EXTENSION);
} else {
update = handle.createUpdate(UPDATE_MYSQL_APP_EXTENSION);
}
update.bind("appId", app.getId().toString()).bind("appName", app.getName()).execute();
}
public static void migrateServiceTypesAndConnections(Handle handle, boolean postgresql) {
LOG.info("Starting service type and connection type migrations");
try {

4
openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json
View File

@ -8,7 +8,7 @@
"appConfig": {
"oneOf": [
{
"$ref": "external/metaPilotAppConfig.json"
"$ref": "external/collateAIAppConfig.json"
},
{
"$ref": "external/automatorAppConfig.json"
@ -27,7 +27,7 @@
"privateConfig": {
"oneOf": [
{
"$ref": "./private/external/metaPilotAppPrivateConfig.json"
"$ref": "private/external/collateAIAppPrivateConfig.json"
}
]
}

23
openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json → openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/collateAIAppConfig.json vendored
View File

@ -1,26 +1,27 @@
{
"$id": "https://open-metadata.org/schema/entity/applications/configuration/external/metaPilotAppConfig.json",
"$id": "https://open-metadata.org/schema/entity/applications/configuration/external/collateAIAppConfig.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "MetaPilotAppConfig",
"description": "Configuration for the MetaPilot External Application.",
"title": "CollateAIAppConfig",
"description": "Configuration for the CollateAI External Application.",
"type": "object",
"javaType": "org.openmetadata.schema.entity.app.external.MetaPilotAppConfig",
"javaType": "org.openmetadata.schema.entity.app.external.CollateAIAppConfig",
"definitions": {
"metaPilotAppType": {
"collateAIAppType": {
"description": "Application type.",
"type": "string",
"enum": ["MetaPilot"],
"default": "MetaPilot"
"enum": ["CollateAI"],
"default": "CollateAI"
}
},
"properties": {
"type": {
"title": "Application Type",
"description": "Application Type",
"$ref": "#/definitions/metaPilotAppType",
"default": "MetaPilot"
"$ref": "#/definitions/collateAIAppType",
"default": "CollateAI"
},
"queryFilter": {
"filter": {
"title": "Filter",
"description": "Query filter to be passed to ES. E.g., `{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"term\":{\"domain.displayName.keyword\":\"DG Anim\"}}]}}]}}}`. This is the same payload as in the Explore page.",
"type": "string"
},
@ -32,5 +33,5 @@
}
},
"additionalProperties": false,
"required": ["queryFilter"]
"required": ["filter"]
}

21
openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json → openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.json vendored
View File

@ -1,21 +1,22 @@
{
"$id": "https://open-metadata.org/schema/entity/applications/configuration/private/external/metaPilotAppConfig.json",
"$id": "https://open-metadata.org/schema/entity/applications/configuration/private/external/collateAIAppConfig.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "MetaPilotAppPrivateConfig",
"description": "PRivate Configuration for the MetaPilot External Application.",
"title": "CollateAIAppPrivateConfig",
"description": "PRivate Configuration for the CollateAI External Application.",
"type": "object",
"javaType": "org.openmetadata.schema.entity.app.external.MetaPilotAppPrivateConfig",
"javaType": "org.openmetadata.schema.entity.app.external.CollateAIAppPrivateConfig",
"definitions": {
"metaPilotLimits": {
"description": "Limits for the MetaPilot Application.",
"collateAILimits": {
"description": "Limits for the CollateAI Application.",
"type": "object",
"javaType": "org.openmetadata.schema.entity.app.external.collateAI.CollateAILimits",
"properties": {
"descriptions": {
"description": "Maximum number of descriptions generated by the MetaPilot",
"description": "Maximum number of descriptions generated by the CollateAI",
"type": "integer"
},
"queries": {
"description": "Maximum number of queries generated by MetaPilot.",
"description": "Maximum number of queries generated by CollateAI.",
"type": "integer"
},
"billingCycleStart": {
@ -47,8 +48,8 @@
},
"limits": {
"title": "Limits",
"description": "Limits for the MetaPilot Application.",
"$ref": "#/definitions/metaPilotLimits"
"description": "Limits for the CollateAI Application.",
"$ref": "#/definitions/collateAILimits"
}
},
"additionalProperties": false,