From 33f4d2ede0dba16a0af9afd924a0f87319be8c29 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Tue, 28 Sep 2021 16:30:49 -0700 Subject: [PATCH] fix(upgrade): Improving NoCodeUpgrade logic to account for Bootstrap logic (#3301) --- .../app/auth/sso/oidc/OidcCallbackLogic.java | 13 ++-- .../upgrade/nocode/DataMigrationStep.java | 3 +- .../nocode/IngestDataPlatformsStep.java | 69 ------------------- .../datahub/upgrade/nocode/NoCodeUpgrade.java | 1 - .../nocode/UpgradeQualificationStep.java | 2 +- docker/mariadb/init.sql | 4 +- docker/mysql-setup/init.sql | 4 +- docker/mysql/init.sql | 4 +- docker/postgres-setup/init.sql | 4 +- docker/postgres/init.sql | 4 +- docker/quickstart/mysql/init.sql | 4 +- .../tests/integration/mysql/setup/setup.sql | 4 +- .../entity/AspectStorageValidationUtil.java | 10 ++- .../boot/steps/IngestDataPlatformsStep.java | 2 +- .../boot/steps/IngestPoliciesStep.java | 5 +- .../java/com/linkedin/metadata/Constants.java | 2 +- 16 files changed, 37 insertions(+), 98 deletions(-) delete mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/IngestDataPlatformsStep.java diff --git a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java index 7a41a42af7..800ce4218e 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java @@ -15,6 +15,7 @@ import com.linkedin.identity.CorpGroupInfo; import com.linkedin.identity.CorpUserEditableInfo; import com.linkedin.identity.CorpUserInfo; import com.linkedin.identity.GroupMembership; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.CorpGroupAspect; import com.linkedin.metadata.aspect.CorpGroupAspectArray; import com.linkedin.metadata.aspect.CorpUserAspect; @@ -64,7 +65,7 @@ import static auth.AuthUtils.*; @Slf4j public class OidcCallbackLogic extends DefaultCallbackLogic { - private static final String SYSTEM_PRINCIPAL = "urn:li:corpuser:system"; + private static final String SYSTEM_ACTOR = Constants.SYSTEM_ACTOR; private final EntityClient _entityClient = GmsClientFactory.getEntitiesClient(); private final SsoManager _ssoManager; @@ -271,7 +272,7 @@ public class OidcCallbackLogic extends DefaultCallbackLogic urnsToFetch = corpGroups.stream().map(CorpGroupSnapshot::getUrn).collect(Collectors.toSet()); - final Map existingGroups = _entityClient.batchGet(urnsToFetch, SYSTEM_PRINCIPAL); + final Map existingGroups = _entityClient.batchGet(urnsToFetch, SYSTEM_ACTOR); log.debug(String.format("Fetched GMS groups with urns %s", existingGroups.keySet())); @@ -335,7 +336,7 @@ public class OidcCallbackLogic extends DefaultCallbackLogic new Entity().setValue(Snapshot.create(groupSnapshot)) - ).collect(Collectors.toSet()), SYSTEM_PRINCIPAL); + ).collect(Collectors.toSet()), SYSTEM_ACTOR); log.debug(String.format("Successfully provisioned groups with urns %s", groupsToCreateUrns)); @@ -349,7 +350,7 @@ public class OidcCallbackLogic extends DefaultCallbackLogic executable() { - return (context) -> { - - context.report().addLine("Preparing to ingest DataPlatforms..."); - - Map urnToInfo = DataPlatformsUtil.getDataPlatformInfoMap(); - - context.report().addLine(String.format("Found %s DataPlatforms", urnToInfo.keySet().size())); - - for (final Map.Entry entry : urnToInfo.entrySet()) { - AuditStamp auditStamp; - try { - auditStamp = new AuditStamp().setActor(Urn.createFromString("urn:li:principal:system")).setTime( - Clock.systemUTC().millis()); - } catch (URISyntaxException e) { - throw new RuntimeException("Failed to create Actor Urn"); - } - - _entityService.ingestAspect( - entry.getKey(), - PegasusUtils.getAspectNameFromSchema(entry.getValue().schema()), - entry.getValue(), - auditStamp - ); - } - - context.report().addLine(String.format("Successfully ingested %s DataPlatforms.", urnToInfo.keySet().size())); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); - }; - } -} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java index 793b7f6db7..705e8b9dbf 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java @@ -66,7 +66,6 @@ public class NoCodeUpgrade implements Upgrade { steps.add(new GMSQualificationStep()); steps.add(new UpgradeQualificationStep(server)); steps.add(new CreateAspectTableStep(server)); - steps.add(new IngestDataPlatformsStep(entityService)); steps.add(new DataMigrationStep(server, entityService, entityRegistry)); steps.add(new GMSEnableWriteModeStep(entityClient)); return steps; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java index 0912cbb4d7..258b331e51 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java @@ -61,7 +61,7 @@ public class UpgradeQualificationStep implements UpgradeStep { boolean v2TableExists = AspectStorageValidationUtil.checkV2TableExists(server); if (v2TableExists) { context.report().addLine("-- V2 table exists"); - long v2TableRowCount = AspectStorageValidationUtil.getV2RowCount(server); + long v2TableRowCount = AspectStorageValidationUtil.getV2NonSystemRowCount(server); if (v2TableRowCount == 0) { context.report().addLine("-- V2 table is empty"); return true; diff --git a/docker/mariadb/init.sql b/docker/mariadb/init.sql index 95aa80dd30..04932b172f 100644 --- a/docker/mariadb/init.sql +++ b/docker/mariadb/init.sql @@ -17,12 +17,12 @@ insert into metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); diff --git a/docker/mysql-setup/init.sql b/docker/mysql-setup/init.sql index cc6e934d66..83ae2e0447 100644 --- a/docker/mysql-setup/init.sql +++ b/docker/mysql-setup/init.sql @@ -23,14 +23,14 @@ INSERT INTO temp_metadata_aspect_v2 (urn, aspect, version, metadata, createdon, 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); -- only add default records if metadata_aspect is empty INSERT INTO metadata_aspect_v2 diff --git a/docker/mysql/init.sql b/docker/mysql/init.sql index 1d1bfe3f39..6d08983e59 100644 --- a/docker/mysql/init.sql +++ b/docker/mysql/init.sql @@ -17,14 +17,14 @@ INSERT INTO metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); -- create metadata index table diff --git a/docker/postgres-setup/init.sql b/docker/postgres-setup/init.sql index 721d55ca36..26020359cf 100644 --- a/docker/postgres-setup/init.sql +++ b/docker/postgres-setup/init.sql @@ -19,14 +19,14 @@ INSERT INTO temp_metadata_aspect_v2 (urn, aspect, version, metadata, createdon, 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); -- only add default records if metadata_aspect is empty INSERT INTO metadata_aspect_v2 diff --git a/docker/postgres/init.sql b/docker/postgres/init.sql index 5c5216181c..162422e25f 100644 --- a/docker/postgres/init.sql +++ b/docker/postgres/init.sql @@ -17,12 +17,12 @@ insert into metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); diff --git a/docker/quickstart/mysql/init.sql b/docker/quickstart/mysql/init.sql index 0f8ec87d01..d94e6bd9b7 100644 --- a/docker/quickstart/mysql/init.sql +++ b/docker/quickstart/mysql/init.sql @@ -17,14 +17,14 @@ INSERT INTO metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'corpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); -- create metadata index table diff --git a/metadata-ingestion/tests/integration/mysql/setup/setup.sql b/metadata-ingestion/tests/integration/mysql/setup/setup.sql index f9cdfbb55b..f86c002e27 100644 --- a/metadata-ingestion/tests/integration/mysql/setup/setup.sql +++ b/metadata-ingestion/tests/integration/mysql/setup/setup.sql @@ -19,14 +19,14 @@ insert into metadata_aspect (urn, aspect, version, metadata, createdon, createdb 0, '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ), ( 'urn:li:corpuser:datahub', 'com.linkedin.identity.CorpUserEditableInfo', 0, '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', now(), - 'urn:li:principal:datahub' + 'urn:li:corpuser:__datahub_system' ); -- create metadata index table diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectStorageValidationUtil.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectStorageValidationUtil.java index 823c515bbb..568af4ab0e 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectStorageValidationUtil.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectStorageValidationUtil.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.entity; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.ebean.EbeanAspectV1; import com.linkedin.metadata.entity.ebean.EbeanAspectV2; import io.ebean.EbeanServer; @@ -7,6 +8,8 @@ import io.ebean.SqlQuery; import io.ebean.SqlRow; import java.util.List; +import static io.ebean.Expr.*; + public class AspectStorageValidationUtil { @@ -18,8 +21,11 @@ public class AspectStorageValidationUtil { return server.find(EbeanAspectV1.class).findCount(); } - public static long getV2RowCount(EbeanServer server) { - return server.find(EbeanAspectV2.class).findCount(); + /** + * Get the number of rows created not by the DataHub system actor (urn:li:corpuser:__datahub_system) + */ + public static long getV2NonSystemRowCount(EbeanServer server) { + return server.find(EbeanAspectV2.class).where(ne("createdby", Constants.SYSTEM_ACTOR)).findCount(); } public static boolean checkV2TableExists(EbeanServer server) { diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java index 86227d6040..3d9e63ad10 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java @@ -57,7 +57,7 @@ public class IngestDataPlatformsStep implements BootstrapStep { RecordUtils.toRecordTemplate(DataPlatformInfo.class, dataPlatform.get("aspect").toString()); final AuditStamp aspectAuditStamp = - new AuditStamp().setActor(Urn.createFromString(Constants.UNKNOWN_ACTOR)).setTime(System.currentTimeMillis()); + new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); _entityService.ingestAspect(urn, PLATFORM_ASPECT_NAME, info, aspectAuditStamp); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java index 5cc4ae6d5a..f4e4010b8e 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.boot.BootstrapStep; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.dao.utils.RecordUtils; @@ -92,7 +93,7 @@ public class IngestPoliciesStep implements BootstrapStep { keyAspectProposal.setEntityUrn(urn); _entityService.ingestProposal(keyAspectProposal, - new AuditStamp().setActor(Urn.createFromString("urn:li:corpuser:system")).setTime(System.currentTimeMillis())); + new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis())); final MetadataChangeProposal proposal = new MetadataChangeProposal(); proposal.setEntityUrn(urn); @@ -102,7 +103,7 @@ public class IngestPoliciesStep implements BootstrapStep { proposal.setChangeType(ChangeType.UPSERT); _entityService.ingestProposal(proposal, - new AuditStamp().setActor(Urn.createFromString("urn:li:corpuser:system")).setTime(System.currentTimeMillis())); + new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis())); } private boolean hasDefaultPolicies() throws URISyntaxException { diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java b/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java index 5f54a1ec93..6b288a9770 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -6,7 +6,7 @@ package com.linkedin.metadata; public class Constants { public static final String ACTOR_HEADER_NAME = "X-DataHub-Actor"; public static final String DATAHUB_ACTOR = "urn:li:corpuser:datahub"; // Super user. - public static final String SYSTEM_ACTOR = "urn:li:principal:datahub"; // DataHub internal service principal. + public static final String SYSTEM_ACTOR = "urn:li:corpuser:__datahub_system"; // DataHub internal service principal. public static final String UNKNOWN_ACTOR = "urn:li:corpuser:UNKNOWN"; // Unknown principal. public static final Long ASPECT_LATEST_VERSION = 0L;