yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-25 17:08:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(config): configurable bootstrap policies file (#8812)

Browse Source 
Co-authored-by: John Joyce <john@acryl.io>
This commit is contained in:
Sergio Gómez Villamor 2023-10-11 03:04:44 +02:00 committed by GitHub
parent a17db676e3
commit dfcea2441e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
metadata-service/configuration/src/main/resources/application.yml
View File

@ -276,6 +276,10 @@ bootstrap:
enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones
backfillBrowsePathsV2:
enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. Deprecating in favor of running through SystemUpdate
policies:
file: ${BOOTSTRAP_POLICIES_FILE:classpath:boot/policies.json}
# eg for local file
# file: "file:///datahub/datahub-gms/resources/custom-policies.json"
servlets:
waitTimeout: ${BOOTSTRAP_SERVLETS_WAITTIMEOUT:60} # Total waiting time in seconds for servlets to initialize

7
metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
View File

@ -31,6 +31,7 @@ import com.linkedin.metadata.models.registry.EntityRegistry;
import com.linkedin.metadata.search.EntitySearchService;
import com.linkedin.metadata.search.SearchService;
import com.linkedin.metadata.search.transformer.SearchDocumentTransformer;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.Nonnull;
@ -41,6 +42,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Scope;
import org.springframework.core.io.Resource;
@Configuration
@ -89,13 +91,16 @@ public class BootstrapManagerFactory {
@Value("${bootstrap.backfillBrowsePathsV2.enabled}")
private Boolean _backfillBrowsePathsV2Enabled;
@Value("${bootstrap.policies.file}")
private Resource _policiesResource;
@Bean(name = "bootstrapManager")
@Scope("singleton")
@Nonnull
protected BootstrapManager createInstance() {
final IngestRootUserStep ingestRootUserStep = new IngestRootUserStep(_entityService);
final IngestPoliciesStep ingestPoliciesStep =
new IngestPoliciesStep(_entityRegistry, _entityService, _entitySearchService, _searchDocumentTransformer);
new IngestPoliciesStep(_entityRegistry, _entityService, _entitySearchService, _searchDocumentTransformer, _policiesResource);
final IngestRolesStep ingestRolesStep = new IngestRolesStep(_entityService, _entityRegistry);
final IngestDataPlatformsStep ingestDataPlatformsStep = new IngestDataPlatformsStep(_entityService);
final IngestDataPlatformInstancesStep ingestDataPlatformInstancesStep =

10
metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java
View File

@ -25,6 +25,7 @@ import com.linkedin.metadata.utils.GenericRecordUtils;
import com.linkedin.mxe.GenericAspect;
import com.linkedin.mxe.MetadataChangeProposal;
import com.linkedin.policy.DataHubPolicyInfo;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Collections;
@ -35,7 +36,8 @@ import java.util.Optional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import static com.linkedin.metadata.Constants.*;
@ -52,6 +54,8 @@ public class IngestPoliciesStep implements BootstrapStep {
private final EntitySearchService _entitySearchService;
private final SearchDocumentTransformer _searchDocumentTransformer;
private final Resource _policiesResource;
@Override
public String name() {
return "IngestPoliciesStep";
@ -66,10 +70,10 @@ public class IngestPoliciesStep implements BootstrapStep {
.maxStringLength(maxSize).build());
// 0. Execute preflight check to see whether we need to ingest policies
log.info("Ingesting default access policies...");
log.info("Ingesting default access policies from: {}...", _policiesResource);
// 1. Read from the file into JSON.
final JsonNode policiesObj = mapper.readTree(new ClassPathResource("./boot/policies.json").getFile());
final JsonNode policiesObj = mapper.readTree(_policiesResource.getFile());
if (!policiesObj.isArray()) {
throw new RuntimeException(