yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-02 19:58:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(bootstrap): Creating dedicated thread pool for executing async bootstrap steps + misc fixes (#5798)

Browse Source
This commit is contained in:
John Joyce 2022-08-31 19:34:17 -07:00 committed by GitHub
parent af1fc8d91d
commit ca29f8b679
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 40 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
datahub-web-react/src/app/auth/LogIn.tsx
View File

@ -71,8 +71,8 @@ export const LogIn: React.VFC<LogInProps> = () => {
analytics.event({ type: EventType.LogInEvent });
return Promise.resolve();
})
.catch((error) => {
message.error(`Failed to log in! ${error}`);
.catch((_) => {
message.error(`Failed to log in! An unexpected error occurred.`);
})
.finally(() => setLoading(false));
},

4
datahub-web-react/src/app/auth/ResetCredentials.tsx
View File

@ -75,8 +75,8 @@ export const ResetCredentials: React.VFC<ResetCredentialsProps> = () => {
analytics.event({ type: EventType.ResetCredentialsEvent });
return Promise.resolve();
})
.catch((error) => {
message.error(`Failed to log in! ${error}`);
.catch((_) => {
message.error(`Failed to log in!`);
})
.finally(() => setLoading(false));
},

4
datahub-web-react/src/app/auth/SignUp.tsx
View File

@ -90,8 +90,8 @@ export const SignUp: React.VFC<SignUpProps> = () => {
analytics.event({ type: EventType.SignUpEvent, title: values.title });
return Promise.resolve();
})
.catch((error) => {
message.error(`Failed to log in! ${error}`);
.catch((_) => {
message.error(`Failed to log in! An unexpected error occurred.`);
})
.finally(() => setLoading(false));
},

13
datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
View File

@ -52,6 +52,7 @@ export function HeaderLinks(props: Props) {
const showSettings = true;
const showIngestion =
isIngestionEnabled && me && me.platformPrivileges.manageIngestion && me.platformPrivileges.manageSecrets;
const showDomains = me?.platformPrivileges.createDomains || me?.platformPrivileges.manageDomains;
return (
<LinksWrapper areLinksHidden={areLinksHidden}>
@ -82,11 +83,13 @@ export function HeaderLinks(props: Props) {
<BookOutlined style={{ fontSize: '14px', fontWeight: 'bold' }} /> Glossary
</Link>
</MenuItem>
<MenuItem key="1">
<Link to="/domains">
<FolderOutlined style={{ fontSize: '14px', fontWeight: 'bold' }} /> Domains
</Link>
</MenuItem>
{showDomains && (
<MenuItem key="1">
<Link to="/domains">
<FolderOutlined style={{ fontSize: '14px', fontWeight: 'bold' }} /> Domains
</Link>
</MenuItem>
)}
</Menu>
}
>

5
metadata-service/auth-impl/src/main/java/com/datahub/authentication/user/NativeUserService.java
View File

@ -61,7 +61,7 @@ public class NativeUserService {
}
public void createNativeUser(@Nonnull String userUrnString, @Nonnull String fullName, @Nonnull String email,
@Nonnull String title, @Nonnull String password, @Nonnull String inviteToken, Authentication authentication)
@Nonnull String title, @Nonnull String password, @Nonnull String inviteToken, @Nonnull Authentication authentication)
throws Exception {
Objects.requireNonNull(userUrnString, "userUrnSting must not be null!");
Objects.requireNonNull(fullName, "fullName must not be null!");
@ -69,6 +69,7 @@ public class NativeUserService {
Objects.requireNonNull(title, "title must not be null!");
Objects.requireNonNull(password, "password must not be null!");
Objects.requireNonNull(inviteToken, "inviteToken must not be null!");
Objects.requireNonNull(inviteToken, "authentication must not be null!");
InviteToken inviteTokenAspect =
(InviteToken) _entityService.getLatestAspect(Urn.createFromString(GLOBAL_INVITE_TOKEN),
@ -125,7 +126,7 @@ public class NativeUserService {
}
void updateCorpUserCredentials(@Nonnull Urn userUrn, @Nonnull String password,
Authentication authentication) throws Exception {
@Nonnull Authentication authentication) throws Exception {
// Construct corpUserCredentials
CorpUserCredentials corpUserCredentials = new CorpUserCredentials();
final byte[] salt = getRandomBytes(SALT_TOKEN_LENGTH);

6
metadata-service/auth-servlet-impl/src/main/java/com/datahub/authentication/AuthServiceController.java
View File

@ -165,11 +165,12 @@ public class AuthServiceController {
String titleString = title.asText();
String passwordString = password.asText();
String inviteTokenString = inviteToken.asText();
Authentication auth = AuthenticationContext.getAuthentication();
log.debug(String.format("Attempting to create credentials for native user %s", userUrnString));
return CompletableFuture.supplyAsync(() -> {
try {
_nativeUserService.createNativeUser(userUrnString, fullNameString, emailString, titleString, passwordString,
inviteTokenString, AuthenticationContext.getAuthentication());
inviteTokenString, auth);
String response = buildSignUpResponse();
return new ResponseEntity<>(response, HttpStatus.OK);
} catch (Exception e) {
@ -225,11 +226,12 @@ public class AuthServiceController {
String userUrnString = userUrn.asText();
String passwordString = password.asText();
String resetTokenString = resetToken.asText();
Authentication auth = AuthenticationContext.getAuthentication();
log.debug(String.format("Attempting to reset credentials for native user %s", userUrnString));
return CompletableFuture.supplyAsync(() -> {
try {
_nativeUserService.resetCorpUserCredentials(userUrnString, passwordString, resetTokenString,
AuthenticationContext.getAuthentication());
auth);
String response = buildResetNativeUserCredentialsResponse();
return new ResponseEntity<>(response, HttpStatus.OK);
} catch (Exception e) {

5
metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapManager.java
View File

@ -2,6 +2,8 @@ package com.linkedin.metadata.boot;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@ -13,6 +15,7 @@ import org.springframework.stereotype.Component;
@Component
public class BootstrapManager {
private final ExecutorService _asyncExecutor = Executors.newFixedThreadPool(5);
private final List<BootstrapStep> _bootSteps;
public BootstrapManager(final List<BootstrapStep> bootSteps) {
@ -42,7 +45,7 @@ public class BootstrapManager {
} catch (Exception e) {
log.error(String.format("Caught exception while executing bootstrap step %s. Continuing...", step.name()), e);
}
});
}, _asyncExecutor);
}
}
}

11
metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestRolesStep.java
View File

@ -15,6 +15,7 @@ import com.linkedin.mxe.GenericAspect;
import com.linkedin.mxe.MetadataChangeProposal;
import com.linkedin.policy.DataHubRoleInfo;
import java.net.URISyntaxException;
import javax.annotation.Nonnull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.ClassPathResource;
@ -25,6 +26,7 @@ import static com.linkedin.metadata.Constants.*;
@Slf4j
@RequiredArgsConstructor
public class IngestRolesStep implements BootstrapStep {
private static final int SLEEP_SECONDS = 60;
private final EntityService _entityService;
@Override
@ -32,10 +34,19 @@ public class IngestRolesStep implements BootstrapStep {
return this.getClass().getSimpleName();
}
@Nonnull
@Override
public ExecutionMode getExecutionMode() {
return ExecutionMode.ASYNC;
}
@Override
public void execute() throws Exception {
final ObjectMapper mapper = new ObjectMapper();
// Sleep to ensure deployment process finishes.
Thread.sleep(SLEEP_SECONDS * 1000);
// 0. Execute preflight check to see whether we need to ingest Roles
log.info("Ingesting default Roles...");

4
metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java
View File

@ -437,8 +437,10 @@ public class JavaEntityClient implements EntityClient {
// TODO: Factor out ingest logic into a util that can be accessed by the java client and the resource
@SneakyThrows
@Override
public String ingestProposal(@Nonnull MetadataChangeProposal metadataChangeProposal,
public String ingestProposal(
@Nonnull final MetadataChangeProposal metadataChangeProposal,
@Nonnull final Authentication authentication) throws RemoteInvocationException {
String actorUrnStr = authentication.getActor() != null ? authentication.getActor().toUrnStr() : Constants.UNKNOWN_ACTOR;
final AuditStamp auditStamp =
new AuditStamp().setTime(_clock.millis()).setActor(Urn.createFromString(actorUrnStr));

7
metadata-service/war/src/main/resources/boot/policies.json
View File

@ -239,9 +239,6 @@
]
},
"privileges":[
"MANAGE_INGESTION",
"MANAGE_SECRETS",
"VIEW_ANALYTICS",
"GENERATE_PERSONAL_ACCESS_TOKENS",
"MANAGE_DOMAINS",
"MANAGE_GLOSSARIES",
@ -314,9 +311,7 @@
"urn:li:dataHubRole:Reader"
]
},
"privileges":[
"VIEW_ANALYTICS"
],
"privileges":[],
"displayName":"Readers - Platform Policy",
"description":"Readers can view analytics.",
"state":"ACTIVE",