yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-05 14:06:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(log): reduce log volume for ingestion and consumers (#11714)

Browse Source
This commit is contained in:
Davi Arnaut 2024-10-26 00:10:40 -04:00 committed by GitHub
parent 6770ee8535
commit a6e1624f48
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 8 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java
View File

@ -164,6 +164,8 @@ public class EntityServiceImpl implements EntityService<ChangeItemImpl> {
private final Integer ebeanMaxTransactionRetry; private final Integer ebeanMaxTransactionRetry;
private final boolean enableBrowseV2; private final boolean enableBrowseV2;
private static final long DB_TIMER_LOG_THRESHOLD_MS = 50;
@Getter @Getter
private final Map<Set<ThrottleType>, ThrottleEvent> throttleEvents = new ConcurrentHashMap<>(); private final Map<Set<ThrottleType>, ThrottleEvent> throttleEvents = new ConcurrentHashMap<>();
@ -997,10 +999,10 @@ public class EntityServiceImpl implements EntityService<ChangeItemImpl> {
if (txContext != null) { if (txContext != null) {
txContext.commitAndContinue(); txContext.commitAndContinue();
} }
long took = ingestToLocalDBTimer.stop(); long took = TimeUnit.NANOSECONDS.toMillis(ingestToLocalDBTimer.stop());
log.info( if (took > DB_TIMER_LOG_THRESHOLD_MS) {
"Ingestion of aspects batch to database took {} ms", log.info("Ingestion of aspects batch to database took {} ms", took);
TimeUnit.NANOSECONDS.toMillis(took)); }
// Retention optimization and tx // Retention optimization and tx
if (retentionService != null) { if (retentionService != null) {

2
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
View File

@ -112,7 +112,7 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI
@Nullable String runId) { @Nullable String runId) {
final String docId = indexBuilders.getIndexConvention().getEntityDocumentId(urn); final String docId = indexBuilders.getIndexConvention().getEntityDocumentId(urn);
log.info( log.debug(
"Appending run id for entity name: {}, doc id: {}, run id: {}", entityName, docId, runId); "Appending run id for entity name: {}, doc id: {}, run id: {}", entityName, docId, runId);
esWriteDAO.applyScriptUpdate( esWriteDAO.applyScriptUpdate(
opContext, opContext,

2
metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java
View File

@ -95,7 +95,7 @@ public class MCLKafkaListener {
// Here - plug in additional "custom processor hooks" // Here - plug in additional "custom processor hooks"
for (MetadataChangeLogHook hook : this.hooks) { for (MetadataChangeLogHook hook : this.hooks) {
log.info( log.debug(
"Invoking MCL hook {} for urn: {}", "Invoking MCL hook {} for urn: {}",
hook.getClass().getSimpleName(), hook.getClass().getSimpleName(),
event.getEntityUrn()); event.getEntityUrn());

9
metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java
View File

@ -243,13 +243,6 @@ public class AspectResource extends CollectionResourceTaskTemplate<String, Versi
@ActionParam(PARAM_PROPOSAL) @Nonnull MetadataChangeProposal metadataChangeProposal, @ActionParam(PARAM_PROPOSAL) @Nonnull MetadataChangeProposal metadataChangeProposal,
@ActionParam(PARAM_ASYNC) @Optional(UNSET) String async) @ActionParam(PARAM_ASYNC) @Optional(UNSET) String async)
throws URISyntaxException { throws URISyntaxException {
String urn = metadataChangeProposal.getEntityUrn() != null ? metadataChangeProposal.getEntityUrn().toString() :
java.util.Optional.ofNullable(metadataChangeProposal.getEntityKeyAspect()).orElse(new GenericAspect())
.getValue().asString(StandardCharsets.UTF_8);
String proposedValue = java.util.Optional.ofNullable(metadataChangeProposal.getAspect()).orElse(new GenericAspect())
.getValue().asString(StandardCharsets.UTF_8);
final boolean asyncBool; final boolean asyncBool;
if (UNSET.equals(async)) { if (UNSET.equals(async)) {
asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME)); asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME));
@ -266,8 +259,6 @@ public class AspectResource extends CollectionResourceTaskTemplate<String, Versi
@ActionParam(PARAM_PROPOSALS) @Nonnull MetadataChangeProposal[] metadataChangeProposals, @ActionParam(PARAM_PROPOSALS) @Nonnull MetadataChangeProposal[] metadataChangeProposals,
@ActionParam(PARAM_ASYNC) @Optional(UNSET) String async) @ActionParam(PARAM_ASYNC) @Optional(UNSET) String async)
throws URISyntaxException { throws URISyntaxException {
log.info("INGEST PROPOSAL BATCH proposals: {}", Arrays.asList(metadataChangeProposals));
final boolean asyncBool; final boolean asyncBool;
if (UNSET.equals(async)) { if (UNSET.equals(async)) {
asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME)); asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME));