yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-25 08:58:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(perf-test): changes for perf testing (#3728)

Browse Source
This commit is contained in:
Aseem Bansal 2021-12-14 12:37:57 +05:30 committed by GitHub
parent f24440eff3
commit a4ac79db48
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 2163 additions and 2076 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4227
docker/monitoring/grafana/dashboards/datahub_dashboard.json
View File

File diff suppressed because it is too large Load Diff

1
docker/monitoring/prometheus.yaml
View File

@ -3,6 +3,7 @@ scrape_configs:
scrape_interval: 10s
static_configs:
- targets:
- 'host.docker.internal:4318'
- 'datahub-gms:4318'
- 'datahub-mae-consumer:4318'
- 'datahub-mce-consumer:4318'

3
metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataAuditEventsProcessor.java
View File

@ -1,5 +1,6 @@
package com.linkedin.metadata.kafka;
import com.codahale.metrics.Timer;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.MetricRegistry;
import com.linkedin.common.urn.Urn;
@ -92,7 +93,7 @@ public class MetadataAuditEventsProcessor {
final GenericRecord record = consumerRecord.value();
log.debug("Got MAE");
try {
try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "maeProcess").time()) {
final MetadataAuditEvent event = EventUtils.avroToPegasusMAE(record);
final RecordTemplate snapshot = RecordUtils.getSelectedRecordTemplateFromUnion(event.getNewSnapshot());

8
perf-test/locustfiles/ingest.py
View File

@ -10,11 +10,15 @@ from datahub.metadata.com.linkedin.pegasus2avro.common import (
)
from datahub.metadata.com.linkedin.pegasus2avro.dataset import DatasetProperties
from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
from locust import HttpUser, between, task
from locust import HttpUser, constant, task
class IngestUser(HttpUser):
wait_time = between(1, 5)
wait_time = constant(1)
@task
def config(self):
self.client.get("/config")
@task
def ingest(self):