diff --git a/metadata-ingestion/README.md b/metadata-ingestion/README.md index 25b9238e1b..840794bfc3 100644 --- a/metadata-ingestion/README.md +++ b/metadata-ingestion/README.md @@ -34,7 +34,7 @@ optional arguments: ``` ## Bootstrapping Data Hub -If you want to quickly ingest lots of sample data and test Data Hub in action, you can run below command: +Leverage the mce-cli to quickly ingest lots of sample data and test Data Hub in action, you can run below command: ``` ➜ python mce_cli.py produce -d bootstrap_mce.dat Producing MetadataChangeEvent records to topic MetadataChangeEvent. ^c to exit. @@ -45,7 +45,7 @@ Flushing records... This will bootstrap Data Hub with sample datasets and sample users. ## Ingest metadata from LDAP server to Data Hub -The ldap_etl.py provides you ETL channel to communicate with your LDAP server. +The ldap_etl provides you ETL channel to communicate with your LDAP server. ``` ➜ Config your LDAP server environmental variable in the file LDAPSERVER # Your server host. diff --git a/metadata-ingestion/ldap_etl.py b/metadata-ingestion/ldap-etl/ldap_etl.py similarity index 96% rename from metadata-ingestion/ldap_etl.py rename to metadata-ingestion/ldap-etl/ldap_etl.py index ce21057f0d..ceaaf14fe4 100644 --- a/metadata-ingestion/ldap_etl.py +++ b/metadata-ingestion/ldap-etl/ldap_etl.py @@ -14,10 +14,10 @@ PAGESIZE = PAGESIZE ATTRLIST = ['cn', 'title', 'mail', 'sAMAccountName', 'department','manager'] SEARCHFILTER='SEARCHFILTER' -AVROLOADPATH = 'AVROLOADPATH' -KAFKATOPIC = 'KAFKATOPIC' -BOOTSTRAP = 'BOOTSTRAP' -SCHEMAREGISTRY = 'SCHEMAREGISTRY' +AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc' +KAFKATOPIC = 'MetadataChangeEvent' +BOOTSTRAP = 'localhost:9092' +SCHEMAREGISTRY = 'http://localhost:8081' def create_controls(pagesize): """ diff --git a/metadata-ingestion/bootstrap_mce.dat b/metadata-ingestion/mce-cli/bootstrap_mce.dat similarity index 100% rename from metadata-ingestion/bootstrap_mce.dat rename to metadata-ingestion/mce-cli/bootstrap_mce.dat diff --git a/metadata-ingestion/mce_cli.py b/metadata-ingestion/mce-cli/mce_cli.py similarity index 96% rename from metadata-ingestion/mce_cli.py rename to metadata-ingestion/mce-cli/mce_cli.py index 381cfcb396..5e6d1d3b93 100644 --- a/metadata-ingestion/mce_cli.py +++ b/metadata-ingestion/mce-cli/mce_cli.py @@ -1,7 +1,8 @@ +#! /usr/bin/python import argparse from confluent_kafka import avro -record_schema = avro.load("../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc") +record_schema = avro.load("../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc") topic = "MetadataChangeEvent" class MetadataChangeEvent(object):