yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 18:07:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

<refactor>[ingestions]: align the default kafka topics with PR #1756 (#1758)

Browse Source
This commit is contained in:
Chris Lee 2020-07-29 20:26:01 -07:00 committed by GitHub
parent 779eaeed70
commit 4143fb901e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/metadata-ingestion/bin/dataset-hive-generator.py
View File

@ -11,7 +11,7 @@ import simplejson as json
HIVESTORE='localhost'
AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
KAFKATOPIC = 'MetadataChangeEvent'
KAFKATOPIC = 'MetadataChangeEvent_v4'
BOOTSTRAP = 'localhost:9092'
SCHEMAREGISTRY = 'http://localhost:8081'

4
contrib/metadata-ingestion/openldap-etl/openldap-etl.py
View File

@ -15,7 +15,7 @@ ATTRLIST = ['cn', 'title', 'mail', 'displayName', 'departmentNumber','manager']
SEARCHFILTER='givenname=Homer'
AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
KAFKATOPIC = 'MetadataChangeEvent'
KAFKATOPIC = 'MetadataChangeEvent_v4'
BOOTSTRAP = 'localhost:9092'
SCHEMAREGISTRY = 'http://localhost:8081'
@ -161,4 +161,4 @@ while True:
break
l.unbind()
sys.exit(0)
sys.exit(0)

6
docker/kafka/Dockerfile
View File

@ -3,6 +3,6 @@ FROM confluentinc/cp-kafka:5.4.0
CMD echo Waiting for Kafka to be ready... && \
cub kafka-ready -b $KAFKA_BOOTSTRAP_SERVER 1 60 && \
kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic MetadataAuditEvent && \
kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic MetadataChangeEvent && \
kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic FailedMetadataChangeEvent
kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic MetadataAuditEvent_v4 && \
kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic MetadataChangeEvent_v4 && \
kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic FailedMetadataChangeEvent_v4

4
metadata-ingestion/kafka-etl/kafka_etl.py
View File

@ -8,7 +8,7 @@ from confluent_kafka.avro import AvroProducer
ZOOKEEPER='localhost:2181'
AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
KAFKATOPIC = 'MetadataChangeEvent'
KAFKATOPIC = 'MetadataChangeEvent_v4'
BOOTSTRAP = 'localhost:9092'
SCHEMAREGISTRY = 'http://localhost:8081'
@ -64,4 +64,4 @@ for dataset_name in topics:
print(topic)
build_kafka_dataset_mce(dataset_name, str(schema), int(schema_version))
sys.exit(0)
sys.exit(0)

4
metadata-ingestion/ldap-etl/ldap_etl.py
View File

@ -17,7 +17,7 @@ ATTRLIST = ['cn', 'title', 'mail', 'sAMAccountName', 'department','manager']
SEARCHFILTER='SEARCHFILTER'
AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
KAFKATOPIC = 'MetadataChangeEvent'
KAFKATOPIC = 'MetadataChangeEvent_v4'
BOOTSTRAP = 'localhost:9092'
SCHEMAREGISTRY = 'http://localhost:8081'
@ -154,4 +154,4 @@ while True:
break
l.unbind()
sys.exit(0)
sys.exit(0)

2
metadata-ingestion/mce-cli/mce_cli.py
View File

@ -7,7 +7,7 @@ from confluent_kafka.avro import AvroProducer
from confluent_kafka.avro.serializer import SerializerError
topic = "MetadataChangeEvent"
topic = "MetadataChangeEvent_v4"
class MetadataChangeEvent(object):

4
metadata-ingestion/mysql-etl/mysql_etl.py
View File

@ -12,7 +12,7 @@ USER = 'USER'
PASSWORD = 'PASSWORD'
AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
KAFKATOPIC = 'MetadataChangeEvent'
KAFKATOPIC = 'MetadataChangeEvent_v4'
BOOTSTRAP = 'localhost:9092'
SCHEMAREGISTRY = 'http://localhost:8081'
@ -77,4 +77,4 @@ try:
except Error as e:
sys.stdout.write('Error while connecting to MySQL %s' % e)
sys.exit(0)
sys.exit(0)

2
metadata-ingestion/sql-etl/common.py
View File

@ -15,7 +15,7 @@ OPTIONS = {} # e.g. {"encoding": "latin1"}
@dataclass
class KafkaConfig:
avsc_path = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc'
kafka_topic = 'MetadataChangeEvent'
kafka_topic = 'MetadataChangeEvent_v4'
bootstrap_server = 'localhost:9092'
schema_registry = 'http://localhost:8081'