diff --git a/ingestion/setup.py b/ingestion/setup.py index 9774dac0d10..16d40684be8 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -371,7 +371,7 @@ test = { VERSIONS["grpc-tools"], VERSIONS["neo4j"], "testcontainers==3.7.1;python_version<'3.9'", - "testcontainers==4.4.0;python_version>='3.9'", + "testcontainers==4.8.0;python_version>='3.9'", "minio==7.2.5", *plugins["mlflow"], *plugins["datalake-s3"], diff --git a/ingestion/src/_openmetadata_testutils/kafka/__init__.py b/ingestion/src/_openmetadata_testutils/kafka/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/src/_openmetadata_testutils/kafka/load_csv_data.py b/ingestion/src/_openmetadata_testutils/kafka/load_csv_data.py new file mode 100644 index 00000000000..f94917d02af --- /dev/null +++ b/ingestion/src/_openmetadata_testutils/kafka/load_csv_data.py @@ -0,0 +1,133 @@ +""" +pip install confluent_kafka pandas requests avro-python3 +""" + +import json +import os +from typing import Dict, List + +import pandas as pd +import requests +from confluent_kafka import Producer +from confluent_kafka.schema_registry import SchemaRegistryClient +from confluent_kafka.schema_registry.avro import AvroSerializer +from confluent_kafka.serialization import MessageField, SerializationContext + + +class Kafka: + def __init__(self, broker: str): + self.broker = broker + + def get_producer(self): + return Producer({"bootstrap.servers": self.broker}) + + +class SchemaRegistry: + def __init__(self, url: str): + self.url = url + + def register_schema(self, schema: str, topic: str): + """Register the schema with the Schema Registry""" + url = f"{self.url}/subjects/{sanitize_name(topic)}-value/versions" + headers = {"Content-Type": "application/vnd.schemaregistry.v1+json"} + data = json.dumps({"schema": schema}) + response = requests.post(url, headers=headers, data=data) + if response.status_code == 200: + print(f"Schema registered for topic {topic}") + else: + print(f"Failed to register schema for topic {topic}: {response.text}") + + def get_avro_serializer(self, schema: str) -> AvroSerializer: + schema_registry_conf = {"url": self.url} + schema_registry_client = SchemaRegistryClient(schema_registry_conf) + return AvroSerializer(schema_registry_client, schema_str=schema) + + +def delivery_report(err, msg): + """Called once for each message produced to indicate delivery result. + Triggered by poll() or flush().""" + if err is not None: + print(f"Message delivery failed: {err}") + else: + print(f"Message delivered to {msg.topic()} [{msg.partition()}]") + + +def sanitize_name(name): + """Sanitize the name to conform to Avro naming rules""" + return name.replace("-", "_").replace(" ", "_") + + +def generate_avro_schema(df: pd.DataFrame, topic: str) -> str: + """Generate an Avro schema from a pandas DataFrame""" + fields: List[Dict[str, str]] = [] + for column in df.columns: + fields.append( + {"name": sanitize_name(column), "type": "string"} + ) # Assuming all columns are of type string + schema_dict: Dict[str, any] = { + "namespace": "example.avro", + "type": "record", + "name": sanitize_name(topic), + "fields": fields, + } + return json.dumps(schema_dict) + + +def send_csv_to_kafka(kafka: Kafka, schema_registry: SchemaRegistry, file_path: str): + # Read the CSV file + df = pd.read_csv(file_path) + + # Sanitize column names + df.columns = [sanitize_name(col) for col in df.columns] + + # Convert all fields to string + df = df.astype(str) + + # Get the file name without extension to use as the topic name + topic = os.path.splitext(os.path.basename(file_path))[0] + + # Generate and register the Avro schema + schema = generate_avro_schema(df, topic) + schema_registry.register_schema(schema, topic) + + # Create an Avro serializer with the generated schema + avro_serializer = schema_registry.get_avro_serializer(schema) + + # Create a Kafka producer + producer = kafka.get_producer() + + # Iterate over the rows of the DataFrame and send each row as an Avro message + for _, row in df.iterrows(): + message = row.to_dict() + try: + producer.produce( + topic=topic, + value=avro_serializer( + message, SerializationContext(topic, MessageField.VALUE) + ), + callback=delivery_report, + ) + except Exception as e: + print(f"Message serialization failed: {e}") + break + + # Wait for any outstanding messages to be delivered and delivery reports to be received + producer.flush() + + +def main(kafka_broker: str, schema_registry_url: str, csv_directory: str): + # Iterate over all files in the directory + kafka = Kafka(kafka_broker) + schema_registry = SchemaRegistry(schema_registry_url) + for file_name in os.listdir(csv_directory): + if file_name.endswith(".csv"): + file_path = os.path.join(csv_directory, file_name) + send_csv_to_kafka(kafka, schema_registry, file_path) + + +if __name__ == "__main__": + main( + kafka_broker="localhost:9092", + schema_registry_url="http://localhost:8081", + csv_directory="./data", + ) diff --git a/ingestion/src/_openmetadata_testutils/kafka/schema_registry_container.py b/ingestion/src/_openmetadata_testutils/kafka/schema_registry_container.py new file mode 100644 index 00000000000..086321d07d9 --- /dev/null +++ b/ingestion/src/_openmetadata_testutils/kafka/schema_registry_container.py @@ -0,0 +1,31 @@ +from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import wait_for_logs + + +class SchemaRegistryContainer(DockerContainer): + def __init__( + self, + image: str = "confluentinc/cp-schema-registry:7.6.0", + port: int = 8081, + schema_registry_kafkastore_bootstrap_servers="PLAINTEXT://localhost:9092", + schema_registry_host_name="localhost", + **kwargs, + ) -> None: + super().__init__(image, **kwargs) + self.with_env( + "SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", + schema_registry_kafkastore_bootstrap_servers, + ) + self.with_env("SCHEMA_REGISTRY_HOST_NAME", schema_registry_host_name) + self.port = port + self.with_exposed_ports(port) + + def start(self, timeout=30) -> "SchemaRegistryContainer": + super().start() + wait_for_logs(self, r".*\Server started.*", timeout=timeout) + return self + + def get_connection_url(self): + return ( + f"http://{self.get_container_host_ip()}:{self.get_exposed_port(self.port)}" + ) diff --git a/ingestion/tests/integration/conftest.py b/ingestion/tests/integration/conftest.py index b3f9fcff1b8..3c987e5aa68 100644 --- a/ingestion/tests/integration/conftest.py +++ b/ingestion/tests/integration/conftest.py @@ -15,7 +15,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.workflow.ingestion import IngestionWorkflow if not sys.version_info >= (3, 9): - collect_ignore = ["trino"] + collect_ignore = ["trino", "kafka"] @pytest.fixture(scope="session", autouse=True) diff --git a/ingestion/tests/integration/kafka/conftest.py b/ingestion/tests/integration/kafka/conftest.py new file mode 100644 index 00000000000..4bcc0f63a13 --- /dev/null +++ b/ingestion/tests/integration/kafka/conftest.py @@ -0,0 +1,140 @@ +import os.path +from textwrap import dedent + +import pytest +import testcontainers.core.network +from docker.types import EndpointConfig +from testcontainers.core.container import DockerContainer +from testcontainers.kafka import KafkaContainer + +from _openmetadata_testutils.kafka import load_csv_data +from _openmetadata_testutils.kafka.schema_registry_container import ( + SchemaRegistryContainer, +) +from metadata.generated.schema.api.services.createMessagingService import ( + CreateMessagingServiceRequest, +) +from metadata.generated.schema.entity.services.connections.messaging.kafkaConnection import ( + KafkaConnection, +) +from metadata.generated.schema.entity.services.messagingService import ( + MessagingConnection, + MessagingServiceType, +) +from metadata.generated.schema.metadataIngestion.messagingServiceMetadataPipeline import ( + MessagingMetadataConfigType, +) + + +def _connect_to_network( + ctr: DockerContainer, network: testcontainers.core.network, alias: str +): + # Needed until https://github.com/testcontainers/testcontainers-python/issues/645 is fixed + ctr.with_kwargs( + network=network.name, + networking_config={network.name: EndpointConfig("1.33", aliases=[alias])}, + ) + + +class CustomKafkaContainer(KafkaContainer): + def __init__(self): + super().__init__() + self.security_protocol_map += ",EXTERNAL:PLAINTEXT" + self.with_env( + "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", self.security_protocol_map + ) + + self.listeners = f"PLAINTEXT://0.0.0.0:29092,BROKER://0.0.0.0:9092,EXTERNAL://0.0.0.0:{self.port}" + self.with_env("KAFKA_LISTENERS", self.listeners) + + def tc_start(self): + listeners = ",".join( + [ + f"EXTERNAL://{self.get_bootstrap_server()}", + f"PLAINTEXT://{self._get_network_alias()}:29092", + "BROKER://$(hostname -i | cut -d' ' -f1):9092", + ] + ) + data = ( + dedent( + f""" + #!/bin/bash + {self.boot_command} + export KAFKA_ADVERTISED_LISTENERS={listeners} + . /etc/confluent/docker/bash-config + /etc/confluent/docker/configure + /etc/confluent/docker/launch + """ + ) + .strip() + .encode("utf-8") + ) + self.create_file(data, KafkaContainer.TC_START_SCRIPT) + + +@pytest.fixture(scope="module") +def docker_network(): + with testcontainers.core.network.Network() as network: + yield network + + +@pytest.fixture(scope="module") +def schema_registry_container(docker_network, kafka_container): + with SchemaRegistryContainer( + schema_registry_kafkastore_bootstrap_servers="PLAINTEXT://kafka:9092", + schema_registry_host_name="schema-registry", + ).with_network(docker_network).with_network_aliases("schema-registry") as container: + load_csv_data.main( + kafka_broker=kafka_container.get_bootstrap_server(), + schema_registry_url=container.get_connection_url(), + csv_directory=os.path.dirname(__file__) + "/data", + ) + yield container + + +@pytest.fixture(scope="module") +def kafka_container(docker_network): + container = CustomKafkaContainer() + _connect_to_network(container, docker_network, "kafka") + with container: + yield container + + +@pytest.fixture(scope="module") +def create_service_request( + kafka_container, schema_registry_container, tmp_path_factory +): + return CreateMessagingServiceRequest( + name="docker_test_" + tmp_path_factory.mktemp("kafka").name, + serviceType=MessagingServiceType.Kafka, + connection=MessagingConnection( + config=KafkaConnection( + bootstrapServers=kafka_container.get_bootstrap_server(), + schemaRegistryURL=schema_registry_container.get_connection_url(), + ) + ), + ) + + +@pytest.fixture(scope="module") +def ingestion_config(db_service, metadata, workflow_config, sink_config): + return { + "source": { + "type": db_service.connection.config.type.value.lower(), + "serviceName": db_service.fullyQualifiedName.root, + "sourceConfig": { + "config": {"type": MessagingMetadataConfigType.MessagingMetadata.value} + }, + "serviceConnection": db_service.connection.model_dump(), + }, + "sink": sink_config, + "workflowConfig": workflow_config, + } + + +@pytest.fixture(scope="module") +def unmask_password(): + def patch_password(service): + return service + + return patch_password diff --git a/ingestion/tests/integration/kafka/data/customers-100.csv b/ingestion/tests/integration/kafka/data/customers-100.csv new file mode 100644 index 00000000000..2041c1594c8 --- /dev/null +++ b/ingestion/tests/integration/kafka/data/customers-100.csv @@ -0,0 +1,101 @@ +Index,Customer Id,First Name,Last Name,Company,City,Country,Phone 1,Phone 2,Email,Subscription Date,Website +1,DD37Cf93aecA6Dc,Sheryl,Baxter,Rasmussen Group,East Leonard,Chile,229.077.5154,397.884.0519x718,zunigavanessa@smith.info,2020-08-24,http://www.stephenson.com/ +2,1Ef7b82A4CAAD10,Preston,Lozano,Vega-Gentry,East Jimmychester,Djibouti,5153435776,686-620-1820x944,vmata@colon.com,2021-04-23,http://www.hobbs.com/ +3,6F94879bDAfE5a6,Roy,Berry,Murillo-Perry,Isabelborough,Antigua and Barbuda,+1-539-402-0259,(496)978-3969x58947,beckycarr@hogan.com,2020-03-25,http://www.lawrence.com/ +4,5Cef8BFA16c5e3c,Linda,Olsen,"Dominguez, Mcmillan and Donovan",Bensonview,Dominican Republic,001-808-617-6467x12895,+1-813-324-8756,stanleyblackwell@benson.org,2020-06-02,http://www.good-lyons.com/ +5,053d585Ab6b3159,Joanna,Bender,"Martin, Lang and Andrade",West Priscilla,Slovakia (Slovak Republic),001-234-203-0635x76146,001-199-446-3860x3486,colinalvarado@miles.net,2021-04-17,https://goodwin-ingram.com/ +6,2d08FB17EE273F4,Aimee,Downs,Steele Group,Chavezborough,Bosnia and Herzegovina,(283)437-3886x88321,999-728-1637,louis27@gilbert.com,2020-02-25,http://www.berger.net/ +7,EA4d384DfDbBf77,Darren,Peck,"Lester, Woodard and Mitchell",Lake Ana,Pitcairn Islands,(496)452-6181x3291,+1-247-266-0963x4995,tgates@cantrell.com,2021-08-24,https://www.le.com/ +8,0e04AFde9f225dE,Brett,Mullen,"Sanford, Davenport and Giles",Kimport,Bulgaria,001-583-352-7197x297,001-333-145-0369,asnow@colon.com,2021-04-12,https://hammond-ramsey.com/ +9,C2dE4dEEc489ae0,Sheryl,Meyers,Browning-Simon,Robersonstad,Cyprus,854-138-4911x5772,+1-448-910-2276x729,mariokhan@ryan-pope.org,2020-01-13,https://www.bullock.net/ +10,8C2811a503C7c5a,Michelle,Gallagher,Beck-Hendrix,Elaineberg,Timor-Leste,739.218.2516x459,001-054-401-0347x617,mdyer@escobar.net,2021-11-08,https://arias.com/ +11,216E205d6eBb815,Carl,Schroeder,"Oconnell, Meza and Everett",Shannonville,Guernsey,637-854-0256x825,114.336.0784x788,kirksalas@webb.com,2021-10-20,https://simmons-hurley.com/ +12,CEDec94deE6d69B,Jenna,Dodson,"Hoffman, Reed and Mcclain",East Andrea,Vietnam,(041)737-3846,+1-556-888-3485x42608,mark42@robbins.com,2020-11-29,http://www.douglas.net/ +13,e35426EbDEceaFF,Tracey,Mata,Graham-Francis,South Joannamouth,Togo,001-949-844-8787,(855)713-8773,alex56@walls.org,2021-12-02,http://www.beck.com/ +14,A08A8aF8BE9FaD4,Kristine,Cox,Carpenter-Cook,Jodyberg,Sri Lanka,786-284-3358x62152,+1-315-627-1796x8074,holdenmiranda@clarke.com,2021-02-08,https://www.brandt.com/ +15,6fEaA1b7cab7B6C,Faith,Lutz,Carter-Hancock,Burchbury,Singapore,(781)861-7180x8306,207-185-3665,cassieparrish@blevins-chapman.net,2022-01-26,http://stevenson.org/ +16,8cad0b4CBceaeec,Miranda,Beasley,Singleton and Sons,Desireeshire,Oman,540.085.3135x185,+1-600-462-6432x21881,vduncan@parks-hardy.com,2022-04-12,http://acosta.org/ +17,a5DC21AE3a21eaA,Caroline,Foley,Winters-Mendoza,West Adriennestad,Western Sahara,936.222.4746x9924,001-469-948-6341x359,holtgwendolyn@watson-davenport.com,2021-03-10,http://www.benson-roth.com/ +18,F8Aa9d6DfcBeeF8,Greg,Mata,Valentine LLC,Lake Leslie,Mozambique,(701)087-2415,(195)156-1861x26241,jaredjuarez@carroll.org,2022-03-26,http://pitts-cherry.com/ +19,F160f5Db3EfE973,Clifford,Jacobson,Simon LLC,Harmonview,South Georgia and the South Sandwich Islands,001-151-330-3524x0469,(748)477-7174,joseph26@jacobson.com,2020-09-24,https://mcconnell.com/ +20,0F60FF3DdCd7aB0,Joanna,Kirk,Mays-Mccormick,Jamesshire,French Polynesia,(266)131-7001x711,(283)312-5579x11543,tuckerangie@salazar.net,2021-09-24,https://www.camacho.net/ +21,9F9AdB7B8A6f7F2,Maxwell,Frye,Patterson Inc,East Carly,Malta,423.262.3059,202-880-0688x7491,fgibson@drake-webb.com,2022-01-12,http://www.roberts.com/ +22,FBd0Ded4F02a742,Kiara,Houston,"Manning, Hester and Arroyo",South Alvin,Netherlands,001-274-040-3582x10611,+1-528-175-0973x4684,blanchardbob@wallace-shannon.com,2020-09-15,https://www.reid-potts.com/ +23,2FB0FAA1d429421,Colleen,Howard,Greer and Sons,Brittanyview,Paraguay,1935085151,(947)115-7711x5488,rsingleton@ryan-cherry.com,2020-08-19,http://paul.biz/ +24,010468dAA11382c,Janet,Valenzuela,Watts-Donaldson,Veronicamouth,Lao People's Democratic Republic,354.259.5062x7538,500.433.2022,stefanie71@spence.com,2020-09-08,https://moreno.biz/ +25,eC1927Ca84E033e,Shane,Wilcox,Tucker LLC,Bryanville,Albania,(429)005-9030x11004,541-116-4501,mariah88@santos.com,2021-04-06,https://www.ramos.com/ +26,09D7D7C8Fe09aea,Marcus,Moody,Giles Ltd,Kaitlyntown,Panama,674-677-8623,909-277-5485x566,donnamullins@norris-barrett.org,2022-05-24,https://www.curry.com/ +27,aBdfcF2c50b0bfD,Dakota,Poole,Simmons Group,Michealshire,Belarus,(371)987-8576x4720,071-152-1376,stacey67@fields.org,2022-02-20,https://sanford-wilcox.biz/ +28,b92EBfdF8a3f0E6,Frederick,Harper,"Hinton, Chaney and Stokes",South Marissatown,Switzerland,+1-077-121-1558x0687,264.742.7149,jacobkhan@bright.biz,2022-05-26,https://callahan.org/ +29,3B5dAAFA41AFa22,Stefanie,Fitzpatrick,Santana-Duran,Acevedoville,Saint Vincent and the Grenadines,(752)776-3286,+1-472-021-4814x85074,wterrell@clark.com,2020-07-30,https://meyers.com/ +30,EDA69ca7a6e96a2,Kent,Bradshaw,Sawyer PLC,North Harold,Tanzania,+1-472-143-5037x884,126.922.6153,qjimenez@boyd.com,2020-04-26,http://maynard-ho.com/ +31,64DCcDFaB9DFd4e,Jack,Tate,"Acosta, Petersen and Morrow",West Samuel,Zimbabwe,965-108-4406x20714,046.906.1442x6784,gfigueroa@boone-zavala.com,2021-09-15,http://www.hawkins-ramsey.com/ +32,679c6c83DD872d6,Tom,Trujillo,Mcgee Group,Cunninghamborough,Denmark,416-338-3758,(775)890-7209,tapiagreg@beard.info,2022-01-13,http://www.daniels-klein.com/ +33,7Ce381e4Afa4ba9,Gabriel,Mejia,Adkins-Salinas,Port Annatown,Liechtenstein,4077245425,646.044.0696x66800,coleolson@jennings.net,2021-04-24,https://patel-hanson.info/ +34,A09AEc6E3bF70eE,Kaitlyn,Santana,Herrera Group,New Kaitlyn,United States of America,6303643286,447-710-6202x07313,georgeross@miles.org,2021-09-21,http://pham.com/ +35,aA9BAFfBc3710fe,Faith,Moon,"Waters, Chase and Aguilar",West Marthaburgh,Bahamas,+1-586-217-0359x6317,+1-818-199-1403,willistonya@randolph-baker.com,2021-11-03,https://spencer-charles.info/ +36,E11dfb2DB8C9f72,Tammie,Haley,"Palmer, Barnes and Houston",East Teresa,Belize,001-276-734-4113x6087,(430)300-8770,harrisisaiah@jenkins.com,2022-01-04,http://evans-simon.com/ +37,889eCf90f68c5Da,Nicholas,Sosa,Jordan Ltd,South Hunter,Uruguay,(661)425-6042,975-998-1519,fwolfe@dorsey.com,2021-08-10,https://www.fleming-richards.com/ +38,7a1Ee69F4fF4B4D,Jordan,Gay,Glover and Sons,South Walter,Solomon Islands,7208417020,8035336772,tiffanydavies@harris-mcfarland.org,2021-02-24,http://www.lee.org/ +39,dca4f1D0A0fc5c9,Bruce,Esparza,Huerta-Mclean,Poolefurt,Montenegro,559-529-4424,001-625-000-7132x0367,preese@frye-vega.com,2021-10-22,http://www.farley.org/ +40,17aD8e2dB3df03D,Sherry,Garza,Anderson Ltd,West John,Poland,001-067-713-6440x158,(978)289-8785x5766,ann48@miller.com,2021-11-01,http://spence.com/ +41,2f79Cd309624Abb,Natalie,Gentry,Monroe PLC,West Darius,Dominican Republic,830.996.8238,499.122.5415,tcummings@fitzpatrick-ashley.com,2020-10-10,http://www.dorsey.biz/ +42,6e5ad5a5e2bB5Ca,Bryan,Dunn,Kaufman and Sons,North Jimstad,Burkina Faso,001-710-802-5565,078.699.8982x13881,woodwardandres@phelps.com,2021-09-08,http://www.butler.com/ +43,7E441b6B228DBcA,Wayne,Simpson,Perkins-Trevino,East Rebekahborough,Bolivia,(344)156-8632x1869,463-445-3702x38463,barbarapittman@holder.com,2020-12-13,https://gillespie-holder.com/ +44,D3fC11A9C235Dc6,Luis,Greer,Cross PLC,North Drew,Bulgaria,001-336-025-6849x701,684.698.2911x6092,bstuart@williamson-mcclure.com,2022-05-15,https://fletcher-nielsen.com/ +45,30Dfa48fe5Ede78,Rhonda,Frost,"Herrera, Shepherd and Underwood",Lake Lindaburgh,Monaco,(127)081-9339,+1-431-028-3337x3492,zkrueger@wolf-chavez.net,2021-12-06,http://www.khan.com/ +46,fD780ED8dbEae7B,Joanne,Montes,"Price, Sexton and Mcdaniel",Gwendolynview,Palau,(897)726-7952,(467)886-9467x5721,juan80@henson.net,2020-07-01,http://ochoa.com/ +47,300A40d3ce24bBA,Geoffrey,Guzman,Short-Wiggins,Zimmermanland,Uzbekistan,975.235.8921x269,(983)188-6873,bauercrystal@gay.com,2020-04-23,https://decker-kline.com/ +48,283DFCD0Dba40aF,Gloria,Mccall,"Brennan, Acosta and Ramos",North Kerriton,Ghana,445-603-6729,001-395-959-4736x4524,bartlettjenna@zuniga-moss.biz,2022-03-11,http://burgess-frank.com/ +49,F4Fc91fEAEad286,Brady,Cohen,Osborne-Erickson,North Eileenville,United Arab Emirates,741.849.0139x524,+1-028-691-7497x0894,mccalltyrone@durham-rose.biz,2022-03-10,http://hammond-barron.com/ +50,80F33Fd2AcebF05,Latoya,Mccann,"Hobbs, Garrett and Sanford",Port Sergiofort,Belarus,(530)287-4548x29481,162-234-0249x32790,bobhammond@barry.biz,2021-12-02,https://www.burton.com/ +51,Aa20BDe68eAb0e9,Gerald,Hawkins,"Phelps, Forbes and Koch",New Alberttown,Canada,+1-323-239-1456x96168,(092)508-0269,uwarner@steele-arias.com,2021-03-19,https://valenzuela.com/ +52,e898eEB1B9FE22b,Samuel,Crawford,"May, Goodwin and Martin",South Jasmine,Algeria,802-242-7457,626.116.9535x8578,xpittman@ritter-carney.net,2021-03-27,https://guerrero.org/ +53,faCEF517ae7D8eB,Patricia,Goodwin,"Christian, Winters and Ellis",Cowanfort,Swaziland,322.549.7139x70040,(111)741-4173,vaughanchristy@lara.biz,2021-03-08,http://clark.info/ +54,c09952De6Cda8aA,Stacie,Richard,Byrd Inc,New Deborah,Madagascar,001-622-948-3641x24810,001-731-168-2893x8891,clinton85@colon-arias.org,2020-10-15,https://kim.com/ +55,f3BEf3Be028166f,Robin,West,"Nixon, Blackwell and Sosa",Wallstown,Ecuador,698.303.4267,001-683-837-7651x525,greenemiranda@zimmerman.com,2022-01-13,https://www.mora.com/ +56,C6F2Fc6a7948a4e,Ralph,Haas,Montes PLC,Lake Ellenchester,Palestinian Territory,2239271999,001-962-434-0867x649,goodmancesar@figueroa.biz,2020-05-25,http://may.com/ +57,c8FE57cBBdCDcb2,Phyllis,Maldonado,Costa PLC,Lake Whitney,Saint Barthelemy,4500370767,001-508-064-6725x017,yhanson@warner-diaz.org,2021-01-25,http://www.bernard.com/ +58,B5acdFC982124F2,Danny,Parrish,Novak LLC,East Jaredbury,United Arab Emirates,(669)384-8597x8794,506.731.5952x571,howelldarren@house-cohen.com,2021-03-17,http://www.parsons-hudson.com/ +59,8c7DdF10798bCC3,Kathy,Hill,"Moore, Mccoy and Glass",Selenabury,South Georgia and the South Sandwich Islands,001-171-716-2175x310,888.625.0654,ncamacho@boone-simmons.org,2020-11-15,http://hayden.com/ +60,C681dDd0cc422f7,Kelli,Hardy,Petty Ltd,Huangfort,Sao Tome and Principe,020.324.2191x2022,424-157-8216,kristopher62@oliver.com,2020-12-20,http://www.kidd.com/ +61,a940cE42e035F28,Lynn,Pham,"Brennan, Camacho and Tapia",East Pennyshire,Portugal,846.468.6834x611,001-248-691-0006,mpham@rios-guzman.com,2020-08-21,https://www.murphy.com/ +62,9Cf5E6AFE0aeBfd,Shelley,Harris,"Prince, Malone and Pugh",Port Jasminborough,Togo,423.098.0315x8373,+1-386-458-8944x15194,zachary96@mitchell-bryant.org,2020-12-10,https://www.ryan.com/ +63,aEcbe5365BbC67D,Eddie,Jimenez,Caldwell Group,West Kristine,Ethiopia,+1-235-657-1073x6306,(026)401-7353x2417,kristiwhitney@bernard.com,2022-03-24,http://cherry.com/ +64,FCBdfCEAe20A8Dc,Chloe,Hutchinson,Simon LLC,South Julia,Netherlands,981-544-9452,+1-288-552-4666x060,leah85@sutton-terrell.com,2022-05-15,https://mitchell.info/ +65,636cBF0835E10ff,Eileen,Lynch,"Knight, Abbott and Hubbard",Helenborough,Liberia,+1-158-951-4131x53578,001-673-779-6713x680,levigiles@vincent.com,2021-01-02,http://mckay.com/ +66,fF1b6c9E8Fbf1ff,Fernando,Lambert,Church-Banks,Lake Nancy,Lithuania,497.829.9038,3863743398,fisherlinda@schaefer.net,2021-04-23,https://www.vang.com/ +67,2A13F74EAa7DA6c,Makayla,Cannon,Henderson Inc,Georgeport,New Caledonia,001-215-801-6392x46009,027-609-6460,scottcurtis@hurley.biz,2020-01-20,http://www.velazquez.net/ +68,a014Ec1b9FccC1E,Tom,Alvarado,Donaldson-Dougherty,South Sophiaberg,Kiribati,(585)606-2980x2258,730-797-3594x5614,nicholsonnina@montgomery.info,2020-08-18,http://odom-massey.com/ +69,421a109cABDf5fa,Virginia,Dudley,Warren Ltd,Hartbury,French Southern Territories,027.846.3705x14184,+1-439-171-1846x4636,zvalencia@phelps.com,2021-01-31,http://hunter-esparza.com/ +70,CC68FD1D3Bbbf22,Riley,Good,Wade PLC,Erikaville,Canada,6977745822,855-436-7641,alex06@galloway.com,2020-02-03,http://conway.org/ +71,CBCd2Ac8E3eBDF9,Alexandria,Buck,Keller-Coffey,Nicolasfort,Iran,078-900-4760x76668,414-112-8700x68751,lee48@manning.com,2021-02-20,https://ramsey.org/ +72,Ef859092FbEcC07,Richard,Roth,Conway-Mcbride,New Jasmineshire,Morocco,581-440-6539,9857827463,aharper@maddox-townsend.org,2020-02-23,https://www.brooks.com/ +73,F560f2d3cDFb618,Candice,Keller,Huynh and Sons,East Summerstad,Zimbabwe,001-927-965-8550x92406,001-243-038-4271x53076,buckleycory@odonnell.net,2020-08-22,https://www.lucero.com/ +74,A3F76Be153Df4a3,Anita,Benson,Parrish Ltd,Skinnerport,Russian Federation,874.617.5668x69878,(399)820-6418x0071,angie04@oconnell.com,2020-02-09,http://oconnor.com/ +75,D01Af0AF7cBbFeA,Regina,Stein,Guzman-Brown,Raystad,Solomon Islands,001-469-848-0724x4407,001-085-360-4426x00357,zrosario@rojas-hardin.net,2022-01-15,http://www.johnston.info/ +76,d40e89dCade7b2F,Debra,Riddle,"Chang, Aguirre and Leblanc",Colinhaven,United States Virgin Islands,+1-768-182-6014x14336,(303)961-4491,shieldskerry@robles.com,2020-07-11,http://kaiser.info/ +77,BF6a1f9bd1bf8DE,Brittany,Zuniga,Mason-Hester,West Reginald,Kyrgyz Republic,(050)136-9025,001-480-851-2496x0157,mchandler@cochran-huerta.org,2021-07-24,http://www.boyle.com/ +78,FfaeFFbbbf280db,Cassidy,Mcmahon,"Mcguire, Huynh and Hopkins",Lake Sherryborough,Myanmar,5040771311,684-682-0021x1326,katrinalane@fitzgerald.com,2020-10-21,https://hurst.com/ +79,CbAE1d1e9a8dCb1,Laurie,Pennington,"Sanchez, Marsh and Hale",Port Katherineville,Dominica,007.155.3406x553,+1-809-862-5566x277,cookejill@powell.com,2020-06-08,http://www.hebert.com/ +80,A7F85c1DE4dB87f,Alejandro,Blair,"Combs, Waller and Durham",Thomasland,Iceland,(690)068-4641x51468,555.509.8691x2329,elizabethbarr@ewing.com,2020-09-19,https://mercado-blevins.com/ +81,D6CEAfb3BDbaa1A,Leslie,Jennings,Blankenship-Arias,Coreybury,Micronesia,629.198.6346,075.256.0829,corey75@wiggins.com,2021-11-13,https://www.juarez.com/ +82,Ebdb6F6F7c90b69,Kathleen,Mckay,"Coffey, Lamb and Johnson",Lake Janiceton,Saint Vincent and the Grenadines,(733)910-9968,(691)247-4128x0665,chloelester@higgins-wilkinson.com,2021-09-12,http://www.owens-mooney.com/ +83,E8E7e8Cfe516ef0,Hunter,Moreno,Fitzpatrick-Lawrence,East Clinton,Isle of Man,(733)833-6754,001-761-013-7121,isaac26@benton-finley.com,2020-12-28,http://walls.info/ +84,78C06E9b6B3DF20,Chad,Davidson,Garcia-Jimenez,South Joshuashire,Oman,8275702958,(804)842-4715,justinwalters@jimenez.com,2021-11-15,http://www.garner-oliver.com/ +85,03A1E62ADdeb31c,Corey,Holt,"Mcdonald, Bird and Ramirez",New Glenda,Fiji,001-439-242-4986x7918,3162708934,maurice46@morgan.com,2020-02-18,http://www.watson.com/ +86,C6763c99d0bd16D,Emma,Cunningham,Stephens Inc,North Jillianview,New Zealand,128-059-0206x60217,(312)164-4545x2284,walter83@juarez.org,2022-05-13,http://www.reid.info/ +87,ebe77E5Bf9476CE,Duane,Woods,Montoya-Miller,Lyonsberg,Maldives,(636)544-7783x7288,(203)287-1003x5932,kmercer@wagner.com,2020-07-21,http://murray.org/ +88,E4Bbcd8AD81fC5f,Alison,Vargas,"Vaughn, Watts and Leach",East Cristinabury,Benin,365-273-8144,053-308-7653x6287,vcantu@norton.com,2020-11-10,http://mason.info/ +89,efeb73245CDf1fF,Vernon,Kane,Carter-Strickland,Thomasfurt,Yemen,114-854-1159x555,499-608-4612,hilljesse@barrett.info,2021-04-15,http://www.duffy-hensley.net/ +90,37Ec4B395641c1E,Lori,Flowers,Decker-Mcknight,North Joeburgh,Namibia,679.415.1210,945-842-3659x4581,tyrone77@valenzuela.info,2021-01-09,http://www.deleon-crosby.com/ +91,5ef6d3eefdD43bE,Nina,Chavez,Byrd-Campbell,Cassidychester,Bhutan,053-344-3205,+1-330-920-5422x571,elliserica@frank.com,2020-03-26,https://www.pugh.com/ +92,98b3aeDcC3B9FF3,Shane,Foley,Rocha-Hart,South Dannymouth,Hungary,+1-822-569-0302,001-626-114-5844x55073,nsteele@sparks.com,2021-07-06,https://www.holt-sparks.com/ +93,aAb6AFc7AfD0fF3,Collin,Ayers,Lamb-Peterson,South Lonnie,Anguilla,404-645-5351x012,001-257-582-8850x8516,dudleyemily@gonzales.biz,2021-06-29,http://www.ruiz.com/ +94,54B5B5Fe9F1B6C5,Sherry,Young,"Lee, Lucero and Johnson",Frankchester,Solomon Islands,158-687-1764,(438)375-6207x003,alan79@gates-mclaughlin.com,2021-04-04,https://travis.net/ +95,BE91A0bdcA49Bbc,Darrell,Douglas,"Newton, Petersen and Mathis",Daisyborough,Mali,001-084-845-9524x1777,001-769-564-6303,grayjean@lowery-good.com,2022-02-17,https://banks.biz/ +96,cb8E23e48d22Eae,Karl,Greer,Carey LLC,East Richard,Guyana,(188)169-1674x58692,001-841-293-3519x614,hhart@jensen.com,2022-01-30,http://hayes-perez.com/ +97,CeD220bdAaCfaDf,Lynn,Atkinson,"Ware, Burns and Oneal",New Bradview,Sri Lanka,+1-846-706-2218,605.413.3198,vkemp@ferrell.com,2021-07-10,https://novak-allison.com/ +98,28CDbC0dFe4b1Db,Fred,Guerra,Schmitt-Jones,Ortegaland,Solomon Islands,+1-753-067-8419x7170,+1-632-666-7507x92121,swagner@kane.org,2021-09-18,https://www.ross.com/ +99,c23d1D9EE8DEB0A,Yvonne,Farmer,Fitzgerald-Harrell,Lake Elijahview,Aruba,(530)311-9786,001-869-452-0943x12424,mccarthystephen@horn-green.biz,2021-08-11,http://watkins.info/ +100,2354a0E336A91A1,Clarence,Haynes,"Le, Nash and Cross",Judymouth,Honduras,(753)813-6941,783.639.1472,colleen91@faulkner.biz,2020-03-11,http://www.hatfield-saunders.net/ diff --git a/ingestion/tests/integration/kafka/data/organizations-100.csv b/ingestion/tests/integration/kafka/data/organizations-100.csv new file mode 100644 index 00000000000..93dcac9f39a --- /dev/null +++ b/ingestion/tests/integration/kafka/data/organizations-100.csv @@ -0,0 +1,101 @@ +Index,Organization Id,Name,Website,Country,Description,Founded,Industry,Number of employees +1,FAB0d41d5b5d22c,Ferrell LLC,https://price.net/,Papua New Guinea,Horizontal empowering knowledgebase,1990,Plastics,3498 +2,6A7EdDEA9FaDC52,"Mckinney, Riley and Day",http://www.hall-buchanan.info/,Finland,User-centric system-worthy leverage,2015,Glass / Ceramics / Concrete,4952 +3,0bFED1ADAE4bcC1,Hester Ltd,http://sullivan-reed.com/,China,Switchable scalable moratorium,1971,Public Safety,5287 +4,2bFC1Be8a4ce42f,Holder-Sellers,https://becker.com/,Turkmenistan,De-engineered systemic artificial intelligence,2004,Automotive,921 +5,9eE8A6a4Eb96C24,Mayer Group,http://www.brewer.com/,Mauritius,Synchronized needs-based challenge,1991,Transportation,7870 +6,cC757116fe1C085,Henry-Thompson,http://morse.net/,Bahamas,Face-to-face well-modulated customer loyalty,1992,Primary / Secondary Education,4914 +7,219233e8aFF1BC3,Hansen-Everett,https://www.kidd.org/,Pakistan,Seamless disintermediate collaboration,2018,Publishing Industry,7832 +8,ccc93DCF81a31CD,Mcintosh-Mora,https://www.brooks.com/,Heard Island and McDonald Islands,Centralized attitude-oriented capability,1970,Import / Export,4389 +9,0B4F93aA06ED03e,Carr Inc,http://ross.com/,Kuwait,Distributed impactful customer loyalty,1996,Plastics,8167 +10,738b5aDe6B1C6A5,Gaines Inc,http://sandoval-hooper.com/,Uzbekistan,Multi-lateral scalable protocol,1997,Outsourcing / Offshoring,9698 +11,AE61b8Ffebbc476,Kidd Group,http://www.lyons.com/,Bouvet Island (Bouvetoya),Proactive foreground paradigm,2001,Primary / Secondary Education,7473 +12,eb3B7D06cCdD609,Crane-Clarke,https://www.sandoval.com/,Denmark,Front-line clear-thinking encryption,2014,Food / Beverages,9011 +13,8D0c29189C9798B,"Keller, Campos and Black",https://www.garner.info/,Liberia,Ameliorated directional emulation,2020,Museums / Institutions,2862 +14,D2c91cc03CA394c,Glover-Pope,http://www.silva.biz/,United Arab Emirates,Persevering contextually-based approach,2013,Medical Practice,9079 +15,C8AC1eaf9C036F4,Pacheco-Spears,https://aguilar.com/,Sweden,Secured logistical synergy,1984,Maritime,769 +16,b5D10A14f7a8AfE,Hodge-Ayers,http://www.archer-elliott.com/,Honduras,Future-proofed radical implementation,1990,Facilities Services,8508 +17,68139b5C4De03B4,"Bowers, Guerra and Krause",http://www.carrillo-nicholson.com/,Uganda,De-engineered transitional strategy,1972,Primary / Secondary Education,6986 +18,5c2EffEfdba2BdF,Mckenzie-Melton,http://montoya-thompson.com/,Hong Kong,Reverse-engineered heuristic alliance,1998,Investment Management / Hedge Fund / Private Equity,4589 +19,ba179F19F7925f5,Branch-Mann,http://www.lozano.com/,Botswana,Adaptive intangible frame,1999,Architecture / Planning,7961 +20,c1Ce9B350BAc66b,Weiss and Sons,https://barrett.com/,Korea,Sharable optimal functionalities,2011,Plastics,5984 +21,8de40AC4e6EaCa4,"Velez, Payne and Coffey",http://burton.com/,Luxembourg,Mandatory coherent synergy,1986,Wholesale,5010 +22,Aad86a4F0385F2d,Harrell LLC,http://www.frey-rosario.com/,Guadeloupe,Reverse-engineered mission-critical moratorium,2018,Construction,2185 +23,22aC3FFd64fD703,"Eaton, Reynolds and Vargas",http://www.freeman.biz/,Monaco,Self-enabling multi-tasking process improvement,2014,Luxury Goods / Jewelry,8987 +24,5Ec4C272bCf085c,Robbins-Cummings,http://donaldson-wilkins.com/,Belgium,Organic non-volatile hierarchy,1991,Pharmaceuticals,5038 +25,5fDBeA8BB91a000,Jenkins Inc,http://www.kirk.biz/,South Africa,Front-line systematic help-desk,2002,Insurance,1215 +26,dFfD6a6F9AC2d9C,"Greene, Benjamin and Novak",http://www.kent.net/,Romania,Centralized leadingedge moratorium,2012,Museums / Institutions,4941 +27,4B217cC5a0674C5,"Dickson, Richmond and Clay",http://everett.com/,Czech Republic,Team-oriented tangible complexity,1980,Real Estate / Mortgage,3122 +28,88b1f1cDcf59a37,Prince-David,http://thompson.com/,Christmas Island,Virtual holistic methodology,1970,Banking / Mortgage,1046 +29,f9F7bBCAEeC360F,Ayala LLC,http://www.zhang.com/,Philippines,Open-source zero administration hierarchy,2021,Legal Services,7664 +30,7Cb3AeFcE4Ba31e,Rivas Group,https://hebert.org/,Australia,Open-architected well-modulated capacity,1998,Logistics / Procurement,4155 +31,ccBcC32adcbc530,"Sloan, Mays and Whitehead",http://lawson.com/,Chad,Face-to-face high-level conglomeration,1997,Civil Engineering,365 +32,f5afd686b3d05F5,"Durham, Allen and Barnes",http://chan-stafford.org/,Zimbabwe,Synergistic web-enabled framework,1993,Mechanical or Industrial Engineering,6135 +33,38C6cfC5074Fa5e,Fritz-Franklin,http://www.lambert.com/,Nepal,Automated 4thgeneration website,1972,Hospitality,4516 +34,5Cd7efccCcba38f,Burch-Ewing,http://cline.net/,Taiwan,User-centric 4thgeneration system engine,1981,Venture Capital / VC,7443 +35,9E6Acb51e3F9d6F,"Glass, Barrera and Turner",https://dunlap.com/,Kyrgyz Republic,Multi-channeled 3rdgeneration open system,2020,Utilities,2610 +36,4D4d7E18321eaeC,Pineda-Cox,http://aguilar.org/,Bolivia,Fundamental asynchronous capability,2010,Human Resources / HR,1312 +37,485f5d06B938F2b,"Baker, Mccann and Macdonald",http://www.anderson-barker.com/,Kenya,Cross-group user-facing focus group,2013,Legislative Office,1638 +38,19E3a5Bf6dBDc4F,Cuevas-Moss,https://dodson-castaneda.net/,Guatemala,Extended human-resource intranet,1994,Music,9995 +39,6883A965c7b68F7,Hahn PLC,http://newman.com/,Belarus,Organic logistical leverage,2012,Electrical / Electronic Manufacturing,3715 +40,AC5B7AA74Aa4A2E,"Valentine, Ferguson and Kramer",http://stuart.net/,Jersey,Centralized secondary time-frame,1997,Non - Profit / Volunteering,3585 +41,decab0D5027CA6a,Arroyo Inc,https://www.turner.com/,Grenada,Managed demand-driven website,2006,Writing / Editing,9067 +42,dF084FbBb613eea,Walls LLC,http://www.reese-vasquez.biz/,Cape Verde,Self-enabling fresh-thinking installation,1989,Investment Management / Hedge Fund / Private Equity,1678 +43,A2D89Ab9bCcAd4e,"Mitchell, Warren and Schneider",https://fox.biz/,Trinidad and Tobago,Enhanced intangible time-frame,2021,Capital Markets / Hedge Fund / Private Equity,3816 +44,77aDc905434a49f,Prince PLC,https://www.watts.com/,Sweden,Profit-focused coherent installation,2016,Individual / Family Services,7645 +45,235fdEFE2cfDa5F,Brock-Blackwell,http://www.small.com/,Benin,Secured foreground emulation,1986,Online Publishing,7034 +46,1eD64cFe986BBbE,Walton-Barnett,https://ashley-schaefer.com/,Western Sahara,Right-sized clear-thinking flexibility,2001,Luxury Goods / Jewelry,1746 +47,CbBbFcdd0eaE2cF,Bartlett-Arroyo,https://cruz.com/,Northern Mariana Islands,Realigned didactic function,1976,Civic / Social Organization,3987 +48,49aECbDaE6aBD53,"Wallace, Madden and Morris",http://www.blevins-fernandez.biz/,Germany,Persistent real-time customer loyalty,2016,Pharmaceuticals,9443 +49,7b3fe6e7E72bFa4,Berg-Sparks,https://cisneros-love.com/,Canada,Stand-alone static implementation,1974,Arts / Crafts,2073 +50,c6DedA82A8aef7E,Gonzales Ltd,http://bird.com/,Tonga,Managed human-resource policy,1988,Consumer Goods,9069 +51,7D9FBF85cdC3871,Lawson and Sons,https://www.wong.com/,French Southern Territories,Compatible analyzing intranet,2021,Arts / Crafts,3527 +52,7dd18Fb7cB07b65,"Mcguire, Mcconnell and Olsen",https://melton-briggs.com/,Korea,Profound client-server frame,1988,Printing,8445 +53,EF5B55FadccB8Fe,Charles-Phillips,https://bowman.com/,Cote d'Ivoire,Monitored client-server implementation,2012,Mental Health Care,3450 +54,f8D4B99e11fAF5D,Odom Ltd,https://www.humphrey-hess.com/,Cote d'Ivoire,Advanced static process improvement,2012,Management Consulting,1825 +55,e24D21BFd3bF1E5,Richard PLC,https://holden-coleman.net/,Mayotte,Object-based optimizing model,1971,Broadcast Media,4942 +56,B9BdfEB6D3Ca44E,Sampson Ltd,https://blevins.com/,Cayman Islands,Intuitive local adapter,2005,Farming,1418 +57,2a74D6f3D3B268e,"Cherry, Le and Callahan",https://waller-delacruz.biz/,Nigeria,Universal human-resource collaboration,2017,Entertainment / Movie Production,7202 +58,Bf3F3f62c8aBC33,Cherry PLC,https://www.avila.info/,Marshall Islands,Persistent tertiary website,1980,Plastics,8245 +59,aeBe26B80a7a23c,Melton-Nichols,https://kennedy.com/,Palau,User-friendly clear-thinking productivity,2021,Legislative Office,8741 +60,aAeb29ad43886C6,Potter-Walsh,http://thomas-french.org/,Turkey,Optional non-volatile open system,2008,Human Resources / HR,6923 +61,bD1bc6bB6d1FeD3,Freeman-Chen,https://mathis.com/,Timor-Leste,Phased next generation adapter,1973,International Trade / Development,346 +62,EB9f456e8b7022a,Soto Group,https://norris.info/,Vietnam,Enterprise-wide executive installation,1988,Business Supplies / Equipment,9097 +63,Dfef38C51D8DAe3,"Poole, Cruz and Whitney",https://reed.info/,Reunion,Balanced analyzing groupware,1978,Marketing / Advertising / Sales,2992 +64,055ffEfB2Dd95B0,Riley Ltd,http://wiley.com/,Brazil,Optional exuding superstructure,1986,Textiles,9315 +65,cBfe4dbAE1699da,"Erickson, Andrews and Bailey",https://www.hobbs-grant.com/,Eritrea,Vision-oriented secondary project,2014,Consumer Electronics,7829 +66,fdFbecbadcdCdf1,"Wilkinson, Charles and Arroyo",http://hunter-mcfarland.com/,United States Virgin Islands,Assimilated 24/7 archive,1996,Building Materials,602 +67,5DCb8A5a5ca03c0,Floyd Ltd,http://www.whitney.com/,Falkland Islands (Malvinas),Function-based fault-tolerant concept,2017,Public Relations / PR,2911 +68,ce57DCbcFD6d618,Newman-Galloway,https://www.scott.com/,Luxembourg,Enhanced foreground collaboration,1987,Information Technology / IT,3934 +69,5aaD187dc929371,Frazier-Butler,https://www.daugherty-farley.info/,Northern Mariana Islands,Persistent interactive circuit,1972,Outsourcing / Offshoring,5130 +70,902D7Ac8b6d476b,Newton Inc,https://www.richmond-manning.info/,Netherlands Antilles,Fundamental stable info-mediaries,1976,Military Industry,563 +71,32BB9Ff4d939788,Duffy-Levy,https://www.potter.com/,Guernsey,Diverse exuding installation,1982,Wireless,6146 +72,adcB0afbE58bAe3,Wagner LLC,https://decker-esparza.com/,Uruguay,Reactive attitude-oriented toolset,1987,International Affairs,6874 +73,dfcA1c84AdB61Ac,Mccall-Holmes,http://www.dean.com/,Benin,Object-based value-added database,2009,Legal Services,696 +74,208044AC2fe52F3,Massey LLC,https://frazier.biz/,Suriname,Configurable zero administration Graphical User Interface,1986,Accounting,5004 +75,f3C365f0c1A0623,Hicks LLC,http://alvarez.biz/,Pakistan,Quality-focused client-server Graphical User Interface,1970,Computer Software / Engineering,8480 +76,ec5Bdd3CBAfaB93,"Cole, Russell and Avery",http://www.blankenship.com/,Mongolia,De-engineered fault-tolerant challenge,2000,Law Enforcement,7012 +77,DDB19Be7eeB56B4,Cummings-Rojas,https://simon-pearson.com/,Svalbard & Jan Mayen Islands,User-centric modular customer loyalty,2012,Financial Services,7529 +78,dd6CA3d0bc3cAfc,"Beasley, Greene and Mahoney",http://www.petersen-lawrence.com/,Togo,Extended content-based methodology,1976,Religious Institutions,869 +79,A0B9d56e61070e3,"Beasley, Sims and Allison",http://burke.info/,Latvia,Secured zero tolerance hub,1972,Facilities Services,6182 +80,cBa7EFe5D05Adaf,Crawford-Rivera,https://black-ramirez.org/,Cuba,Persevering exuding budgetary management,1999,Online Publishing,7805 +81,Ea3f6D52Ec73563,Montes-Hensley,https://krueger.org/,Liechtenstein,Multi-tiered secondary productivity,2009,Printing,8433 +82,bC0CEd48A8000E0,Velazquez-Odom,https://stokes.com/,Djibouti,Streamlined 6thgeneration function,2002,Alternative Dispute Resolution,4044 +83,c89b9b59BC4baa1,Eaton-Morales,https://www.reeves-graham.com/,Micronesia,Customer-focused explicit frame,1990,Capital Markets / Hedge Fund / Private Equity,7013 +84,FEC51bce8421a7b,"Roberson, Pennington and Palmer",http://www.keith-fisher.com/,Cameroon,Adaptive bi-directional hierarchy,1993,Telecommunications,5571 +85,e0E8e27eAc9CAd5,"George, Russo and Guerra",https://drake.com/,Sweden,Centralized non-volatile capability,1989,Military Industry,2880 +86,B97a6CF9bf5983C,Davila Inc,https://mcconnell.info/,Cocos (Keeling) Islands,Profit-focused dedicated frame,2017,Consumer Electronics,2215 +87,a0a6f9b3DbcBEb5,Mays-Preston,http://www.browning-key.com/,Mali,User-centric heuristic focus group,2006,Military Industry,5786 +88,8cC1bDa330a5871,Pineda-Morton,https://www.carr.com/,United States Virgin Islands,Grass-roots methodical info-mediaries,1991,Printing,6168 +89,ED889CB2FE9cbd3,Huang and Sons,https://www.bolton.com/,Eritrea,Re-contextualized dynamic hierarchy,1981,Semiconductors,7484 +90,F4Dc1417BC6cb8f,Gilbert-Simon,https://www.bradford.biz/,Burundi,Grass-roots radical parallelism,1973,Newspapers / Journalism,1927 +91,7ABc3c7ecA03B34,Sampson-Griffith,http://hendricks.org/,Benin,Multi-layered composite paradigm,1972,Textiles,3881 +92,4e0719FBE38e0aB,Miles-Dominguez,http://www.turner.com/,Gibraltar,Organized empowering forecast,1996,Civic / Social Organization,897 +93,dEbDAAeDfaed00A,Rowe and Sons,https://www.simpson.org/,El Salvador,Balanced multimedia knowledgebase,1978,Facilities Services,8172 +94,61BDeCfeFD0cEF5,"Valenzuela, Holmes and Rowland",https://www.dorsey.net/,Taiwan,Persistent tertiary focus group,1999,Transportation,1483 +95,4e91eD25f486110,"Best, Wade and Shepard",https://zimmerman.com/,Zimbabwe,Innovative background definition,1991,Gambling / Casinos,4873 +96,0a0bfFbBbB8eC7c,Holmes Group,https://mcdowell.org/,Ethiopia,Right-sized zero tolerance focus group,1975,Photography,2988 +97,BA6Cd9Dae2Efd62,Good Ltd,http://duffy.com/,Anguilla,Reverse-engineered composite moratorium,1971,Consumer Services,4292 +98,E7df80C60Abd7f9,Clements-Espinoza,http://www.flowers.net/,Falkland Islands (Malvinas),Progressive modular hub,1991,Broadcast Media,236 +99,AFc285dbE2fEd24,Mendez Inc,https://www.burke.net/,Kyrgyz Republic,User-friendly exuding migration,1993,Education Management,339 +100,e9eB5A60Cef8354,Watkins-Kaiser,http://www.herring.com/,Togo,Synergistic background access,2009,Financial Services,2785 diff --git a/ingestion/tests/integration/kafka/data/people-100.csv b/ingestion/tests/integration/kafka/data/people-100.csv new file mode 100644 index 00000000000..b8d39f2c51f --- /dev/null +++ b/ingestion/tests/integration/kafka/data/people-100.csv @@ -0,0 +1,101 @@ +Index,User Id,First Name,Last Name,Sex,Email,Phone,Date of birth,Job Title +1,88F7B33d2bcf9f5,Shelby,Terrell,Male,elijah57@example.net,001-084-906-7849x73518,1945-10-26,Games developer +2,f90cD3E76f1A9b9,Phillip,Summers,Female,bethany14@example.com,214.112.6044x4913,1910-03-24,Phytotherapist +3,DbeAb8CcdfeFC2c,Kristine,Travis,Male,bthompson@example.com,277.609.7938,1992-07-02,Homeopath +4,A31Bee3c201ef58,Yesenia,Martinez,Male,kaitlinkaiser@example.com,584.094.6111,2017-08-03,Market researcher +5,1bA7A3dc874da3c,Lori,Todd,Male,buchananmanuel@example.net,689-207-3558x7233,1938-12-01,Veterinary surgeon +6,bfDD7CDEF5D865B,Erin,Day,Male,tconner@example.org,001-171-649-9856x5553,2015-10-28,Waste management officer +7,bE9EEf34cB72AF7,Katherine,Buck,Female,conniecowan@example.com,+1-773-151-6685x49162,1989-01-22,Intelligence analyst +8,2EFC6A4e77FaEaC,Ricardo,Hinton,Male,wyattbishop@example.com,001-447-699-7998x88612,1924-03-26,Hydrogeologist +9,baDcC4DeefD8dEB,Dave,Farrell,Male,nmccann@example.net,603-428-2429x27392,2018-10-06,Lawyer +10,8e4FB470FE19bF0,Isaiah,Downs,Male,virginiaterrell@example.org,+1-511-372-1544x8206,1964-09-20,"Engineer, site" +11,BF0BbA03C29Bb3b,Sheila,Ross,Female,huangcathy@example.com,895.881.4746,2008-03-20,Advertising account executive +12,F738c69fB34E62E,Stacy,Newton,Male,rayleroy@example.org,710.673.3213x80335,1980-10-20,Warden/ranger +13,C03fDADdAadAdCe,Mandy,Blake,Male,jefferynoble@example.org,(992)466-1305x4947,2007-12-08,"Scientist, clinical (histocompatibility and immunogenetics)" +14,b759b74BD1dE80d,Bridget,Nash,Female,mercedes44@example.com,(216)627-8359,2004-06-28,Social worker +15,1F0B7D65A00DAF9,Crystal,Farmer,Male,pmiranda@example.org,+1-024-377-5391,1992-03-09,Agricultural consultant +16,50Bb061cB30B461,Thomas,Knight,Female,braunpriscilla@example.net,+1-360-880-0766,2006-02-18,Sport and exercise psychologist +17,D6dbA5308fEC4BC,Maurice,Rangel,Male,sheenabanks@example.com,(246)187-4969,2004-08-20,Secretary/administrator +18,311D775990f066d,Frank,Meadows,Male,gbrewer@example.org,429.965.3902x4447,2008-09-16,Audiological scientist +19,7F7E1BAcb0C9AFf,Alvin,Paul,Male,gilbertdonaldson@example.com,219.436.0887x07551,1949-05-12,"Teacher, adult education" +20,88473e15D5c3cD0,Jared,Mitchell,Female,jcortez@example.com,+1-958-849-6781,1921-01-18,Paediatric nurse +21,b31D271F8c200AB,Jacqueline,Norton,Female,carias@example.net,819.309.7679x59173,1952-10-09,"Scientist, marine" +22,42F4BdA841aBadC,Colleen,Hatfield,Female,fknox@example.org,638.584.1090,1949-10-14,Commercial horticulturist +23,cBbBcA0FCA3C4Bc,Randy,Barnes,Male,huangbill@example.org,001-960-629-7164x67214,1947-12-30,Outdoor activities/education manager +24,f1f89173353aD90,Janice,Rhodes,Female,juarezdominique@example.net,001-249-314-9742x6996,1999-11-01,Drilling engineer +25,c5B09fb33e8bA0A,Alfred,Mcneil,Female,cassandramorris@example.com,(468)276-9509x53058,1993-05-28,Systems analyst +26,c9F2282C40BEC1E,Sean,Levine,Male,sallymiller@example.net,4915828504,2010-10-09,"Conservation officer, nature" +27,9c1bc7EC53Fb7cE,Louis,Payne,Male,bsullivan@example.net,6232695307,1916-01-29,Counsellor +28,ddEc50e2A2e3a2B,Brittney,Vega,Female,ayalajose@example.net,945-739-8686,1932-10-31,Recycling officer +29,66F096D36Ebae11,Judy,Buckley,Male,irosales@example.net,001-654-208-1241x52830,1963-07-28,Art gallery manager +30,F0fE2faAd78F8b5,Norman,Weber,Male,mconrad@example.com,223.002.0429,1957-05-21,Gaffer +31,5d2feAfbdCAA6B5,Isaiah,Camacho,Female,jimblake@example.org,001-536-544-3367,1966-04-07,Food technologist +32,cDa5F303fCd6dEa,Jacqueline,Gallagher,Male,nsampson@example.net,(247)762-8934,1999-02-25,Building services engineer +33,8Ef7DBfcaB02b6B,Bonnie,Andrews,Female,caitlin24@example.net,+1-253-987-2776x9161,1953-12-21,Seismic interpreter +34,6Dec5b5542F8ed8,Brandon,Schmidt,Female,mconley@example.net,+1-386-673-1465x006,1931-05-12,"Engineer, biomedical" +35,3Fb8a7f68e12784,Jackson,Sparks,Female,reynoldsdarryl@example.net,(137)908-3129x65035,1980-11-18,Set designer +36,035eff50B9A0F24,Melody,Cook,Male,jeannovak@example.org,(826)792-7381,1963-06-25,Research scientist (life sciences) +37,aa614aAE4B7Cf0C,Leonard,Hurst,Male,clinton78@example.org,941-038-0427x38800,1938-03-13,"Accountant, chartered management" +38,ACcde95AAe3e6cC,Gene,Rich,Female,luisdeleon@example.org,+1-356-818-6604x89537,1946-08-22,"Surveyor, quantity" +39,b6a35de5CB6fc25,Cynthia,Wiggins,Female,rosariodave@example.org,(110)858-2437x70190,1984-01-27,Outdoor activities/education manager +40,e92A191E345fA3A,Tanya,Mckinney,Female,vickihouston@example.com,(830)774-9002x086,2003-03-12,Information systems manager +41,7D0AcBF6CCac3fd,Matthew,Stone,Female,evelyn31@example.org,952-381-6360,2017-08-23,"Scientist, clinical (histocompatibility and immunogenetics)" +42,CEFA7BBCef013AE,Kirk,Walsh,Female,stephenfuller@example.org,001-826-496-5529x8661,2009-04-08,Accounting technician +43,9edBC94aE7cA22a,Willie,Vang,Female,haleymathews@example.net,741.168.6854x067,1978-02-02,Management consultant +44,fFe7BAA737aDbe2,Miguel,Hill,Female,tyrone56@example.org,5247842945,1930-08-26,Make +45,5F2f3fAca8B0946,Darren,Andrews,Male,lhernandez@example.com,(975)799-4261,1997-10-04,Retail banker +46,6bFcfc3cc1BC6B4,Haley,Pugh,Female,molly03@example.org,(746)182-6137x2453,1980-09-16,Commissioning editor +47,f3BD2cBF7eEb6df,Danielle,Estrada,Female,jvang@example.org,(890)374-9518x772,1930-07-09,"Accountant, chartered management" +48,Ee4eB129dC7913A,Becky,Brady,Male,erikmueller@example.org,(390)002-0863,1957-06-27,Seismic interpreter +49,dBCEf340C3657Eb,Caitlyn,Frey,Male,rivasdominique@example.org,805-021-3965x46344,1968-01-26,Jewellery designer +50,E47FB71DD9ACCd9,Joshua,Sweeney,Male,daisymcgee@example.net,875.994.2100x535,1954-07-28,"Education officer, museum" +51,eA3fDd79BE9f0E7,Heidi,Escobar,Female,staffordtravis@example.net,601-155-3065x1131,1931-09-25,Estate manager/land agent +52,aF0eE4547Bc025c,Brian,Oconnell,Female,saralong@example.net,952-283-1423x733,1911-10-23,Physiotherapist +53,9F5DeD7aD228F5a,Beverly,Esparza,Female,iphelps@example.net,+1-327-578-8754x6771,1930-12-09,Passenger transport manager +54,D3Fa0220dDE4d36,Nathaniel,Rivas,Female,roberto29@example.com,(655)887-2040x37888,1908-11-17,Call centre manager +55,60FdBFd5e7BE8fF,Debra,Payne,Female,yolanda07@example.org,001-731-525-8400x52593,1927-08-20,Special educational needs teacher +56,D8bF5Ab2b98caff,Mackenzie,Rocha,Female,abbottyvette@example.net,4225525458,1980-10-21,Museum/gallery exhibitions officer +57,CD8d33aA25bc8BB,Courtney,Watkins,Female,ochang@example.org,210.683.2761x5883,2003-12-07,Pension scheme manager +58,Fac3BfFf0A3d03c,Fred,Olsen,Female,amyanderson@example.com,497-774-3053,1910-04-10,Archaeologist +59,e552D7ddafe1FFb,Ryan,Nelson,Female,qnorman@example.org,956.330.2951,1924-05-02,Historic buildings inspector/conservation officer +60,0f8deedb629A5f6,Grace,Phelps,Male,clarkeangela@example.net,(034)867-8827x6777,1909-10-15,Petroleum engineer +61,bB9e49E506F65ed,Shari,Daugherty,Male,kalvarado@example.org,001-951-655-4798x6124,1944-11-24,Curator +62,Ed724605A403D91,Kelli,Garner,Male,jodyvincent@example.org,995.000.4213x0982,2010-01-17,Retail banker +63,0aBE5ACb18E0c10,Jackie,Bennett,Male,hutchinsonkirk@example.com,001-740-937-0846x0087,1915-11-11,Neurosurgeon +64,5D2cb63CaAF53f6,Leslie,Conway,Female,floreschristina@example.org,795.782.4384x555,1983-11-06,Chiropractor +65,Ee6974f90eeCe18,Harold,Barnett,Female,nathan65@example.org,+1-026-265-6392,1943-03-15,"Biochemist, clinical" +66,cEf02C076afa07f,Larry,Harper,Male,maria32@example.org,+1-244-630-3792x4121,2021-05-05,"Scientist, water quality" +67,9Df5Ba591bF3EFf,Mike,Ward,Female,imccullough@example.com,116-729-5046,1967-11-09,Hydrologist +68,3faB1CBfEFBDdD4,Brittney,Rubio,Female,corey92@example.com,593.976.2528,1959-12-24,"Biochemist, clinical" +69,Ebcefdf75eCb0a9,Frank,Pineda,Male,daltoncalvin@example.net,(035)961-5060x9182,1926-03-10,Hospital pharmacist +70,e75e5DBfcb68887,Sandra,Wu,Male,ubanks@example.com,+1-096-606-6454x067,1925-04-28,Warehouse manager +71,6a53a8D41dDF6de,Ryan,Benton,Male,lopezdebbie@example.org,+1-695-557-9948x485,2020-10-06,Physiological scientist +72,F0d3bD1aaf9E3Bc,Tamara,Hull,Male,meagan39@example.net,017.665.3744x7944,1933-01-31,English as a second language teacher +73,5bC87340799FBD0,Jean,Ritter,Female,kristina76@example.com,(954)060-1066,1985-08-06,Financial trader +74,dBfA17Aaf16b4ab,Veronica,Briggs,Female,weissbridget@example.com,+1-521-589-2387x48490,1974-06-08,Structural engineer +75,c935b7Eb6FA0B0F,Kim,Andrews,Female,wpetersen@example.org,7677125383,1990-11-15,"Biochemist, clinical" +76,b3e15e65Ca2CcBf,Tina,Cunningham,Male,wongmary@example.org,079-907-5051,1956-11-29,Race relations officer +77,dade3452F0c32FD,Jonathon,Atkinson,Male,gailfrench@example.net,874-037-2032x932,2011-07-19,"Psychologist, forensic" +78,AdEd6cfD85DeC46,Jermaine,Reid,Female,vpaul@example.com,(742)214-8691,1974-08-18,Newspaper journalist +79,DAf111987098ae4,Regina,Stevens,Male,xpoole@example.net,891-359-2684,2011-11-28,Public house manager +80,6e6a5b885F6496d,Terrence,Huff,Male,cassandra80@example.org,221.800.6408x5416,1944-02-27,Careers information officer +81,12DCb4ED8E01D5C,Tyler,Foley,Female,johnathan72@example.org,001-386-469-3075x8030,1908-09-19,Economist +82,E1cB5cA8CA7CC0a,Andrew,Waters,Male,nhall@example.net,+1-376-865-2765x3351,1948-05-14,Jewellery designer +83,AedDfaE8Cf49F07,Reginald,Stephenson,Male,erikaball@example.net,+1-832-500-6044x475,2010-02-08,Contracting civil engineer +84,bff9853aFAeF772,Douglas,Reese,Female,nixonvanessa@example.net,001-834-660-8312x9864,1961-11-11,Higher education lecturer +85,E883773cA5219Be,Helen,Williamson,Female,melvin08@example.net,001-377-726-4229,1911-08-11,"Lecturer, further education" +86,CB19EafEbBfF9eC,Mario,Vaughn,Male,oblake@example.com,160-144-5039x12276,1990-07-08,Research scientist (life sciences) +87,5834700fbEd2771,Chelsea,Dickson,Male,johnnyhendricks@example.net,001-698-651-0138x18588,1958-05-13,"Teacher, early years/pre" +88,2b0Ab1Dc9E01D7E,Dustin,Bailey,Male,pbarron@example.net,+1-965-621-1157x345,1908-08-22,Travel agency manager +89,3f3a3D89ad042Dd,Harry,Medina,Female,olsenmalik@example.net,+1-451-099-5805,1947-08-24,Technical sales engineer +90,9425E2F38C408ef,Kathy,Haney,Female,teresa37@example.com,(164)105-8456,1955-09-02,Charity fundraiser +91,F0aeC9c2759F3C6,Alison,Nixon,Female,zmiles@example.net,3506680871,1941-07-10,Patent attorney +92,d6EA619A7C4aA95,Jamie,Hardy,Female,sheenadouglas@example.com,(900)803-9295x11533,1994-07-17,"Conservator, furniture" +93,2A33E7Cad1bb0F5,Melody,Cox,Female,evan90@example.org,(626)520-5080x3511,1974-07-30,Dance movement psychotherapist +94,d181FFB7d3E68bb,Xavier,Cole,Male,nicolas90@example.org,8164259975,1938-11-29,Financial planner +95,feaBf8dAE0C0d6F,Dillon,Guzman,Female,angelanavarro@example.net,971-992-4521,1942-04-01,Air broker +96,5eFda7caAeB260E,Dennis,Barnes,Female,bmartin@example.org,001-095-524-2112x257,1954-07-30,Software engineer +97,CCbFce93d3720bE,Steve,Patterson,Female,latasha46@example.net,001-865-478-5157,1932-04-29,Barrister +98,2fEc528aFAF0b69,Wesley,Bray,Male,regina11@example.org,995-542-3004x76800,1994-12-28,Police officer +99,Adc7ad9B6e4A1Fe,Summer,Oconnell,Female,alexiscantrell@example.org,001-273-685-6932x092,2012-04-12,Broadcast journalist +100,b8D0aD3490FC7e1,Mariah,Bernard,Male,pcopeland@example.org,(341)594-6554x44657,2016-11-15,IT sales professional diff --git a/ingestion/tests/integration/kafka/test_kafka.py b/ingestion/tests/integration/kafka/test_kafka.py deleted file mode 100644 index 66f8362c877..00000000000 --- a/ingestion/tests/integration/kafka/test_kafka.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2021 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Test Kafka using the topology -""" - - -from unittest import TestCase - -from confluent_kafka import Producer -from testcontainers.kafka import KafkaContainer - -from metadata.generated.schema.entity.data.topic import Topic -from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( - AuthProvider, - OpenMetadataConnection, -) -from metadata.generated.schema.security.client.openMetadataJWTClientConfig import ( - OpenMetadataJWTClientConfig, -) -from metadata.ingestion.models.custom_pydantic import CustomSecretStr -from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.workflow.metadata import MetadataWorkflow - - -def produce_and_consume_kafka_message(container): - topic = "test-topic" - bootstrap_server = container.get_bootstrap_server() - - producer = Producer({"bootstrap.servers": bootstrap_server}) - producer.produce(topic, b"verification message") - producer.flush() - - -OM_JWT = "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - - -def get_ingestion_config(port: str): - return { - "source": { - "type": "kafka", - "serviceName": "TEST_KAFKA", - "serviceConnection": { - "config": { - "type": "Kafka", - "bootstrapServers": f"localhost:{port}", - } - }, - "sourceConfig": {"config": {"type": "MessagingMetadata"}}, - }, - "sink": {"type": "metadata-rest", "config": {}}, - "workflowConfig": { - "loggerLevel": "DEBUG", - "openMetadataServerConfig": { - "hostPort": "http://localhost:8585/api", - "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, - }, - }, - } - - -def int_admin_ometa(url: str = "http://localhost:8585/api") -> OpenMetadata: - """Initialize the ometa connection with default admin:admin creds""" - server_config = OpenMetadataConnection( - hostPort=url, - authProvider=AuthProvider.openmetadata, - securityConfig=OpenMetadataJWTClientConfig(jwtToken=CustomSecretStr(OM_JWT)), - ) - metadata = OpenMetadata(server_config) - assert metadata.health_check() - return metadata - - -class KafkaUnitTest(TestCase): - @classmethod - def setUpClass(cls) -> None: - cls.metadata = int_admin_ometa() - cls.kafka_container = KafkaContainer() - cls.kafka_container.start() - cls.ingestion_config = get_ingestion_config( - cls.kafka_container.get_exposed_port(9093) - ) - produce_and_consume_kafka_message(cls.kafka_container) - - ingestion_workflow = MetadataWorkflow.create( - cls.ingestion_config, - ) - ingestion_workflow.execute() - ingestion_workflow.raise_from_status() - ingestion_workflow.stop() - - def test_topic(self): - topic = self.metadata.get_by_name( - entity=Topic, - fqn="TEST_KAFKA.test-topic", - ) - self.assertIsNotNone(topic) diff --git a/ingestion/tests/integration/kafka/test_metadata.py b/ingestion/tests/integration/kafka/test_metadata.py new file mode 100644 index 00000000000..41e97b089dd --- /dev/null +++ b/ingestion/tests/integration/kafka/test_metadata.py @@ -0,0 +1,37 @@ +import sys + +import pytest + +from metadata.generated.schema.entity.data.topic import Topic +from metadata.workflow.metadata import MetadataWorkflow + +if not sys.version_info >= (3, 9): + pytest.skip("requires python 3.9+", allow_module_level=True) + + +def test_ingest_metadata( + patch_passwords_for_db_services, run_workflow, ingestion_config, metadata_assertions +): + run_workflow(MetadataWorkflow, ingestion_config) + metadata_assertions() + + +@pytest.fixture( + scope="module", + params=[ + "customers-100", + "organizations-100", + "people-100", + ], +) +def metadata_assertions(metadata, db_service, request): + def _assertions(): + topic: Topic = metadata.get_by_name( + entity=Topic, + fqn=f"{db_service.fullyQualifiedName.root}.{request.param}", + fields=["*"], + nullable=False, + ) + assert topic.messageSchema is not None + + return _assertions