MINOR: kafka integration tests (#17457)

* tests: kafka integration

kafka integration tests with schema registry

* added ignore kafka for python 3.8

* fixed tests

ingestion/setup.py

@@ -371,7 +371,7 @@ test = {
     VERSIONS["grpc-tools"],
     VERSIONS["neo4j"],
     "testcontainers==3.7.1;python_version<'3.9'",
-    "testcontainers==4.4.0;python_version>='3.9'",
+    "testcontainers==4.8.0;python_version>='3.9'",
     "minio==7.2.5",
     *plugins["mlflow"],
     *plugins["datalake-s3"],

ingestion/src/_openmetadata_testutils/kafka/load_csv_data.py

@@ -0,0 +1,133 @@
+"""
+pip install confluent_kafka pandas requests avro-python3
+"""
+
+import json
+import os
+from typing import Dict, List
+
+import pandas as pd
+import requests
+from confluent_kafka import Producer
+from confluent_kafka.schema_registry import SchemaRegistryClient
+from confluent_kafka.schema_registry.avro import AvroSerializer
+from confluent_kafka.serialization import MessageField, SerializationContext
+
+
+class Kafka:
+    def __init__(self, broker: str):
+        self.broker = broker
+
+    def get_producer(self):
+        return Producer({"bootstrap.servers": self.broker})
+
+
+class SchemaRegistry:
+    def __init__(self, url: str):
+        self.url = url
+
+    def register_schema(self, schema: str, topic: str):
+        """Register the schema with the Schema Registry"""
+        url = f"{self.url}/subjects/{sanitize_name(topic)}-value/versions"
+        headers = {"Content-Type": "application/vnd.schemaregistry.v1+json"}
+        data = json.dumps({"schema": schema})
+        response = requests.post(url, headers=headers, data=data)
+        if response.status_code == 200:
+            print(f"Schema registered for topic {topic}")
+        else:
+            print(f"Failed to register schema for topic {topic}: {response.text}")
+
+    def get_avro_serializer(self, schema: str) -> AvroSerializer:
+        schema_registry_conf = {"url": self.url}
+        schema_registry_client = SchemaRegistryClient(schema_registry_conf)
+        return AvroSerializer(schema_registry_client, schema_str=schema)
+
+
+def delivery_report(err, msg):
+    """Called once for each message produced to indicate delivery result.
+    Triggered by poll() or flush()."""
+    if err is not None:
+        print(f"Message delivery failed: {err}")
+    else:
+        print(f"Message delivered to {msg.topic()} [{msg.partition()}]")
+
+
+def sanitize_name(name):
+    """Sanitize the name to conform to Avro naming rules"""
+    return name.replace("-", "_").replace(" ", "_")
+
+
+def generate_avro_schema(df: pd.DataFrame, topic: str) -> str:
+    """Generate an Avro schema from a pandas DataFrame"""
+    fields: List[Dict[str, str]] = []
+    for column in df.columns:
+        fields.append(
+            {"name": sanitize_name(column), "type": "string"}
+        )  # Assuming all columns are of type string
+    schema_dict: Dict[str, any] = {
+        "namespace": "example.avro",
+        "type": "record",
+        "name": sanitize_name(topic),
+        "fields": fields,
+    }
+    return json.dumps(schema_dict)
+
+
+def send_csv_to_kafka(kafka: Kafka, schema_registry: SchemaRegistry, file_path: str):
+    # Read the CSV file
+    df = pd.read_csv(file_path)
+
+    # Sanitize column names
+    df.columns = [sanitize_name(col) for col in df.columns]
+
+    # Convert all fields to string
+    df = df.astype(str)
+
+    # Get the file name without extension to use as the topic name
+    topic = os.path.splitext(os.path.basename(file_path))[0]
+
+    # Generate and register the Avro schema
+    schema = generate_avro_schema(df, topic)
+    schema_registry.register_schema(schema, topic)
+
+    # Create an Avro serializer with the generated schema
+    avro_serializer = schema_registry.get_avro_serializer(schema)
+
+    # Create a Kafka producer
+    producer = kafka.get_producer()
+
+    # Iterate over the rows of the DataFrame and send each row as an Avro message
+    for _, row in df.iterrows():
+        message = row.to_dict()
+        try:
+            producer.produce(
+                topic=topic,
+                value=avro_serializer(
+                    message, SerializationContext(topic, MessageField.VALUE)
+                ),
+                callback=delivery_report,
+            )
+        except Exception as e:
+            print(f"Message serialization failed: {e}")
+            break
+
+    # Wait for any outstanding messages to be delivered and delivery reports to be received
+    producer.flush()
+
+
+def main(kafka_broker: str, schema_registry_url: str, csv_directory: str):
+    # Iterate over all files in the directory
+    kafka = Kafka(kafka_broker)
+    schema_registry = SchemaRegistry(schema_registry_url)
+    for file_name in os.listdir(csv_directory):
+        if file_name.endswith(".csv"):
+            file_path = os.path.join(csv_directory, file_name)
+            send_csv_to_kafka(kafka, schema_registry, file_path)
+
+
+if __name__ == "__main__":
+    main(
+        kafka_broker="localhost:9092",
+        schema_registry_url="http://localhost:8081",
+        csv_directory="./data",
+    )

ingestion/src/_openmetadata_testutils/kafka/schema_registry_container.py

@@ -0,0 +1,31 @@
+from testcontainers.core.container import DockerContainer
+from testcontainers.core.waiting_utils import wait_for_logs
+
+
+class SchemaRegistryContainer(DockerContainer):
+    def __init__(
+        self,
+        image: str = "confluentinc/cp-schema-registry:7.6.0",
+        port: int = 8081,
+        schema_registry_kafkastore_bootstrap_servers="PLAINTEXT://localhost:9092",
+        schema_registry_host_name="localhost",
+        **kwargs,
+    ) -> None:
+        super().__init__(image, **kwargs)
+        self.with_env(
+            "SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS",
+            schema_registry_kafkastore_bootstrap_servers,
+        )
+        self.with_env("SCHEMA_REGISTRY_HOST_NAME", schema_registry_host_name)
+        self.port = port
+        self.with_exposed_ports(port)
+
+    def start(self, timeout=30) -> "SchemaRegistryContainer":
+        super().start()
+        wait_for_logs(self, r".*\Server started.*", timeout=timeout)
+        return self
+
+    def get_connection_url(self):
+        return (
+            f"http://{self.get_container_host_ip()}:{self.get_exposed_port(self.port)}"
+        )

ingestion/tests/integration/conftest.py

@@ -15,7 +15,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.workflow.ingestion import IngestionWorkflow
 
 if not sys.version_info >= (3, 9):
-    collect_ignore = ["trino"]
+    collect_ignore = ["trino", "kafka"]
 
 
 @pytest.fixture(scope="session", autouse=True)

ingestion/tests/integration/kafka/conftest.py

@@ -0,0 +1,140 @@
+import os.path
+from textwrap import dedent
+
+import pytest
+import testcontainers.core.network
+from docker.types import EndpointConfig
+from testcontainers.core.container import DockerContainer
+from testcontainers.kafka import KafkaContainer
+
+from _openmetadata_testutils.kafka import load_csv_data
+from _openmetadata_testutils.kafka.schema_registry_container import (
+    SchemaRegistryContainer,
+)
+from metadata.generated.schema.api.services.createMessagingService import (
+    CreateMessagingServiceRequest,
+)
+from metadata.generated.schema.entity.services.connections.messaging.kafkaConnection import (
+    KafkaConnection,
+)
+from metadata.generated.schema.entity.services.messagingService import (
+    MessagingConnection,
+    MessagingServiceType,
+)
+from metadata.generated.schema.metadataIngestion.messagingServiceMetadataPipeline import (
+    MessagingMetadataConfigType,
+)
+
+
+def _connect_to_network(
+    ctr: DockerContainer, network: testcontainers.core.network, alias: str
+):
+    # Needed until https://github.com/testcontainers/testcontainers-python/issues/645 is fixed
+    ctr.with_kwargs(
+        network=network.name,
+        networking_config={network.name: EndpointConfig("1.33", aliases=[alias])},
+    )
+
+
+class CustomKafkaContainer(KafkaContainer):
+    def __init__(self):
+        super().__init__()
+        self.security_protocol_map += ",EXTERNAL:PLAINTEXT"
+        self.with_env(
+            "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", self.security_protocol_map
+        )
+
+        self.listeners = f"PLAINTEXT://0.0.0.0:29092,BROKER://0.0.0.0:9092,EXTERNAL://0.0.0.0:{self.port}"
+        self.with_env("KAFKA_LISTENERS", self.listeners)
+
+    def tc_start(self):
+        listeners = ",".join(
+            [
+                f"EXTERNAL://{self.get_bootstrap_server()}",
+                f"PLAINTEXT://{self._get_network_alias()}:29092",
+                "BROKER://$(hostname -i | cut -d' ' -f1):9092",
+            ]
+        )
+        data = (
+            dedent(
+                f"""
+                #!/bin/bash
+                {self.boot_command}
+                export KAFKA_ADVERTISED_LISTENERS={listeners}
+                . /etc/confluent/docker/bash-config
+                /etc/confluent/docker/configure
+                /etc/confluent/docker/launch
+                """
+            )
+            .strip()
+            .encode("utf-8")
+        )
+        self.create_file(data, KafkaContainer.TC_START_SCRIPT)
+
+
+@pytest.fixture(scope="module")
+def docker_network():
+    with testcontainers.core.network.Network() as network:
+        yield network
+
+
+@pytest.fixture(scope="module")
+def schema_registry_container(docker_network, kafka_container):
+    with SchemaRegistryContainer(
+        schema_registry_kafkastore_bootstrap_servers="PLAINTEXT://kafka:9092",
+        schema_registry_host_name="schema-registry",
+    ).with_network(docker_network).with_network_aliases("schema-registry") as container:
+        load_csv_data.main(
+            kafka_broker=kafka_container.get_bootstrap_server(),
+            schema_registry_url=container.get_connection_url(),
+            csv_directory=os.path.dirname(__file__) + "/data",
+        )
+        yield container
+
+
+@pytest.fixture(scope="module")
+def kafka_container(docker_network):
+    container = CustomKafkaContainer()
+    _connect_to_network(container, docker_network, "kafka")
+    with container:
+        yield container
+
+
+@pytest.fixture(scope="module")
+def create_service_request(
+    kafka_container, schema_registry_container, tmp_path_factory
+):
+    return CreateMessagingServiceRequest(
+        name="docker_test_" + tmp_path_factory.mktemp("kafka").name,
+        serviceType=MessagingServiceType.Kafka,
+        connection=MessagingConnection(
+            config=KafkaConnection(
+                bootstrapServers=kafka_container.get_bootstrap_server(),
+                schemaRegistryURL=schema_registry_container.get_connection_url(),
+            )
+        ),
+    )
+
+
+@pytest.fixture(scope="module")
+def ingestion_config(db_service, metadata, workflow_config, sink_config):
+    return {
+        "source": {
+            "type": db_service.connection.config.type.value.lower(),
+            "serviceName": db_service.fullyQualifiedName.root,
+            "sourceConfig": {
+                "config": {"type": MessagingMetadataConfigType.MessagingMetadata.value}
+            },
+            "serviceConnection": db_service.connection.model_dump(),
+        },
+        "sink": sink_config,
+        "workflowConfig": workflow_config,
+    }
+
+
+@pytest.fixture(scope="module")
+def unmask_password():
+    def patch_password(service):
+        return service
+
+    return patch_password

ingestion/tests/integration/kafka/data/customers-100.csv

@@ -0,0 +1,101 @@
+Index,Customer Id,First Name,Last Name,Company,City,Country,Phone 1,Phone 2,Email,Subscription Date,Website
+1,DD37Cf93aecA6Dc,Sheryl,Baxter,Rasmussen Group,East Leonard,Chile,229.077.5154,397.884.0519x718,zunigavanessa@smith.info,2020-08-24,http://www.stephenson.com/
+2,1Ef7b82A4CAAD10,Preston,Lozano,Vega-Gentry,East Jimmychester,Djibouti,5153435776,686-620-1820x944,vmata@colon.com,2021-04-23,http://www.hobbs.com/
+3,6F94879bDAfE5a6,Roy,Berry,Murillo-Perry,Isabelborough,Antigua and Barbuda,+1-539-402-0259,(496)978-3969x58947,beckycarr@hogan.com,2020-03-25,http://www.lawrence.com/
+4,5Cef8BFA16c5e3c,Linda,Olsen,"Dominguez, Mcmillan and Donovan",Bensonview,Dominican Republic,001-808-617-6467x12895,+1-813-324-8756,stanleyblackwell@benson.org,2020-06-02,http://www.good-lyons.com/
+5,053d585Ab6b3159,Joanna,Bender,"Martin, Lang and Andrade",West Priscilla,Slovakia (Slovak Republic),001-234-203-0635x76146,001-199-446-3860x3486,colinalvarado@miles.net,2021-04-17,https://goodwin-ingram.com/
+6,2d08FB17EE273F4,Aimee,Downs,Steele Group,Chavezborough,Bosnia and Herzegovina,(283)437-3886x88321,999-728-1637,louis27@gilbert.com,2020-02-25,http://www.berger.net/
+7,EA4d384DfDbBf77,Darren,Peck,"Lester, Woodard and Mitchell",Lake Ana,Pitcairn Islands,(496)452-6181x3291,+1-247-266-0963x4995,tgates@cantrell.com,2021-08-24,https://www.le.com/
+8,0e04AFde9f225dE,Brett,Mullen,"Sanford, Davenport and Giles",Kimport,Bulgaria,001-583-352-7197x297,001-333-145-0369,asnow@colon.com,2021-04-12,https://hammond-ramsey.com/
+9,C2dE4dEEc489ae0,Sheryl,Meyers,Browning-Simon,Robersonstad,Cyprus,854-138-4911x5772,+1-448-910-2276x729,mariokhan@ryan-pope.org,2020-01-13,https://www.bullock.net/
+10,8C2811a503C7c5a,Michelle,Gallagher,Beck-Hendrix,Elaineberg,Timor-Leste,739.218.2516x459,001-054-401-0347x617,mdyer@escobar.net,2021-11-08,https://arias.com/
+11,216E205d6eBb815,Carl,Schroeder,"Oconnell, Meza and Everett",Shannonville,Guernsey,637-854-0256x825,114.336.0784x788,kirksalas@webb.com,2021-10-20,https://simmons-hurley.com/
+12,CEDec94deE6d69B,Jenna,Dodson,"Hoffman, Reed and Mcclain",East Andrea,Vietnam,(041)737-3846,+1-556-888-3485x42608,mark42@robbins.com,2020-11-29,http://www.douglas.net/
+13,e35426EbDEceaFF,Tracey,Mata,Graham-Francis,South Joannamouth,Togo,001-949-844-8787,(855)713-8773,alex56@walls.org,2021-12-02,http://www.beck.com/
+14,A08A8aF8BE9FaD4,Kristine,Cox,Carpenter-Cook,Jodyberg,Sri Lanka,786-284-3358x62152,+1-315-627-1796x8074,holdenmiranda@clarke.com,2021-02-08,https://www.brandt.com/
+15,6fEaA1b7cab7B6C,Faith,Lutz,Carter-Hancock,Burchbury,Singapore,(781)861-7180x8306,207-185-3665,cassieparrish@blevins-chapman.net,2022-01-26,http://stevenson.org/
+16,8cad0b4CBceaeec,Miranda,Beasley,Singleton and Sons,Desireeshire,Oman,540.085.3135x185,+1-600-462-6432x21881,vduncan@parks-hardy.com,2022-04-12,http://acosta.org/
+17,a5DC21AE3a21eaA,Caroline,Foley,Winters-Mendoza,West Adriennestad,Western Sahara,936.222.4746x9924,001-469-948-6341x359,holtgwendolyn@watson-davenport.com,2021-03-10,http://www.benson-roth.com/
+18,F8Aa9d6DfcBeeF8,Greg,Mata,Valentine LLC,Lake Leslie,Mozambique,(701)087-2415,(195)156-1861x26241,jaredjuarez@carroll.org,2022-03-26,http://pitts-cherry.com/
+19,F160f5Db3EfE973,Clifford,Jacobson,Simon LLC,Harmonview,South Georgia and the South Sandwich Islands,001-151-330-3524x0469,(748)477-7174,joseph26@jacobson.com,2020-09-24,https://mcconnell.com/
+20,0F60FF3DdCd7aB0,Joanna,Kirk,Mays-Mccormick,Jamesshire,French Polynesia,(266)131-7001x711,(283)312-5579x11543,tuckerangie@salazar.net,2021-09-24,https://www.camacho.net/
+21,9F9AdB7B8A6f7F2,Maxwell,Frye,Patterson Inc,East Carly,Malta,423.262.3059,202-880-0688x7491,fgibson@drake-webb.com,2022-01-12,http://www.roberts.com/
+22,FBd0Ded4F02a742,Kiara,Houston,"Manning, Hester and Arroyo",South Alvin,Netherlands,001-274-040-3582x10611,+1-528-175-0973x4684,blanchardbob@wallace-shannon.com,2020-09-15,https://www.reid-potts.com/
+23,2FB0FAA1d429421,Colleen,Howard,Greer and Sons,Brittanyview,Paraguay,1935085151,(947)115-7711x5488,rsingleton@ryan-cherry.com,2020-08-19,http://paul.biz/
+24,010468dAA11382c,Janet,Valenzuela,Watts-Donaldson,Veronicamouth,Lao People's Democratic Republic,354.259.5062x7538,500.433.2022,stefanie71@spence.com,2020-09-08,https://moreno.biz/
+25,eC1927Ca84E033e,Shane,Wilcox,Tucker LLC,Bryanville,Albania,(429)005-9030x11004,541-116-4501,mariah88@santos.com,2021-04-06,https://www.ramos.com/
+26,09D7D7C8Fe09aea,Marcus,Moody,Giles Ltd,Kaitlyntown,Panama,674-677-8623,909-277-5485x566,donnamullins@norris-barrett.org,2022-05-24,https://www.curry.com/
+27,aBdfcF2c50b0bfD,Dakota,Poole,Simmons Group,Michealshire,Belarus,(371)987-8576x4720,071-152-1376,stacey67@fields.org,2022-02-20,https://sanford-wilcox.biz/
+28,b92EBfdF8a3f0E6,Frederick,Harper,"Hinton, Chaney and Stokes",South Marissatown,Switzerland,+1-077-121-1558x0687,264.742.7149,jacobkhan@bright.biz,2022-05-26,https://callahan.org/
+29,3B5dAAFA41AFa22,Stefanie,Fitzpatrick,Santana-Duran,Acevedoville,Saint Vincent and the Grenadines,(752)776-3286,+1-472-021-4814x85074,wterrell@clark.com,2020-07-30,https://meyers.com/
+30,EDA69ca7a6e96a2,Kent,Bradshaw,Sawyer PLC,North Harold,Tanzania,+1-472-143-5037x884,126.922.6153,qjimenez@boyd.com,2020-04-26,http://maynard-ho.com/
+31,64DCcDFaB9DFd4e,Jack,Tate,"Acosta, Petersen and Morrow",West Samuel,Zimbabwe,965-108-4406x20714,046.906.1442x6784,gfigueroa@boone-zavala.com,2021-09-15,http://www.hawkins-ramsey.com/
+32,679c6c83DD872d6,Tom,Trujillo,Mcgee Group,Cunninghamborough,Denmark,416-338-3758,(775)890-7209,tapiagreg@beard.info,2022-01-13,http://www.daniels-klein.com/
+33,7Ce381e4Afa4ba9,Gabriel,Mejia,Adkins-Salinas,Port Annatown,Liechtenstein,4077245425,646.044.0696x66800,coleolson@jennings.net,2021-04-24,https://patel-hanson.info/
+34,A09AEc6E3bF70eE,Kaitlyn,Santana,Herrera Group,New Kaitlyn,United States of America,6303643286,447-710-6202x07313,georgeross@miles.org,2021-09-21,http://pham.com/
+35,aA9BAFfBc3710fe,Faith,Moon,"Waters, Chase and Aguilar",West Marthaburgh,Bahamas,+1-586-217-0359x6317,+1-818-199-1403,willistonya@randolph-baker.com,2021-11-03,https://spencer-charles.info/
+36,E11dfb2DB8C9f72,Tammie,Haley,"Palmer, Barnes and Houston",East Teresa,Belize,001-276-734-4113x6087,(430)300-8770,harrisisaiah@jenkins.com,2022-01-04,http://evans-simon.com/
+37,889eCf90f68c5Da,Nicholas,Sosa,Jordan Ltd,South Hunter,Uruguay,(661)425-6042,975-998-1519,fwolfe@dorsey.com,2021-08-10,https://www.fleming-richards.com/
+38,7a1Ee69F4fF4B4D,Jordan,Gay,Glover and Sons,South Walter,Solomon Islands,7208417020,8035336772,tiffanydavies@harris-mcfarland.org,2021-02-24,http://www.lee.org/
+39,dca4f1D0A0fc5c9,Bruce,Esparza,Huerta-Mclean,Poolefurt,Montenegro,559-529-4424,001-625-000-7132x0367,preese@frye-vega.com,2021-10-22,http://www.farley.org/
+40,17aD8e2dB3df03D,Sherry,Garza,Anderson Ltd,West John,Poland,001-067-713-6440x158,(978)289-8785x5766,ann48@miller.com,2021-11-01,http://spence.com/
+41,2f79Cd309624Abb,Natalie,Gentry,Monroe PLC,West Darius,Dominican Republic,830.996.8238,499.122.5415,tcummings@fitzpatrick-ashley.com,2020-10-10,http://www.dorsey.biz/
+42,6e5ad5a5e2bB5Ca,Bryan,Dunn,Kaufman and Sons,North Jimstad,Burkina Faso,001-710-802-5565,078.699.8982x13881,woodwardandres@phelps.com,2021-09-08,http://www.butler.com/
+43,7E441b6B228DBcA,Wayne,Simpson,Perkins-Trevino,East Rebekahborough,Bolivia,(344)156-8632x1869,463-445-3702x38463,barbarapittman@holder.com,2020-12-13,https://gillespie-holder.com/
+44,D3fC11A9C235Dc6,Luis,Greer,Cross PLC,North Drew,Bulgaria,001-336-025-6849x701,684.698.2911x6092,bstuart@williamson-mcclure.com,2022-05-15,https://fletcher-nielsen.com/
+45,30Dfa48fe5Ede78,Rhonda,Frost,"Herrera, Shepherd and Underwood",Lake Lindaburgh,Monaco,(127)081-9339,+1-431-028-3337x3492,zkrueger@wolf-chavez.net,2021-12-06,http://www.khan.com/
+46,fD780ED8dbEae7B,Joanne,Montes,"Price, Sexton and Mcdaniel",Gwendolynview,Palau,(897)726-7952,(467)886-9467x5721,juan80@henson.net,2020-07-01,http://ochoa.com/
+47,300A40d3ce24bBA,Geoffrey,Guzman,Short-Wiggins,Zimmermanland,Uzbekistan,975.235.8921x269,(983)188-6873,bauercrystal@gay.com,2020-04-23,https://decker-kline.com/
+48,283DFCD0Dba40aF,Gloria,Mccall,"Brennan, Acosta and Ramos",North Kerriton,Ghana,445-603-6729,001-395-959-4736x4524,bartlettjenna@zuniga-moss.biz,2022-03-11,http://burgess-frank.com/
+49,F4Fc91fEAEad286,Brady,Cohen,Osborne-Erickson,North Eileenville,United Arab Emirates,741.849.0139x524,+1-028-691-7497x0894,mccalltyrone@durham-rose.biz,2022-03-10,http://hammond-barron.com/
+50,80F33Fd2AcebF05,Latoya,Mccann,"Hobbs, Garrett and Sanford",Port Sergiofort,Belarus,(530)287-4548x29481,162-234-0249x32790,bobhammond@barry.biz,2021-12-02,https://www.burton.com/
+51,Aa20BDe68eAb0e9,Gerald,Hawkins,"Phelps, Forbes and Koch",New Alberttown,Canada,+1-323-239-1456x96168,(092)508-0269,uwarner@steele-arias.com,2021-03-19,https://valenzuela.com/
+52,e898eEB1B9FE22b,Samuel,Crawford,"May, Goodwin and Martin",South Jasmine,Algeria,802-242-7457,626.116.9535x8578,xpittman@ritter-carney.net,2021-03-27,https://guerrero.org/
+53,faCEF517ae7D8eB,Patricia,Goodwin,"Christian, Winters and Ellis",Cowanfort,Swaziland,322.549.7139x70040,(111)741-4173,vaughanchristy@lara.biz,2021-03-08,http://clark.info/
+54,c09952De6Cda8aA,Stacie,Richard,Byrd Inc,New Deborah,Madagascar,001-622-948-3641x24810,001-731-168-2893x8891,clinton85@colon-arias.org,2020-10-15,https://kim.com/
+55,f3BEf3Be028166f,Robin,West,"Nixon, Blackwell and Sosa",Wallstown,Ecuador,698.303.4267,001-683-837-7651x525,greenemiranda@zimmerman.com,2022-01-13,https://www.mora.com/
+56,C6F2Fc6a7948a4e,Ralph,Haas,Montes PLC,Lake Ellenchester,Palestinian Territory,2239271999,001-962-434-0867x649,goodmancesar@figueroa.biz,2020-05-25,http://may.com/
+57,c8FE57cBBdCDcb2,Phyllis,Maldonado,Costa PLC,Lake Whitney,Saint Barthelemy,4500370767,001-508-064-6725x017,yhanson@warner-diaz.org,2021-01-25,http://www.bernard.com/
+58,B5acdFC982124F2,Danny,Parrish,Novak LLC,East Jaredbury,United Arab Emirates,(669)384-8597x8794,506.731.5952x571,howelldarren@house-cohen.com,2021-03-17,http://www.parsons-hudson.com/
+59,8c7DdF10798bCC3,Kathy,Hill,"Moore, Mccoy and Glass",Selenabury,South Georgia and the South Sandwich Islands,001-171-716-2175x310,888.625.0654,ncamacho@boone-simmons.org,2020-11-15,http://hayden.com/
+60,C681dDd0cc422f7,Kelli,Hardy,Petty Ltd,Huangfort,Sao Tome and Principe,020.324.2191x2022,424-157-8216,kristopher62@oliver.com,2020-12-20,http://www.kidd.com/
+61,a940cE42e035F28,Lynn,Pham,"Brennan, Camacho and Tapia",East Pennyshire,Portugal,846.468.6834x611,001-248-691-0006,mpham@rios-guzman.com,2020-08-21,https://www.murphy.com/
+62,9Cf5E6AFE0aeBfd,Shelley,Harris,"Prince, Malone and Pugh",Port Jasminborough,Togo,423.098.0315x8373,+1-386-458-8944x15194,zachary96@mitchell-bryant.org,2020-12-10,https://www.ryan.com/
+63,aEcbe5365BbC67D,Eddie,Jimenez,Caldwell Group,West Kristine,Ethiopia,+1-235-657-1073x6306,(026)401-7353x2417,kristiwhitney@bernard.com,2022-03-24,http://cherry.com/
+64,FCBdfCEAe20A8Dc,Chloe,Hutchinson,Simon LLC,South Julia,Netherlands,981-544-9452,+1-288-552-4666x060,leah85@sutton-terrell.com,2022-05-15,https://mitchell.info/
+65,636cBF0835E10ff,Eileen,Lynch,"Knight, Abbott and Hubbard",Helenborough,Liberia,+1-158-951-4131x53578,001-673-779-6713x680,levigiles@vincent.com,2021-01-02,http://mckay.com/
+66,fF1b6c9E8Fbf1ff,Fernando,Lambert,Church-Banks,Lake Nancy,Lithuania,497.829.9038,3863743398,fisherlinda@schaefer.net,2021-04-23,https://www.vang.com/
+67,2A13F74EAa7DA6c,Makayla,Cannon,Henderson Inc,Georgeport,New Caledonia,001-215-801-6392x46009,027-609-6460,scottcurtis@hurley.biz,2020-01-20,http://www.velazquez.net/
+68,a014Ec1b9FccC1E,Tom,Alvarado,Donaldson-Dougherty,South Sophiaberg,Kiribati,(585)606-2980x2258,730-797-3594x5614,nicholsonnina@montgomery.info,2020-08-18,http://odom-massey.com/
+69,421a109cABDf5fa,Virginia,Dudley,Warren Ltd,Hartbury,French Southern Territories,027.846.3705x14184,+1-439-171-1846x4636,zvalencia@phelps.com,2021-01-31,http://hunter-esparza.com/
+70,CC68FD1D3Bbbf22,Riley,Good,Wade PLC,Erikaville,Canada,6977745822,855-436-7641,alex06@galloway.com,2020-02-03,http://conway.org/
+71,CBCd2Ac8E3eBDF9,Alexandria,Buck,Keller-Coffey,Nicolasfort,Iran,078-900-4760x76668,414-112-8700x68751,lee48@manning.com,2021-02-20,https://ramsey.org/
+72,Ef859092FbEcC07,Richard,Roth,Conway-Mcbride,New Jasmineshire,Morocco,581-440-6539,9857827463,aharper@maddox-townsend.org,2020-02-23,https://www.brooks.com/
+73,F560f2d3cDFb618,Candice,Keller,Huynh and Sons,East Summerstad,Zimbabwe,001-927-965-8550x92406,001-243-038-4271x53076,buckleycory@odonnell.net,2020-08-22,https://www.lucero.com/
+74,A3F76Be153Df4a3,Anita,Benson,Parrish Ltd,Skinnerport,Russian Federation,874.617.5668x69878,(399)820-6418x0071,angie04@oconnell.com,2020-02-09,http://oconnor.com/
+75,D01Af0AF7cBbFeA,Regina,Stein,Guzman-Brown,Raystad,Solomon Islands,001-469-848-0724x4407,001-085-360-4426x00357,zrosario@rojas-hardin.net,2022-01-15,http://www.johnston.info/
+76,d40e89dCade7b2F,Debra,Riddle,"Chang, Aguirre and Leblanc",Colinhaven,United States Virgin Islands,+1-768-182-6014x14336,(303)961-4491,shieldskerry@robles.com,2020-07-11,http://kaiser.info/
+77,BF6a1f9bd1bf8DE,Brittany,Zuniga,Mason-Hester,West Reginald,Kyrgyz Republic,(050)136-9025,001-480-851-2496x0157,mchandler@cochran-huerta.org,2021-07-24,http://www.boyle.com/
+78,FfaeFFbbbf280db,Cassidy,Mcmahon,"Mcguire, Huynh and Hopkins",Lake Sherryborough,Myanmar,5040771311,684-682-0021x1326,katrinalane@fitzgerald.com,2020-10-21,https://hurst.com/
+79,CbAE1d1e9a8dCb1,Laurie,Pennington,"Sanchez, Marsh and Hale",Port Katherineville,Dominica,007.155.3406x553,+1-809-862-5566x277,cookejill@powell.com,2020-06-08,http://www.hebert.com/
+80,A7F85c1DE4dB87f,Alejandro,Blair,"Combs, Waller and Durham",Thomasland,Iceland,(690)068-4641x51468,555.509.8691x2329,elizabethbarr@ewing.com,2020-09-19,https://mercado-blevins.com/
+81,D6CEAfb3BDbaa1A,Leslie,Jennings,Blankenship-Arias,Coreybury,Micronesia,629.198.6346,075.256.0829,corey75@wiggins.com,2021-11-13,https://www.juarez.com/
+82,Ebdb6F6F7c90b69,Kathleen,Mckay,"Coffey, Lamb and Johnson",Lake Janiceton,Saint Vincent and the Grenadines,(733)910-9968,(691)247-4128x0665,chloelester@higgins-wilkinson.com,2021-09-12,http://www.owens-mooney.com/
+83,E8E7e8Cfe516ef0,Hunter,Moreno,Fitzpatrick-Lawrence,East Clinton,Isle of Man,(733)833-6754,001-761-013-7121,isaac26@benton-finley.com,2020-12-28,http://walls.info/
+84,78C06E9b6B3DF20,Chad,Davidson,Garcia-Jimenez,South Joshuashire,Oman,8275702958,(804)842-4715,justinwalters@jimenez.com,2021-11-15,http://www.garner-oliver.com/
+85,03A1E62ADdeb31c,Corey,Holt,"Mcdonald, Bird and Ramirez",New Glenda,Fiji,001-439-242-4986x7918,3162708934,maurice46@morgan.com,2020-02-18,http://www.watson.com/
+86,C6763c99d0bd16D,Emma,Cunningham,Stephens Inc,North Jillianview,New Zealand,128-059-0206x60217,(312)164-4545x2284,walter83@juarez.org,2022-05-13,http://www.reid.info/
+87,ebe77E5Bf9476CE,Duane,Woods,Montoya-Miller,Lyonsberg,Maldives,(636)544-7783x7288,(203)287-1003x5932,kmercer@wagner.com,2020-07-21,http://murray.org/
+88,E4Bbcd8AD81fC5f,Alison,Vargas,"Vaughn, Watts and Leach",East Cristinabury,Benin,365-273-8144,053-308-7653x6287,vcantu@norton.com,2020-11-10,http://mason.info/
+89,efeb73245CDf1fF,Vernon,Kane,Carter-Strickland,Thomasfurt,Yemen,114-854-1159x555,499-608-4612,hilljesse@barrett.info,2021-04-15,http://www.duffy-hensley.net/
+90,37Ec4B395641c1E,Lori,Flowers,Decker-Mcknight,North Joeburgh,Namibia,679.415.1210,945-842-3659x4581,tyrone77@valenzuela.info,2021-01-09,http://www.deleon-crosby.com/
+91,5ef6d3eefdD43bE,Nina,Chavez,Byrd-Campbell,Cassidychester,Bhutan,053-344-3205,+1-330-920-5422x571,elliserica@frank.com,2020-03-26,https://www.pugh.com/
+92,98b3aeDcC3B9FF3,Shane,Foley,Rocha-Hart,South Dannymouth,Hungary,+1-822-569-0302,001-626-114-5844x55073,nsteele@sparks.com,2021-07-06,https://www.holt-sparks.com/
+93,aAb6AFc7AfD0fF3,Collin,Ayers,Lamb-Peterson,South Lonnie,Anguilla,404-645-5351x012,001-257-582-8850x8516,dudleyemily@gonzales.biz,2021-06-29,http://www.ruiz.com/
+94,54B5B5Fe9F1B6C5,Sherry,Young,"Lee, Lucero and Johnson",Frankchester,Solomon Islands,158-687-1764,(438)375-6207x003,alan79@gates-mclaughlin.com,2021-04-04,https://travis.net/
+95,BE91A0bdcA49Bbc,Darrell,Douglas,"Newton, Petersen and Mathis",Daisyborough,Mali,001-084-845-9524x1777,001-769-564-6303,grayjean@lowery-good.com,2022-02-17,https://banks.biz/
+96,cb8E23e48d22Eae,Karl,Greer,Carey LLC,East Richard,Guyana,(188)169-1674x58692,001-841-293-3519x614,hhart@jensen.com,2022-01-30,http://hayes-perez.com/
+97,CeD220bdAaCfaDf,Lynn,Atkinson,"Ware, Burns and Oneal",New Bradview,Sri Lanka,+1-846-706-2218,605.413.3198,vkemp@ferrell.com,2021-07-10,https://novak-allison.com/
+98,28CDbC0dFe4b1Db,Fred,Guerra,Schmitt-Jones,Ortegaland,Solomon Islands,+1-753-067-8419x7170,+1-632-666-7507x92121,swagner@kane.org,2021-09-18,https://www.ross.com/
+99,c23d1D9EE8DEB0A,Yvonne,Farmer,Fitzgerald-Harrell,Lake Elijahview,Aruba,(530)311-9786,001-869-452-0943x12424,mccarthystephen@horn-green.biz,2021-08-11,http://watkins.info/
+100,2354a0E336A91A1,Clarence,Haynes,"Le, Nash and Cross",Judymouth,Honduras,(753)813-6941,783.639.1472,colleen91@faulkner.biz,2020-03-11,http://www.hatfield-saunders.net/

ingestion/tests/integration/kafka/data/organizations-100.csv

@@ -0,0 +1,101 @@
+Index,Organization Id,Name,Website,Country,Description,Founded,Industry,Number of employees
+1,FAB0d41d5b5d22c,Ferrell LLC,https://price.net/,Papua New Guinea,Horizontal empowering knowledgebase,1990,Plastics,3498
+2,6A7EdDEA9FaDC52,"Mckinney, Riley and Day",http://www.hall-buchanan.info/,Finland,User-centric system-worthy leverage,2015,Glass / Ceramics / Concrete,4952
+3,0bFED1ADAE4bcC1,Hester Ltd,http://sullivan-reed.com/,China,Switchable scalable moratorium,1971,Public Safety,5287
+4,2bFC1Be8a4ce42f,Holder-Sellers,https://becker.com/,Turkmenistan,De-engineered systemic artificial intelligence,2004,Automotive,921
+5,9eE8A6a4Eb96C24,Mayer Group,http://www.brewer.com/,Mauritius,Synchronized needs-based challenge,1991,Transportation,7870
+6,cC757116fe1C085,Henry-Thompson,http://morse.net/,Bahamas,Face-to-face well-modulated customer loyalty,1992,Primary / Secondary Education,4914
+7,219233e8aFF1BC3,Hansen-Everett,https://www.kidd.org/,Pakistan,Seamless disintermediate collaboration,2018,Publishing Industry,7832
+8,ccc93DCF81a31CD,Mcintosh-Mora,https://www.brooks.com/,Heard Island and McDonald Islands,Centralized attitude-oriented capability,1970,Import / Export,4389
+9,0B4F93aA06ED03e,Carr Inc,http://ross.com/,Kuwait,Distributed impactful customer loyalty,1996,Plastics,8167
+10,738b5aDe6B1C6A5,Gaines Inc,http://sandoval-hooper.com/,Uzbekistan,Multi-lateral scalable protocol,1997,Outsourcing / Offshoring,9698
+11,AE61b8Ffebbc476,Kidd Group,http://www.lyons.com/,Bouvet Island (Bouvetoya),Proactive foreground paradigm,2001,Primary / Secondary Education,7473
+12,eb3B7D06cCdD609,Crane-Clarke,https://www.sandoval.com/,Denmark,Front-line clear-thinking encryption,2014,Food / Beverages,9011
+13,8D0c29189C9798B,"Keller, Campos and Black",https://www.garner.info/,Liberia,Ameliorated directional emulation,2020,Museums / Institutions,2862
+14,D2c91cc03CA394c,Glover-Pope,http://www.silva.biz/,United Arab Emirates,Persevering contextually-based approach,2013,Medical Practice,9079
+15,C8AC1eaf9C036F4,Pacheco-Spears,https://aguilar.com/,Sweden,Secured logistical synergy,1984,Maritime,769
+16,b5D10A14f7a8AfE,Hodge-Ayers,http://www.archer-elliott.com/,Honduras,Future-proofed radical implementation,1990,Facilities Services,8508
+17,68139b5C4De03B4,"Bowers, Guerra and Krause",http://www.carrillo-nicholson.com/,Uganda,De-engineered transitional strategy,1972,Primary / Secondary Education,6986
+18,5c2EffEfdba2BdF,Mckenzie-Melton,http://montoya-thompson.com/,Hong Kong,Reverse-engineered heuristic alliance,1998,Investment Management / Hedge Fund / Private Equity,4589
+19,ba179F19F7925f5,Branch-Mann,http://www.lozano.com/,Botswana,Adaptive intangible frame,1999,Architecture / Planning,7961
+20,c1Ce9B350BAc66b,Weiss and Sons,https://barrett.com/,Korea,Sharable optimal functionalities,2011,Plastics,5984
+21,8de40AC4e6EaCa4,"Velez, Payne and Coffey",http://burton.com/,Luxembourg,Mandatory coherent synergy,1986,Wholesale,5010
+22,Aad86a4F0385F2d,Harrell LLC,http://www.frey-rosario.com/,Guadeloupe,Reverse-engineered mission-critical moratorium,2018,Construction,2185
+23,22aC3FFd64fD703,"Eaton, Reynolds and Vargas",http://www.freeman.biz/,Monaco,Self-enabling multi-tasking process improvement,2014,Luxury Goods / Jewelry,8987
+24,5Ec4C272bCf085c,Robbins-Cummings,http://donaldson-wilkins.com/,Belgium,Organic non-volatile hierarchy,1991,Pharmaceuticals,5038
+25,5fDBeA8BB91a000,Jenkins Inc,http://www.kirk.biz/,South Africa,Front-line systematic help-desk,2002,Insurance,1215
+26,dFfD6a6F9AC2d9C,"Greene, Benjamin and Novak",http://www.kent.net/,Romania,Centralized leadingedge moratorium,2012,Museums / Institutions,4941
+27,4B217cC5a0674C5,"Dickson, Richmond and Clay",http://everett.com/,Czech Republic,Team-oriented tangible complexity,1980,Real Estate / Mortgage,3122
+28,88b1f1cDcf59a37,Prince-David,http://thompson.com/,Christmas Island,Virtual holistic methodology,1970,Banking / Mortgage,1046
+29,f9F7bBCAEeC360F,Ayala LLC,http://www.zhang.com/,Philippines,Open-source zero administration hierarchy,2021,Legal Services,7664
+30,7Cb3AeFcE4Ba31e,Rivas Group,https://hebert.org/,Australia,Open-architected well-modulated capacity,1998,Logistics / Procurement,4155
+31,ccBcC32adcbc530,"Sloan, Mays and Whitehead",http://lawson.com/,Chad,Face-to-face high-level conglomeration,1997,Civil Engineering,365
+32,f5afd686b3d05F5,"Durham, Allen and Barnes",http://chan-stafford.org/,Zimbabwe,Synergistic web-enabled framework,1993,Mechanical or Industrial Engineering,6135
+33,38C6cfC5074Fa5e,Fritz-Franklin,http://www.lambert.com/,Nepal,Automated 4thgeneration website,1972,Hospitality,4516
+34,5Cd7efccCcba38f,Burch-Ewing,http://cline.net/,Taiwan,User-centric 4thgeneration system engine,1981,Venture Capital / VC,7443
+35,9E6Acb51e3F9d6F,"Glass, Barrera and Turner",https://dunlap.com/,Kyrgyz Republic,Multi-channeled 3rdgeneration open system,2020,Utilities,2610
+36,4D4d7E18321eaeC,Pineda-Cox,http://aguilar.org/,Bolivia,Fundamental asynchronous capability,2010,Human Resources / HR,1312
+37,485f5d06B938F2b,"Baker, Mccann and Macdonald",http://www.anderson-barker.com/,Kenya,Cross-group user-facing focus group,2013,Legislative Office,1638
+38,19E3a5Bf6dBDc4F,Cuevas-Moss,https://dodson-castaneda.net/,Guatemala,Extended human-resource intranet,1994,Music,9995
+39,6883A965c7b68F7,Hahn PLC,http://newman.com/,Belarus,Organic logistical leverage,2012,Electrical / Electronic Manufacturing,3715
+40,AC5B7AA74Aa4A2E,"Valentine, Ferguson and Kramer",http://stuart.net/,Jersey,Centralized secondary time-frame,1997,Non - Profit / Volunteering,3585
+41,decab0D5027CA6a,Arroyo Inc,https://www.turner.com/,Grenada,Managed demand-driven website,2006,Writing / Editing,9067
+42,dF084FbBb613eea,Walls LLC,http://www.reese-vasquez.biz/,Cape Verde,Self-enabling fresh-thinking installation,1989,Investment Management / Hedge Fund / Private Equity,1678
+43,A2D89Ab9bCcAd4e,"Mitchell, Warren and Schneider",https://fox.biz/,Trinidad and Tobago,Enhanced intangible time-frame,2021,Capital Markets / Hedge Fund / Private Equity,3816
+44,77aDc905434a49f,Prince PLC,https://www.watts.com/,Sweden,Profit-focused coherent installation,2016,Individual / Family Services,7645
+45,235fdEFE2cfDa5F,Brock-Blackwell,http://www.small.com/,Benin,Secured foreground emulation,1986,Online Publishing,7034
+46,1eD64cFe986BBbE,Walton-Barnett,https://ashley-schaefer.com/,Western Sahara,Right-sized clear-thinking flexibility,2001,Luxury Goods / Jewelry,1746
+47,CbBbFcdd0eaE2cF,Bartlett-Arroyo,https://cruz.com/,Northern Mariana Islands,Realigned didactic function,1976,Civic / Social Organization,3987
+48,49aECbDaE6aBD53,"Wallace, Madden and Morris",http://www.blevins-fernandez.biz/,Germany,Persistent real-time customer loyalty,2016,Pharmaceuticals,9443
+49,7b3fe6e7E72bFa4,Berg-Sparks,https://cisneros-love.com/,Canada,Stand-alone static implementation,1974,Arts / Crafts,2073
+50,c6DedA82A8aef7E,Gonzales Ltd,http://bird.com/,Tonga,Managed human-resource policy,1988,Consumer Goods,9069
+51,7D9FBF85cdC3871,Lawson and Sons,https://www.wong.com/,French Southern Territories,Compatible analyzing intranet,2021,Arts / Crafts,3527
+52,7dd18Fb7cB07b65,"Mcguire, Mcconnell and Olsen",https://melton-briggs.com/,Korea,Profound client-server frame,1988,Printing,8445
+53,EF5B55FadccB8Fe,Charles-Phillips,https://bowman.com/,Cote d'Ivoire,Monitored client-server implementation,2012,Mental Health Care,3450
+54,f8D4B99e11fAF5D,Odom Ltd,https://www.humphrey-hess.com/,Cote d'Ivoire,Advanced static process improvement,2012,Management Consulting,1825
+55,e24D21BFd3bF1E5,Richard PLC,https://holden-coleman.net/,Mayotte,Object-based optimizing model,1971,Broadcast Media,4942
+56,B9BdfEB6D3Ca44E,Sampson Ltd,https://blevins.com/,Cayman Islands,Intuitive local adapter,2005,Farming,1418
+57,2a74D6f3D3B268e,"Cherry, Le and Callahan",https://waller-delacruz.biz/,Nigeria,Universal human-resource collaboration,2017,Entertainment / Movie Production,7202
+58,Bf3F3f62c8aBC33,Cherry PLC,https://www.avila.info/,Marshall Islands,Persistent tertiary website,1980,Plastics,8245
+59,aeBe26B80a7a23c,Melton-Nichols,https://kennedy.com/,Palau,User-friendly clear-thinking productivity,2021,Legislative Office,8741
+60,aAeb29ad43886C6,Potter-Walsh,http://thomas-french.org/,Turkey,Optional non-volatile open system,2008,Human Resources / HR,6923
+61,bD1bc6bB6d1FeD3,Freeman-Chen,https://mathis.com/,Timor-Leste,Phased next generation adapter,1973,International Trade / Development,346
+62,EB9f456e8b7022a,Soto Group,https://norris.info/,Vietnam,Enterprise-wide executive installation,1988,Business Supplies / Equipment,9097
+63,Dfef38C51D8DAe3,"Poole, Cruz and Whitney",https://reed.info/,Reunion,Balanced analyzing groupware,1978,Marketing / Advertising / Sales,2992
+64,055ffEfB2Dd95B0,Riley Ltd,http://wiley.com/,Brazil,Optional exuding superstructure,1986,Textiles,9315
+65,cBfe4dbAE1699da,"Erickson, Andrews and Bailey",https://www.hobbs-grant.com/,Eritrea,Vision-oriented secondary project,2014,Consumer Electronics,7829
+66,fdFbecbadcdCdf1,"Wilkinson, Charles and Arroyo",http://hunter-mcfarland.com/,United States Virgin Islands,Assimilated 24/7 archive,1996,Building Materials,602
+67,5DCb8A5a5ca03c0,Floyd Ltd,http://www.whitney.com/,Falkland Islands (Malvinas),Function-based fault-tolerant concept,2017,Public Relations / PR,2911
+68,ce57DCbcFD6d618,Newman-Galloway,https://www.scott.com/,Luxembourg,Enhanced foreground collaboration,1987,Information Technology / IT,3934
+69,5aaD187dc929371,Frazier-Butler,https://www.daugherty-farley.info/,Northern Mariana Islands,Persistent interactive circuit,1972,Outsourcing / Offshoring,5130
+70,902D7Ac8b6d476b,Newton Inc,https://www.richmond-manning.info/,Netherlands Antilles,Fundamental stable info-mediaries,1976,Military Industry,563
+71,32BB9Ff4d939788,Duffy-Levy,https://www.potter.com/,Guernsey,Diverse exuding installation,1982,Wireless,6146
+72,adcB0afbE58bAe3,Wagner LLC,https://decker-esparza.com/,Uruguay,Reactive attitude-oriented toolset,1987,International Affairs,6874
+73,dfcA1c84AdB61Ac,Mccall-Holmes,http://www.dean.com/,Benin,Object-based value-added database,2009,Legal Services,696
+74,208044AC2fe52F3,Massey LLC,https://frazier.biz/,Suriname,Configurable zero administration Graphical User Interface,1986,Accounting,5004
+75,f3C365f0c1A0623,Hicks LLC,http://alvarez.biz/,Pakistan,Quality-focused client-server Graphical User Interface,1970,Computer Software / Engineering,8480
+76,ec5Bdd3CBAfaB93,"Cole, Russell and Avery",http://www.blankenship.com/,Mongolia,De-engineered fault-tolerant challenge,2000,Law Enforcement,7012
+77,DDB19Be7eeB56B4,Cummings-Rojas,https://simon-pearson.com/,Svalbard & Jan Mayen Islands,User-centric modular customer loyalty,2012,Financial Services,7529
+78,dd6CA3d0bc3cAfc,"Beasley, Greene and Mahoney",http://www.petersen-lawrence.com/,Togo,Extended content-based methodology,1976,Religious Institutions,869
+79,A0B9d56e61070e3,"Beasley, Sims and Allison",http://burke.info/,Latvia,Secured zero tolerance hub,1972,Facilities Services,6182
+80,cBa7EFe5D05Adaf,Crawford-Rivera,https://black-ramirez.org/,Cuba,Persevering exuding budgetary management,1999,Online Publishing,7805
+81,Ea3f6D52Ec73563,Montes-Hensley,https://krueger.org/,Liechtenstein,Multi-tiered secondary productivity,2009,Printing,8433
+82,bC0CEd48A8000E0,Velazquez-Odom,https://stokes.com/,Djibouti,Streamlined 6thgeneration function,2002,Alternative Dispute Resolution,4044
+83,c89b9b59BC4baa1,Eaton-Morales,https://www.reeves-graham.com/,Micronesia,Customer-focused explicit frame,1990,Capital Markets / Hedge Fund / Private Equity,7013
+84,FEC51bce8421a7b,"Roberson, Pennington and Palmer",http://www.keith-fisher.com/,Cameroon,Adaptive bi-directional hierarchy,1993,Telecommunications,5571
+85,e0E8e27eAc9CAd5,"George, Russo and Guerra",https://drake.com/,Sweden,Centralized non-volatile capability,1989,Military Industry,2880
+86,B97a6CF9bf5983C,Davila Inc,https://mcconnell.info/,Cocos (Keeling) Islands,Profit-focused dedicated frame,2017,Consumer Electronics,2215
+87,a0a6f9b3DbcBEb5,Mays-Preston,http://www.browning-key.com/,Mali,User-centric heuristic focus group,2006,Military Industry,5786
+88,8cC1bDa330a5871,Pineda-Morton,https://www.carr.com/,United States Virgin Islands,Grass-roots methodical info-mediaries,1991,Printing,6168
+89,ED889CB2FE9cbd3,Huang and Sons,https://www.bolton.com/,Eritrea,Re-contextualized dynamic hierarchy,1981,Semiconductors,7484
+90,F4Dc1417BC6cb8f,Gilbert-Simon,https://www.bradford.biz/,Burundi,Grass-roots radical parallelism,1973,Newspapers / Journalism,1927
+91,7ABc3c7ecA03B34,Sampson-Griffith,http://hendricks.org/,Benin,Multi-layered composite paradigm,1972,Textiles,3881
+92,4e0719FBE38e0aB,Miles-Dominguez,http://www.turner.com/,Gibraltar,Organized empowering forecast,1996,Civic / Social Organization,897
+93,dEbDAAeDfaed00A,Rowe and Sons,https://www.simpson.org/,El Salvador,Balanced multimedia knowledgebase,1978,Facilities Services,8172
+94,61BDeCfeFD0cEF5,"Valenzuela, Holmes and Rowland",https://www.dorsey.net/,Taiwan,Persistent tertiary focus group,1999,Transportation,1483
+95,4e91eD25f486110,"Best, Wade and Shepard",https://zimmerman.com/,Zimbabwe,Innovative background definition,1991,Gambling / Casinos,4873
+96,0a0bfFbBbB8eC7c,Holmes Group,https://mcdowell.org/,Ethiopia,Right-sized zero tolerance focus group,1975,Photography,2988
+97,BA6Cd9Dae2Efd62,Good Ltd,http://duffy.com/,Anguilla,Reverse-engineered composite moratorium,1971,Consumer Services,4292
+98,E7df80C60Abd7f9,Clements-Espinoza,http://www.flowers.net/,Falkland Islands (Malvinas),Progressive modular hub,1991,Broadcast Media,236
+99,AFc285dbE2fEd24,Mendez Inc,https://www.burke.net/,Kyrgyz Republic,User-friendly exuding migration,1993,Education Management,339
+100,e9eB5A60Cef8354,Watkins-Kaiser,http://www.herring.com/,Togo,Synergistic background access,2009,Financial Services,2785

ingestion/tests/integration/kafka/data/people-100.csv

@@ -0,0 +1,101 @@
+Index,User Id,First Name,Last Name,Sex,Email,Phone,Date of birth,Job Title
+1,88F7B33d2bcf9f5,Shelby,Terrell,Male,elijah57@example.net,001-084-906-7849x73518,1945-10-26,Games developer
+2,f90cD3E76f1A9b9,Phillip,Summers,Female,bethany14@example.com,214.112.6044x4913,1910-03-24,Phytotherapist
+3,DbeAb8CcdfeFC2c,Kristine,Travis,Male,bthompson@example.com,277.609.7938,1992-07-02,Homeopath
+4,A31Bee3c201ef58,Yesenia,Martinez,Male,kaitlinkaiser@example.com,584.094.6111,2017-08-03,Market researcher
+5,1bA7A3dc874da3c,Lori,Todd,Male,buchananmanuel@example.net,689-207-3558x7233,1938-12-01,Veterinary surgeon
+6,bfDD7CDEF5D865B,Erin,Day,Male,tconner@example.org,001-171-649-9856x5553,2015-10-28,Waste management officer
+7,bE9EEf34cB72AF7,Katherine,Buck,Female,conniecowan@example.com,+1-773-151-6685x49162,1989-01-22,Intelligence analyst
+8,2EFC6A4e77FaEaC,Ricardo,Hinton,Male,wyattbishop@example.com,001-447-699-7998x88612,1924-03-26,Hydrogeologist
+9,baDcC4DeefD8dEB,Dave,Farrell,Male,nmccann@example.net,603-428-2429x27392,2018-10-06,Lawyer
+10,8e4FB470FE19bF0,Isaiah,Downs,Male,virginiaterrell@example.org,+1-511-372-1544x8206,1964-09-20,"Engineer, site"
+11,BF0BbA03C29Bb3b,Sheila,Ross,Female,huangcathy@example.com,895.881.4746,2008-03-20,Advertising account executive
+12,F738c69fB34E62E,Stacy,Newton,Male,rayleroy@example.org,710.673.3213x80335,1980-10-20,Warden/ranger
+13,C03fDADdAadAdCe,Mandy,Blake,Male,jefferynoble@example.org,(992)466-1305x4947,2007-12-08,"Scientist, clinical (histocompatibility and immunogenetics)"
+14,b759b74BD1dE80d,Bridget,Nash,Female,mercedes44@example.com,(216)627-8359,2004-06-28,Social worker
+15,1F0B7D65A00DAF9,Crystal,Farmer,Male,pmiranda@example.org,+1-024-377-5391,1992-03-09,Agricultural consultant
+16,50Bb061cB30B461,Thomas,Knight,Female,braunpriscilla@example.net,+1-360-880-0766,2006-02-18,Sport and exercise psychologist
+17,D6dbA5308fEC4BC,Maurice,Rangel,Male,sheenabanks@example.com,(246)187-4969,2004-08-20,Secretary/administrator
+18,311D775990f066d,Frank,Meadows,Male,gbrewer@example.org,429.965.3902x4447,2008-09-16,Audiological scientist
+19,7F7E1BAcb0C9AFf,Alvin,Paul,Male,gilbertdonaldson@example.com,219.436.0887x07551,1949-05-12,"Teacher, adult education"
+20,88473e15D5c3cD0,Jared,Mitchell,Female,jcortez@example.com,+1-958-849-6781,1921-01-18,Paediatric nurse
+21,b31D271F8c200AB,Jacqueline,Norton,Female,carias@example.net,819.309.7679x59173,1952-10-09,"Scientist, marine"
+22,42F4BdA841aBadC,Colleen,Hatfield,Female,fknox@example.org,638.584.1090,1949-10-14,Commercial horticulturist
+23,cBbBcA0FCA3C4Bc,Randy,Barnes,Male,huangbill@example.org,001-960-629-7164x67214,1947-12-30,Outdoor activities/education manager
+24,f1f89173353aD90,Janice,Rhodes,Female,juarezdominique@example.net,001-249-314-9742x6996,1999-11-01,Drilling engineer
+25,c5B09fb33e8bA0A,Alfred,Mcneil,Female,cassandramorris@example.com,(468)276-9509x53058,1993-05-28,Systems analyst
+26,c9F2282C40BEC1E,Sean,Levine,Male,sallymiller@example.net,4915828504,2010-10-09,"Conservation officer, nature"
+27,9c1bc7EC53Fb7cE,Louis,Payne,Male,bsullivan@example.net,6232695307,1916-01-29,Counsellor
+28,ddEc50e2A2e3a2B,Brittney,Vega,Female,ayalajose@example.net,945-739-8686,1932-10-31,Recycling officer
+29,66F096D36Ebae11,Judy,Buckley,Male,irosales@example.net,001-654-208-1241x52830,1963-07-28,Art gallery manager
+30,F0fE2faAd78F8b5,Norman,Weber,Male,mconrad@example.com,223.002.0429,1957-05-21,Gaffer
+31,5d2feAfbdCAA6B5,Isaiah,Camacho,Female,jimblake@example.org,001-536-544-3367,1966-04-07,Food technologist
+32,cDa5F303fCd6dEa,Jacqueline,Gallagher,Male,nsampson@example.net,(247)762-8934,1999-02-25,Building services engineer
+33,8Ef7DBfcaB02b6B,Bonnie,Andrews,Female,caitlin24@example.net,+1-253-987-2776x9161,1953-12-21,Seismic interpreter
+34,6Dec5b5542F8ed8,Brandon,Schmidt,Female,mconley@example.net,+1-386-673-1465x006,1931-05-12,"Engineer, biomedical"
+35,3Fb8a7f68e12784,Jackson,Sparks,Female,reynoldsdarryl@example.net,(137)908-3129x65035,1980-11-18,Set designer
+36,035eff50B9A0F24,Melody,Cook,Male,jeannovak@example.org,(826)792-7381,1963-06-25,Research scientist (life sciences)
+37,aa614aAE4B7Cf0C,Leonard,Hurst,Male,clinton78@example.org,941-038-0427x38800,1938-03-13,"Accountant, chartered management"
+38,ACcde95AAe3e6cC,Gene,Rich,Female,luisdeleon@example.org,+1-356-818-6604x89537,1946-08-22,"Surveyor, quantity"
+39,b6a35de5CB6fc25,Cynthia,Wiggins,Female,rosariodave@example.org,(110)858-2437x70190,1984-01-27,Outdoor activities/education manager
+40,e92A191E345fA3A,Tanya,Mckinney,Female,vickihouston@example.com,(830)774-9002x086,2003-03-12,Information systems manager
+41,7D0AcBF6CCac3fd,Matthew,Stone,Female,evelyn31@example.org,952-381-6360,2017-08-23,"Scientist, clinical (histocompatibility and immunogenetics)"
+42,CEFA7BBCef013AE,Kirk,Walsh,Female,stephenfuller@example.org,001-826-496-5529x8661,2009-04-08,Accounting technician
+43,9edBC94aE7cA22a,Willie,Vang,Female,haleymathews@example.net,741.168.6854x067,1978-02-02,Management consultant
+44,fFe7BAA737aDbe2,Miguel,Hill,Female,tyrone56@example.org,5247842945,1930-08-26,Make
+45,5F2f3fAca8B0946,Darren,Andrews,Male,lhernandez@example.com,(975)799-4261,1997-10-04,Retail banker
+46,6bFcfc3cc1BC6B4,Haley,Pugh,Female,molly03@example.org,(746)182-6137x2453,1980-09-16,Commissioning editor
+47,f3BD2cBF7eEb6df,Danielle,Estrada,Female,jvang@example.org,(890)374-9518x772,1930-07-09,"Accountant, chartered management"
+48,Ee4eB129dC7913A,Becky,Brady,Male,erikmueller@example.org,(390)002-0863,1957-06-27,Seismic interpreter
+49,dBCEf340C3657Eb,Caitlyn,Frey,Male,rivasdominique@example.org,805-021-3965x46344,1968-01-26,Jewellery designer
+50,E47FB71DD9ACCd9,Joshua,Sweeney,Male,daisymcgee@example.net,875.994.2100x535,1954-07-28,"Education officer, museum"
+51,eA3fDd79BE9f0E7,Heidi,Escobar,Female,staffordtravis@example.net,601-155-3065x1131,1931-09-25,Estate manager/land agent
+52,aF0eE4547Bc025c,Brian,Oconnell,Female,saralong@example.net,952-283-1423x733,1911-10-23,Physiotherapist
+53,9F5DeD7aD228F5a,Beverly,Esparza,Female,iphelps@example.net,+1-327-578-8754x6771,1930-12-09,Passenger transport manager
+54,D3Fa0220dDE4d36,Nathaniel,Rivas,Female,roberto29@example.com,(655)887-2040x37888,1908-11-17,Call centre manager
+55,60FdBFd5e7BE8fF,Debra,Payne,Female,yolanda07@example.org,001-731-525-8400x52593,1927-08-20,Special educational needs teacher
+56,D8bF5Ab2b98caff,Mackenzie,Rocha,Female,abbottyvette@example.net,4225525458,1980-10-21,Museum/gallery exhibitions officer
+57,CD8d33aA25bc8BB,Courtney,Watkins,Female,ochang@example.org,210.683.2761x5883,2003-12-07,Pension scheme manager
+58,Fac3BfFf0A3d03c,Fred,Olsen,Female,amyanderson@example.com,497-774-3053,1910-04-10,Archaeologist
+59,e552D7ddafe1FFb,Ryan,Nelson,Female,qnorman@example.org,956.330.2951,1924-05-02,Historic buildings inspector/conservation officer
+60,0f8deedb629A5f6,Grace,Phelps,Male,clarkeangela@example.net,(034)867-8827x6777,1909-10-15,Petroleum engineer
+61,bB9e49E506F65ed,Shari,Daugherty,Male,kalvarado@example.org,001-951-655-4798x6124,1944-11-24,Curator
+62,Ed724605A403D91,Kelli,Garner,Male,jodyvincent@example.org,995.000.4213x0982,2010-01-17,Retail banker
+63,0aBE5ACb18E0c10,Jackie,Bennett,Male,hutchinsonkirk@example.com,001-740-937-0846x0087,1915-11-11,Neurosurgeon
+64,5D2cb63CaAF53f6,Leslie,Conway,Female,floreschristina@example.org,795.782.4384x555,1983-11-06,Chiropractor
+65,Ee6974f90eeCe18,Harold,Barnett,Female,nathan65@example.org,+1-026-265-6392,1943-03-15,"Biochemist, clinical"
+66,cEf02C076afa07f,Larry,Harper,Male,maria32@example.org,+1-244-630-3792x4121,2021-05-05,"Scientist, water quality"
+67,9Df5Ba591bF3EFf,Mike,Ward,Female,imccullough@example.com,116-729-5046,1967-11-09,Hydrologist
+68,3faB1CBfEFBDdD4,Brittney,Rubio,Female,corey92@example.com,593.976.2528,1959-12-24,"Biochemist, clinical"
+69,Ebcefdf75eCb0a9,Frank,Pineda,Male,daltoncalvin@example.net,(035)961-5060x9182,1926-03-10,Hospital pharmacist
+70,e75e5DBfcb68887,Sandra,Wu,Male,ubanks@example.com,+1-096-606-6454x067,1925-04-28,Warehouse manager
+71,6a53a8D41dDF6de,Ryan,Benton,Male,lopezdebbie@example.org,+1-695-557-9948x485,2020-10-06,Physiological scientist
+72,F0d3bD1aaf9E3Bc,Tamara,Hull,Male,meagan39@example.net,017.665.3744x7944,1933-01-31,English as a second language teacher
+73,5bC87340799FBD0,Jean,Ritter,Female,kristina76@example.com,(954)060-1066,1985-08-06,Financial trader
+74,dBfA17Aaf16b4ab,Veronica,Briggs,Female,weissbridget@example.com,+1-521-589-2387x48490,1974-06-08,Structural engineer
+75,c935b7Eb6FA0B0F,Kim,Andrews,Female,wpetersen@example.org,7677125383,1990-11-15,"Biochemist, clinical"
+76,b3e15e65Ca2CcBf,Tina,Cunningham,Male,wongmary@example.org,079-907-5051,1956-11-29,Race relations officer
+77,dade3452F0c32FD,Jonathon,Atkinson,Male,gailfrench@example.net,874-037-2032x932,2011-07-19,"Psychologist, forensic"
+78,AdEd6cfD85DeC46,Jermaine,Reid,Female,vpaul@example.com,(742)214-8691,1974-08-18,Newspaper journalist
+79,DAf111987098ae4,Regina,Stevens,Male,xpoole@example.net,891-359-2684,2011-11-28,Public house manager
+80,6e6a5b885F6496d,Terrence,Huff,Male,cassandra80@example.org,221.800.6408x5416,1944-02-27,Careers information officer
+81,12DCb4ED8E01D5C,Tyler,Foley,Female,johnathan72@example.org,001-386-469-3075x8030,1908-09-19,Economist
+82,E1cB5cA8CA7CC0a,Andrew,Waters,Male,nhall@example.net,+1-376-865-2765x3351,1948-05-14,Jewellery designer
+83,AedDfaE8Cf49F07,Reginald,Stephenson,Male,erikaball@example.net,+1-832-500-6044x475,2010-02-08,Contracting civil engineer
+84,bff9853aFAeF772,Douglas,Reese,Female,nixonvanessa@example.net,001-834-660-8312x9864,1961-11-11,Higher education lecturer
+85,E883773cA5219Be,Helen,Williamson,Female,melvin08@example.net,001-377-726-4229,1911-08-11,"Lecturer, further education"
+86,CB19EafEbBfF9eC,Mario,Vaughn,Male,oblake@example.com,160-144-5039x12276,1990-07-08,Research scientist (life sciences)
+87,5834700fbEd2771,Chelsea,Dickson,Male,johnnyhendricks@example.net,001-698-651-0138x18588,1958-05-13,"Teacher, early years/pre"
+88,2b0Ab1Dc9E01D7E,Dustin,Bailey,Male,pbarron@example.net,+1-965-621-1157x345,1908-08-22,Travel agency manager
+89,3f3a3D89ad042Dd,Harry,Medina,Female,olsenmalik@example.net,+1-451-099-5805,1947-08-24,Technical sales engineer
+90,9425E2F38C408ef,Kathy,Haney,Female,teresa37@example.com,(164)105-8456,1955-09-02,Charity fundraiser
+91,F0aeC9c2759F3C6,Alison,Nixon,Female,zmiles@example.net,3506680871,1941-07-10,Patent attorney
+92,d6EA619A7C4aA95,Jamie,Hardy,Female,sheenadouglas@example.com,(900)803-9295x11533,1994-07-17,"Conservator, furniture"
+93,2A33E7Cad1bb0F5,Melody,Cox,Female,evan90@example.org,(626)520-5080x3511,1974-07-30,Dance movement psychotherapist
+94,d181FFB7d3E68bb,Xavier,Cole,Male,nicolas90@example.org,8164259975,1938-11-29,Financial planner
+95,feaBf8dAE0C0d6F,Dillon,Guzman,Female,angelanavarro@example.net,971-992-4521,1942-04-01,Air broker
+96,5eFda7caAeB260E,Dennis,Barnes,Female,bmartin@example.org,001-095-524-2112x257,1954-07-30,Software engineer
+97,CCbFce93d3720bE,Steve,Patterson,Female,latasha46@example.net,001-865-478-5157,1932-04-29,Barrister
+98,2fEc528aFAF0b69,Wesley,Bray,Male,regina11@example.org,995-542-3004x76800,1994-12-28,Police officer
+99,Adc7ad9B6e4A1Fe,Summer,Oconnell,Female,alexiscantrell@example.org,001-273-685-6932x092,2012-04-12,Broadcast journalist
+100,b8D0aD3490FC7e1,Mariah,Bernard,Male,pcopeland@example.org,(341)594-6554x44657,2016-11-15,IT sales professional

ingestion/tests/integration/kafka/test_kafka.py

@@ -1,109 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""
-Test Kafka using the topology
-"""
-
-
-from unittest import TestCase
-
-from confluent_kafka import Producer
-from testcontainers.kafka import KafkaContainer
-
-from metadata.generated.schema.entity.data.topic import Topic
-from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
-    AuthProvider,
-    OpenMetadataConnection,
-)
-from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
-    OpenMetadataJWTClientConfig,
-)
-from metadata.ingestion.models.custom_pydantic import CustomSecretStr
-from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.workflow.metadata import MetadataWorkflow
-
-
-def produce_and_consume_kafka_message(container):
-    topic = "test-topic"
-    bootstrap_server = container.get_bootstrap_server()
-
-    producer = Producer({"bootstrap.servers": bootstrap_server})
-    producer.produce(topic, b"verification message")
-    producer.flush()
-
-
-OM_JWT = "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
-
-
-def get_ingestion_config(port: str):
-    return {
-        "source": {
-            "type": "kafka",
-            "serviceName": "TEST_KAFKA",
-            "serviceConnection": {
-                "config": {
-                    "type": "Kafka",
-                    "bootstrapServers": f"localhost:{port}",
-                }
-            },
-            "sourceConfig": {"config": {"type": "MessagingMetadata"}},
-        },
-        "sink": {"type": "metadata-rest", "config": {}},
-        "workflowConfig": {
-            "loggerLevel": "DEBUG",
-            "openMetadataServerConfig": {
-                "hostPort": "http://localhost:8585/api",
-                "authProvider": "openmetadata",
-                "securityConfig": {
-                    "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
-                },
-            },
-        },
-    }
-
-
-def int_admin_ometa(url: str = "http://localhost:8585/api") -> OpenMetadata:
-    """Initialize the ometa connection with default admin:admin creds"""
-    server_config = OpenMetadataConnection(
-        hostPort=url,
-        authProvider=AuthProvider.openmetadata,
-        securityConfig=OpenMetadataJWTClientConfig(jwtToken=CustomSecretStr(OM_JWT)),
-    )
-    metadata = OpenMetadata(server_config)
-    assert metadata.health_check()
-    return metadata
-
-
-class KafkaUnitTest(TestCase):
-    @classmethod
-    def setUpClass(cls) -> None:
-        cls.metadata = int_admin_ometa()
-        cls.kafka_container = KafkaContainer()
-        cls.kafka_container.start()
-        cls.ingestion_config = get_ingestion_config(
-            cls.kafka_container.get_exposed_port(9093)
-        )
-        produce_and_consume_kafka_message(cls.kafka_container)
-
-        ingestion_workflow = MetadataWorkflow.create(
-            cls.ingestion_config,
-        )
-        ingestion_workflow.execute()
-        ingestion_workflow.raise_from_status()
-        ingestion_workflow.stop()
-
-    def test_topic(self):
-        topic = self.metadata.get_by_name(
-            entity=Topic,
-            fqn="TEST_KAFKA.test-topic",
-        )
-        self.assertIsNotNone(topic)

ingestion/tests/integration/kafka/test_metadata.py

@@ -0,0 +1,37 @@
+import sys
+
+import pytest
+
+from metadata.generated.schema.entity.data.topic import Topic
+from metadata.workflow.metadata import MetadataWorkflow
+
+if not sys.version_info >= (3, 9):
+    pytest.skip("requires python 3.9+", allow_module_level=True)
+
+
+def test_ingest_metadata(
+    patch_passwords_for_db_services, run_workflow, ingestion_config, metadata_assertions
+):
+    run_workflow(MetadataWorkflow, ingestion_config)
+    metadata_assertions()
+
+
+@pytest.fixture(
+    scope="module",
+    params=[
+        "customers-100",
+        "organizations-100",
+        "people-100",
+    ],
+)
+def metadata_assertions(metadata, db_service, request):
+    def _assertions():
+        topic: Topic = metadata.get_by_name(
+            entity=Topic,
+            fqn=f"{db_service.fullyQualifiedName.root}.{request.param}",
+            fields=["*"],
+            nullable=False,
+        )
+        assert topic.messageSchema is not None
+
+    return _assertions