mirror of
https://github.com/datahub-project/datahub.git
synced 2025-10-30 18:26:58 +00:00
70 lines
3.4 KiB
Python
70 lines
3.4 KiB
Python
from gometa.ingestion.sink.datahub_kafka import DatahubKafkaSink, KafkaCallback
|
|
from gometa.ingestion.api.sink import WriteCallback, SinkReport
|
|
|
|
import unittest
|
|
from unittest.mock import patch, MagicMock
|
|
from gometa.ingestion.api.common import RecordEnvelope
|
|
|
|
|
|
class KafkaSinkTest(unittest.TestCase):
|
|
@patch("gometa.ingestion.sink.datahub_kafka.PipelineContext")
|
|
@patch("gometa.ingestion.sink.datahub_kafka.SerializingProducer")
|
|
def test_kafka_sink_config(self, mock_producer, mock_context):
|
|
kafka_sink = DatahubKafkaSink.create(
|
|
{"connection": {"bootstrap": "foobar:9092"}}, mock_context
|
|
)
|
|
kafka_sink.close()
|
|
assert mock_producer.call_count == 1 # constructor should be called
|
|
|
|
def validate_kafka_callback(self, mock_k_callback, record_envelope, write_callback):
|
|
assert mock_k_callback.call_count == 1 # KafkaCallback constructed
|
|
constructor_args, constructor_kwargs = mock_k_callback.call_args
|
|
assert constructor_args[1] == record_envelope
|
|
assert constructor_args[2] == write_callback
|
|
|
|
@patch("gometa.ingestion.sink.datahub_kafka.PipelineContext")
|
|
@patch("gometa.ingestion.sink.datahub_kafka.SerializingProducer")
|
|
@patch("gometa.ingestion.sink.datahub_kafka.KafkaCallback")
|
|
def test_kafka_sink_write(self, mock_k_callback, mock_producer, mock_context):
|
|
mock_producer_instance = mock_producer.return_value
|
|
mock_k_callback_instance = mock_k_callback.return_value
|
|
callback = MagicMock(spec=WriteCallback)
|
|
kafka_sink = DatahubKafkaSink.create(
|
|
{"connection": {"bootstrap": "foobar:9092"}}, mock_context
|
|
)
|
|
re = RecordEnvelope(record="test", metadata={})
|
|
kafka_sink.write_record_async(re, callback)
|
|
assert mock_producer_instance.poll.call_count == 1 # poll() called once
|
|
self.validate_kafka_callback(
|
|
mock_k_callback, re, callback
|
|
) # validate kafka callback was constructed appropriately
|
|
|
|
# validate that confluent_kafka.Producer.produce was called with the right arguments
|
|
args, kwargs = mock_producer_instance.produce.call_args
|
|
created_callback = kwargs["on_delivery"]
|
|
assert created_callback == mock_k_callback_instance.kafka_callback
|
|
|
|
# TODO: Test that kafka producer is configured correctly
|
|
|
|
@patch("gometa.ingestion.sink.datahub_kafka.PipelineContext")
|
|
@patch("gometa.ingestion.sink.datahub_kafka.SerializingProducer")
|
|
def test_kafka_sink_close(self, mock_producer, mock_context):
|
|
mock_producer_instance = mock_producer.return_value
|
|
kafka_sink = DatahubKafkaSink.create({}, mock_context)
|
|
kafka_sink.close()
|
|
mock_producer_instance.flush.assert_called_once()
|
|
|
|
@patch("gometa.ingestion.sink.datahub_kafka.RecordEnvelope")
|
|
@patch("gometa.ingestion.sink.datahub_kafka.WriteCallback")
|
|
def test_kafka_callback_class(self, mock_w_callback, mock_re):
|
|
callback = KafkaCallback(
|
|
SinkReport(), record_envelope=mock_re, write_callback=mock_w_callback
|
|
)
|
|
mock_error = MagicMock()
|
|
mock_message = MagicMock()
|
|
callback.kafka_callback(mock_error, mock_message)
|
|
assert mock_w_callback.on_failure.call_count == 1
|
|
mock_w_callback.on_failure.called_with(mock_re, None, {"error", mock_error})
|
|
callback.kafka_callback(None, mock_message)
|
|
mock_w_callback.on_success.called_once_with(mock_re, {"msg", mock_message})
|