From 994b49d05512daab493ae5b4cf3a3f40810651d8 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 23 Aug 2021 14:59:39 -0700 Subject: [PATCH] Fix #281: Ingestion: Add a sample topics connector --- .../resources/topics/TopicResource.java | 4 +- .../json/schema/api/data/createTopic.json | 2 +- .../json/schema/entity/data/topic.json | 2 +- ingestion/examples/kafka_schemas/service.json | 11 +++ ingestion/examples/kafka_schemas/topics.json | 57 ++++++++++++ ingestion/pipelines/sample_topics.json | 28 ++++++ ingestion/setup.py | 3 +- .../metadata/generated/data/tags/__init__.py | 2 +- .../generated/data/tags/personalDataTags.py | 2 +- .../metadata/generated/data/tags/piiTags.py | 2 +- .../metadata/generated/data/tags/tierTags.py | 2 +- .../metadata/generated/data/tags/userTags.py | 2 +- .../metadata/generated/schema/api/__init__.py | 2 +- .../generated/schema/api/catalogVersion.py | 2 +- .../generated/schema/api/data/__init__.py | 2 +- .../schema/api/data/createDatabase.py | 2 +- .../generated/schema/api/data/createTable.py | 2 +- .../generated/schema/api/data/createTopic.py | 5 +- .../generated/schema/api/feed/__init__.py | 2 +- .../generated/schema/api/feed/createThread.py | 2 +- .../generated/schema/api/services/__init__.py | 2 +- .../api/services/createDatabaseService.py | 2 +- .../api/services/createMessagingService.py | 2 +- .../api/services/updateDatabaseService.py | 2 +- .../api/services/updateMessagingService.py | 2 +- .../metadata/generated/schema/api/setOwner.py | 2 +- .../generated/schema/api/tags/__init__.py | 2 +- .../generated/schema/api/tags/createTag.py | 2 +- .../schema/api/tags/createTagCategory.py | 2 +- .../generated/schema/api/teams/__init__.py | 2 +- .../generated/schema/api/teams/createTeam.py | 2 +- .../generated/schema/api/teams/createUser.py | 2 +- .../generated/schema/entity/__init__.py | 2 +- .../metadata/generated/schema/entity/bots.py | 2 +- .../generated/schema/entity/data/__init__.py | 2 +- .../generated/schema/entity/data/dashboard.py | 2 +- .../generated/schema/entity/data/database.py | 2 +- .../generated/schema/entity/data/metrics.py | 2 +- .../generated/schema/entity/data/pipeline.py | 2 +- .../generated/schema/entity/data/report.py | 2 +- .../generated/schema/entity/data/table.py | 2 +- .../generated/schema/entity/data/topic.py | 5 +- .../generated/schema/entity/feed/__init__.py | 2 +- .../generated/schema/entity/feed/thread.py | 2 +- .../schema/entity/services/__init__.py | 2 +- .../schema/entity/services/databaseService.py | 2 +- .../entity/services/messagingService.py | 2 +- .../generated/schema/entity/tags/__init__.py | 2 +- .../schema/entity/tags/tagCategory.py | 2 +- .../generated/schema/entity/teams/__init__.py | 2 +- .../generated/schema/entity/teams/team.py | 2 +- .../generated/schema/entity/teams/user.py | 2 +- .../generated/schema/type/__init__.py | 2 +- .../generated/schema/type/auditLog.py | 2 +- .../metadata/generated/schema/type/basic.py | 2 +- .../schema/type/collectionDescriptor.py | 2 +- .../generated/schema/type/dailyCount.py | 2 +- .../generated/schema/type/entityReference.py | 2 +- .../generated/schema/type/entityUsage.py | 2 +- .../generated/schema/type/jdbcConnection.py | 2 +- .../metadata/generated/schema/type/profile.py | 2 +- .../generated/schema/type/schedule.py | 2 +- .../generated/schema/type/tagLabel.py | 2 +- .../generated/schema/type/usageDetails.py | 2 +- .../src/metadata/ingestion/source/kafka.py | 4 +- .../ingestion/source/sample_topics.py | 89 +++++++++++++++++++ 66 files changed, 252 insertions(+), 68 deletions(-) create mode 100644 ingestion/examples/kafka_schemas/service.json create mode 100644 ingestion/examples/kafka_schemas/topics.json create mode 100644 ingestion/pipelines/sample_topics.json create mode 100644 ingestion/src/metadata/ingestion/source/sample_topics.py diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java index 706ed3168bc..bf0935781ae 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java @@ -239,7 +239,7 @@ public class TopicResource { Topic topic = new Topic().withId(UUID.randomUUID()).withName(create.getName()).withDescription(create.getDescription()) .withService(create.getService()).withPartitions(create.getPartitions()) - .withSchema(create.getSchema()).withSchemaType(create.getSchemaType()) + .withSchemaText(create.getSchemaText()).withSchemaType(create.getSchemaType()) .withCleanupPolicies(create.getCleanupPolicies()) .withMaximumMessageSize(create.getMaximumMessageSize()) .withMinimumInSyncReplicas(create.getMinimumInSyncReplicas()) @@ -289,7 +289,7 @@ public class TopicResource { Topic topic = new Topic().withId(UUID.randomUUID()).withName(create.getName()).withDescription(create.getDescription()) .withService(create.getService()).withPartitions(create.getPartitions()) - .withSchema(create.getSchema()).withSchemaType(create.getSchemaType()) + .withSchemaText(create.getSchemaText()).withSchemaType(create.getSchemaType()) .withCleanupPolicies(create.getCleanupPolicies()) .withMaximumMessageSize(create.getMaximumMessageSize()) .withMinimumInSyncReplicas(create.getMinimumInSyncReplicas()) diff --git a/catalog-rest-service/src/main/resources/json/schema/api/data/createTopic.json b/catalog-rest-service/src/main/resources/json/schema/api/data/createTopic.json index 8a1864c8585..b1ae7d396c4 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/data/createTopic.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/data/createTopic.json @@ -22,7 +22,7 @@ "type" : "integer", "minimum": "1" }, - "schema" : { + "schemaText" : { "description" : "Schema used for message serialization. Optional as some topics may not have associated schemas.", "type" : "string" }, diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json index 6e133ee2c8e..4588ca85c5b 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json @@ -72,7 +72,7 @@ "type" : "integer", "minimum": "1" }, - "schema" : { + "schemaText" : { "description" : "Schema used for message serialization. Optional as some topics may not have associated schemas.", "type" : "string" }, diff --git a/ingestion/examples/kafka_schemas/service.json b/ingestion/examples/kafka_schemas/service.json new file mode 100644 index 00000000000..df5e70876ab --- /dev/null +++ b/ingestion/examples/kafka_schemas/service.json @@ -0,0 +1,11 @@ +{ + "id": "a6fb4f54-ba3d-4a16-97f0-766713199189", + "name": "sample_kafka", + "serviceType": "Kafka", + "description": "Kafka messaging queue service", + "href": "null", + "brokers": [ + "localhost:9092" + ], + "schemaRegistry": "http://localhost:8081" +} diff --git a/ingestion/examples/kafka_schemas/topics.json b/ingestion/examples/kafka_schemas/topics.json new file mode 100644 index 00000000000..d89fb5f981b --- /dev/null +++ b/ingestion/examples/kafka_schemas/topics.json @@ -0,0 +1,57 @@ +{ + "topics": [{ + "name": "customer_events", + "description": "Kafka topic to capture the customer events such as location updates or profile updates", + "partitions": 56, + "retentionSize": 322122382273, + "cleanupPolicies": ["delete"], + "schemaType": "Avro", + "schemaText": "{\"namespace\":\"org.open-metadata.kafka\",\"name\":\"Customer\",\"type\":\"record\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"},{\"name\":\"email\",\"type\":\"string\"},{\"name\":\"address_line_1\",\"type\":\"string\"},{\"name\":\"address_line_2\",\"type\":\"string\"},{\"name\":\"post_code\",\"type\":\"string\"},{\"name\":\"country\",\"type\":\"string\"}]}" + }, + { + "name": "product_events", + "description": "Kafka topic to capture the product events. This topic will get updates on products decription, price etc.", + "partitions": 128, + "retentionSize": 322122382273, + "cleanupPolicies": ["delete"], + "schemaType": "Avro", + "schemaText": "{\"namespace\":\"org.open-metadata.kafka\",\"type\":\"record\",\"name\":\"Product\",\"fields\":[{\"name\":\"product_id\",\"type\":\"int\"},{\"name\":\"title\",\"type\":\"string\"},{\"name\":\"price\",\"type\":\"double\"},{\"name\":\"sku\",\"type\":\"string\"},{\"name\":\"barcode\",\"type\":\"string\"},{\"name\":\"shop_id\",\"type\":\"int\"}]}" + }, + { + "name": "shop_updates", + "description": "Kafka topic to get any shop updates such as new products, location", + "partitions": 16, + "retentionSize": 322122382273, + "cleanupPolicies": ["delete"], + "schemaType": "Avro", + "schemaText": "{\"namespace\":\"org.open-metadata.kafka\",\"type\":\"record\",\"name\":\"Shop\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"domain\",\"type\":\"string\"},{\"name\":\"user_id\",\"type\":\"int\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"},{\"name\":\"email\",\"type\":\"string\"}]}" + }, + { + "name": "shop_products", + "description": "Kafka topic to get products in a shop. This is constantly updating", + "partitions": 128, + "retentionSize": 3222122382273, + "cleanupPolicies": ["delete"], + "schemaType": "Avro", + "schemaText": "{\"namespace\":\"org.open-metadata.kafka\",\"type\":\"record\",\"name\":\"Shop Products\",\"fields\":[{\"name\":\"prodcut_id\",\"type\":\"int\"},{\"name\":\"product_variant_id\",\"type\":\"int\"},{\"name\":\"shop_id\",\"type\":\"int\"},{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"product_title\",\"type\":\"string\"},{\"name\":\"price\",\"type\":\"double\"},{\"name\":\"quantity\",\"type\":\"int\"},{\"name\":\"product_vendor\",\"type\":\"int\"},{\"name\":\"fulfillable_quantity\",\"type\":\"int\"},{\"name\":\"fulfilment_service\",\"type\":\"string\"}]}" + }, + { + "name": "orders", + "description": "All the order events on our online store", + "partitions": 128, + "retentionSize": 3222122382273, + "cleanupPolicies": ["delete"], + "schemaType": "Avro", + "schemaText": "{\"namespace\":\"org.open-metadata.kafka\",\"type\":\"record\",\"name\":\"Order\",\"fields\":[{\"name\":\"order_id\",\"type\":\"int\"},{\"name\":\"api_client_id\",\"type\":\"int\"},{\"name\":\"billing_address_id\",\"type\":\"int\"},{\"name\":\"customer_id\",\"type\":\"int\"},{\"name\":\"location_id\",\"type\":\"int\"},{\"name\":\"shipping_address_id\",\"type\":\"int\"},{\"name\":\"user_id\",\"type\":\"int\"},{\"name\":\"total_price\",\"type\":\"double\"},{\"name\":\"discount_code\",\"type\":\"string\"},{\"name\":\"processed_at\",\"type\":\"int\"}]}" + }, + { + "name": "sales", + "description": "All sales related events gets captured in this topic", + "partitions": 128, + "retentionSize": 3222122382273, + "cleanupPolicies": ["delete"], + "schemaType": "Avro", + "schemaText": "{\"namespace\":\"org.open-metadata.kafka\",\"type\":\"record\",\"name\":\"Order\",\"fields\":[{\"name\":\"sale_id\",\"type\":\"int\"},{\"name\":\"billing_address_id\",\"type\":\"int\"},{\"name\":\"billing_address_id\",\"type\":\"int\"},{\"name\":\"api_client_id\",\"type\":\"int\"},{\"name\":\"customer_id\",\"type\":\"int\"},{\"name\":\"product_id\",\"type\":\"int\"},{\"name\":\"location_id\",\"type\":\"int\"},{\"name\":\"order_id\",\"type\":\"double\"}]}" + } + ] +} \ No newline at end of file diff --git a/ingestion/pipelines/sample_topics.json b/ingestion/pipelines/sample_topics.json new file mode 100644 index 00000000000..0a23aaa34dc --- /dev/null +++ b/ingestion/pipelines/sample_topics.json @@ -0,0 +1,28 @@ +{ + "source": { + "type": "sample-topics", + "config": { + "service_name": "sample_kafka", + "service_type": "Kafka", + "sample_schema_folder": "./examples/kafka_schemas/" + } + }, + "sink": { + "type": "metadata-rest-topics", + "config": {} + }, + "metadata_server": { + "type": "metadata-server", + "config": { + "api_endpoint": "http://localhost:8585/api", + "auth_provider_type": "no-auth" + } + }, + "cron": { + "minute": "*/5", + "hour": null, + "day": null, + "month": null, + "day_of_week": null + } +} diff --git a/ingestion/setup.py b/ingestion/setup.py index 54c5dfe8770..7ba7c22e550 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -92,7 +92,8 @@ plugins: Dict[str, Set[str]] = { "scheduler": scheduler_requirements, "snowflake": {"snowflake-sqlalchemy<=1.2.4"}, "snowflake-usage": {"snowflake-sqlalchemy<=1.2.4"}, - "sample-tables": {"faker~=8.1.1", } + "sample-tables": {"faker~=8.1.1", }, + "sample-topics": {} } build_options = {"includes": ["_cffi_backend"]} diff --git a/ingestion/src/metadata/generated/data/tags/__init__.py b/ingestion/src/metadata/generated/data/tags/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/data/tags/__init__.py +++ b/ingestion/src/metadata/generated/data/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/data/tags/personalDataTags.py b/ingestion/src/metadata/generated/data/tags/personalDataTags.py index 34660544135..690dfcdef47 100644 --- a/ingestion/src/metadata/generated/data/tags/personalDataTags.py +++ b/ingestion/src/metadata/generated/data/tags/personalDataTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/personalDataTags.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/piiTags.py b/ingestion/src/metadata/generated/data/tags/piiTags.py index 6ad2a3716ae..4db15680f12 100644 --- a/ingestion/src/metadata/generated/data/tags/piiTags.py +++ b/ingestion/src/metadata/generated/data/tags/piiTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/piiTags.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/tierTags.py b/ingestion/src/metadata/generated/data/tags/tierTags.py index a5fc6fd5d24..5895552cb1b 100644 --- a/ingestion/src/metadata/generated/data/tags/tierTags.py +++ b/ingestion/src/metadata/generated/data/tags/tierTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/tierTags.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/userTags.py b/ingestion/src/metadata/generated/data/tags/userTags.py index b0f6f61a214..701315009cb 100644 --- a/ingestion/src/metadata/generated/data/tags/userTags.py +++ b/ingestion/src/metadata/generated/data/tags/userTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/userTags.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/__init__.py b/ingestion/src/metadata/generated/schema/api/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/api/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/catalogVersion.py b/ingestion/src/metadata/generated/schema/api/catalogVersion.py index 2d95803f3a4..cee4126e4e5 100644 --- a/ingestion/src/metadata/generated/schema/api/catalogVersion.py +++ b/ingestion/src/metadata/generated/schema/api/catalogVersion.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/catalogVersion.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/__init__.py b/ingestion/src/metadata/generated/schema/api/data/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/api/data/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/data/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/data/createDatabase.py b/ingestion/src/metadata/generated/schema/api/data/createDatabase.py index 1b1f5e02fe4..9cfe5a527f9 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createDatabase.py +++ b/ingestion/src/metadata/generated/schema/api/data/createDatabase.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createDatabase.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createTable.py b/ingestion/src/metadata/generated/schema/api/data/createTable.py index 9bb90b6b13a..64f63cb1b94 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createTable.py +++ b/ingestion/src/metadata/generated/schema/api/data/createTable.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createTable.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createTopic.py b/ingestion/src/metadata/generated/schema/api/data/createTopic.py index cdc825a9ecd..283979acf5f 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createTopic.py +++ b/ingestion/src/metadata/generated/schema/api/data/createTopic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createTopic.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations @@ -26,9 +26,8 @@ class CreateTopic(BaseModel): partitions: conint(ge=1) = Field( ..., description='Number of partitions into which the topic is divided.' ) - schema_: Optional[str] = Field( + schemaText: Optional[str] = Field( None, - alias='schema', description='Schema used for message serialization. Optional as some topics may not have associated schemas.', ) schemaType: Optional[topic.SchemaType] = Field( diff --git a/ingestion/src/metadata/generated/schema/api/feed/__init__.py b/ingestion/src/metadata/generated/schema/api/feed/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/api/feed/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/feed/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/feed/createThread.py b/ingestion/src/metadata/generated/schema/api/feed/createThread.py index 01f99f89dd8..f0f8554c532 100644 --- a/ingestion/src/metadata/generated/schema/api/feed/createThread.py +++ b/ingestion/src/metadata/generated/schema/api/feed/createThread.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/feed/createThread.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/__init__.py b/ingestion/src/metadata/generated/schema/api/services/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/api/services/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/services/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py b/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py index 375dbe444b5..498df94986a 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createDatabaseService.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py b/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py index 55093eff400..cbd379ff823 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createMessagingService.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py b/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py index 9caf1777f64..4cae8fda4c8 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateDatabaseService.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py b/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py index 5bf37e5f1eb..3ab50f8489d 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateMessagingService.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/setOwner.py b/ingestion/src/metadata/generated/schema/api/setOwner.py index 11c0df99f07..f60f51a0856 100644 --- a/ingestion/src/metadata/generated/schema/api/setOwner.py +++ b/ingestion/src/metadata/generated/schema/api/setOwner.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/setOwner.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/tags/__init__.py b/ingestion/src/metadata/generated/schema/api/tags/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/tags/createTag.py b/ingestion/src/metadata/generated/schema/api/tags/createTag.py index 78cea9d8310..a90ecb11aaa 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/createTag.py +++ b/ingestion/src/metadata/generated/schema/api/tags/createTag.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/tags/createTag.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py b/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py index 01e442530cd..985777c2d75 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py +++ b/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/tags/createTagCategory.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/teams/__init__.py b/ingestion/src/metadata/generated/schema/api/teams/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/teams/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/teams/createTeam.py b/ingestion/src/metadata/generated/schema/api/teams/createTeam.py index d34654d7d78..e8fd74e6325 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/createTeam.py +++ b/ingestion/src/metadata/generated/schema/api/teams/createTeam.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/teams/createTeam.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/teams/createUser.py b/ingestion/src/metadata/generated/schema/api/teams/createUser.py index 2961579cb7e..f29c2677fd3 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/createUser.py +++ b/ingestion/src/metadata/generated/schema/api/teams/createUser.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/teams/createUser.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/__init__.py b/ingestion/src/metadata/generated/schema/entity/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/entity/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/bots.py b/ingestion/src/metadata/generated/schema/entity/bots.py index 2e3aaf9d63f..3ff76dbe6fe 100644 --- a/ingestion/src/metadata/generated/schema/entity/bots.py +++ b/ingestion/src/metadata/generated/schema/entity/bots.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/bots.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/__init__.py b/ingestion/src/metadata/generated/schema/entity/data/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/data/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/data/dashboard.py b/ingestion/src/metadata/generated/schema/entity/data/dashboard.py index b9cd5dac632..3a1a2374ebe 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/dashboard.py +++ b/ingestion/src/metadata/generated/schema/entity/data/dashboard.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/dashboard.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/database.py b/ingestion/src/metadata/generated/schema/entity/data/database.py index 621fd7c3148..8ecacadc01f 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/database.py +++ b/ingestion/src/metadata/generated/schema/entity/data/database.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/database.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/metrics.py b/ingestion/src/metadata/generated/schema/entity/data/metrics.py index 576df597667..91b8a661f64 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/metrics.py +++ b/ingestion/src/metadata/generated/schema/entity/data/metrics.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/metrics.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/pipeline.py b/ingestion/src/metadata/generated/schema/entity/data/pipeline.py index 5842cdb4cab..9dbba0e28b3 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/pipeline.py +++ b/ingestion/src/metadata/generated/schema/entity/data/pipeline.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/pipeline.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/report.py b/ingestion/src/metadata/generated/schema/entity/data/report.py index 0a4e4fe2ff3..d1a251e8f24 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/report.py +++ b/ingestion/src/metadata/generated/schema/entity/data/report.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/report.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/table.py b/ingestion/src/metadata/generated/schema/entity/data/table.py index ce9a59c47e9..bbba6172d76 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/table.py +++ b/ingestion/src/metadata/generated/schema/entity/data/table.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/table.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/topic.py b/ingestion/src/metadata/generated/schema/entity/data/topic.py index 664739636ee..bfd142855e2 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/topic.py +++ b/ingestion/src/metadata/generated/schema/entity/data/topic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/topic.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations @@ -49,9 +49,8 @@ class Topic(BaseModel): partitions: conint(ge=1) = Field( ..., description='Number of partitions into which the topic is divided.' ) - schema_: Optional[str] = Field( + schemaText: Optional[str] = Field( None, - alias='schema', description='Schema used for message serialization. Optional as some topics may not have associated schemas.', ) schemaType: Optional[SchemaType] = Field( diff --git a/ingestion/src/metadata/generated/schema/entity/feed/__init__.py b/ingestion/src/metadata/generated/schema/entity/feed/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/entity/feed/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/feed/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/feed/thread.py b/ingestion/src/metadata/generated/schema/entity/feed/thread.py index df5ffecebd2..8c7bfdcfb56 100644 --- a/ingestion/src/metadata/generated/schema/entity/feed/thread.py +++ b/ingestion/src/metadata/generated/schema/entity/feed/thread.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/feed/thread.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/__init__.py b/ingestion/src/metadata/generated/schema/entity/services/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/services/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py index 99b9b4f331b..73a4ecb8756 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/databaseService.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/messagingService.py b/ingestion/src/metadata/generated/schema/entity/services/messagingService.py index e0240ad9144..5f1dfb74f55 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/messagingService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/messagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/messagingService.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/tags/__init__.py b/ingestion/src/metadata/generated/schema/entity/tags/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/entity/tags/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py b/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py index 3ef2dec1f55..408cc450cf8 100644 --- a/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py +++ b/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/tags/tagCategory.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/teams/__init__.py b/ingestion/src/metadata/generated/schema/entity/teams/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/teams/team.py b/ingestion/src/metadata/generated/schema/entity/teams/team.py index 69f971caac1..4bb1ce3079d 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/team.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/team.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/teams/team.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/teams/user.py b/ingestion/src/metadata/generated/schema/entity/teams/user.py index a40f74ec037..4a6cda7e14c 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/user.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/user.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/teams/user.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/__init__.py b/ingestion/src/metadata/generated/schema/type/__init__.py index 627b75246fb..32b56879984 100644 --- a/ingestion/src/metadata/generated/schema/type/__init__.py +++ b/ingestion/src/metadata/generated/schema/type/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 diff --git a/ingestion/src/metadata/generated/schema/type/auditLog.py b/ingestion/src/metadata/generated/schema/type/auditLog.py index 19afa068e33..c0c1ab5a648 100644 --- a/ingestion/src/metadata/generated/schema/type/auditLog.py +++ b/ingestion/src/metadata/generated/schema/type/auditLog.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/auditLog.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/basic.py b/ingestion/src/metadata/generated/schema/type/basic.py index 6a6dd205f8b..6852d3d58c6 100644 --- a/ingestion/src/metadata/generated/schema/type/basic.py +++ b/ingestion/src/metadata/generated/schema/type/basic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/basic.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py b/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py index bd853751935..39ea0b27ca2 100644 --- a/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py +++ b/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/collectionDescriptor.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/dailyCount.py b/ingestion/src/metadata/generated/schema/type/dailyCount.py index 9c5104b5309..64c3a4cdc3f 100644 --- a/ingestion/src/metadata/generated/schema/type/dailyCount.py +++ b/ingestion/src/metadata/generated/schema/type/dailyCount.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/dailyCount.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityReference.py b/ingestion/src/metadata/generated/schema/type/entityReference.py index d28bac24327..a1df9025068 100644 --- a/ingestion/src/metadata/generated/schema/type/entityReference.py +++ b/ingestion/src/metadata/generated/schema/type/entityReference.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityReference.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityUsage.py b/ingestion/src/metadata/generated/schema/type/entityUsage.py index 3dc48b0c7bf..7d97e281875 100644 --- a/ingestion/src/metadata/generated/schema/type/entityUsage.py +++ b/ingestion/src/metadata/generated/schema/type/entityUsage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityUsage.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/jdbcConnection.py b/ingestion/src/metadata/generated/schema/type/jdbcConnection.py index 08624857abd..6fea642e119 100644 --- a/ingestion/src/metadata/generated/schema/type/jdbcConnection.py +++ b/ingestion/src/metadata/generated/schema/type/jdbcConnection.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/jdbcConnection.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/profile.py b/ingestion/src/metadata/generated/schema/type/profile.py index c66745d4dfb..345fccbb39d 100644 --- a/ingestion/src/metadata/generated/schema/type/profile.py +++ b/ingestion/src/metadata/generated/schema/type/profile.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/profile.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/schedule.py b/ingestion/src/metadata/generated/schema/type/schedule.py index b8d44c7baf6..2246ed1b2b1 100644 --- a/ingestion/src/metadata/generated/schema/type/schedule.py +++ b/ingestion/src/metadata/generated/schema/type/schedule.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/schedule.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/tagLabel.py b/ingestion/src/metadata/generated/schema/type/tagLabel.py index b644a9aa615..b90c8253e6f 100644 --- a/ingestion/src/metadata/generated/schema/type/tagLabel.py +++ b/ingestion/src/metadata/generated/schema/type/tagLabel.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/tagLabel.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/usageDetails.py b/ingestion/src/metadata/generated/schema/type/usageDetails.py index 315a633bd8f..f8c1a5d9643 100644 --- a/ingestion/src/metadata/generated/schema/type/usageDetails.py +++ b/ingestion/src/metadata/generated/schema/type/usageDetails.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/usageDetails.json -# timestamp: 2021-08-22T05:45:37+00:00 +# timestamp: 2021-08-23T15:40:00+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/ingestion/source/kafka.py b/ingestion/src/metadata/ingestion/source/kafka.py index 5da2135b006..e86d28d5104 100644 --- a/ingestion/src/metadata/ingestion/source/kafka.py +++ b/ingestion/src/metadata/ingestion/source/kafka.py @@ -80,7 +80,7 @@ class KafkaSource(Source): def prepare(self): pass - def next_record(self) -> Iterable[Topic]: + def next_record(self) -> Iterable[CreateTopic]: topics = self.admin_client.list_topics().topics for t in topics: if self.config.filter_pattern.included(t): @@ -90,7 +90,7 @@ class KafkaSource(Source): service=EntityReference(id=self.service.id, type="messagingService"), partitions=1) if topic_schema is not None: - topic.schema_ = topic_schema.schema_str + topic.schemaText = topic_schema.schema_str if topic_schema.schema_type == "AVRO": topic.schemaType = SchemaType.Avro.name elif topic_schema.schema_type == "PROTOBUF": diff --git a/ingestion/src/metadata/ingestion/source/sample_topics.py b/ingestion/src/metadata/ingestion/source/sample_topics.py new file mode 100644 index 00000000000..60f36470a62 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/sample_topics.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +from dataclasses import dataclass, field +from typing import Iterable, List +from metadata.config.common import ConfigModel +from metadata.generated.schema.api.data.createTopic import CreateTopic +from metadata.generated.schema.api.services.createMessagingService import CreateMessagingServiceEntityRequest +from metadata.generated.schema.entity.services.messagingService import MessagingService +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.ingestion.api.source import SourceStatus, Source +from metadata.ingestion.ometa.auth_provider import MetadataServerConfig +from metadata.ingestion.ometa.client import REST + + +def get_service_or_create(service_json, metadata_config) -> MessagingService: + client = REST(metadata_config) + service = client.get_messaging_service(service_json['name']) + if service is not None: + return service + else: + created_service = client.create_messaging_service(CreateMessagingServiceEntityRequest(**service_json)) + return created_service + + +class SampleTopicSourceConfig(ConfigModel): + sample_schema_folder: str + service_name: str + service_type: str = "Kafka" + + def get_sample_schema_folder(self): + return self.sample_schema_folder + + +@dataclass +class SampleTopicSourceStatus(SourceStatus): + topics_scanned: List[str] = field(default_factory=list) + + def report_topic_scanned(self, topic_name: str) -> None: + self.topics_scanned.append(topic_name) + + +class SampleTopicsSource(Source): + + def __init__(self, config: SampleTopicSourceConfig, metadata_config: MetadataServerConfig, ctx): + super().__init__(ctx) + self.status = SampleTopicSourceStatus() + self.config = config + self.metadata_config = metadata_config + self.client = REST(metadata_config) + self.service_json = json.load(open(config.sample_schema_folder + "/service.json", 'r')) + self.topics = json.load(open(config.sample_schema_folder + "/topics.json", 'r')) + self.service = get_service_or_create(self.service_json, metadata_config) + + @classmethod + def create(cls, config_dict, metadata_config_dict, ctx): + config = SampleTopicSourceConfig.parse_obj(config_dict) + metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict) + return cls(config, metadata_config, ctx) + + def prepare(self): + pass + + def next_record(self) -> Iterable[CreateTopic]: + for topic in self.topics['topics']: + topic['service'] = EntityReference(id=self.service.id, type="messagingService") + create_topic = CreateTopic(**topic) + self.status.scanned(create_topic.name.__root__) + yield create_topic + + def close(self): + pass + + def get_status(self): + return self.status