yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-24 22:18:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix Issue-404 replicationFactor and Message Maximum Byte added (#419)

Browse Source 
* fix Issue-404 replicationFactor and Message Maximum Byte added

* Cleanup Policy values changed in sample topic data
This commit is contained in:
Ayush Shah 2021-09-08 00:52:51 +05:30 committed by GitHub
parent 3965b030a9
commit 2b61c8ea8c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 17 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java
View File

@ -228,6 +228,7 @@ public class TopicResource {
.withMaximumMessageSize(create.getMaximumMessageSize())
.withMinimumInSyncReplicas(create.getMinimumInSyncReplicas())
.withRetentionSize(create.getRetentionSize()).withRetentionTime(create.getRetentionTime())
.withReplicationFactor(create.getReplicationFactor())
.withTags(create.getTags());
topic = addHref(uriInfo, dao.create(topic, create.getService(), create.getOwner()));
return Response.created(topic.getHref()).entity(topic).build();
@ -278,6 +279,7 @@ public class TopicResource {
.withMaximumMessageSize(create.getMaximumMessageSize())
.withMinimumInSyncReplicas(create.getMinimumInSyncReplicas())
.withRetentionSize(create.getRetentionSize()).withRetentionTime(create.getRetentionTime())
.withReplicationFactor(create.getReplicationFactor())
.withTags(create.getTags());
PutResponse<Topic> response = dao.createOrUpdate(topic, create.getService(), create.getOwner());
topic = addHref(uriInfo, response.getEntity());

4
catalog-rest-service/src/main/resources/json/schema/api/data/createTopic.json
View File

@ -37,6 +37,10 @@
"$ref" : "../../entity/data/topic.json#/definitions/cleanupPolicy"
}
},
"replicationFactor": {
"description": "Replication Factor in integer (more than 1).",
"type" : "integer"
},
"retentionTime": {
"description": "Retention time in milliseconds. For Kafka - `retention.ms` configuration.",
"type" : "number"

4
catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json
View File

@ -91,6 +91,10 @@
"description": "Retention time in milliseconds. For Kafka - `retention.ms` configuration.",
"type" : "number"
},
"replicationFactor": {
"description": "Replication Factor in integer (more than 1).",
"type" : "integer"
},
"maximumMessageSize" : {
"description": "Maximum message size in bytes. For Kafka - `max.message.bytes` configuration.",
"type" : "integer"

6
ingestion/examples/sample_data/topics/topics.json
View File

@ -31,7 +31,7 @@
"description": "Kafka topic to get products in a shop. This is constantly updating",
"partitions": 128,
"retentionSize": 3222122382273,
"cleanupPolicies": ["delete"],
"cleanupPolicies": ["compact,delete"],
"schemaType": "Avro",
"schemaText": "{\"namespace\":\"org.open-metadata.kafka\",\"type\":\"record\",\"name\":\"Shop Products\",\"fields\":[{\"name\":\"prodcut_id\",\"type\":\"int\"},{\"name\":\"product_variant_id\",\"type\":\"int\"},{\"name\":\"shop_id\",\"type\":\"int\"},{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"product_title\",\"type\":\"string\"},{\"name\":\"price\",\"type\":\"double\"},{\"name\":\"quantity\",\"type\":\"int\"},{\"name\":\"product_vendor\",\"type\":\"int\"},{\"name\":\"fulfillable_quantity\",\"type\":\"int\"},{\"name\":\"fulfilment_service\",\"type\":\"string\"}]}"
},
@ -40,7 +40,7 @@
"description": "All the order events on our online store",
"partitions": 128,
"retentionSize": 3222122382273,
"cleanupPolicies": ["delete"],
"cleanupPolicies": ["compact"],
"schemaType": "Avro",
"schemaText": "{\"namespace\":\"org.open-metadata.kafka\",\"type\":\"record\",\"name\":\"Order\",\"fields\":[{\"name\":\"order_id\",\"type\":\"int\"},{\"name\":\"api_client_id\",\"type\":\"int\"},{\"name\":\"billing_address_id\",\"type\":\"int\"},{\"name\":\"customer_id\",\"type\":\"int\"},{\"name\":\"location_id\",\"type\":\"int\"},{\"name\":\"shipping_address_id\",\"type\":\"int\"},{\"name\":\"user_id\",\"type\":\"int\"},{\"name\":\"total_price\",\"type\":\"double\"},{\"name\":\"discount_code\",\"type\":\"string\"},{\"name\":\"processed_at\",\"type\":\"int\"}]}"
},
@ -49,7 +49,7 @@
"description": "All sales related events gets captured in this topic",
"partitions": 128,
"retentionSize": 3222122382273,
"cleanupPolicies": ["delete"],
"cleanupPolicies": ["compact,delete"],
"schemaType": "Avro",
"schemaText": "{\"namespace\":\"org.open-metadata.kafka\",\"type\":\"record\",\"name\":\"Order\",\"fields\":[{\"name\":\"sale_id\",\"type\":\"int\"},{\"name\":\"billing_address_id\",\"type\":\"int\"},{\"name\":\"billing_address_id\",\"type\":\"int\"},{\"name\":\"api_client_id\",\"type\":\"int\"},{\"name\":\"customer_id\",\"type\":\"int\"},{\"name\":\"product_id\",\"type\":\"int\"},{\"name\":\"location_id\",\"type\":\"int\"},{\"name\":\"order_id\",\"type\":\"double\"}]}"
}

4
ingestion/src/metadata/generated/schema/api/data/createTopic.py
View File

@ -37,6 +37,10 @@ class CreateTopic(BaseModel):
None,
description='Topic clean up policy. For Kafka - `cleanup.policy` configuration.',
)
replicationFactor: Optional[int] = Field(
None,
description='Replication Factor in integer (more than 1).',
)
retentionTime: Optional[float] = Field(
None,
description='Retention time in milliseconds. For Kafka - `retention.ms` configuration.',