mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-29 08:29:09 +00:00
a9135e8db5
* Filter out PII queries * Mask topic PII * Mask topic PII * Update sample data test * Format * Moved logic to repository or PIIMasker * Fix test * chore(ui): update sample data api endpoint for topic --------- Co-authored-by: Sachin Chaurasiya <sachinchaurasiyachotey87@gmail.com>
116 lines
8.6 KiB
JSON
116 lines
8.6 KiB
JSON
{
|
|
"topics": [
|
|
{
|
|
"name": "customer_events",
|
|
"description": "Kafka topic to capture the customer events such as location updates or profile updates",
|
|
"partitions": 56,
|
|
"retentionSize": 455858109,
|
|
"replicationFactor":2,
|
|
"maximumMessageSize":167,
|
|
"cleanupPolicies": ["delete"],
|
|
"schemaType": "Avro",
|
|
"schemaText": "{\"namespace\":\"openmetadata.kafka\",\"name\":\"Customer\",\"type\":\"record\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"},{\"name\":\"email\",\"type\":\"string\"},{\"name\":\"address_line_1\",\"type\":\"string\"},{\"name\":\"address_line_2\",\"type\":\"string\"},{\"name\":\"post_code\",\"type\":\"string\"},{\"name\":\"country\",\"type\":\"string\"}]}",
|
|
"sampleData": [
|
|
"{\"id\": 1, \"first_name\": \"Lévy\", \"country\": \"ES\"}",
|
|
"{\"id\": 2, \"first_name\": \"Lima\", \"country\": \"ES\"}"
|
|
]
|
|
},
|
|
{
|
|
"name": "product_events",
|
|
"description": "Kafka topic to capture the product events. This topic will get updates on products decription, price etc.",
|
|
"partitions": 128,
|
|
"retentionSize": 1068320655,
|
|
"replicationFactor":4,
|
|
"maximumMessageSize":208,
|
|
"cleanupPolicies": ["delete"],
|
|
"schemaType": "Avro",
|
|
"schemaText": "{\"namespace\":\"openmetadata.kafka\",\"type\":\"record\",\"name\":\"Product\",\"fields\":[{\"name\":\"product_id\",\"type\":\"int\"},{\"name\":\"title\",\"type\":\"string\"},{\"name\":\"price\",\"type\":\"double\"},{\"name\":\"sku\",\"type\":\"string\"},{\"name\":\"barcode\",\"type\":\"string\"},{\"name\":\"shop_id\",\"type\":\"int\"}]}"
|
|
},
|
|
{
|
|
"name": "shop_updates",
|
|
"description": "Kafka topic to get any shop updates such as new products, location",
|
|
"partitions": 16,
|
|
"retentionSize": 213590548,
|
|
"replicationFactor":3,
|
|
"maximumMessageSize":226,
|
|
"cleanupPolicies": ["delete"],
|
|
"schemaType": "Avro",
|
|
"schemaText": "{\"namespace\":\"openmetadata.kafka\",\"type\":\"record\",\"name\":\"Shop\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"domain\",\"type\":\"string\"},{\"name\":\"user_id\",\"type\":\"int\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"},{\"name\":\"email\",\"type\":\"string\"}]}"
|
|
},
|
|
{
|
|
"name": "shop_products",
|
|
"description": "Kafka topic to get products in a shop. This is constantly updating",
|
|
"partitions": 128,
|
|
"retentionSize": 1475079463,
|
|
"replicationFactor":2,
|
|
"maximumMessageSize":244,
|
|
"cleanupPolicies": ["compact","delete"],
|
|
"schemaType": "Avro",
|
|
"schemaText": "{\"namespace\":\"openmetadata.kafka\",\"type\":\"record\",\"name\":\"Shop\",\"fields\":[{\"name\":\"prodcut_id\",\"type\":\"int\"},{\"name\":\"product_variant_id\",\"type\":\"int\"},{\"name\":\"shop_id\",\"type\":\"int\"},{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"product_title\",\"type\":\"string\"},{\"name\":\"price\",\"type\":\"double\"},{\"name\":\"quantity\",\"type\":\"int\"},{\"name\":\"product_vendor\",\"type\":\"int\"},{\"name\":\"fulfillable_quantity\",\"type\":\"int\"},{\"name\":\"fulfilment_service\",\"type\":\"string\"}]}"
|
|
},
|
|
{
|
|
"name": "orders",
|
|
"description": "All the order events on our online store",
|
|
"partitions": 128,
|
|
"retentionSize": 291959935,
|
|
"replicationFactor":3,
|
|
"maximumMessageSize":61,
|
|
"cleanupPolicies": ["compact"],
|
|
"schemaType": "Avro",
|
|
"schemaText": "{\"namespace\":\"openmetadata.kafka\",\"type\":\"record\",\"name\":\"Order\",\"fields\":[{\"name\":\"order_id\",\"type\":\"int\"},{\"name\":\"api_client_id\",\"type\":\"int\"},{\"name\":\"billing_address_id\",\"type\":\"int\"},{\"name\":\"customer_id\",\"type\":\"int\"},{\"name\":\"location_id\",\"type\":\"int\"},{\"name\":\"shipping_address_id\",\"type\":\"int\"},{\"name\":\"user_id\",\"type\":\"int\"},{\"name\":\"total_price\",\"type\":\"double\"},{\"name\":\"discount_code\",\"type\":\"string\"},{\"name\":\"processed_at\",\"type\":\"int\"}]}"
|
|
},
|
|
{
|
|
"name": "sales",
|
|
"description": "All sales related events gets captured in this topic",
|
|
"partitions": 128,
|
|
"retentionSize": 1931232624,
|
|
"replicationFactor":4,
|
|
"maximumMessageSize":249,
|
|
"cleanupPolicies": ["compact","delete"],
|
|
"schemaType": "Avro",
|
|
"schemaText": "{\"namespace\":\"openmetadata.kafka\",\"type\":\"record\",\"name\":\"Order\",\"fields\":[{\"name\":\"sale_id\",\"type\":\"int\"},{\"name\":\"billing_address_id\",\"type\":\"int\"},{\"name\":\"billing_address_id_2\",\"type\":\"int\"},{\"name\":\"api_client_id\",\"type\":\"int\"},{\"name\":\"customer_id\",\"type\":\"int\"},{\"name\":\"product_id\",\"type\":\"int\"},{\"name\":\"location_id\",\"type\":\"int\"},{\"name\":\"order_id\",\"type\":\"double\"}]}"
|
|
},
|
|
{
|
|
"name": "avro_record",
|
|
"description": "All Avro record related events gets captured in this topic",
|
|
"partitions": 128,
|
|
"retentionSize": 1931232624,
|
|
"replicationFactor":4,
|
|
"maximumMessageSize":249,
|
|
"cleanupPolicies": ["compact","delete"],
|
|
"schemaType": "Avro",
|
|
"schemaText": "{\n \"namespace\": \"openmetadata.kafka\",\n \"name\": \"level\",\n \"type\": \"record\",\n \"doc\": \"This is a first level record\",\n \"fields\": [\n {\n \"name\": \"uid\",\n \"type\": \"int\",\n \"doc\": \"The field represents unique id\"\n },\n {\n \"name\": \"somefield\",\n \"type\": \"string\"\n },\n {\n \"name\": \"options\",\n \"doc\": \"The field represents options array\",\n \"type\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"record\",\n \"name\": \"lvl2_record\",\n \"doc\": \"The field represents a level 2 record\",\n \"fields\": [\n {\n \"name\": \"item1_lvl2\",\n \"type\": \"string\"\n },\n {\n \"name\": \"item2_lvl2\",\n \"type\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"record\",\n \"name\": \"lvl3_record\",\n \"fields\": [\n {\n \"name\": \"item1_lvl3\",\n \"type\": \"string\",\n \"doc\": \"The field represents level3 item\"\n },\n {\n \"name\": \"item2_lvl3\",\n \"type\": \"string\"\n }\n ]\n }\n }\n }\n ]\n }\n }\n }\n ]\n}"
|
|
},
|
|
{
|
|
"name": "json_schema_record",
|
|
"description": "All Json Schema record related events gets captured in this topic",
|
|
"partitions": 128,
|
|
"retentionSize": 1931232624,
|
|
"replicationFactor":4,
|
|
"maximumMessageSize":249,
|
|
"cleanupPolicies": ["compact","delete"],
|
|
"schemaType": "JSON",
|
|
"schemaText": "{\"type\":\"object\",\"required\":[\"name\",\"age\",\"club_name\"],\"properties\":{\"name\":{\"type\":\"object\",\"required\":[\"first_name\",\"last_name\"],\"properties\":{\"first_name\":{\"type\":\"string\"},\"last_name\":{\"type\":\"string\"}}},\"age\":{\"type\":\"integer\"},\"club_name\":{\"type\":\"string\"}}}"
|
|
},
|
|
{
|
|
"name": "address_book",
|
|
"description": "All Protobuf record related events gets captured in this topic",
|
|
"partitions": 128,
|
|
"retentionSize": 1931232624,
|
|
"replicationFactor":4,
|
|
"maximumMessageSize":249,
|
|
"cleanupPolicies": ["compact","delete"],
|
|
"schemaType": "Protobuf",
|
|
"schemaText": "syntax = \"proto2\";\n\npackage tutorial;\n\nmessage Person {\n optional string name = 1;\n optional int32 id = 2;\n optional string email = 3;\n\n enum PhoneType {\n MOBILE = 0;\n HOME = 1;\n WORK = 2;\n }\n\n message PhoneNumber {\n optional string number = 1;\n optional PhoneType type = 2 [default = HOME];\n }\n\n repeated PhoneNumber phones = 4;\n}\n\nmessage AddressBook {\n repeated Person people = 1;\n}"
|
|
},
|
|
{
|
|
"name": "customer_contacts",
|
|
"description": "Kafka topic to capture the customer contacts such as email, phone",
|
|
"partitions": 56,
|
|
"retentionSize": 455858109,
|
|
"replicationFactor":2,
|
|
"maximumMessageSize":167,
|
|
"cleanupPolicies": ["delete"]
|
|
}
|
|
]
|
|
} |