mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-13 08:37:03 +00:00
SampleData Usage Fix (#4398)
* SampleData Test Connection & Usage Fix * Fixed Pytest
This commit is contained in:
Mayur Singal
GitHub
parent
6bd587be8d
commit
450fb2b132
@ -823,3 +823,193 @@ query
|
||||
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from shopify.fact_sale join shopify.dim_staff on fact_sale.user_id = dim_staff.user_id"
|
||||
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from shopify.fact_sale join shopify.dim_staff on fact_sale.user_id = dim_staff.user_id"
|
||||
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from shopify.fact_sale join shopify.dim_staff on fact_sale.user_id = dim_staff.user_id"
|
||||
"select address_id from dim_address"
|
||||
"select shop_id from dim_address"
|
||||
"select first_name from dim_address"
|
||||
"select last_name from dim_address"
|
||||
"select address1 from dim_address"
|
||||
"select address2 from dim_address"
|
||||
"select company from dim_address"
|
||||
"select city from dim_address"
|
||||
"select region from dim_address"
|
||||
"select zip from dim_address"
|
||||
"select country from dim_address"
|
||||
"select phone from dim_address"
|
||||
"select api_client_id from dim_api_client"
|
||||
"select title from dim_api_client"
|
||||
"select customer_id from dim_customer"
|
||||
"select shop_id from dim_customer"
|
||||
"select average_order_size from dim_customer"
|
||||
"select total_order_count from dim_customer"
|
||||
"select total_order_value from dim_customer"
|
||||
"select first_order_date from dim_customer"
|
||||
"select last_order_date from dim_customer"
|
||||
"select rank from dim_customer"
|
||||
"select new from dim_customer"
|
||||
"select returning from dim_customer"
|
||||
"select loyal from dim_customer"
|
||||
"select at_risk from dim_customer"
|
||||
"select dormant from dim_customer"
|
||||
"select promising from dim_customer"
|
||||
"select predicted_average_number_of_days_between_orders from dim_customer"
|
||||
"select expected_purchase_value_in_next_30_days from dim_customer"
|
||||
"select first_name from dim_customer"
|
||||
"select last_name from dim_customer"
|
||||
"select city from dim_customer"
|
||||
"select region from dim_customer"
|
||||
"select country from dim_customer"
|
||||
"select email from dim_customer"
|
||||
"select phone from dim_customer"
|
||||
"select verified_email from dim_customer"
|
||||
"select created_at from dim_customer"
|
||||
"select accepts_marketing from dim_customer"
|
||||
"select customer from dim_customer"
|
||||
"select shipping_address from dim_customer"
|
||||
"select orders from dim_customer"
|
||||
"select location_id from dim_location"
|
||||
"select shop_id from dim_location"
|
||||
"select name from dim_location"
|
||||
"select address from dim_location"
|
||||
"select product_id from dim.product"
|
||||
"select shop_id from dim.product"
|
||||
"select title from dim.product"
|
||||
"select vendor from dim.product"
|
||||
"select created_at from dim.product"
|
||||
"select deleted_at from dim.product"
|
||||
"select product_variant_id from dim.product_variant"
|
||||
"select product_id from dim.product_variant"
|
||||
"select shop_id from dim.product_variant"
|
||||
"select title from dim.product_variant"
|
||||
"select barcode from dim.product_variant"
|
||||
"select sku from dim.product_variant"
|
||||
"select price from dim.product_variant"
|
||||
"select grams from dim.product_variant"
|
||||
"select created_at from dim.product_variant"
|
||||
"select deleted_at from dim.product_variant"
|
||||
"select shop_id from dim.shop"
|
||||
"select name from dim.shop"
|
||||
"select domain from dim.shop"
|
||||
"select user_id from dim_staff"
|
||||
"select shop_id from dim_staff"
|
||||
"select first_name from dim_staff"
|
||||
"select last_name from dim_staff"
|
||||
"select email from dim_staff"
|
||||
"select line_item_id from fact_line_item"
|
||||
"select billing_address_id from fact_line_item"
|
||||
"select order_id from fact_line_item"
|
||||
"select product_id from fact_line_item"
|
||||
"select product_variant_id from fact_line_item"
|
||||
"select shop_id from fact_line_item"
|
||||
"select name from fact_line_item"
|
||||
"select product_title from fact_line_item"
|
||||
"select price from fact_line_item"
|
||||
"select quantity from fact_line_item"
|
||||
"select requires_shipping from fact_line_item"
|
||||
"select taxable from fact_line_item"
|
||||
"select gift_card from fact_line_item"
|
||||
"select grams from fact_line_item"
|
||||
"select product_vendor from fact_line_item"
|
||||
"select fulfillable_quantity from fact_line_item"
|
||||
"select fulfillment_service from fact_line_item"
|
||||
"select order_id from fact_order"
|
||||
"select api_client_id from fact_order"
|
||||
"select billing_address_id from fact_order"
|
||||
"select customer_id from fact_order"
|
||||
"select location_id from fact_order"
|
||||
"select shipping_address_id from fact_order"
|
||||
"select shop_id from fact_order"
|
||||
"select user_id from fact_order"
|
||||
"select name from fact_order"
|
||||
"select total_price from fact_order"
|
||||
"select discount_code from fact_order"
|
||||
"select processed_at from fact_order"
|
||||
"select canceled_at from fact_order"
|
||||
"select deleted_at from fact_order"
|
||||
"select test from fact_order"
|
||||
"select sale_id from fact_sale"
|
||||
"select billing_address_id from fact_sale"
|
||||
"select api_client_id from fact_sale"
|
||||
"select customer_id from fact_sale"
|
||||
"select line_item_id from fact_sale"
|
||||
"select location_id from fact_sale"
|
||||
"select order_id from fact_sale"
|
||||
"select product_id from fact_sale"
|
||||
"select product_variant_id from fact_sale"
|
||||
"select shipping_address_id from fact_sale"
|
||||
"select shop_id from fact_sale"
|
||||
"select user_id from fact_sale"
|
||||
"select gross_sales from fact_sale"
|
||||
"select net_sales from fact_sale"
|
||||
"select total_sales from fact_sale"
|
||||
"select returns from fact_sale"
|
||||
"select discounts from fact_sale"
|
||||
"select shipping from fact_sale"
|
||||
"select taxes from fact_sale"
|
||||
"select gift_card_discounts from fact_sale"
|
||||
"select gift_card_gross_sales from fact_sale"
|
||||
"select gift_cards_issued from fact_sale"
|
||||
"select quantity from fact_sale"
|
||||
"select currency from fact_sale"
|
||||
"select is_deleted from fact_sale"
|
||||
"select test from fact_sale"
|
||||
"select happened_at from fact_sale"
|
||||
"select derived_session_token from fact_session"
|
||||
"select shop_id from fact_session"
|
||||
"select session_duration from fact_session"
|
||||
"select count_of_pageviews from fact_session"
|
||||
"select session_started_at from fact_session"
|
||||
"select session_token from fact_session"
|
||||
"select user_token from fact_session"
|
||||
"select landing_page_url from fact_session"
|
||||
"select exit_page_path from fact_session"
|
||||
"select exit_page_url from fact_session"
|
||||
"select referrer_tld from fact_session"
|
||||
"select ua_browser from fact_session"
|
||||
"select ua_raw from fact_session"
|
||||
"select count_of_orders_completed from fact_session"
|
||||
"select completed_first_order_at from fact_session"
|
||||
"select hit_first_checkout_at from fact_session"
|
||||
"select started_first_checkout_at from fact_session"
|
||||
"select count_of_cart_additions from fact_session"
|
||||
"select count_of_distinct_products_added_to_cart from fact_session"
|
||||
"select count_of_distinct_product_variants_added_to_cart from fact_session"
|
||||
"select had_error from fact_session"
|
||||
"select had_payment_error from fact_session"
|
||||
"select had_out_of_stock_warning from fact_session"
|
||||
"select had_credit_card_info_error from fact_session"
|
||||
"select had_discount from fact_session"
|
||||
"select had_free_shipping from fact_session"
|
||||
"select location_city from fact_session"
|
||||
"select location_region from fact_session"
|
||||
"select location_region_code from fact_session"
|
||||
"select location_country from fact_session"
|
||||
"select location_country_code from fact_session"
|
||||
"select comments from raw_customer"
|
||||
"select creditcard from raw_customer"
|
||||
"select membership from raw_customer"
|
||||
"select orders from raw_customer"
|
||||
"select platform from raw_customer"
|
||||
"select preference from raw_customer"
|
||||
"select shipping_address from raw_customer"
|
||||
"select shipping_date from raw_customer"
|
||||
"select transaction_date from raw_customer"
|
||||
"select customer from raw_customer"
|
||||
"select comments from raw_order"
|
||||
"select creditcard from raw_order"
|
||||
"select membership from raw_order"
|
||||
"select orders from raw_order"
|
||||
"select platform from raw_order"
|
||||
"select preference from raw_order"
|
||||
"select shipping_address from raw_order"
|
||||
"select shipping_date from raw_order"
|
||||
"select transaction_date from raw_order"
|
||||
"select total_order_count from raw_order"
|
||||
"select total_order_value from raw_order"
|
||||
"select first_order_date from raw_order"
|
||||
"select last_order_date from raw_order"
|
||||
"select comments from raw_product_catalog"
|
||||
"select products from raw_product_catalog"
|
||||
"select platform from raw_product_catalog"
|
||||
"select store_address from raw_product_catalog"
|
||||
"select first_order_date from raw_product_catalog"
|
||||
"select last_order_date from raw_product_catalog"
|
||||
@ -1035,56 +1035,6 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select address_id from dim_address",
|
||||
"duration": 0.731
|
||||
},
|
||||
{
|
||||
"query": "select shop_id from dim_address",
|
||||
"duration": 0.155
|
||||
},
|
||||
{
|
||||
"query": "select first_name from dim_address",
|
||||
"duration": 0.075
|
||||
},
|
||||
{
|
||||
"query": "select last_name from dim_address",
|
||||
"duration": 0.398
|
||||
},
|
||||
{
|
||||
"query": "select address1 from dim_address",
|
||||
"duration": 0.698
|
||||
},
|
||||
{
|
||||
"query": "select address2 from dim_address",
|
||||
"duration": 0.671
|
||||
},
|
||||
{
|
||||
"query": "select company from dim_address",
|
||||
"duration": 0.661
|
||||
},
|
||||
{
|
||||
"query": "select city from dim_address",
|
||||
"duration": 0.713
|
||||
},
|
||||
{
|
||||
"query": "select region from dim_address",
|
||||
"duration": 0.368
|
||||
},
|
||||
{
|
||||
"query": "select zip from dim_address",
|
||||
"duration": 0.345
|
||||
},
|
||||
{
|
||||
"query": "select country from dim_address",
|
||||
"duration": 0.762
|
||||
},
|
||||
{
|
||||
"query": "select phone from dim_address",
|
||||
"duration": 0.71
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -1431,16 +1381,6 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select api_client_id from dim_api_client",
|
||||
"duration": 0.641
|
||||
},
|
||||
{
|
||||
"query": "select title from dim_api_client",
|
||||
"duration": 0.539
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -3737,124 +3677,6 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select customer_id from dim_customer",
|
||||
"duration": 0.229
|
||||
},
|
||||
{
|
||||
"query": "select shop_id from dim_customer",
|
||||
"duration": 0.014
|
||||
},
|
||||
{
|
||||
"query": "select average_order_size from dim_customer",
|
||||
"duration": 0.552
|
||||
},
|
||||
{
|
||||
"query": "select total_order_count from dim_customer",
|
||||
"duration": 0.56
|
||||
},
|
||||
{
|
||||
"query": "select total_order_value from dim_customer",
|
||||
"duration": 0.207
|
||||
},
|
||||
{
|
||||
"query": "select first_order_date from dim_customer",
|
||||
"duration": 0.37
|
||||
},
|
||||
{
|
||||
"query": "select last_order_date from dim_customer",
|
||||
"duration": 0.077
|
||||
},
|
||||
{
|
||||
"query": "select rank from dim_customer",
|
||||
"duration": 0.274
|
||||
},
|
||||
{
|
||||
"query": "select new from dim_customer",
|
||||
"duration": 0.181
|
||||
},
|
||||
{
|
||||
"query": "select returning from dim_customer",
|
||||
"duration": 0.267
|
||||
},
|
||||
{
|
||||
"query": "select loyal from dim_customer",
|
||||
"duration": 0.775
|
||||
},
|
||||
{
|
||||
"query": "select at_risk from dim_customer",
|
||||
"duration": 0.633
|
||||
},
|
||||
{
|
||||
"query": "select dormant from dim_customer",
|
||||
"duration": 0.055
|
||||
},
|
||||
{
|
||||
"query": "select promising from dim_customer",
|
||||
"duration": 0.666
|
||||
},
|
||||
{
|
||||
"query": "select predicted_average_number_of_days_between_orders from dim_customer",
|
||||
"duration": 0.287
|
||||
},
|
||||
{
|
||||
"query": "select expected_purchase_value_in_next_30_days from dim_customer",
|
||||
"duration": 0.204
|
||||
},
|
||||
{
|
||||
"query": "select first_name from dim_customer",
|
||||
"duration": 0.717
|
||||
},
|
||||
{
|
||||
"query": "select last_name from dim_customer",
|
||||
"duration": 0.484
|
||||
},
|
||||
{
|
||||
"query": "select city from dim_customer",
|
||||
"duration": 0.597
|
||||
},
|
||||
{
|
||||
"query": "select region from dim_customer",
|
||||
"duration": 0.212
|
||||
},
|
||||
{
|
||||
"query": "select country from dim_customer",
|
||||
"duration": 0.01
|
||||
},
|
||||
{
|
||||
"query": "select email from dim_customer",
|
||||
"duration": 0.278
|
||||
},
|
||||
{
|
||||
"query": "select phone from dim_customer",
|
||||
"duration": 0.323
|
||||
},
|
||||
{
|
||||
"query": "select verified_email from dim_customer",
|
||||
"duration": 0.844
|
||||
},
|
||||
{
|
||||
"query": "select created_at from dim_customer",
|
||||
"duration": 0.666
|
||||
},
|
||||
{
|
||||
"query": "select accepts_marketing from dim_customer",
|
||||
"duration": 0.108
|
||||
},
|
||||
{
|
||||
"query": "select customer from dim_customer",
|
||||
"duration": 0.498
|
||||
},
|
||||
{
|
||||
"query": "select shipping_address from dim_customer",
|
||||
"duration": 0.206
|
||||
},
|
||||
{
|
||||
"query": "select orders from dim_customer",
|
||||
"duration": 0.197
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -4183,24 +4005,6 @@
|
||||
]
|
||||
]
|
||||
},
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select location_id from dim_location",
|
||||
"duration": 0.462
|
||||
},
|
||||
{
|
||||
"query": "select shop_id from dim_location",
|
||||
"duration": 0.584
|
||||
},
|
||||
{
|
||||
"query": "select name from dim_location",
|
||||
"duration": 0.891
|
||||
},
|
||||
{
|
||||
"query": "select address from dim_location",
|
||||
"duration": 0.488
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -4587,32 +4391,6 @@
|
||||
]
|
||||
]
|
||||
},
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select product_id from dim.product",
|
||||
"duration": 0.598
|
||||
},
|
||||
{
|
||||
"query": "select shop_id from dim.product",
|
||||
"duration": 0.205
|
||||
},
|
||||
{
|
||||
"query": "select title from dim.product",
|
||||
"duration": 0.165
|
||||
},
|
||||
{
|
||||
"query": "select vendor from dim.product",
|
||||
"duration": 0.847
|
||||
},
|
||||
{
|
||||
"query": "select created_at from dim.product",
|
||||
"duration": 0.114
|
||||
},
|
||||
{
|
||||
"query": "select deleted_at from dim.product",
|
||||
"duration": 0.559
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -5088,48 +4866,6 @@
|
||||
]
|
||||
]
|
||||
},
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select product_variant_id from dim.product_variant",
|
||||
"duration": 0.716
|
||||
},
|
||||
{
|
||||
"query": "select product_id from dim.product_variant",
|
||||
"duration": 0.636
|
||||
},
|
||||
{
|
||||
"query": "select shop_id from dim.product_variant",
|
||||
"duration": 0.015
|
||||
},
|
||||
{
|
||||
"query": "select title from dim.product_variant",
|
||||
"duration": 0.459
|
||||
},
|
||||
{
|
||||
"query": "select barcode from dim.product_variant",
|
||||
"duration": 0.828
|
||||
},
|
||||
{
|
||||
"query": "select sku from dim.product_variant",
|
||||
"duration": 0.561
|
||||
},
|
||||
{
|
||||
"query": "select price from dim.product_variant",
|
||||
"duration": 0.488
|
||||
},
|
||||
{
|
||||
"query": "select grams from dim.product_variant",
|
||||
"duration": 0.669
|
||||
},
|
||||
{
|
||||
"query": "select created_at from dim.product_variant",
|
||||
"duration": 0.029
|
||||
},
|
||||
{
|
||||
"query": "select deleted_at from dim.product_variant",
|
||||
"duration": 0.368
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -5413,20 +5149,6 @@
|
||||
]
|
||||
]
|
||||
},
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select shop_id from dim.shop",
|
||||
"duration": 0.566
|
||||
},
|
||||
{
|
||||
"query": "select name from dim.shop",
|
||||
"duration": 0.513
|
||||
},
|
||||
{
|
||||
"query": "select domain from dim.shop",
|
||||
"duration": 0.452
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -5792,28 +5514,6 @@
|
||||
]
|
||||
]
|
||||
},
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select user_id from dim_staff",
|
||||
"duration": 0.717
|
||||
},
|
||||
{
|
||||
"query": "select shop_id from dim_staff",
|
||||
"duration": 0.875
|
||||
},
|
||||
{
|
||||
"query": "select first_name from dim_staff",
|
||||
"duration": 0.492
|
||||
},
|
||||
{
|
||||
"query": "select last_name from dim_staff",
|
||||
"duration": 0.566
|
||||
},
|
||||
{
|
||||
"query": "select email from dim_staff",
|
||||
"duration": 0.294
|
||||
}
|
||||
],
|
||||
"tableProfile": [
|
||||
{
|
||||
"profileDate": "2021-10-16",
|
||||
@ -6587,76 +6287,6 @@
|
||||
]
|
||||
]
|
||||
},
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select line_item_id from fact_line_item",
|
||||
"duration": 0.418
|
||||
},
|
||||
{
|
||||
"query": "select billing_address_id from fact_line_item",
|
||||
"duration": 0.061
|
||||
},
|
||||
{
|
||||
"query": "select order_id from fact_line_item",
|
||||
"duration": 0.323
|
||||
},
|
||||
{
|
||||
"query": "select product_id from fact_line_item",
|
||||
"duration": 0.214
|
||||
},
|
||||
{
|
||||
"query": "select product_variant_id from fact_line_item",
|
||||
"duration": 0.709
|
||||
},
|
||||
{
|
||||
"query": "select shop_id from fact_line_item",
|
||||
"duration": 0.284
|
||||
},
|
||||
{
|
||||
"query": "select name from fact_line_item",
|
||||
"duration": 0.911
|
||||
},
|
||||
{
|
||||
"query": "select product_title from fact_line_item",
|
||||
"duration": 0.358
|
||||
},
|
||||
{
|
||||
"query": "select price from fact_line_item",
|
||||
"duration": 0.287
|
||||
},
|
||||
{
|
||||
"query": "select quantity from fact_line_item",
|
||||
"duration": 0.58
|
||||
},
|
||||
{
|
||||
"query": "select requires_shipping from fact_line_item",
|
||||
"duration": 0.101
|
||||
},
|
||||
{
|
||||
"query": "select taxable from fact_line_item",
|
||||
"duration": 0.281
|
||||
},
|
||||
{
|
||||
"query": "select gift_card from fact_line_item",
|
||||
"duration": 0.439
|
||||
},
|
||||
{
|
||||
"query": "select grams from fact_line_item",
|
||||
"duration": 0.514
|
||||
},
|
||||
{
|
||||
"query": "select product_vendor from fact_line_item",
|
||||
"duration": 0.42
|
||||
},
|
||||
{
|
||||
"query": "select fulfillable_quantity from fact_line_item",
|
||||
"duration": 0.449
|
||||
},
|
||||
{
|
||||
"query": "select fulfillment_service from fact_line_item",
|
||||
"duration": 0.509
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -7453,68 +7083,6 @@
|
||||
]
|
||||
]
|
||||
},
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select order_id from fact_order",
|
||||
"duration": 0.775
|
||||
},
|
||||
{
|
||||
"query": "select api_client_id from fact_order",
|
||||
"duration": 0.563
|
||||
},
|
||||
{
|
||||
"query": "select billing_address_id from fact_order",
|
||||
"duration": 0.636
|
||||
},
|
||||
{
|
||||
"query": "select customer_id from fact_order",
|
||||
"duration": 0.919
|
||||
},
|
||||
{
|
||||
"query": "select location_id from fact_order",
|
||||
"duration": 0.27
|
||||
},
|
||||
{
|
||||
"query": "select shipping_address_id from fact_order",
|
||||
"duration": 0.37
|
||||
},
|
||||
{
|
||||
"query": "select shop_id from fact_order",
|
||||
"duration": 0.648
|
||||
},
|
||||
{
|
||||
"query": "select user_id from fact_order",
|
||||
"duration": 0.449
|
||||
},
|
||||
{
|
||||
"query": "select name from fact_order",
|
||||
"duration": 0.008
|
||||
},
|
||||
{
|
||||
"query": "select total_price from fact_order",
|
||||
"duration": 0.111
|
||||
},
|
||||
{
|
||||
"query": "select discount_code from fact_order",
|
||||
"duration": 0.19
|
||||
},
|
||||
{
|
||||
"query": "select processed_at from fact_order",
|
||||
"duration": 0.029
|
||||
},
|
||||
{
|
||||
"query": "select canceled_at from fact_order",
|
||||
"duration": 0.65
|
||||
},
|
||||
{
|
||||
"query": "select deleted_at from fact_order",
|
||||
"duration": 0.51
|
||||
},
|
||||
{
|
||||
"query": "select test from fact_order",
|
||||
"duration": 0.943
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -9953,116 +9521,6 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select sale_id from fact_sale",
|
||||
"duration": 0.308
|
||||
},
|
||||
{
|
||||
"query": "select billing_address_id from fact_sale",
|
||||
"duration": 0.446
|
||||
},
|
||||
{
|
||||
"query": "select api_client_id from fact_sale",
|
||||
"duration": 0.104
|
||||
},
|
||||
{
|
||||
"query": "select customer_id from fact_sale",
|
||||
"duration": 0.289
|
||||
},
|
||||
{
|
||||
"query": "select line_item_id from fact_sale",
|
||||
"duration": 0.254
|
||||
},
|
||||
{
|
||||
"query": "select location_id from fact_sale",
|
||||
"duration": 0.925
|
||||
},
|
||||
{
|
||||
"query": "select order_id from fact_sale",
|
||||
"duration": 0.727
|
||||
},
|
||||
{
|
||||
"query": "select product_id from fact_sale",
|
||||
"duration": 0.729
|
||||
},
|
||||
{
|
||||
"query": "select product_variant_id from fact_sale",
|
||||
"duration": 0.895
|
||||
},
|
||||
{
|
||||
"query": "select shipping_address_id from fact_sale",
|
||||
"duration": 0.676
|
||||
},
|
||||
{
|
||||
"query": "select shop_id from fact_sale",
|
||||
"duration": 0.694
|
||||
},
|
||||
{
|
||||
"query": "select user_id from fact_sale",
|
||||
"duration": 0.689
|
||||
},
|
||||
{
|
||||
"query": "select gross_sales from fact_sale",
|
||||
"duration": 0.426
|
||||
},
|
||||
{
|
||||
"query": "select net_sales from fact_sale",
|
||||
"duration": 0.187
|
||||
},
|
||||
{
|
||||
"query": "select total_sales from fact_sale",
|
||||
"duration": 0.027
|
||||
},
|
||||
{
|
||||
"query": "select returns from fact_sale",
|
||||
"duration": 0.63
|
||||
},
|
||||
{
|
||||
"query": "select discounts from fact_sale",
|
||||
"duration": 0.443
|
||||
},
|
||||
{
|
||||
"query": "select shipping from fact_sale",
|
||||
"duration": 0.319
|
||||
},
|
||||
{
|
||||
"query": "select taxes from fact_sale",
|
||||
"duration": 0.176
|
||||
},
|
||||
{
|
||||
"query": "select gift_card_discounts from fact_sale",
|
||||
"duration": 0.355
|
||||
},
|
||||
{
|
||||
"query": "select gift_card_gross_sales from fact_sale",
|
||||
"duration": 0.412
|
||||
},
|
||||
{
|
||||
"query": "select gift_cards_issued from fact_sale",
|
||||
"duration": 0.296
|
||||
},
|
||||
{
|
||||
"query": "select quantity from fact_sale",
|
||||
"duration": 0.016
|
||||
},
|
||||
{
|
||||
"query": "select currency from fact_sale",
|
||||
"duration": 0.468
|
||||
},
|
||||
{
|
||||
"query": "select is_deleted from fact_sale",
|
||||
"duration": 0.22
|
||||
},
|
||||
{
|
||||
"query": "select test from fact_sale",
|
||||
"duration": 0.096
|
||||
},
|
||||
{
|
||||
"query": "select happened_at from fact_sale",
|
||||
"duration": 0.532
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -12324,132 +11782,6 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select derived_session_token from fact_session",
|
||||
"duration": 0.109
|
||||
},
|
||||
{
|
||||
"query": "select shop_id from fact_session",
|
||||
"duration": 0.71
|
||||
},
|
||||
{
|
||||
"query": "select session_duration from fact_session",
|
||||
"duration": 0.436
|
||||
},
|
||||
{
|
||||
"query": "select count_of_pageviews from fact_session",
|
||||
"duration": 0.371
|
||||
},
|
||||
{
|
||||
"query": "select session_started_at from fact_session",
|
||||
"duration": 0.849
|
||||
},
|
||||
{
|
||||
"query": "select session_token from fact_session",
|
||||
"duration": 0.073
|
||||
},
|
||||
{
|
||||
"query": "select user_token from fact_session",
|
||||
"duration": 0.655
|
||||
},
|
||||
{
|
||||
"query": "select landing_page_url from fact_session",
|
||||
"duration": 0.489
|
||||
},
|
||||
{
|
||||
"query": "select exit_page_path from fact_session",
|
||||
"duration": 0.831
|
||||
},
|
||||
{
|
||||
"query": "select exit_page_url from fact_session",
|
||||
"duration": 0.826
|
||||
},
|
||||
{
|
||||
"query": "select referrer_tld from fact_session",
|
||||
"duration": 0.371
|
||||
},
|
||||
{
|
||||
"query": "select ua_browser from fact_session",
|
||||
"duration": 0.712
|
||||
},
|
||||
{
|
||||
"query": "select ua_raw from fact_session",
|
||||
"duration": 0.811
|
||||
},
|
||||
{
|
||||
"query": "select count_of_orders_completed from fact_session",
|
||||
"duration": 0.246
|
||||
},
|
||||
{
|
||||
"query": "select completed_first_order_at from fact_session",
|
||||
"duration": 0.593
|
||||
},
|
||||
{
|
||||
"query": "select hit_first_checkout_at from fact_session",
|
||||
"duration": 0.246
|
||||
},
|
||||
{
|
||||
"query": "select started_first_checkout_at from fact_session",
|
||||
"duration": 0.358
|
||||
},
|
||||
{
|
||||
"query": "select count_of_cart_additions from fact_session",
|
||||
"duration": 0.999
|
||||
},
|
||||
{
|
||||
"query": "select count_of_distinct_products_added_to_cart from fact_session",
|
||||
"duration": 0.526
|
||||
},
|
||||
{
|
||||
"query": "select count_of_distinct_product_variants_added_to_cart from fact_session",
|
||||
"duration": 0.376
|
||||
},
|
||||
{
|
||||
"query": "select had_error from fact_session",
|
||||
"duration": 0.515
|
||||
},
|
||||
{
|
||||
"query": "select had_payment_error from fact_session",
|
||||
"duration": 0.699
|
||||
},
|
||||
{
|
||||
"query": "select had_out_of_stock_warning from fact_session",
|
||||
"duration": 0.322
|
||||
},
|
||||
{
|
||||
"query": "select had_credit_card_info_error from fact_session",
|
||||
"duration": 0.815
|
||||
},
|
||||
{
|
||||
"query": "select had_discount from fact_session",
|
||||
"duration": 0.771
|
||||
},
|
||||
{
|
||||
"query": "select had_free_shipping from fact_session",
|
||||
"duration": 0.21
|
||||
},
|
||||
{
|
||||
"query": "select location_city from fact_session",
|
||||
"duration": 0.506
|
||||
},
|
||||
{
|
||||
"query": "select location_region from fact_session",
|
||||
"duration": 0.088
|
||||
},
|
||||
{
|
||||
"query": "select location_region_code from fact_session",
|
||||
"duration": 0.853
|
||||
},
|
||||
{
|
||||
"query": "select location_country from fact_session",
|
||||
"duration": 0.711
|
||||
},
|
||||
{
|
||||
"query": "select location_country_code from fact_session",
|
||||
"duration": 0.779
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -13343,48 +12675,6 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select comments from raw_customer",
|
||||
"duration": 0.413
|
||||
},
|
||||
{
|
||||
"query": "select creditcard from raw_customer",
|
||||
"duration": 0.674
|
||||
},
|
||||
{
|
||||
"query": "select membership from raw_customer",
|
||||
"duration": 0.806
|
||||
},
|
||||
{
|
||||
"query": "select orders from raw_customer",
|
||||
"duration": 0.851
|
||||
},
|
||||
{
|
||||
"query": "select platform from raw_customer",
|
||||
"duration": 0.898
|
||||
},
|
||||
{
|
||||
"query": "select preference from raw_customer",
|
||||
"duration": 0.016
|
||||
},
|
||||
{
|
||||
"query": "select shipping_address from raw_customer",
|
||||
"duration": 0.4
|
||||
},
|
||||
{
|
||||
"query": "select shipping_date from raw_customer",
|
||||
"duration": 0.42
|
||||
},
|
||||
{
|
||||
"query": "select transaction_date from raw_customer",
|
||||
"duration": 0.637
|
||||
},
|
||||
{
|
||||
"query": "select customer from raw_customer",
|
||||
"duration": 0.287
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -14434,60 +13724,6 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select comments from raw_order",
|
||||
"duration": 0.827
|
||||
},
|
||||
{
|
||||
"query": "select creditcard from raw_order",
|
||||
"duration": 0.776
|
||||
},
|
||||
{
|
||||
"query": "select membership from raw_order",
|
||||
"duration": 0.123
|
||||
},
|
||||
{
|
||||
"query": "select orders from raw_order",
|
||||
"duration": 0.227
|
||||
},
|
||||
{
|
||||
"query": "select platform from raw_order",
|
||||
"duration": 0.775
|
||||
},
|
||||
{
|
||||
"query": "select preference from raw_order",
|
||||
"duration": 0.655
|
||||
},
|
||||
{
|
||||
"query": "select shipping_address from raw_order",
|
||||
"duration": 0.75
|
||||
},
|
||||
{
|
||||
"query": "select shipping_date from raw_order",
|
||||
"duration": 0.629
|
||||
},
|
||||
{
|
||||
"query": "select transaction_date from raw_order",
|
||||
"duration": 0.207
|
||||
},
|
||||
{
|
||||
"query": "select total_order_count from raw_order",
|
||||
"duration": 0.848
|
||||
},
|
||||
{
|
||||
"query": "select total_order_value from raw_order",
|
||||
"duration": 0.536
|
||||
},
|
||||
{
|
||||
"query": "select first_order_date from raw_order",
|
||||
"duration": 0.55
|
||||
},
|
||||
{
|
||||
"query": "select last_order_date from raw_order",
|
||||
"duration": 0.775
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
@ -15084,32 +14320,6 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"tableQueries": [
|
||||
{
|
||||
"query": "select comments from raw_product_catalog",
|
||||
"duration": 0.845
|
||||
},
|
||||
{
|
||||
"query": "select products from raw_product_catalog",
|
||||
"duration": 0.309
|
||||
},
|
||||
{
|
||||
"query": "select platform from raw_product_catalog",
|
||||
"duration": 0.278
|
||||
},
|
||||
{
|
||||
"query": "select store_address from raw_product_catalog",
|
||||
"duration": 0.333
|
||||
},
|
||||
{
|
||||
"query": "select first_order_date from raw_product_catalog",
|
||||
"duration": 0.125
|
||||
},
|
||||
{
|
||||
"query": "select last_order_date from raw_product_catalog",
|
||||
"duration": 0.381
|
||||
}
|
||||
],
|
||||
"databaseSchema": {
|
||||
"id": "d7be1e2c-b3dc-11ec-b909-0242ac120002",
|
||||
"type": "databaseSchema",
|
||||
|
||||
@ -81,6 +81,7 @@ class MetadataUsageBulkSink(BulkSink):
|
||||
for record in usage_records:
|
||||
table_usage = TableUsageCount(**json.loads(record))
|
||||
table_entities = []
|
||||
self.service_name = table_usage.service_name
|
||||
if "." in table_usage.table:
|
||||
(
|
||||
table_usage.database_schema,
|
||||
@ -100,7 +101,6 @@ class MetadataUsageBulkSink(BulkSink):
|
||||
search_index="table_search_index",
|
||||
)
|
||||
table_entities = es_result
|
||||
self.service_name = table_usage.service_name
|
||||
for table_entity in table_entities:
|
||||
if table_entity is not None:
|
||||
if not table_usage_map.get(table_entity.id.__root__):
|
||||
|
||||
@ -66,5 +66,5 @@ class ESMixin(Generic[T]):
|
||||
)
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"Elasticsearch failed for query: {generate_es_string}")
|
||||
logger.warning(f"Elasticsearch failed for query: {generate_es_string}")
|
||||
return multiple_entities
|
||||
|
||||
@ -244,5 +244,5 @@ class OMetaLineageMixin(Generic[T]):
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.debug(str(err))
|
||||
logger.error(f"Ingesting lineage failed")
|
||||
logger.warning(f"Ingesting lineage failed")
|
||||
return False
|
||||
|
||||
@ -12,7 +12,6 @@
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import traceback
|
||||
import uuid
|
||||
@ -457,24 +456,7 @@ class SampleDataSource(Source[Entity]):
|
||||
)
|
||||
resp = self.metadata.list_entities(entity=User, limit=5)
|
||||
self.user_entity = resp.entities
|
||||
user_entity_len = min(len(self.user_entity), 5)
|
||||
for table in self.tables["tables"]:
|
||||
try:
|
||||
for sql_object in table["tableQueries"]:
|
||||
user_entity = self.user_entity[
|
||||
random.choice(range(user_entity_len))
|
||||
]
|
||||
user_dict = {
|
||||
"id": user_entity.id.__root__,
|
||||
"name": user_entity.name.__root__,
|
||||
"displayName": user_entity.displayName,
|
||||
"href": user_entity.href,
|
||||
"description": user_entity.description,
|
||||
}
|
||||
sql_object["user"] = EntityReference(**user_dict, type="user")
|
||||
except Exception as err:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.debug(err)
|
||||
table_metadata = Table(**table)
|
||||
table_and_db = OMetaDatabaseAndTable(
|
||||
table=table_metadata, database=db, database_schema=schema
|
||||
|
||||
@ -101,7 +101,7 @@ def create_generic_connection(connection, verbose: bool = False):
|
||||
@singledispatch
|
||||
def get_connection(
|
||||
connection, verbose: bool = False
|
||||
) -> Union[Engine, DynamoClient, GlueClient]:
|
||||
) -> Union[Engine, DynamoClient, GlueClient, SalesforceClient]:
|
||||
"""
|
||||
Given an SQL configuration, build the SQLAlchemy Engine
|
||||
"""
|
||||
|
||||
@ -67,7 +67,7 @@ class QueryParserTest(TestCase):
|
||||
"shopify.dim_address": 100,
|
||||
"shopify.dim_shop": 190,
|
||||
"shopify.dim_customer": 125,
|
||||
"dim_customer": 9,
|
||||
"dim_customer": 38,
|
||||
"shopify.dim_location": 75,
|
||||
"dim_location.shop_id": 25,
|
||||
"dim_shop.shop_id": 105,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user