yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-21 15:38:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ingestion: add sample usage connector

Browse Source
This commit is contained in:
Suresh Srinivas 2021-08-13 01:06:10 -07:00
parent a2403b4570
commit 597da5b24e
3 changed files with 35 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
ingestion/examples/shopify_schemas/query_log
View File

@ -1,26 +1,26 @@
query, database
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
@ -58,7 +58,7 @@ join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_staff.user_id,dim_staff.first_name, dim_staff.last_name, dim_shop.shop_id from dim_staff join dim_shop on
dim_staff.shop_id=dim_shop.shop_id", "shopify"
"select dim_staff.user_id,dim_staff.first_name, dim_staff.last_name, dim_shop.shop_id from dim_staff join dim_shop on
@ -190,27 +190,27 @@ dim_staff.shop_id=dim_shop.shop_id", "shopify"
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from fact_sale join dim_staff on fact_sale.user_id = dim_staff.user_id", "shopify"
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from fact_sale join dim_staff on fact_sale.user_id = dim_staff.user_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
@ -248,7 +248,7 @@ join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_staff.user_id,dim_staff.first_name, dim_staff.last_name, dim_shop.shop_id from dim_staff join dim_shop on
dim_staff.shop_id=dim_shop.shop_id", "shopify"
"select dim_staff.user_id,dim_staff.first_name, dim_staff.last_name, dim_shop.shop_id from dim_staff join dim_shop on
@ -380,27 +380,27 @@ dim_staff.shop_id=dim_shop.shop_id", "shopify"
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from fact_sale join dim_staff on fact_sale.user_id = dim_staff.user_id", "shopify"
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from fact_sale join dim_staff on fact_sale.user_id = dim_staff.user_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
@ -438,7 +438,7 @@ join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_staff.user_id,dim_staff.first_name, dim_staff.last_name, dim_shop.shop_id from dim_staff join dim_shop on
dim_staff.shop_id=dim_shop.shop_id", "shopify"
"select dim_staff.user_id,dim_staff.first_name, dim_staff.last_name, dim_shop.shop_id from dim_staff join dim_shop on
@ -570,27 +570,27 @@ dim_staff.shop_id=dim_shop.shop_id", "shopify"
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from fact_sale join dim_staff on fact_sale.user_id = dim_staff.user_id", "shopify"
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from fact_sale join dim_staff on fact_sale.user_id = dim_staff.user_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
@ -628,7 +628,7 @@ join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_staff.user_id,dim_staff.first_name, dim_staff.last_name, dim_shop.shop_id from dim_staff join dim_shop on
dim_staff.shop_id=dim_shop.shop_id", "shopify"
"select dim_staff.user_id,dim_staff.first_name, dim_staff.last_name, dim_shop.shop_id from dim_staff join dim_shop on
@ -760,27 +760,27 @@ dim_staff.shop_id=dim_shop.shop_id", "shopify"
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from fact_sale join dim_staff on fact_sale.user_id = dim_staff.user_id", "shopify"
"select fact_sale.gross_sale, dim_staff.first_name, dim_staff.last_name from fact_sale join dim_staff on fact_sale.user_id = dim_staff.user_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
@ -818,7 +818,7 @@ join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_customer.customer_id, dim_customer.total_order_count, dim_customer.total_order_value from dim_customer left
join dim_shop on dim_customer.shop_id=dim_shop.shop_id", "shopify"
"select dim_address.address_id, dim_address.shop_id, dim_address.first_name,dim_address.last_name, dim_shop.name
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.id", "shopify"
from dim_address left join dim_shop on dim_address.shop_id = dim_shop.shop_id", "shopify"
"select dim_staff.user_id,dim_staff.first_name, dim_staff.last_name, dim_shop.shop_id from dim_staff join dim_shop on
dim_staff.shop_id=dim_shop.shop_id", "shopify"
"select dim_staff.user_id,dim_staff.first_name, dim_staff.last_name, dim_shop.shop_id from dim_staff join dim_shop on

1
ingestion/src/metadata/ingestion/bulksink/metadata_usage_rest.py
View File

@ -125,7 +125,6 @@ class MetadataUsageBulkSink(BulkSink):
return table_joins
def __get_column_fqdn(self, table_column: TableColumn):
print(table_column.table)
if table_column.table not in self.tables_dict:
return None
table_entity = self.tables_dict[table_column.table]

1
ingestion/src/metadata/ingestion/source/sample_data_generator.py
View File

@ -257,7 +257,6 @@ class SampleUserMetadataGenerator:
def get_service_or_create(service_json, metadata_config) -> DatabaseServiceEntity:
client = REST(metadata_config)
service = client.get_database_service(service_json['name'])
print(service)
if service is not None:
return service
else: