diff --git a/ingestion/examples/shopify_schemas/tables.json b/ingestion/examples/shopify_schemas/tables.json index adcbd7cb97e..de47bf0e995 100644 --- a/ingestion/examples/shopify_schemas/tables.json +++ b/ingestion/examples/shopify_schemas/tables.json @@ -92,234 +92,6 @@ "ordinalPosition": 12 } ], - "sampleData": { - "columns": [ - "shop_id", - "address1", - "city", - "company", - "last_name", - "country", - "phone", - "first_name", - "address2", - "region", - "zip", - "address_id" - ], - "rows": [ - [ - "cc79f7b0-e5ed-4223-8ab8-ab4dd3e60ecc", - "67fc33a4-67f8-42fc-8693-f756de908e8d", - "Production option.", - "White officer staff.", - "Sell explain best.", - "Meeting agreement.", - "Design already.", - "Matter television.", - "Miss training.", - "Cup yourself.", - "Determine in air.", - "Give marriage day." - ], - [ - "378631ba-5b51-4a88-a78d-d8ca9fde5ffc", - "7819c541-32c7-4641-b1db-7e1c1bc540d7", - "Five occur deal.", - "Republican class.", - "Senior before ten.", - "Peace paper final.", - "Impact director.", - "Choose sometimes.", - "Current race.", - "Service yet series.", - "Agency together.", - "Score store class." - ], - [ - "57434be2-fae1-4101-9dff-f181d54fd229", - "0c1d1850-906f-4820-9e8e-7864a3d19af5", - "Machine consumer.", - "Cause explain.", - "Some wide cut.", - "Hope sit amount.", - "Protect senior.", - "Explain mission a.", - "Seven relate.", - "Local read grow.", - "Management happy.", - "Level share fund." - ], - [ - "fff72b3f-9184-43d4-bffa-920495175c22", - "0c918475-d684-492a-bb51-d3a17fccc114", - "Spend national dog.", - "Watch least arm.", - "Whole form chance.", - "Real hospital.", - "Television data.", - "Reveal growth.", - "Present glass it.", - "Possible free.", - "Shake message place.", - "Draw smile history." - ], - [ - "bec1e09b-4c7d-4822-af07-89ea05f64186", - "c8ae8d2f-7b21-4afb-a94d-10cf0ee9df2c", - "Hour option threat.", - "Blood public health.", - "Clear place set.", - "Claim realize.", - "Can tell pattern.", - "Hour capital ball.", - "Home often majority.", - "Issue American.", - "Little store she.", - "Really follow few." - ], - [ - "f61ed11c-8948-40e9-8ab7-5fcaeebb32c0", - "d1874f2e-3cd9-44ae-a737-e0d9b574f790", - "Product say manager.", - "Attack relationship.", - "Old court.", - "Room writer draw.", - "Ability group short.", - "Whole age purpose.", - "Either section.", - "International.", - "Young world you.", - "Start treatment." - ], - [ - "8cdf1b2d-8817-434e-bc9e-07f61023490d", - "2fe692a8-d75b-4a0b-bbd5-7a2cb97822cf", - "Quickly bag.", - "Agree value single.", - "Produce operation.", - "Capital consider.", - "Ball benefit health.", - "Behavior exist.", - "Control rather.", - "Room fund half rule.", - "Woman next run.", - "It body explain." - ], - [ - "f2a8a1d8-5471-4597-a676-0a3d10f08281", - "ab078153-f27b-41bb-bdcd-09bf62e4cee9", - "But PM air wear.", - "Seven ok subject.", - "Trade offer book.", - "Central bank will.", - "Keep Congress.", - "Only father baby.", - "Debate item every.", - "Floor interest.", - "Month professor.", - "Stage peace history." - ], - [ - "336fb3f3-19e1-49f9-94b6-00bc7098aec3", - "c28355de-a6b3-4236-b7ad-947143351c1d", - "Bank lawyer history.", - "Win sit join.", - "Especially effort.", - "Energy service.", - "Avoid forward must.", - "Report about head.", - "Try reality people.", - "Change clear wait.", - "Difference certain.", - "Specific resource." - ], - [ - "a5914de1-4fb8-4912-adc7-2f6253de2d41", - "13d59c87-9450-4c4a-b2aa-5472556a80f0", - "Half me power.", - "Inside magazine.", - "Game product voice.", - "Congress son accept.", - "Security discover.", - "Everyone quality.", - "Way admit nice save.", - "Within civil.", - "Strong west.", - "Least simply." - ], - [ - "516ad154-95a1-49ed-a5bf-7304080338f7", - "c392fc0e-a96a-4cf9-b4cf-1df41fa0dd1f", - "Response radio.", - "Quickly table.", - "Health.", - "Include ago police.", - "Prepare there.", - "Type safe range.", - "Feeling strong less.", - "For fast nothing.", - "Decade skill.", - "They money spend." - ], - [ - "e8f42c8b-28d5-472f-8380-6c21e997874a", - "9ca18289-53e6-44ec-aa3d-3d1acc9ad464", - "Stage bad property.", - "Over phone Mr foot.", - "From water hot.", - "Participant parent.", - "Move theory whether.", - "Face his child.", - "Much ago sound fact.", - "Else morning.", - "Suggest win nation.", - "Meeting respond." - ], - [ - "90a7e5c9-ee17-4fac-9405-a9b34e416513", - "09b1e988-bfc2-4dc4-8bae-1fe99eea2c76", - "Remain myself guess.", - "Article weight.", - "Marriage official.", - "Hundred source.", - "Animal parent great.", - "Him peace science.", - "Need run Democrat.", - "Pick according.", - "Safe state teacher.", - "Bag cell four keep." - ], - [ - "ee0866ba-2177-4fa5-80c3-f58801dd910e", - "9f924e39-6571-4925-a9bc-fea33002009d", - "Read team record.", - "Treat where hotel.", - "Soon police allow.", - "Strategy Mrs beat.", - "Condition sister.", - "Blue growth exactly.", - "Young science side.", - "Citizen hospital.", - "News air charge.", - "Senior Democrat." - ], - [ - "281c55b0-d7b8-4d06-84ed-060ffa346bf5", - "19f84462-c6dc-493d-b82a-0d66d3762e4d", - "Type most recently.", - "Hundred country.", - "Indicate must.", - "Air health job.", - "Themselves owner.", - "Life beyond central.", - "Note involve.", - "Beyond story look.", - "System weight ever.", - "Less mother key." - ] - ] - }, "database": { "id": "89229e26-7f74-4443-a568-85512eaeaa07", "type": "database" @@ -348,26 +120,7 @@ "ordinalPosition": 2 } ], - "sampleData": { - "columns": ["title", "api_client_id"], - "rows": [ - ["1fbff7bc-2321-4e4c-9e99-0f4f93793069", "Certainly."], - ["62c732dd-80b6-4905-92f8-1b5f7957009a", "Push family stand."], - ["f8a3ff22-ff1e-4e4c-b9cc-6f405e84417e", "Usually perhaps."], - ["63c83b4e-433a-474c-9d3e-64c8db212c17", "Trouble money."], - ["750d1e45-a512-4876-8d7e-18eb342630a7", "Speech three."], - ["182e21d9-706c-464b-9ffa-3e290b66c1a8", "Difference example."], - ["1ac0e9f5-bd8f-444a-ba42-c546c7c0354c", "Hour open before."], - ["b35f6284-7915-48ac-8612-41bf38a45d95", "Serious decision."], - ["c728b0fb-2dd1-4490-8426-518b53490109", "Cause until policy."], - ["a899c3f2-64c0-4763-89f6-84fcd36efd1d", "Ago wall than."], - ["6718b242-75ac-4122-8bce-031fe1ea14a4", "Entire memory note."], - ["9e2d7a80-6417-4e21-a169-00c25e58e246", "Matter bad feeling."], - ["97732da7-ada5-4f98-9621-b6b494d482ea", "Agree interview."], - ["0e700838-1237-4748-97df-ee5b3b93cb98", "His military."], - ["5e6be1aa-ab8c-49ef-8c94-23916ba36f3e", "Audience issue."] - ] - }, + "database": { "id": "89229e26-7f74-4443-a568-85512eaeaa07", "type": "database" @@ -565,458 +318,6 @@ "ordinalPosition": 26 } ], - "sampleData": { - "columns": [ - "customer_id", - "accepts_marketing", - "expected_purchase_value_in_next_30_days", - "returning", - "predicted_average_number_of_days_between_orders", - "average_order_size", - "first_order_date", - "created_at", - "email", - "shop_id", - "last_name", - "promising", - "region", - "verified_email", - "at_risk", - "dormant", - "new", - "last_order_date", - "phone", - "country", - "total_order_count", - "city", - "rank", - "total_order_value", - "first_name", - "loyal" - ], - "rows": [ - [ - "2571bc3a-186e-4745-a140-4699001d8a3c", - "6f08d500-3148-4322-bb20-93ff81e7e92e", - 2653, - 6622, - 9846, - 845325730, - 907425617, - 2585, - false, - true, - true, - false, - false, - true, - 7050, - 3616, - "Cell thought argue.", - "Decision use may.", - "Central class PM.", - "Teach body pressure.", - "Enough never.", - "Establish standard.", - "Mother news.", - false, - 1166404411, - false - ], - [ - "62171523-fca4-486d-9364-8b09bffe418f", - "10f9fcbe-8387-49e6-a6f7-39651d818378", - 3208, - 6680, - 7749, - 920479881, - 1020365255, - 2572, - true, - false, - true, - false, - true, - false, - 7579, - 5351, - "Assume Mrs.", - "Dream behind check.", - "Effort many exactly.", - "Particularly son.", - "Seek prepare myself.", - "Hard us phone very.", - "Check next marriage.", - true, - 695249818, - true - ], - [ - "feec3c47-1f22-4df9-b2df-f1009908e6c0", - "4b2ecb4c-50d9-4e72-a461-c201c1509967", - 7352, - 2699, - 4669, - 64378196, - 532979642, - 7281, - true, - true, - true, - false, - false, - false, - 5933, - 6493, - "Environment.", - "Information control.", - "Least white along.", - "Use value another.", - "Sister member month.", - "Different live it.", - "Increase still nice.", - true, - 231196109, - true - ], - [ - "299ebddb-c428-4280-8185-5bcf1feb404d", - "ae9cca8c-49ca-4605-b050-c6a5165a959d", - 1077, - 501, - 9836, - 481247239, - 370597405, - 3481, - false, - true, - false, - true, - true, - false, - 7332, - 5344, - "Top leader yeah car.", - "Somebody specific.", - "Strong anything.", - "Notice everybody.", - "Scientist event.", - "Main difficult.", - "Decision leg boy.", - false, - 1522006114, - true - ], - [ - "09f21d9c-8bf4-4a40-bee6-860c98acb26e", - "419750ef-5ff7-4396-8be9-5810b19a1ff8", - 1336, - 3691, - 2111, - 158279442, - 791785452, - 6688, - false, - true, - true, - false, - true, - true, - 3825, - 3404, - "Give glass follow.", - "All type worry.", - "Have involve high.", - "Friend material.", - "Consider give give.", - "Myself south level.", - "Tend growth dinner.", - false, - 894742403, - true - ], - [ - "a0ffbe34-938d-4864-8911-36a587b73d15", - "189b62f3-e100-49ff-accf-f450fef33e88", - 5605, - 5436, - 152, - 1039660118, - 1221443794, - 7293, - true, - true, - true, - true, - true, - false, - 1178, - 758, - "Than these live pay.", - "Lawyer interest all.", - "Film beyond.", - "Industry keep story.", - "Article despite.", - "Role voice whole.", - "Wall current bar.", - true, - 1312422513, - false - ], - [ - "45f30d62-8bbb-41aa-9746-9c004b4f109a", - "f995ca51-8bdc-4ccd-abff-3f207a351a45", - 6808, - 5276, - 680, - 1623386094, - 98073116, - 6507, - true, - false, - false, - true, - true, - false, - 6995, - 59, - "Into enjoy two late.", - "Number writer gas.", - "War think positive.", - "Around establish.", - "Need consumer piece.", - "Fish American all.", - "Morning president.", - false, - 676462489, - false - ], - [ - "f9363436-19a3-4e6f-813c-b2a196ff7889", - "0d9513ad-60d1-449e-87a9-e2519c66b849", - 7706, - 6727, - 9272, - 900824121, - 234234835, - 707, - true, - false, - false, - true, - false, - false, - 4587, - 7674, - "You dark this day.", - "Night wear these to.", - "Floor degree this.", - "Very board blood.", - "Form total poor.", - "Particularly.", - "Stock continue see.", - false, - 773691910, - true - ], - [ - "09127b56-1293-4187-b7a2-c2188c930ce6", - "45aec9da-853e-4690-88aa-0f8d004aa4ed", - 9852, - 5978, - 3153, - 471658497, - 66214494, - 4448, - false, - true, - false, - false, - true, - false, - 5750, - 3172, - "Entire home.", - "Product street red.", - "Window all way low.", - "Sense option.", - "Practice pressure.", - "Here computer.", - "Scene make left.", - true, - 1569647649, - false - ], - [ - "4b388728-3d90-41f2-b921-5e02aa1b74ee", - "049583e0-be4a-4b7a-aa0b-0576e3ab9250", - 6246, - 5759, - 146, - 175428254, - 737310444, - 4242, - true, - false, - false, - false, - true, - false, - 5224, - 2763, - "Per leg close sign.", - "Eat individual lose.", - "Admit who man home.", - "Entire listen my.", - "Land wife picture.", - "Police two part.", - "Meeting stand.", - true, - 1384260351, - true - ], - [ - "8b9942c8-1937-400c-975c-97a9d568a899", - "a9bcf781-3f19-428b-948f-53513b03855c", - 7462, - 4590, - 3999, - 505880429, - 1328722228, - 6372, - true, - true, - true, - false, - false, - true, - 6613, - 3033, - "Whom miss reflect.", - "Outside democratic.", - "Training tough none.", - "Why analysis still.", - "Thus share sport.", - "Behind ahead.", - "Agency huge picture.", - false, - 916919845, - false - ], - [ - "205748be-7a5a-4e10-b8f4-fd511134c337", - "8ddb4850-a0b3-4378-b9a7-027b175f787d", - 3143, - 4283, - 7924, - 207010479, - 799065176, - 3048, - true, - true, - true, - false, - false, - false, - 454, - 5276, - "Spend such offer.", - "Even evidence law.", - "Benefit five assume.", - "Away it activity.", - "Scientist why many.", - "Hundred close.", - "One but price.", - true, - 1067003109, - true - ], - [ - "7e241e85-703c-4edf-9cd5-38d77620174e", - "44d712f4-b250-451c-849a-832e10af6aa3", - 3389, - 4526, - 1641, - 438970549, - 1217830635, - 4821, - true, - true, - false, - false, - false, - false, - 7061, - 9167, - "Half but.", - "Put suddenly effort.", - "Coach stop company.", - "Actually box agent.", - "Animal region.", - "Dark suggest effect.", - "Discover somebody.", - false, - 871071029, - true - ], - [ - "75190851-34e0-4d73-8748-bacfb8cee770", - "38eaee14-f3d6-4a76-be80-2981e703665e", - 9884, - 7043, - 193, - 106705399, - 991535360, - 4827, - true, - true, - false, - false, - false, - false, - 5792, - 7674, - "People different.", - "Purpose land key.", - "Which million test.", - "Technology suddenly.", - "Computer positive.", - "Always quickly.", - "Team huge long.", - false, - 767294086, - false - ], - [ - "0cbf19a0-172a-49a2-a2b2-2d4c48cdaf40", - "50c20fc9-d5b7-419a-90b7-4d2bcfff99c5", - 9154, - 5986, - 9567, - 541734621, - 925425989, - 329, - true, - true, - false, - true, - false, - true, - 7972, - 5818, - "Seek discover sign.", - "Sister early blue.", - "Pass Mrs call civil.", - "Government.", - "Carry item over.", - "Raise treat travel.", - "Chance challenge.", - false, - 1319102928, - false - ] - ] - }, "database": { "id": "89229e26-7f74-4443-a568-85512eaeaa07", "type": "database" diff --git a/ingestion/src/metadata/ingestion/api/workflow.py b/ingestion/src/metadata/ingestion/api/workflow.py index 8452223bb89..5347a384e0b 100644 --- a/ingestion/src/metadata/ingestion/api/workflow.py +++ b/ingestion/src/metadata/ingestion/api/workflow.py @@ -63,10 +63,8 @@ class Workflow: self.config = config self.ctx = WorkflowContext(workflow_id=self.config.run_id) source_type = self.config.source.type - print(source_type) source_class = self.get('metadata.ingestion.source.{}.{}Source'.format( self.typeClassFetch(source_type, True), self.typeClassFetch(source_type, False))) - print(source_class) metadata_config = self.config.metadata_server.dict().get("config", {}) self.source: Source = source_class.create( self.config.source.dict().get("config", {}), metadata_config, self.ctx diff --git a/ingestion/src/metadata/ingestion/source/sample_tables.py b/ingestion/src/metadata/ingestion/source/sample_tables.py index 16bb52a462c..93d53040dd5 100644 --- a/ingestion/src/metadata/ingestion/source/sample_tables.py +++ b/ingestion/src/metadata/ingestion/source/sample_tables.py @@ -177,24 +177,49 @@ class GenerateFakeSampleData: @classmethod def check_columns(self, columns): fake = Faker() - colList = set() colData = [] - for i in range(15): + colList = [column['name'] for column in columns] + for i in range(5): row = [] for column in columns: - colList.add(column['name']) - if "id" in column['name']: - row.append(uuid.uuid4()) - elif column['columnDataType'] == 'VARCHAR': - row.append(fake.text(max_nb_chars=20)) - elif column['columnDataType'] == 'NUMERIC' and "id" not in column['name']: - row.append(fake.pyint()) - elif column['columnDataType'] == 'BOOLEAN': - row.append(fake.pybool()) - elif column['columnDataType'] == 'TIMESTAMP': - row.append(fake.unix_time()) + col_name = column['name'] + value = None + if "id" in col_name: + value = uuid.uuid4() + elif "price" in col_name: + value = fake.pricetag() + elif "barcode" in col_name: + value = fake.ean(length=13) + elif "phone" in col_name: + value = fake.phone_number() + elif "zip" in col_name: + value = fake.postcode() + elif "address" in col_name: + value = fake.street_address() + elif "company" in col_name: + value = fake.company() + elif "region" in col_name: + value = fake.street_address() + elif "name" in col_name: + value = fake.first_name() + elif "city" in col_name: + value = fake.city() + elif "country" in col_name: + value = fake.country() + if value is None: + if "TIMESTAMP" in column['columnDataType'] or "date" in col_name: + value = fake.unix_time() + elif "BOOLEAN" in column['columnDataType']: + value = fake.pybool() + elif "NUMERIC" in column['columnDataType']: + value = fake.pyint() + elif "VARCHAR" in column['columnDataType']: + value = fake.text(max_nb_chars=20) + else: + value = None + row.append(value) colData.append(row) - return {"columns": list(colList), "rows": colData} + return {"columns": colList, "rows": colData} class SampleTablesSource(Source):