yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-02 19:48:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

ISSUE-243: Generate sample data

Browse Source
This commit is contained in:
Suresh Srinivas 2021-08-19 19:39:57 -07:00
parent 1ce36a3f64
commit 6b5dd36672
3 changed files with 8 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ingestion/examples/workflows/redshift.json
View File

@ -2,9 +2,9 @@
"source": {
"type": "redshift",
"config": {
"host_port": "cluster.name.region.redshift.amazonaws.com:5439",
"username": "username",
"password": "strong_password",
"host_port": "redshift-cluster-1.clot5cqn1cnb.us-west-2.redshift.amazonaws.com:5439",
"username": "awsuser",
"password": "focguC-kaqqe5-nepsok",
"database": "warehouse",
"service_name": "aws_redshift",
"service_type": "Redshift",

2
ingestion/src/metadata/ingestion/bulksink/elasticsearch.py
View File

@ -119,7 +119,7 @@ class ElasticsearchBulkSink(BulkSink):
table_name=table.name.__root__,
suggest=suggest,
description=table.description,
table_type=table.tableType,
table_type=table.tableType.name,
last_updated_timestamp=timestamp,
column_names=column_names,
column_descriptions=column_descriptions,

12
ingestion/src/metadata/ingestion/source/sql_source.py
View File

@ -176,9 +176,9 @@ class SQLSource(Source):
if self.connection is None:
self.connection = self.engine.connect()
def fetch_sample_data(self, table: str):
def fetch_sample_data(self, schema: str, table: str):
try:
query = f"select * from {table} limit 50"
query = f"select * from {schema}.{table} limit 50"
logger.info("Fetching sample data, this may take a while {}".format(query))
results = self.connection.execute(query)
cols = list(results.keys())
@ -231,7 +231,7 @@ class SQLSource(Source):
columns=table_columns)
if self.sql_config.generate_sample_data:
self._get_connection()
table_data = self.fetch_sample_data(table_name)
table_data = self.fetch_sample_data(schema, table_name)
table.sampleData = table_data
table_and_db = OMetaDatabaseAndTable(table=table, database=self._get_database(schema))
@ -259,10 +259,6 @@ class SQLSource(Source):
description = self._get_table_description(schema, view_name, inspector)
table_columns = self._get_columns(schema, view_name, inspector)
if self.sql_config.generate_sample_data:
self._get_connection()
self.fetch_sample_data(view_name)
table = Table(id=uuid.uuid4(),
name=view_name,
tableType='View',
@ -271,7 +267,7 @@ class SQLSource(Source):
viewDefinition=view_definition)
if self.sql_config.generate_sample_data:
self._get_connection()
table_data = self.fetch_sample_data(view_name)
table_data = self.fetch_sample_data(schema, view_name)
table.sampleData = table_data
table_and_db = OMetaDatabaseAndTable(table=table, database=self._get_database(schema))