From 40e7d37244f5f15c1fe19a810bf344bd0fed2b82 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Wed, 9 Mar 2022 17:55:37 +0530 Subject: [PATCH] Snowflake optional config (#3319) * Update AirflowUtils.java * Update databaseServiceMetadataPipeline.json * Updated Account and schema description * snowfliks ui support (#3318) * added lineage for snowflake * Formatted file * initialized metadata_config Co-authored-by: Shailesh Parmar Co-authored-by: Ayush Shah --- .../catalog/airflow/AirflowUtils.java | 8 ++++ .../databaseServiceMetadataPipeline.json | 10 ++++ .../ingestion/source/snowflake_usage.py | 10 +++- .../AddServiceModal/AddServiceModal.tsx | 46 ++++++++++++++++++- .../pipelines/createAirflowPipeline.ts | 12 ++++- .../operations/pipelines/airflowPipeline.ts | 12 ++++- .../databaseServiceMetadataPipeline.ts | 10 +++- 7 files changed, 101 insertions(+), 7 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/airflow/AirflowUtils.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/airflow/AirflowUtils.java index e26eaa5a271..157908385e5 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/airflow/AirflowUtils.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/airflow/AirflowUtils.java @@ -58,6 +58,8 @@ public final class AirflowUtils { public static final String INGESTION_CONNECTION_ARGS = "connect_args"; public static final String INGESTION_USAGE_STAGE_FILE_PATH = "filename"; public static final String INGESTION_STATUS = "status"; + public static final String INGESTION_ACCOUNT = "account"; + public static final String INGESTION_WAREHOUSE = "warehouse"; private AirflowUtils() {} @@ -98,6 +100,12 @@ public final class AirflowUtils { if (databaseServiceMetadataPipeline.getTableFilterPattern() != null) { dbConfig.put(INGESTION_TABLE_FILTER_PATTERN, databaseServiceMetadataPipeline.getTableFilterPattern()); } + if (databaseServiceMetadataPipeline.getAccount() != null) { + dbConfig.put(INGESTION_ACCOUNT, databaseServiceMetadataPipeline.getAccount()); + } + if (databaseServiceMetadataPipeline.getWarehouse() != null) { + dbConfig.put(INGESTION_WAREHOUSE, databaseServiceMetadataPipeline.getWarehouse()); + } dbConfig.put(INGESTION_MARK_DELETED_TABLES, databaseServiceMetadataPipeline.getMarkDeletedTables()); dbConfig.put(INGESTION_DBT_CATALOG_FILE_PATH, databaseServiceMetadataPipeline.getDbtCatalogFilePath()); dbConfig.put(INGESTION_DBT_MANIFEST_FILE_PATH, databaseServiceMetadataPipeline.getDbtManifestFilePath()); diff --git a/catalog-rest-service/src/main/resources/json/schema/operations/pipelines/databaseServiceMetadataPipeline.json b/catalog-rest-service/src/main/resources/json/schema/operations/pipelines/databaseServiceMetadataPipeline.json index afd52992930..925378ff981 100644 --- a/catalog-rest-service/src/main/resources/json/schema/operations/pipelines/databaseServiceMetadataPipeline.json +++ b/catalog-rest-service/src/main/resources/json/schema/operations/pipelines/databaseServiceMetadataPipeline.json @@ -50,6 +50,16 @@ "type": "string", "default": "select * from {}.{} limit 50" }, + "warehouse": { + "description": "Optional Warehouse.", + "type": "string", + "default": null + }, + "account": { + "description": "Optional Account.", + "type": "string", + "default": null + }, "enableDataProfiler": { "description": "Run data profiler as part of this metadata ingestion to get table profile data.", "type": "boolean", diff --git a/ingestion/src/metadata/ingestion/source/snowflake_usage.py b/ingestion/src/metadata/ingestion/source/snowflake_usage.py index e47b9077c97..fd7e27260a1 100644 --- a/ingestion/src/metadata/ingestion/source/snowflake_usage.py +++ b/ingestion/src/metadata/ingestion/source/snowflake_usage.py @@ -27,7 +27,7 @@ from metadata.ingestion.source.sql_alchemy_helper import ( SQLAlchemyHelper, SQLSourceStatus, ) -from metadata.utils.helpers import get_start_and_end +from metadata.utils.helpers import get_start_and_end, ingest_lineage from metadata.utils.sql_queries import SNOWFLAKE_SQL_STATEMENT @@ -75,6 +75,7 @@ class SnowflakeUsageSource(Source[TableQuery]): self.config = config start, end = get_start_and_end(config.duration) self.analysis_date = start + self.metadata_config = metadata_config self.sql_stmt = SnowflakeUsageSource.SQL_STATEMENT.format( start_date=start, end_date=end ) @@ -126,6 +127,13 @@ class SnowflakeUsageSource(Source[TableQuery]): else: self.report.scanned(f"{row['database_name']}") yield table_query + query_info = { + "sql": table_query.sql, + "from_type": "table", + "to_type": "table", + "service_name": self.config.service_name, + } + ingest_lineage(query_info, self.metadata_config) def get_report(self): """ diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx index 4812cd25040..7bdb2ed9e1a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx @@ -42,7 +42,10 @@ import { } from '../../../generated/api/operations/pipelines/createAirflowPipeline'; import { DashboardServiceType } from '../../../generated/entity/services/dashboardService'; // import { DashboardService } from '../../../generated/entity/services/dashboardService'; -import { DatabaseService } from '../../../generated/entity/services/databaseService'; +import { + DatabaseService, + DatabaseServiceType, +} from '../../../generated/entity/services/databaseService'; import { MessagingService, MessagingServiceType, @@ -336,6 +339,9 @@ export const AddServiceModal: FunctionComponent = ({ DynamicFormFieldType[] >(getKeyValuePair(data?.databaseConnection?.connectionArguments || {}) || []); + const [warehouse, setWarehouse] = useState(''); + const [account, setAccount] = useState(''); + const markdownRef = useRef(); const getBrokerUrlPlaceholder = (): string => { @@ -548,6 +554,8 @@ export const AddServiceModal: FunctionComponent = ({ pipelineConfig: { schema: Schema.DatabaseServiceMetadataPipeline, config: { + warehouse: warehouse || undefined, + account: account || undefined, includeViews: value.includeView, generateSampleData: value.ingestSampleData, enableDataProfiler: value.enableDataProfiler, @@ -870,6 +878,42 @@ export const AddServiceModal: FunctionComponent = ({ /> + {/* optional filed for snowflik */} + {selectService === DatabaseServiceType.Snowflake && ( +
+
+ + setWarehouse(e.target.value)} + /> +
+ +
+ + setAccount(e.target.value)} + /> +
+
+ )}

Connection Options

diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/operations/pipelines/createAirflowPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/operations/pipelines/createAirflowPipeline.ts index 64e21e1dafb..b1bf3e577a0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/operations/pipelines/createAirflowPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/operations/pipelines/createAirflowPipeline.ts @@ -127,11 +127,15 @@ export interface EntityReference { * OpenMetadata Pipeline Config. */ export interface PipelineConfig { - config?: any[] | boolean | number | null | ConfigObject | string; + config?: any[] | boolean | ConfigClass | number | null | string; schema?: Schema; } -export interface ConfigObject { +export interface ConfigClass { + /** + * Sample data extraction query. + */ + account?: string; /** * DBT catalog file to extract dbt models with their column schemas. */ @@ -169,6 +173,10 @@ export interface ConfigObject { * Regex exclude tables or databases that matches the pattern. */ tableFilterPattern?: FilterPattern; + /** + * Sample data extraction query. + */ + warehouse?: string; /** * Configuration to tune how far we want to look back in query logs to process usage data. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/operations/pipelines/airflowPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/operations/pipelines/airflowPipeline.ts index bb749121f23..d65f5d76369 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/operations/pipelines/airflowPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/operations/pipelines/airflowPipeline.ts @@ -210,11 +210,15 @@ export interface EntityReference { * OpenMetadata Pipeline Config. */ export interface PipelineConfig { - config?: any[] | boolean | number | null | ConfigObject | string; + config?: any[] | boolean | ConfigClass | number | null | string; schema?: Schema; } -export interface ConfigObject { +export interface ConfigClass { + /** + * Sample data extraction query. + */ + account?: string; /** * DBT catalog file to extract dbt models with their column schemas. */ @@ -252,6 +256,10 @@ export interface ConfigObject { * Regex exclude tables or databases that matches the pattern. */ tableFilterPattern?: FilterPattern; + /** + * Sample data extraction query. + */ + warehouse?: string; /** * Configuration to tune how far we want to look back in query logs to process usage data. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/operations/pipelines/databaseServiceMetadataPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/operations/pipelines/databaseServiceMetadataPipeline.ts index 2665901ff3a..e9a71ab8dc9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/operations/pipelines/databaseServiceMetadataPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/operations/pipelines/databaseServiceMetadataPipeline.ts @@ -12,7 +12,11 @@ * limitations under the License. */ -export interface DatabaseServiceMetadataPipelineObject { +export interface DatabaseServiceMetadataPipelineClass { + /** + * Sample data extraction query. + */ + account?: string; /** * DBT catalog file to extract dbt models with their column schemas. */ @@ -50,6 +54,10 @@ export interface DatabaseServiceMetadataPipelineObject { * Regex exclude tables or databases that matches the pattern. */ tableFilterPattern?: FilterPattern; + /** + * Sample data extraction query. + */ + warehouse?: string; } /**