yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-26 09:22:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add include views to profiler (#12974)

Browse Source
This commit is contained in:
Ayush Shah 2023-08-23 17:30:44 +05:30 committed by GitHub
parent 4862b0df28
commit 4464df81ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 15 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ingestion/src/metadata/profiler/api/workflow.py
View File

@ -23,7 +23,7 @@ from pydantic import ValidationError
from metadata.config.common import WorkflowExecutionError from metadata.config.common import WorkflowExecutionError
from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.data.table import Table, TableType
from metadata.generated.schema.entity.services.connections.database.datalakeConnection import ( from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
DatalakeConnection, DatalakeConnection,
) )
@ -194,7 +194,15 @@ class ProfilerWorkflow(WorkflowStatusMixin):
"Table pattern not allowed", "Table pattern not allowed",
) )
continue continue
if (
table.tableType == TableType.View
and not self.source_config.includeViews
):
self.source_status.filter(
table.fullyQualifiedName.__root__,
"View filtered out",
)
continue
yield table yield table
except Exception as exc: except Exception as exc:
error = ( error = (

5
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json
View File

@ -30,6 +30,11 @@
"description": "Regex to only fetch databases that matches the pattern.", "description": "Regex to only fetch databases that matches the pattern.",
"$ref": "../type/filterPattern.json#/definitions/filterPattern" "$ref": "../type/filterPattern.json#/definitions/filterPattern"
}, },
"includeViews": {
"description": "Optional configuration to turn off fetching metadata for views.",
"type": "boolean",
"default": true
},
"processPiiSensitive": { "processPiiSensitive": {
"description": "Optional configuration to automatically tag columns that might contain sensitive information", "description": "Optional configuration to automatically tag columns that might contain sensitive information",
"type": "boolean", "type": "boolean",