yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-25 17:04:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add include views to profiler (#12974)

Browse Source
This commit is contained in:
Ayush Shah 2023-08-23 17:30:44 +05:30 committed by GitHub
parent 4862b0df28
commit 4464df81ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 15 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ingestion/src/metadata/profiler/api/workflow.py
View File

@ -23,7 +23,7 @@ from pydantic import ValidationError
from metadata.config.common import WorkflowExecutionError
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.table import Table
from metadata.generated.schema.entity.data.table import Table, TableType
from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
DatalakeConnection,
)
@ -194,7 +194,15 @@ class ProfilerWorkflow(WorkflowStatusMixin):
"Table pattern not allowed",
)
continue
if (
table.tableType == TableType.View
and not self.source_config.includeViews
):
self.source_status.filter(
table.fullyQualifiedName.__root__,
"View filtered out",
)
continue
yield table
except Exception as exc:
error = (

5
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json
View File

@ -30,6 +30,11 @@
"description": "Regex to only fetch databases that matches the pattern.",
"$ref": "../type/filterPattern.json#/definitions/filterPattern"
},
"includeViews": {
"description": "Optional configuration to turn off fetching metadata for views.",
"type": "boolean",
"default": true
},
"processPiiSensitive": {
"description": "Optional configuration to automatically tag columns that might contain sensitive information",
"type": "boolean",