mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-07-30 20:59:57 +00:00
78b5f8c8e2
* Added database filter in workflow * Removed association between profiler and data quality * fixed tests with removed association * Fixed sonar code smells and bugs * Updated profiler workflow to: - support only running profiler (removed test run) - support column inclusion and exclusion - added back support for partitioned table and sample * moved status to workflow * Fixed tests * removed test logic from profiler sink * Added logic to return sample from workflow sample value * Added profiler examples * Updated documentation for profiler * Fixed code smells
40 lines
1.1 KiB
YAML
40 lines
1.1 KiB
YAML
source:
|
|
type: postgres
|
|
serviceName: local_postgres
|
|
serviceConnection:
|
|
config:
|
|
type: Postgres
|
|
username: openmetadata_user
|
|
password: openmetadata_password
|
|
hostPort: localhost:5432
|
|
sourceConfig:
|
|
config:
|
|
type: Profiler
|
|
generateSampleData: <true | false>
|
|
profileSample: <number between 0 and 99>
|
|
databaseFilterPattern: <table FQN filtering regex>
|
|
schemaFilterPattern: <table FQN filtering regex>
|
|
tableFilterPattern: <table FQN filtering regex>
|
|
processor:
|
|
type: orm-profiler
|
|
config:
|
|
tableConfig:
|
|
- fullyQualifiedName: <table fqn>
|
|
profileSample: <number between 0 and 99>
|
|
columnConfig:
|
|
profileQuery: <query to use for sampling data for the profiler>
|
|
excludeColumns:
|
|
- <column name>
|
|
includeColumns:
|
|
- columnName: <column name>
|
|
- metrics:
|
|
- MEAN
|
|
- MEDIAN
|
|
- ...
|
|
sink:
|
|
type: metadata-rest
|
|
config: {}
|
|
workflowConfig:
|
|
openMetadataServerConfig:
|
|
hostPort: <OpenMetadata host and port>
|
|
authProvider: <OpenMetadata auth provider> |