mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-27 23:48:19 +00:00
78b5f8c8e2
* Added database filter in workflow * Removed association between profiler and data quality * fixed tests with removed association * Fixed sonar code smells and bugs * Updated profiler workflow to: - support only running profiler (removed test run) - support column inclusion and exclusion - added back support for partitioned table and sample * moved status to workflow * Fixed tests * removed test logic from profiler sink * Added logic to return sample from workflow sample value * Added profiler examples * Updated documentation for profiler * Fixed code smells
52 lines
1.3 KiB
YAML
52 lines
1.3 KiB
YAML
source:
|
|
type: bigquery
|
|
serviceName: local_bigquery
|
|
serviceConnection:
|
|
config:
|
|
type: BigQuery
|
|
credentials:
|
|
gcsConfig:
|
|
type: service_account
|
|
projectId: my-project-id-1234
|
|
privateKeyId: privateKeyID
|
|
privateKey: "-----BEGIN PRIVATE KEY-----\nmySuperSecurePrivateKey==\n-----END PRIVATE KEY-----\n"
|
|
clientEmail: client@email.secure
|
|
clientId: 1234567890
|
|
authUri: authUri
|
|
tokenUri: tokenUri
|
|
authProviderX509CertUrl: authProvider
|
|
clientX509CertUrl: clientCertUrl
|
|
sourceConfig:
|
|
config:
|
|
type: Profiler
|
|
generateSampleData: true
|
|
databaseFilterPattern:
|
|
includes:
|
|
- hello-world-1234
|
|
schemaFilterPattern:
|
|
includes:
|
|
- super_schema
|
|
tableFilterPattern:
|
|
includes:
|
|
- abc
|
|
|
|
processor:
|
|
type: "orm-profiler"
|
|
config:
|
|
tableConfig:
|
|
- fullyQualifiedName: local_bigquery.hello-world-1234.super_schema.abc
|
|
profileSample: 85
|
|
partitionConfig:
|
|
partitionQueryDuration: 180
|
|
columnConfig:
|
|
excludeColumns:
|
|
- a
|
|
- b
|
|
|
|
sink:
|
|
type: metadata-rest
|
|
config: {}
|
|
workflowConfig:
|
|
openMetadataServerConfig:
|
|
hostPort: http://localhost:8585/api
|
|
authProvider: no-auth |