yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-20 05:15:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bigquery support for env without passing any config (#2160)

Browse Source 
* Bigquery support for env without passing any config

* Bigquery File formatted
This commit is contained in:
Ayush Shah 2022-01-23 11:55:48 +05:30 committed by GitHub
parent a784bc217d
commit 6f5a4eafe7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ingestion/examples/workflows/bigquery.json
View File

@ -2,9 +2,6 @@
"source": { "source": {
"type": "bigquery", "type": "bigquery",
"config": { "config": {
"project_id": "project_id",
"host_port": "bigquery.googleapis.com",
"username": "gcpuser@project_id.iam.gserviceaccount.com",
"service_name": "gcp_bigquery", "service_name": "gcp_bigquery",
"options":{ "options":{
"credentials":{ "credentials":{

8
ingestion/src/metadata/ingestion/source/bigquery.py
View File

@ -54,6 +54,8 @@ _types.get_columns = get_columns
class BigQueryConfig(SQLConnectionConfig): class BigQueryConfig(SQLConnectionConfig):
scheme = "bigquery" scheme = "bigquery"
host_port: Optional[str] = "bigquery.googleapis.com"
username: Optional[str] = None
project_id: Optional[str] = None project_id: Optional[str] = None
duration: int = 1 duration: int = 1
service_type = "BigQuery" service_type = "BigQuery"
@ -72,7 +74,11 @@ class BigquerySource(SQLSource):
def create(cls, config_dict, metadata_config_dict, ctx): def create(cls, config_dict, metadata_config_dict, ctx):
config: SQLConnectionConfig = BigQueryConfig.parse_obj(config_dict) config: SQLConnectionConfig = BigQueryConfig.parse_obj(config_dict)
metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict) metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict)
if config.options.get("credentials", None): if config.options.get("credentials_path"):
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = config.options[
"credentials_path"
]
elif config.options.get("credentials", None):
cred_path = create_credential_temp_file(config.options.get("credentials")) cred_path = create_credential_temp_file(config.options.get("credentials"))
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cred_path os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cred_path
config.options["credentials_path"] = cred_path config.options["credentials_path"] = cred_path