yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-12 15:57:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bigquery credential configuration updated (#2041)

Browse Source 
* bigquery-credentials-in-bigquery-json

* ui-changes-reverted

* credentials-moved-to-options

* Update bigquery.json

* Update bigquery.py

* delete-temp-code-added

* delete-temp-code-added

* removed-init-from-config

* Update datatypes_test.py

* Update column_helpers.py

* code-formatted

Co-authored-by: Abhishek <abhishek@Abhisheks-MacBook-Air.local>
Co-authored-by: = <=>
Co-authored-by: Ayush Shah <ayush@getcollate.io>
This commit is contained in:
codingwithabhi 2022-01-21 17:23:02 +05:30 committed by GitHub
parent cd65037ac7
commit ddf4ae261b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 36 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
ingestion/examples/workflows/bigquery.json
View File

@ -6,8 +6,19 @@
"host_port": "bigquery.googleapis.com",
"username": "gcpuser@project_id.iam.gserviceaccount.com",
"service_name": "gcp_bigquery",
"options": {
"credentials_path": "examples/creds/bigquery-cred.json"
"options":{
"credentials":{
"type": "service_account",
"project_id": "project_id",
"private_key_id": "private_key_id",
"private_key": "",
"client_email": "gcpuser@project_id.iam.gserviceaccount.com",
"client_id": "",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": ""
}
},
"table_filter_pattern": {
"excludes": [

23
ingestion/src/metadata/ingestion/source/bigquery.py
View File

@ -10,7 +10,8 @@
# limitations under the License.
import os
from typing import Optional, Tuple
from typing import Optional, Tuple, Any
import json, tempfile, logging
from sqlalchemy_bigquery import _types
from sqlalchemy_bigquery._struct import STRUCT
@ -72,11 +73,18 @@ class BigquerySource(SQLSource):
def create(cls, config_dict, metadata_config_dict, ctx):
config: SQLConnectionConfig = BigQueryConfig.parse_obj(config_dict)
metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict)
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = config.options[
"credentials_path"
]
if config.options.get("credentials", None):
cred_path = create_credential_temp_file(config.options.get("credentials"))
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cred_path
del config.options["credentials"]
config.options["credentials_path"] = cred_path
return cls(config, metadata_config, ctx)
def close(self):
super().close()
if self.config.options["credentials_path"]:
os.unlink(self.config.options["credentials_path"])
def standardize_schema_table_names(
self, schema: str, table: str
) -> Tuple[str, str]:
@ -89,3 +97,10 @@ class BigquerySource(SQLSource):
def parse_raw_data_type(self, raw_data_type):
return raw_data_type.replace(", ", ",").replace(" ", ":").lower()
def create_credential_temp_file(credentials: dict) -> str:
with tempfile.NamedTemporaryFile(delete=False) as fp:
cred_json = json.dumps(credentials, indent=4, separators=(",", ": "))
fp.write(cred_json.encode())
return fp.name

1
ingestion/src/metadata/utils/column_helpers.py
View File

@ -127,6 +127,7 @@ _column_string_mapping = {
"XML": "BINARY",
"XMLTYPE": "BINARY",
"CURSOR": "BINARY",
"TIMESTAMP_NTZ": "TIMESTAMP",
"TIMESTAMP_LTZ": "TIMESTAMP",
"TIMESTAMP_TZ": "TIMESTAMP",
}

3
ingestion/tests/unit/datatypes_test.py
View File

@ -90,6 +90,9 @@ SQLTYPES = [
"XML",
"XMLTYPE",
"YEAR",
"TIMESTAMP_NTZ",
"TIMESTAMP_LTZ",
"TIMESTAMP_TZ",
]