yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-06 21:43:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Hive Bytes issue, add athena yaml, fix bigquerymultiple project id token issue (#13640)

Browse Source
This commit is contained in:
Ayush Shah 2023-10-18 23:48:21 +05:30 committed by GitHub
parent d15f415460
commit f94e2dbb47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 29 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/py-cli-e2e-tests.yml vendored
View File

@ -137,7 +137,7 @@ jobs:
E2E_QUICKSIGHT_REGION: ${{ secrets.E2E_QUICKSIGHT_REGION }} E2E_QUICKSIGHT_REGION: ${{ secrets.E2E_QUICKSIGHT_REGION }}
E2E_DATALAKE_S3_BUCKET_NAME: ${{ secrets.E2E_DATALAKE_S3_BUCKET_NAME }} E2E_DATALAKE_S3_BUCKET_NAME: ${{ secrets.E2E_DATALAKE_S3_BUCKET_NAME }}
E2E_DATALAKE_S3_PREFIX: ${{ secrets.E2E_DATALAKE_S3_PREFIX }} E2E_DATALAKE_S3_PREFIX: ${{ secrets.E2E_DATALAKE_S3_PREFIX }}
E2E_DATALAKE_S3_REGION: ${{ secrets.E2E_DATALAKE_S3_REGION }} E2E_DATALAKE_S3_REGION: ${{ secrets.E2E_AWS_REGION }}
E2E_POSTGRES_USERNAME: ${{ secrets.E2E_POSTGRES_USERNAME }} E2E_POSTGRES_USERNAME: ${{ secrets.E2E_POSTGRES_USERNAME }}
E2E_POSTGRES_PASSWORD: ${{ secrets.E2E_POSTGRES_PASSWORD }} E2E_POSTGRES_PASSWORD: ${{ secrets.E2E_POSTGRES_PASSWORD }}
E2E_POSTGRES_HOSTPORT: ${{ secrets.E2E_POSTGRES_HOSTPORT }} E2E_POSTGRES_HOSTPORT: ${{ secrets.E2E_POSTGRES_HOSTPORT }}
@ -146,7 +146,7 @@ jobs:
E2E_ORACLE_USERNAME: ${{ secrets.E2E_ORACLE_USERNAME }} E2E_ORACLE_USERNAME: ${{ secrets.E2E_ORACLE_USERNAME }}
E2E_ORACLE_PASSWORD: ${{ secrets.E2E_ORACLE_PASSWORD }} E2E_ORACLE_PASSWORD: ${{ secrets.E2E_ORACLE_PASSWORD }}
E2E_ORACLE_SERVICE_NAME: ${{ secrets.E2E_ORACLE_SERVICE_NAME }} E2E_ORACLE_SERVICE_NAME: ${{ secrets.E2E_ORACLE_SERVICE_NAME }}
E2E_ATHENA_REGION: ${{ secrets.E2E_ATHENA_REGION }} E2E_ATHENA_REGION: ${{ secrets.E2E_AWS_REGION }}
E2E_ATHENA_S3STAGINGDIR: ${{ secrets.E2E_ATHENA_S3STAGINGDIR }} E2E_ATHENA_S3STAGINGDIR: ${{ secrets.E2E_ATHENA_S3STAGINGDIR }}
E2E_ATHENA_WORKGROUP: ${{ secrets.E2E_ATHENA_WORKGROUP }} E2E_ATHENA_WORKGROUP: ${{ secrets.E2E_ATHENA_WORKGROUP }}
run: | run: |

2
ingestion/setup.py
View File

@ -54,7 +54,7 @@ VERSIONS = {
"looker-sdk": "looker-sdk>=22.20.0", "looker-sdk": "looker-sdk>=22.20.0",
"lkml": "lkml~=1.3", "lkml": "lkml~=1.3",
"tableau": "tableau-api-lib~=0.1", "tableau": "tableau-api-lib~=0.1",
"pyhive": "pyhive~=0.6", "pyhive": "pyhive~=0.7",
"mongo": "pymongo~=4.3", "mongo": "pymongo~=4.3",
"redshift": "sqlalchemy-redshift==0.8.12", "redshift": "sqlalchemy-redshift==0.8.12",
"snowflake": "snowflake-sqlalchemy~=1.4", "snowflake": "snowflake-sqlalchemy~=1.4",

2
ingestion/src/metadata/profiler/orm/registry.py
View File

@ -60,7 +60,7 @@ class Dialects(Enum):
DynamoDB = "dynamoDB" DynamoDB = "dynamoDB"
Glue = "glue" Glue = "glue"
Hana = "hana" Hana = "hana"
Hive = b"hive" # Hive requires bytes Hive = "hive"
Impala = "impala" Impala = "impala"
IbmDbSa = "ibm_db_sa" IbmDbSa = "ibm_db_sa"
MariaDB = "mariadb" MariaDB = "mariadb"

24
ingestion/tests/cli_e2e/database/athena/athena.yaml Normal file
View File

@ -0,0 +1,24 @@
source:
type: athena
serviceName: e2e_athena
serviceConnection:
config:
type: Athena
databaseName: database_name
awsConfig:
awsRegion: $E2E_AWS_REGION
s3StagingDir: $E2E_ATHENA_S3STAGINGDIR
workgroup: $E2E_ATHENA_WORKGROUP
sourceConfig:
config:
type: DatabaseMetadata
sink:
type: metadata-rest
config: {}
workflowConfig:
loggerLevel: DEBUG
openMetadataServerConfig:
hostPort: http://localhost:8585/api
authProvider: openmetadata
securityConfig:
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

2
ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml
View File

@ -30,4 +30,4 @@ workflowConfig:
hostPort: http://localhost:8585/api hostPort: http://localhost:8585/api
authProvider: openmetadata authProvider: openmetadata
securityConfig: securityConfig:
jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJvcGVuLW1ldGFkYXRhLm9yZyIsInN1YiI6ImluZ2VzdGlvbi1ib3QiLCJlbWFpbCI6ImluZ2VzdGlvbi1ib3RAb3Blbm1ldGFkYXRhLm9yZyIsImlzQm90Ijp0cnVlLCJ0b2tlblR5cGUiOiJCT1QiLCJpYXQiOjE2OTY5NDEzODcsImV4cCI6bnVsbH0.M804qRIxWO5_62qeg32Wg5LWxKBXN2fajCZZ42SAU9e86-yGMqy9Mo2ckgQZctrVlhqeJU3XiMUcheew0lQk-sEOPZN9vhaUm15YGmZq4810chxU67yGCFolDDATmejMK5RiPclTyZF2OQAZWFQxq7tHuxnFl8id4vsa-AdTBLjC3XyT9pspaBI44szIxA9TaO_zMhGeh75X68SeEani3KMaFfv6Y-M3jo2snVxzdDn8RmF3cXzILZuXfhYPl4M2Lf657on_9Uccch_pHVqAClcwtJSfuELWt52mfW44-4zFdKtUDs9Z7sYHiQyL6YaiUYizPhKEDUMpmDS83quinw" jwtToken: eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg