From f94e2dbb476666d90c8d1a036d4ee84693bc8955 Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Wed, 18 Oct 2023 23:48:21 +0530 Subject: [PATCH] Fix Hive Bytes issue, add athena yaml, fix bigquerymultiple project id token issue (#13640) --- .github/workflows/py-cli-e2e-tests.yml | 4 ++-- ingestion/setup.py | 2 +- .../src/metadata/profiler/orm/registry.py | 2 +- .../tests/cli_e2e/database/athena/athena.yaml | 24 +++++++++++++++++++ .../bigquery_multiple_project.yaml | 2 +- 5 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 ingestion/tests/cli_e2e/database/athena/athena.yaml diff --git a/.github/workflows/py-cli-e2e-tests.yml b/.github/workflows/py-cli-e2e-tests.yml index 4ad02dde53d..ea13b84bcaf 100644 --- a/.github/workflows/py-cli-e2e-tests.yml +++ b/.github/workflows/py-cli-e2e-tests.yml @@ -137,7 +137,7 @@ jobs: E2E_QUICKSIGHT_REGION: ${{ secrets.E2E_QUICKSIGHT_REGION }} E2E_DATALAKE_S3_BUCKET_NAME: ${{ secrets.E2E_DATALAKE_S3_BUCKET_NAME }} E2E_DATALAKE_S3_PREFIX: ${{ secrets.E2E_DATALAKE_S3_PREFIX }} - E2E_DATALAKE_S3_REGION: ${{ secrets.E2E_DATALAKE_S3_REGION }} + E2E_DATALAKE_S3_REGION: ${{ secrets.E2E_AWS_REGION }} E2E_POSTGRES_USERNAME: ${{ secrets.E2E_POSTGRES_USERNAME }} E2E_POSTGRES_PASSWORD: ${{ secrets.E2E_POSTGRES_PASSWORD }} E2E_POSTGRES_HOSTPORT: ${{ secrets.E2E_POSTGRES_HOSTPORT }} @@ -146,7 +146,7 @@ jobs: E2E_ORACLE_USERNAME: ${{ secrets.E2E_ORACLE_USERNAME }} E2E_ORACLE_PASSWORD: ${{ secrets.E2E_ORACLE_PASSWORD }} E2E_ORACLE_SERVICE_NAME: ${{ secrets.E2E_ORACLE_SERVICE_NAME }} - E2E_ATHENA_REGION: ${{ secrets.E2E_ATHENA_REGION }} + E2E_ATHENA_REGION: ${{ secrets.E2E_AWS_REGION }} E2E_ATHENA_S3STAGINGDIR: ${{ secrets.E2E_ATHENA_S3STAGINGDIR }} E2E_ATHENA_WORKGROUP: ${{ secrets.E2E_ATHENA_WORKGROUP }} run: | diff --git a/ingestion/setup.py b/ingestion/setup.py index 1eb00102fa2..3485f33aff1 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -54,7 +54,7 @@ VERSIONS = { "looker-sdk": "looker-sdk>=22.20.0", "lkml": "lkml~=1.3", "tableau": "tableau-api-lib~=0.1", - "pyhive": "pyhive~=0.6", + "pyhive": "pyhive~=0.7", "mongo": "pymongo~=4.3", "redshift": "sqlalchemy-redshift==0.8.12", "snowflake": "snowflake-sqlalchemy~=1.4", diff --git a/ingestion/src/metadata/profiler/orm/registry.py b/ingestion/src/metadata/profiler/orm/registry.py index 6fdaada0781..46a4ea562ae 100644 --- a/ingestion/src/metadata/profiler/orm/registry.py +++ b/ingestion/src/metadata/profiler/orm/registry.py @@ -60,7 +60,7 @@ class Dialects(Enum): DynamoDB = "dynamoDB" Glue = "glue" Hana = "hana" - Hive = b"hive" # Hive requires bytes + Hive = "hive" Impala = "impala" IbmDbSa = "ibm_db_sa" MariaDB = "mariadb" diff --git a/ingestion/tests/cli_e2e/database/athena/athena.yaml b/ingestion/tests/cli_e2e/database/athena/athena.yaml new file mode 100644 index 00000000000..ec7b7ddf0c3 --- /dev/null +++ b/ingestion/tests/cli_e2e/database/athena/athena.yaml @@ -0,0 +1,24 @@ +source: + type: athena + serviceName: e2e_athena + serviceConnection: + config: + type: Athena + databaseName: database_name + awsConfig: + awsRegion: $E2E_AWS_REGION + s3StagingDir: $E2E_ATHENA_S3STAGINGDIR + workgroup: $E2E_ATHENA_WORKGROUP + sourceConfig: + config: + type: DatabaseMetadata +sink: + type: metadata-rest + config: {} +workflowConfig: + loggerLevel: DEBUG + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" diff --git a/ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml b/ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml index ad7d8e23184..46daf6dc150 100644 --- a/ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml +++ b/ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml @@ -30,4 +30,4 @@ workflowConfig: hostPort: http://localhost:8585/api authProvider: openmetadata securityConfig: - jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJvcGVuLW1ldGFkYXRhLm9yZyIsInN1YiI6ImluZ2VzdGlvbi1ib3QiLCJlbWFpbCI6ImluZ2VzdGlvbi1ib3RAb3Blbm1ldGFkYXRhLm9yZyIsImlzQm90Ijp0cnVlLCJ0b2tlblR5cGUiOiJCT1QiLCJpYXQiOjE2OTY5NDEzODcsImV4cCI6bnVsbH0.M804qRIxWO5_62qeg32Wg5LWxKBXN2fajCZZ42SAU9e86-yGMqy9Mo2ckgQZctrVlhqeJU3XiMUcheew0lQk-sEOPZN9vhaUm15YGmZq4810chxU67yGCFolDDATmejMK5RiPclTyZF2OQAZWFQxq7tHuxnFl8id4vsa-AdTBLjC3XyT9pspaBI44szIxA9TaO_zMhGeh75X68SeEani3KMaFfv6Y-M3jo2snVxzdDn8RmF3cXzILZuXfhYPl4M2Lf657on_9Uccch_pHVqAClcwtJSfuELWt52mfW44-4zFdKtUDs9Z7sYHiQyL6YaiUYizPhKEDUMpmDS83quinw" \ No newline at end of file + jwtToken: eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg \ No newline at end of file