From a6a4e08af112ee83cf99d4950fba868e15cc50c1 Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Tue, 30 Aug 2022 13:33:23 +0530 Subject: [PATCH] Fix Profiler for json or variant types (#7033) --- ingestion/Dockerfile_local | 12 ++++++++---- .../metadata/ingestion/source/database/snowflake.py | 4 ++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ingestion/Dockerfile_local b/ingestion/Dockerfile_local index 072e748e271..ea2e0199da6 100644 --- a/ingestion/Dockerfile_local +++ b/ingestion/Dockerfile_local @@ -1,7 +1,7 @@ FROM python:3.9-slim as base ENV AIRFLOW_HOME=/airflow RUN apt-get update && \ - apt-get install -y gcc libsasl2-modules libsasl2-dev build-essential libssl-dev libffi-dev librdkafka-dev unixodbc-dev python3.9-dev openjdk-11-jre unixodbc freetds-dev freetds-bin tdsodbc libevent-dev wget openssl --no-install-recommends && \ + apt-get install -y gcc libsasl2-modules libxml2 libsasl2-dev build-essential libssl-dev libffi-dev librdkafka-dev unixodbc-dev python3.9-dev openjdk-11-jre unixodbc freetds-dev freetds-bin tdsodbc libevent-dev wget openssl --no-install-recommends && \ rm -rf /var/lib/apt/lists/* # Manually fix security vulnerability from curl @@ -14,7 +14,8 @@ RUN wget https://curl.se/download/curl-7.84.0.tar.gz && \ FROM base as airflow ENV AIRFLOW_VERSION=2.3.3 -#install odbc driver + +# install odbc driver RUN apt-get update && \ apt-get install -y gnupg && \ curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ @@ -22,17 +23,20 @@ RUN apt-get update && \ apt-get update && \ ACCEPT_EULA=Y apt-get install -y msodbcsql18 && \ rm -rf /var/lib/apt/lists/* + + ENV CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.9.txt" # Add docker provider for the DockerOperator RUN pip install "apache-airflow[docker]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" +RUN pip install "openmetadata-ingestion[all]" +RUN pip uninstall openmetadata-ingestion -y FROM airflow as apis WORKDIR /openmetadata-airflow-apis COPY openmetadata-airflow-apis /openmetadata-airflow-apis -RUN pip install "." "openmetadata-ingestion[all]" -RUN pip uninstall openmetadata-ingestion -y +RUN pip install "." FROM apis as ingestion WORKDIR /ingestion diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake.py b/ingestion/src/metadata/ingestion/source/database/snowflake.py index c29e5636df1..dd86f1f5027 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake.py @@ -8,6 +8,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import traceback from typing import Iterable @@ -50,6 +51,9 @@ ischema_names["GEOGRAPHY"] = GEOGRAPHY logger = ingestion_logger() +SnowflakeDialect._json_deserializer = json.loads + + def get_table_names(self, connection, schema, **kw): cursor = connection.execute(SNOWFLAKE_GET_TABLE_NAMES.format(schema)) result = [self.normalize_name(row[0]) for row in cursor]