yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-26 17:34:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Profiler for json or variant types (#7033)

Browse Source
This commit is contained in:
Ayush Shah 2022-08-30 13:33:23 +05:30 committed by GitHub
parent b7351c220a
commit a6a4e08af1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ingestion/Dockerfile_local
View File

@ -1,7 +1,7 @@
FROM python:3.9-slim as base FROM python:3.9-slim as base
ENV AIRFLOW_HOME=/airflow ENV AIRFLOW_HOME=/airflow
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y gcc libsasl2-modules libsasl2-dev build-essential libssl-dev libffi-dev librdkafka-dev unixodbc-dev python3.9-dev openjdk-11-jre unixodbc freetds-dev freetds-bin tdsodbc libevent-dev wget openssl --no-install-recommends && \ apt-get install -y gcc libsasl2-modules libxml2 libsasl2-dev build-essential libssl-dev libffi-dev librdkafka-dev unixodbc-dev python3.9-dev openjdk-11-jre unixodbc freetds-dev freetds-bin tdsodbc libevent-dev wget openssl --no-install-recommends && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*
# Manually fix security vulnerability from curl # Manually fix security vulnerability from curl
@ -14,7 +14,8 @@ RUN wget https://curl.se/download/curl-7.84.0.tar.gz && \
FROM base as airflow FROM base as airflow
ENV AIRFLOW_VERSION=2.3.3 ENV AIRFLOW_VERSION=2.3.3
#install odbc driver
# install odbc driver
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y gnupg && \ apt-get install -y gnupg && \
curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
@ -22,17 +23,20 @@ RUN apt-get update && \
apt-get update && \ apt-get update && \
ACCEPT_EULA=Y apt-get install -y msodbcsql18 && \ ACCEPT_EULA=Y apt-get install -y msodbcsql18 && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*
ENV CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.9.txt" ENV CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.9.txt"
# Add docker provider for the DockerOperator # Add docker provider for the DockerOperator
RUN pip install "apache-airflow[docker]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" RUN pip install "apache-airflow[docker]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
RUN pip install "openmetadata-ingestion[all]"
RUN pip uninstall openmetadata-ingestion -y
FROM airflow as apis FROM airflow as apis
WORKDIR /openmetadata-airflow-apis WORKDIR /openmetadata-airflow-apis
COPY openmetadata-airflow-apis /openmetadata-airflow-apis COPY openmetadata-airflow-apis /openmetadata-airflow-apis
RUN pip install "." "openmetadata-ingestion[all]" RUN pip install "."
RUN pip uninstall openmetadata-ingestion -y
FROM apis as ingestion FROM apis as ingestion
WORKDIR /ingestion WORKDIR /ingestion

4
ingestion/src/metadata/ingestion/source/database/snowflake.py
View File

@ -8,6 +8,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import json
import traceback import traceback
from typing import Iterable from typing import Iterable
@ -50,6 +51,9 @@ ischema_names["GEOGRAPHY"] = GEOGRAPHY
logger = ingestion_logger() logger = ingestion_logger()
SnowflakeDialect._json_deserializer = json.loads
def get_table_names(self, connection, schema, **kw): def get_table_names(self, connection, schema, **kw):
cursor = connection.execute(SNOWFLAKE_GET_TABLE_NAMES.format(schema)) cursor = connection.execute(SNOWFLAKE_GET_TABLE_NAMES.format(schema))
result = [self.normalize_name(row[0]) for row in cursor] result = [self.normalize_name(row[0]) for row in cursor]