diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index f85eaa39007..393dfeb0034 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -33,6 +33,7 @@ RUN apt-get -qq update \ unixodbc \ unixodbc-dev \ unzip \ + git \ wget --no-install-recommends \ # Accept MSSQL ODBC License && ACCEPT_EULA=Y apt-get install -y msodbcsql18 \ diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index 6931b899f21..7cc7cf25d69 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -33,6 +33,7 @@ RUN apt-get -qq update \ unixodbc-dev \ unzip \ vim \ + git \ wget --no-install-recommends \ # Accept MSSQL ODBC License && ACCEPT_EULA=Y apt-get -qq install -y msodbcsql18 \ diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index 001290f557c..3ffe683cbf6 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -33,6 +33,7 @@ RUN apt-get -qq update \ unixodbc \ unixodbc-dev \ unzip \ + git \ wget --no-install-recommends \ # Accept MSSQL ODBC License && ACCEPT_EULA=Y apt-get -qq install -y msodbcsql18 \ diff --git a/ingestion/operators/docker/Dockerfile-dev b/ingestion/operators/docker/Dockerfile-dev index d4f069405d5..4b74be51630 100644 --- a/ingestion/operators/docker/Dockerfile-dev +++ b/ingestion/operators/docker/Dockerfile-dev @@ -34,6 +34,7 @@ RUN apt-get -qq update \ unixodbc-dev \ unzip \ vim \ + git \ wget --no-install-recommends \ # Accept MSSQL ODBC License && ACCEPT_EULA=Y apt-get -qq install -y msodbcsql18 \ diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/bulk_parser.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/bulk_parser.py index e3c33936d84..af3c32aaa83 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/bulk_parser.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/bulk_parser.py @@ -112,7 +112,6 @@ class BulkLkmlParser(metaclass=Singleton): if cached_view: return cached_view - logger.warning(f"BulkLkmlParser::find_view: can't find view {view_name}") return None def __repr__(self): diff --git a/ingestion/src/metadata/readers/file/local.py b/ingestion/src/metadata/readers/file/local.py index f4b055ef04e..1709d4bf121 100644 --- a/ingestion/src/metadata/readers/file/local.py +++ b/ingestion/src/metadata/readers/file/local.py @@ -11,6 +11,7 @@ """ Local Reader """ +import os import traceback from pathlib import Path from typing import List, Optional, Union @@ -60,3 +61,20 @@ class LocalReader(Reader): str(path).replace(str(self.base_path) + "/", "") for path in Path(self.base_path).rglob("*") ] + + def get_local_files( + self, search_key: str, excluded_files: Optional[List[str]] = None + ) -> List[str]: + """Scan through local path recursively + and retuns file path based on `search_key`""" + + if excluded_files is None: + excluded_files = [] + + file_paths = [] + for root, _, file in os.walk(self.base_path): + for f in file: + if search_key in f and f not in excluded_files: + file_paths.append(f"{root}/{f}") + + return file_paths