From d4d8f3655f2d79956a5d453d995983ceda8d9c53 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 14 Dec 2023 15:46:35 +0100 Subject: [PATCH] #14043 - Fix metadata backup memory profile (#14378) * Update looker docs * Fix db dump --- ingestion/src/metadata/cli/db_dump.py | 9 ++++++--- ingestion/src/metadata/cmd.py | 5 +---- .../content/v1.2.x/connectors/dashboard/looker/index.md | 3 +++ .../content/v1.2.x/connectors/dashboard/looker/yaml.md | 3 +++ .../v1.3.x-SNAPSHOT/connectors/dashboard/looker/index.md | 3 +++ .../v1.3.x-SNAPSHOT/connectors/dashboard/looker/yaml.md | 3 +++ 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/ingestion/src/metadata/cli/db_dump.py b/ingestion/src/metadata/cli/db_dump.py index 576520bc711..ddfffd5b1de 100644 --- a/ingestion/src/metadata/cli/db_dump.py +++ b/ingestion/src/metadata/cli/db_dump.py @@ -123,9 +123,12 @@ def get_hash_column_name(engine: Engine, table_name: str) -> Optional[str]: def run_query_iter(engine: Engine, query: str) -> Iterable[Row]: """Return a generator of rows, one row at a time, with a limit of 100 in-mem rows""" - - for row in engine.execute(text(query)).yield_per(100): - yield row + with engine.connect() as conn: + result = conn.execution_options( + stream_results=True, max_row_buffer=100 + ).execute(text(query)) + for row in result: + yield row def dump_json(tables: List[str], engine: Engine, output: Path) -> None: diff --git a/ingestion/src/metadata/cmd.py b/ingestion/src/metadata/cmd.py index 94dec5134d7..7a264cd81bd 100644 --- a/ingestion/src/metadata/cmd.py +++ b/ingestion/src/metadata/cmd.py @@ -435,12 +435,9 @@ def metadata(args=None): contains_args = vars(get_parser(args)) metadata_workflow = contains_args.get("command") config_file = contains_args.get("config") + path = None if config_file: path = Path(config_file).expanduser() - else: - raise ValueError( - "Could not load config file! Please specify the config path with `-c` or `--config`." - ) if contains_args.get("debug"): set_loggers_level(logging.DEBUG) elif contains_args.get("log_level"): diff --git a/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/index.md b/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/index.md index 3d0bdbc6d09..4ec7bfbbc13 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/index.md +++ b/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/index.md @@ -45,6 +45,9 @@ with read only access to the repository. You can follow these steps from the Git The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata out of LookML Views, including their lineage to the source databases. +Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL. +We do not yet support liquid variables. + {% /note %} ## Metadata Ingestion diff --git a/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/yaml.md b/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/yaml.md index 0eb93bad995..3ba545ebc0d 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/yaml.md +++ b/openmetadata-docs/content/v1.2.x/connectors/dashboard/looker/yaml.md @@ -50,6 +50,9 @@ with read only access to the repository. You can follow these steps from the Git The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata out of LookML Views, including their lineage to the source databases. +Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL. +We do not yet support liquid variables. + {% /note %} ### Python Requirements diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/index.md index 5f1be054064..1c68de2d25a 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/index.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/index.md @@ -45,6 +45,9 @@ with read only access to the repository. You can follow these steps from the Git The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata out of LookML Views, including their lineage to the source databases. +Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL. +We do not yet support liquid variables. + {% /note %} ## Metadata Ingestion diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/yaml.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/yaml.md index 310246955a6..9deaee1f12f 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/yaml.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/dashboard/looker/yaml.md @@ -50,6 +50,9 @@ with read only access to the repository. You can follow these steps from the Git The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata out of LookML Views, including their lineage to the source databases. +Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL. +We do not yet support liquid variables. + {% /note %} ### Python Requirements