#14043 - Fix metadata backup memory profile (#14378)

* Update looker docs

* Fix db dump
This commit is contained in:
Pere Miquel Brull 2023-12-14 15:46:35 +01:00 committed by GitHub
parent 74df616679
commit d4d8f3655f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 19 additions and 7 deletions

View File

@ -123,9 +123,12 @@ def get_hash_column_name(engine: Engine, table_name: str) -> Optional[str]:
def run_query_iter(engine: Engine, query: str) -> Iterable[Row]:
"""Return a generator of rows, one row at a time, with a limit of 100 in-mem rows"""
for row in engine.execute(text(query)).yield_per(100):
yield row
with engine.connect() as conn:
result = conn.execution_options(
stream_results=True, max_row_buffer=100
).execute(text(query))
for row in result:
yield row
def dump_json(tables: List[str], engine: Engine, output: Path) -> None:

View File

@ -435,12 +435,9 @@ def metadata(args=None):
contains_args = vars(get_parser(args))
metadata_workflow = contains_args.get("command")
config_file = contains_args.get("config")
path = None
if config_file:
path = Path(config_file).expanduser()
else:
raise ValueError(
"Could not load config file! Please specify the config path with `-c` or `--config`."
)
if contains_args.get("debug"):
set_loggers_level(logging.DEBUG)
elif contains_args.get("log_level"):

View File

@ -45,6 +45,9 @@ with read only access to the repository. You can follow these steps from the Git
The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata
out of LookML Views, including their lineage to the source databases.
Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL.
We do not yet support liquid variables.
{% /note %}
## Metadata Ingestion

View File

@ -50,6 +50,9 @@ with read only access to the repository. You can follow these steps from the Git
The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata
out of LookML Views, including their lineage to the source databases.
Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL.
We do not yet support liquid variables.
{% /note %}
### Python Requirements

View File

@ -45,6 +45,9 @@ with read only access to the repository. You can follow these steps from the Git
The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata
out of LookML Views, including their lineage to the source databases.
Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL.
We do not yet support liquid variables.
{% /note %}
## Metadata Ingestion

View File

@ -50,6 +50,9 @@ with read only access to the repository. You can follow these steps from the Git
The GitHub credentials are completely optional. Just note that without them, we won't be able to ingest metadata
out of LookML Views, including their lineage to the source databases.
Moreover, Looker lineage only supports LookML views configured with `sql_table_name` and `derived_table` in plain SQL.
We do not yet support liquid variables.
{% /note %}
### Python Requirements