From fb9a35b58a8ca7ec5288e341833dc0457c927f19 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 23 May 2023 06:18:51 +0530 Subject: [PATCH] feat(cli): support `datahub ingest mcps` (#7871) --- .../src/datahub/cli/docker_cli.py | 2 +- .../src/datahub/cli/ingest_cli.py | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index ada45c03a7..44dfab3695 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -951,7 +951,7 @@ def ingest_sample_data(path: Optional[str], token: Optional[str]) -> None: "source": { "type": "file", "config": { - "filename": path, + "path": path, }, }, "sink": { diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py index 2cb8bd87cb..38f1dddeab 100644 --- a/metadata-ingestion/src/datahub/cli/ingest_cli.py +++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py @@ -234,6 +234,31 @@ def parse_restli_response(response): return rows +@ingest.command() +@click.argument("path", type=click.Path(exists=True)) +def mcps(path: str) -> None: + """ + Ingest metadata from a mcp json file or directory of files. + + This requires that you've run `datahub init` to set up your config. + """ + + click.echo("Starting ingestion...") + recipe: dict = { + "source": { + "type": "file", + "config": { + "path": path, + }, + }, + } + + pipeline = Pipeline.create(recipe) + pipeline.run() + ret = pipeline.pretty_print_summary() + sys.exit(ret) + + @ingest.command() @click.argument("page_offset", type=int, default=0) @click.argument("page_size", type=int, default=100)