feat(cli): support datahub ingest mcps (#7871)

This commit is contained in:
Harshal Sheth 2023-05-23 06:18:51 +05:30 committed by GitHub
parent a43903bf6d
commit fb9a35b58a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 26 additions and 1 deletions

View File

@ -951,7 +951,7 @@ def ingest_sample_data(path: Optional[str], token: Optional[str]) -> None:
"source": {
"type": "file",
"config": {
"filename": path,
"path": path,
},
},
"sink": {

View File

@ -234,6 +234,31 @@ def parse_restli_response(response):
return rows
@ingest.command()
@click.argument("path", type=click.Path(exists=True))
def mcps(path: str) -> None:
"""
Ingest metadata from a mcp json file or directory of files.
This requires that you've run `datahub init` to set up your config.
"""
click.echo("Starting ingestion...")
recipe: dict = {
"source": {
"type": "file",
"config": {
"path": path,
},
},
}
pipeline = Pipeline.create(recipe)
pipeline.run()
ret = pipeline.pretty_print_summary()
sys.exit(ret)
@ingest.command()
@click.argument("page_offset", type=int, default=0)
@click.argument("page_size", type=int, default=100)