mirror of
https://github.com/datahub-project/datahub.git
synced 2025-11-02 03:39:03 +00:00
feat(cli): support datahub ingest mcps (#7871)
This commit is contained in:
parent
a43903bf6d
commit
fb9a35b58a
@ -951,7 +951,7 @@ def ingest_sample_data(path: Optional[str], token: Optional[str]) -> None:
|
||||
"source": {
|
||||
"type": "file",
|
||||
"config": {
|
||||
"filename": path,
|
||||
"path": path,
|
||||
},
|
||||
},
|
||||
"sink": {
|
||||
|
||||
@ -234,6 +234,31 @@ def parse_restli_response(response):
|
||||
return rows
|
||||
|
||||
|
||||
@ingest.command()
|
||||
@click.argument("path", type=click.Path(exists=True))
|
||||
def mcps(path: str) -> None:
|
||||
"""
|
||||
Ingest metadata from a mcp json file or directory of files.
|
||||
|
||||
This requires that you've run `datahub init` to set up your config.
|
||||
"""
|
||||
|
||||
click.echo("Starting ingestion...")
|
||||
recipe: dict = {
|
||||
"source": {
|
||||
"type": "file",
|
||||
"config": {
|
||||
"path": path,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pipeline = Pipeline.create(recipe)
|
||||
pipeline.run()
|
||||
ret = pipeline.pretty_print_summary()
|
||||
sys.exit(ret)
|
||||
|
||||
|
||||
@ingest.command()
|
||||
@click.argument("page_offset", type=int, default=0)
|
||||
@click.argument("page_size", type=int, default=100)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user