mirror of
https://github.com/datahub-project/datahub.git
synced 2025-11-13 09:52:46 +00:00
feat(cli): support datahub ingest mcps (#7871)
This commit is contained in:
parent
a43903bf6d
commit
fb9a35b58a
@ -951,7 +951,7 @@ def ingest_sample_data(path: Optional[str], token: Optional[str]) -> None:
|
|||||||
"source": {
|
"source": {
|
||||||
"type": "file",
|
"type": "file",
|
||||||
"config": {
|
"config": {
|
||||||
"filename": path,
|
"path": path,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"sink": {
|
"sink": {
|
||||||
|
|||||||
@ -234,6 +234,31 @@ def parse_restli_response(response):
|
|||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
@ingest.command()
|
||||||
|
@click.argument("path", type=click.Path(exists=True))
|
||||||
|
def mcps(path: str) -> None:
|
||||||
|
"""
|
||||||
|
Ingest metadata from a mcp json file or directory of files.
|
||||||
|
|
||||||
|
This requires that you've run `datahub init` to set up your config.
|
||||||
|
"""
|
||||||
|
|
||||||
|
click.echo("Starting ingestion...")
|
||||||
|
recipe: dict = {
|
||||||
|
"source": {
|
||||||
|
"type": "file",
|
||||||
|
"config": {
|
||||||
|
"path": path,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
pipeline = Pipeline.create(recipe)
|
||||||
|
pipeline.run()
|
||||||
|
ret = pipeline.pretty_print_summary()
|
||||||
|
sys.exit(ret)
|
||||||
|
|
||||||
|
|
||||||
@ingest.command()
|
@ingest.command()
|
||||||
@click.argument("page_offset", type=int, default=0)
|
@click.argument("page_offset", type=int, default=0)
|
||||||
@click.argument("page_size", type=int, default=100)
|
@click.argument("page_size", type=int, default=100)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user