yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-28 18:38:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): fix browsepaths and ownership urns (#2935)

Browse Source
This commit is contained in:
Kevin Hu 2021-07-22 15:26:10 -07:00 committed by GitHub
parent 4e0da7c319
commit 757abfc6f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 21 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/src/datahub/ingestion/source/sagemaker_processors/feature_groups.py
View File

@ -81,7 +81,7 @@ class FeatureGroupProcessor:
feature_group_snapshot = MLFeatureTableSnapshot(
urn=builder.make_ml_feature_table_urn("sagemaker", feature_group_name),
aspects=[
BrowsePathsClass(paths=[f"sagemaker/{feature_group_name}"]),
BrowsePathsClass(paths=[f"/sagemaker/{feature_group_name}"]),
],
)

2
metadata-ingestion/src/datahub/ingestion/source/sagemaker_processors/jobs.py
View File

@ -546,7 +546,7 @@ class JobProcessor:
"jobType": job_type,
},
),
BrowsePathsClass(paths=[f"{job_type}/{name}"]),
BrowsePathsClass(paths=[f"/{job_type}/{name}"]),
],
)

8
metadata-ingestion/src/datahub/ingestion/source/sagemaker_processors/models.py
View File

@ -236,7 +236,7 @@ class ModelProcessor:
if group_details.get("CreatedBy", {}).get("UserProfileName") is not None:
owners.append(
OwnerClass(
owner=group_details["CreatedBy"]["UserProfileName"],
owner=f"urn:li:corpuser:{group_details['CreatedBy']['UserProfileName']}",
type=OwnershipTypeClass.DATAOWNER,
)
)
@ -257,7 +257,7 @@ class ModelProcessor:
},
),
OwnershipClass(owners),
BrowsePathsClass(paths=[f"sagemaker/{group_name}"]),
BrowsePathsClass(paths=[f"/sagemaker/{group_name}"]),
],
)
@ -399,12 +399,12 @@ class ModelProcessor:
]
model_browsepaths = [
f"sagemaker/{x}/{model_details['ModelName']}" for x in model_group_names
f"/sagemaker/{x}/{model_details['ModelName']}" for x in model_group_names
]
# if model is not in any groups, set a single browsepath with the model as the first entity
if not model_browsepaths:
model_browsepaths.append(f"sagemaker/{model_details['ModelName']}")
model_browsepaths.append(f"/sagemaker/{model_details['ModelName']}")
model_snapshot = MLModelSnapshot(
urn=builder.make_ml_model_urn(

30
metadata-ingestion/tests/unit/sagemaker/sagemaker_mces_golden.json
View File

@ -74,7 +74,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"sagemaker/test-2"
"/sagemaker/test-2"
]
}
},
@ -185,7 +185,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"sagemaker/test-1"
"/sagemaker/test-1"
]
}
},
@ -278,7 +278,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"sagemaker/test"
"/sagemaker/test"
]
}
},
@ -869,7 +869,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"auto_ml/an-auto-ml-job"
"/auto_ml/an-auto-ml-job"
]
}
},
@ -949,7 +949,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"compilation/a-compilation-job"
"/compilation/a-compilation-job"
]
}
},
@ -1030,7 +1030,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"edge_packaging/an-edge-packaging-job"
"/edge_packaging/an-edge-packaging-job"
]
}
},
@ -1108,7 +1108,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"hyper_parameter_tuning/a-hyper-parameter-tuning-job"
"/hyper_parameter_tuning/a-hyper-parameter-tuning-job"
]
}
},
@ -1186,7 +1186,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"labeling/a-labeling-job"
"/labeling/a-labeling-job"
]
}
},
@ -1273,7 +1273,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"processing/a-processing-job"
"/processing/a-processing-job"
]
}
},
@ -1378,7 +1378,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"training/a-training-job"
"/training/a-training-job"
]
}
},
@ -1468,7 +1468,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"transform/a-transform-job"
"/transform/a-transform-job"
]
}
},
@ -1574,7 +1574,7 @@
"com.linkedin.pegasus2avro.common.Ownership": {
"owners": [
{
"owner": "some-user",
"owner": "urn:li:corpuser:some-user",
"type": "DATAOWNER",
"source": null
}
@ -1589,7 +1589,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"sagemaker/a-model-package-group"
"/sagemaker/a-model-package-group"
]
}
}
@ -1671,7 +1671,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"sagemaker/a-model-package-group/the-first-model"
"/sagemaker/a-model-package-group/the-first-model"
]
}
}
@ -1756,7 +1756,7 @@
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"sagemaker/a-model-package-group/the-second-model"
"/sagemaker/a-model-package-group/the-second-model"
]
}
}