From f91bcc03f63cd193d40a21ce25a398cddb389fa4 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 21 Oct 2021 23:51:38 +0200 Subject: [PATCH] [Issue-877] - High Level API (#890) * First approach * Test model * Validate endpoints * Update schema codegen * Prepare reqs dev * Rename titles and codegen * Update README * Keep lineage title as addLineage * Update pydantic classes * Update API wrapper & tests * Handle services methods * Prepare tests --- Makefile | 6 + .../json/schema/api/data/createTopic.json | 2 +- .../json/schema/api/feed/createThread.json | 2 +- .../json/schema/api/tags/createTag.json | 2 +- .../schema/api/tags/createTagCategory.json | 2 +- .../json/schema/api/teams/createTeam.json | 2 +- .../json/schema/api/teams/createUser.json | 2 +- ingestion/README.md | 6 + ingestion/requirements-dev.txt | 1 + .../metadata/generated/data/tags/__init__.py | 2 +- .../generated/data/tags/personalDataTags.py | 2 +- .../metadata/generated/data/tags/piiTags.py | 2 +- .../metadata/generated/data/tags/tierTags.py | 2 +- .../metadata/generated/data/tags/userTags.py | 2 +- .../metadata/generated/schema/api/__init__.py | 2 +- .../generated/schema/api/catalogVersion.py | 2 +- .../generated/schema/api/data/__init__.py | 2 +- .../generated/schema/api/data/createChart.py | 2 +- .../schema/api/data/createDashboard.py | 2 +- .../schema/api/data/createDatabase.py | 2 +- .../generated/schema/api/data/createModel.py | 2 +- .../schema/api/data/createPipeline.py | 2 +- .../generated/schema/api/data/createTable.py | 2 +- .../generated/schema/api/data/createTask.py | 2 +- .../generated/schema/api/data/createTopic.py | 4 +- .../generated/schema/api/feed/__init__.py | 2 +- .../generated/schema/api/feed/createThread.py | 4 +- .../generated/schema/api/lineage/__init__.py | 2 +- .../schema/api/lineage/addLineage.py | 2 +- .../generated/schema/api/services/__init__.py | 2 +- .../api/services/createDashboardService.py | 2 +- .../api/services/createDatabaseService.py | 2 +- .../api/services/createMessagingService.py | 2 +- .../api/services/createPipelineService.py | 2 +- .../api/services/updateDashboardService.py | 2 +- .../api/services/updateDatabaseService.py | 2 +- .../api/services/updateMessagingService.py | 2 +- .../api/services/updatePipelineService.py | 2 +- .../metadata/generated/schema/api/setOwner.py | 2 +- .../generated/schema/api/tags/__init__.py | 2 +- .../generated/schema/api/tags/createTag.py | 4 +- .../schema/api/tags/createTagCategory.py | 4 +- .../generated/schema/api/teams/__init__.py | 2 +- .../generated/schema/api/teams/createTeam.py | 4 +- .../generated/schema/api/teams/createUser.py | 4 +- .../generated/schema/entity/__init__.py | 2 +- .../metadata/generated/schema/entity/bots.py | 10 +- .../generated/schema/entity/data/__init__.py | 2 +- .../generated/schema/entity/data/chart.py | 10 +- .../generated/schema/entity/data/dashboard.py | 10 +- .../generated/schema/entity/data/database.py | 13 +- .../generated/schema/entity/data/metrics.py | 20 +- .../generated/schema/entity/data/model.py | 14 +- .../generated/schema/entity/data/pipeline.py | 18 +- .../generated/schema/entity/data/report.py | 14 +- .../generated/schema/entity/data/table.py | 34 +- .../generated/schema/entity/data/task.py | 20 +- .../generated/schema/entity/data/topic.py | 14 +- .../generated/schema/entity/feed/__init__.py | 2 +- .../generated/schema/entity/feed/thread.py | 2 +- .../schema/entity/services/__init__.py | 2 +- .../entity/services/dashboardService.py | 13 +- .../schema/entity/services/databaseService.py | 13 +- .../entity/services/messagingService.py | 14 +- .../schema/entity/services/pipelineService.py | 16 +- .../generated/schema/entity/tags/__init__.py | 2 +- .../schema/entity/tags/tagCategory.py | 21 +- .../generated/schema/entity/teams/__init__.py | 2 +- .../generated/schema/entity/teams/team.py | 10 +- .../generated/schema/entity/teams/user.py | 10 +- .../generated/schema/type/__init__.py | 2 +- .../generated/schema/type/auditLog.py | 2 +- .../metadata/generated/schema/type/basic.py | 25 +- .../schema/type/collectionDescriptor.py | 2 +- .../generated/schema/type/dailyCount.py | 2 +- .../generated/schema/type/entityLineage.py | 4 +- .../generated/schema/type/entityReference.py | 5 +- .../generated/schema/type/entityUsage.py | 2 +- .../generated/schema/type/jdbcConnection.py | 2 +- .../metadata/generated/schema/type/paging.py | 2 +- .../metadata/generated/schema/type/profile.py | 2 +- .../generated/schema/type/schedule.py | 2 +- .../generated/schema/type/tagLabel.py | 2 +- .../generated/schema/type/usageDetails.py | 2 +- .../src/metadata/ingestion/ometa/ometa_api.py | 329 ++++++++++++++++++ .../ingestion/ometa/openmetadata_rest.py | 6 +- .../metadata/ingestion/sink/metadata_rest.py | 6 +- .../src/metadata/ingestion/source/kafka.py | 6 +- .../metadata/ingestion/source/sample_data.py | 6 +- .../ingestion/source/sample_entity.py | 6 +- .../ometa/test_ometa_database_api.py | 177 ++++++++++ .../integration/ometa/test_ometa_model_api.py | 138 ++++++++ ingestion/tests/unit/test_ometa_endpoints.py | 120 +++++++ 93 files changed, 1113 insertions(+), 134 deletions(-) create mode 100644 ingestion/requirements-dev.txt create mode 100644 ingestion/src/metadata/ingestion/ometa/ometa_api.py create mode 100644 ingestion/tests/integration/ometa/test_ometa_database_api.py create mode 100644 ingestion/tests/integration/ometa/test_ometa_model_api.py create mode 100644 ingestion/tests/unit/test_ometa_endpoints.py diff --git a/Makefile b/Makefile index b6c2836d37b..4dfcd6511da 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,9 @@ install: install_test: pip install -r ingestion/requirements-test.txt +install_dev: + pip install -r ingestion/requirements-dev.txt + precommit_install: @echo "Installing pre-commit hooks" @echo "Make sure to first run `make install_test`" @@ -29,3 +32,6 @@ black: black_check: black --check --diff $(PY_SOURCE) --exclude $(PY_SOURCE)/metadata/generated + +generate: + datamodel-codegen --input catalog-rest-service/src/main/resources/json --input-file-type jsonschema --output ingestion/src/metadata/generated diff --git a/catalog-rest-service/src/main/resources/json/schema/api/data/createTopic.json b/catalog-rest-service/src/main/resources/json/schema/api/data/createTopic.json index fb69cdbfa45..569e3b1de99 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/data/createTopic.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/data/createTopic.json @@ -1,7 +1,7 @@ { "$id": "https://github.com/open-metadata/OpenMetadata/blob/main/catalog-rest-service/src/main/resources/json/schema/api/data/createTopic.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Create topic", + "title": "Create topic entity request", "description": "Create a topic entity request", "type": "object", "properties": { diff --git a/catalog-rest-service/src/main/resources/json/schema/api/feed/createThread.json b/catalog-rest-service/src/main/resources/json/schema/api/feed/createThread.json index 118112b32a9..8019f0207f3 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/feed/createThread.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/feed/createThread.json @@ -1,7 +1,7 @@ { "$id": "https://github.com/open-metadata/OpenMetadata/blob/main/catalog-rest-service/src/main/resources/json/schema/api/feed/createThread.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Create thread request", + "title": "Create thread entity request", "description": "Create thread request", "type": "object", diff --git a/catalog-rest-service/src/main/resources/json/schema/api/tags/createTag.json b/catalog-rest-service/src/main/resources/json/schema/api/tags/createTag.json index 8308ff8bb6d..64da9297f24 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/tags/createTag.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/tags/createTag.json @@ -1,7 +1,7 @@ { "$id": "https://github.com/open-metadata/OpenMetadata/blob/main/catalog-rest-service/src/main/resources/json/schema/api/tags/createTag.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Create tag API request", + "title": "Create tag entity request", "description": "Create tag API request", "type": "object", "javaType": "org.openmetadata.catalog.type.CreateTag", diff --git a/catalog-rest-service/src/main/resources/json/schema/api/tags/createTagCategory.json b/catalog-rest-service/src/main/resources/json/schema/api/tags/createTagCategory.json index 5fdb47c194f..f48af306b47 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/tags/createTagCategory.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/tags/createTagCategory.json @@ -1,7 +1,7 @@ { "$id": "https://github.com/open-metadata/OpenMetadata/blob/main/catalog-rest-service/src/main/resources/json/schema/api/tags/createTagCategory.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Create tag category request", + "title": "Create tag category entity request", "description": "Create tag category request", "type": "object", "javaType": "org.openmetadata.catalog.type.CreateTagCategory", diff --git a/catalog-rest-service/src/main/resources/json/schema/api/teams/createTeam.json b/catalog-rest-service/src/main/resources/json/schema/api/teams/createTeam.json index 54e81a8ad7d..39f1d056259 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/teams/createTeam.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/teams/createTeam.json @@ -1,7 +1,7 @@ { "$id": "https://github.com/open-metadata/OpenMetadata/blob/main/catalog-rest-service/src/main/resources/json/schema/api/teams/createTeam.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Team entity", + "title": "Create team entity request", "description": "Team entity", "type": "object", diff --git a/catalog-rest-service/src/main/resources/json/schema/api/teams/createUser.json b/catalog-rest-service/src/main/resources/json/schema/api/teams/createUser.json index 61f0af2721b..d9c07e5417d 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/teams/createUser.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/teams/createUser.json @@ -1,7 +1,7 @@ { "$id": "https://github.com/open-metadata/OpenMetadata/blob/main/catalog-rest-service/src/main/resources/json/schema/api/teams/createUser.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Request to create User entity", + "title": "Create user entity request", "description": "Request to create User entity", "type": "object", diff --git a/ingestion/README.md b/ingestion/README.md index efe0f1bad6e..6075feefbb3 100644 --- a/ingestion/README.md +++ b/ingestion/README.md @@ -70,3 +70,9 @@ docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elas ```text metadata ingest -c ./pipelines/metadata_to_es.json ``` + +## Generated sources + +We are using `datamodel-codegen` to get some `pydantic` classes inside the `generated` module from the JSON Schemas defining the API and Entities. + +This tool bases the class name on the `title` of the JSON Schema (vs. Java POJO, which uses the file name). Note that this convention is important for us, as having a standardized approach in creating the titles helps us create generic code capable of tackling multiple Type Variables. diff --git a/ingestion/requirements-dev.txt b/ingestion/requirements-dev.txt new file mode 100644 index 00000000000..26832211873 --- /dev/null +++ b/ingestion/requirements-dev.txt @@ -0,0 +1 @@ +datamodel-code-generator==0.11.14 diff --git a/ingestion/src/metadata/generated/data/tags/__init__.py b/ingestion/src/metadata/generated/data/tags/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/data/tags/__init__.py +++ b/ingestion/src/metadata/generated/data/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/data/tags/personalDataTags.py b/ingestion/src/metadata/generated/data/tags/personalDataTags.py index f7fb728e2b2..c4f1ba3209e 100644 --- a/ingestion/src/metadata/generated/data/tags/personalDataTags.py +++ b/ingestion/src/metadata/generated/data/tags/personalDataTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/personalDataTags.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/piiTags.py b/ingestion/src/metadata/generated/data/tags/piiTags.py index 6dc7c3bdfe4..a4b20444639 100644 --- a/ingestion/src/metadata/generated/data/tags/piiTags.py +++ b/ingestion/src/metadata/generated/data/tags/piiTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/piiTags.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/tierTags.py b/ingestion/src/metadata/generated/data/tags/tierTags.py index 7039af017de..b463eac9d37 100644 --- a/ingestion/src/metadata/generated/data/tags/tierTags.py +++ b/ingestion/src/metadata/generated/data/tags/tierTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/tierTags.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/userTags.py b/ingestion/src/metadata/generated/data/tags/userTags.py index 14cc1a48074..cea8c089714 100644 --- a/ingestion/src/metadata/generated/data/tags/userTags.py +++ b/ingestion/src/metadata/generated/data/tags/userTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/userTags.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/__init__.py b/ingestion/src/metadata/generated/schema/api/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/api/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/catalogVersion.py b/ingestion/src/metadata/generated/schema/api/catalogVersion.py index 235b3bf740d..00ecc57df9f 100644 --- a/ingestion/src/metadata/generated/schema/api/catalogVersion.py +++ b/ingestion/src/metadata/generated/schema/api/catalogVersion.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/catalogVersion.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/__init__.py b/ingestion/src/metadata/generated/schema/api/data/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/api/data/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/data/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/data/createChart.py b/ingestion/src/metadata/generated/schema/api/data/createChart.py index 1dbb3ebb7d7..535daf83ba5 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createChart.py +++ b/ingestion/src/metadata/generated/schema/api/data/createChart.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createChart.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createDashboard.py b/ingestion/src/metadata/generated/schema/api/data/createDashboard.py index bbcc4b0b336..76339e8d7ff 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createDashboard.py +++ b/ingestion/src/metadata/generated/schema/api/data/createDashboard.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createDashboard.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createDatabase.py b/ingestion/src/metadata/generated/schema/api/data/createDatabase.py index 0bd6b5fcdde..820ebea01c4 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createDatabase.py +++ b/ingestion/src/metadata/generated/schema/api/data/createDatabase.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createDatabase.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createModel.py b/ingestion/src/metadata/generated/schema/api/data/createModel.py index c6cd3a65ad1..d61ce6c7136 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createModel.py +++ b/ingestion/src/metadata/generated/schema/api/data/createModel.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createModel.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createPipeline.py b/ingestion/src/metadata/generated/schema/api/data/createPipeline.py index 7d115df3fbd..15000106276 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createPipeline.py +++ b/ingestion/src/metadata/generated/schema/api/data/createPipeline.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createPipeline.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createTable.py b/ingestion/src/metadata/generated/schema/api/data/createTable.py index 4dc33f6029d..e06e2beb7a5 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createTable.py +++ b/ingestion/src/metadata/generated/schema/api/data/createTable.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createTable.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createTask.py b/ingestion/src/metadata/generated/schema/api/data/createTask.py index ea7c2788a6d..8ba2b72ef79 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createTask.py +++ b/ingestion/src/metadata/generated/schema/api/data/createTask.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createTask.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createTopic.py b/ingestion/src/metadata/generated/schema/api/data/createTopic.py index 51cfd61270c..ed6b69e11f6 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createTopic.py +++ b/ingestion/src/metadata/generated/schema/api/data/createTopic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createTopic.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -12,7 +12,7 @@ from ...entity.data import topic from ...type import entityReference, tagLabel -class CreateTopic(BaseModel): +class CreateTopicEntityRequest(BaseModel): name: topic.TopicName = Field( ..., description='Name that identifies this topic instance uniquely.' ) diff --git a/ingestion/src/metadata/generated/schema/api/feed/__init__.py b/ingestion/src/metadata/generated/schema/api/feed/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/api/feed/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/feed/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/feed/createThread.py b/ingestion/src/metadata/generated/schema/api/feed/createThread.py index 48f1812c123..40bb9f38462 100644 --- a/ingestion/src/metadata/generated/schema/api/feed/createThread.py +++ b/ingestion/src/metadata/generated/schema/api/feed/createThread.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/feed/createThread.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field from ...type import basic -class CreateThreadRequest(BaseModel): +class CreateThreadEntityRequest(BaseModel): message: str = Field(..., description='Message') from_: basic.Uuid = Field( ..., diff --git a/ingestion/src/metadata/generated/schema/api/lineage/__init__.py b/ingestion/src/metadata/generated/schema/api/lineage/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/api/lineage/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/lineage/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py b/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py index 73b302e4718..92abd3dda57 100644 --- a/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py +++ b/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/lineage/addLineage.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/__init__.py b/ingestion/src/metadata/generated/schema/api/services/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/api/services/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/services/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py b/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py index f18052653d1..c51e8399e96 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createDashboardService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py b/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py index b89a87d2a8e..28bb9fd4ed0 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createDatabaseService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py b/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py index c83a87bd3a1..4a44006774c 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createMessagingService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py b/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py index f31b0a390fc..9e2320b66c5 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createPipelineService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py b/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py index 49fe031831c..64795bfc24d 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateDashboardService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py b/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py index 5ac7f4b44e4..f9b038abeef 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateDatabaseService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py b/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py index 6e05a591c59..f305b5b0eb9 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateMessagingService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py b/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py index 03e60ea3f44..00277af0fb9 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updatePipelineService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/setOwner.py b/ingestion/src/metadata/generated/schema/api/setOwner.py index 0fada066393..813466c3965 100644 --- a/ingestion/src/metadata/generated/schema/api/setOwner.py +++ b/ingestion/src/metadata/generated/schema/api/setOwner.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/setOwner.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/tags/__init__.py b/ingestion/src/metadata/generated/schema/api/tags/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/tags/createTag.py b/ingestion/src/metadata/generated/schema/api/tags/createTag.py index 4184102311f..21812b1ed1b 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/createTag.py +++ b/ingestion/src/metadata/generated/schema/api/tags/createTag.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/tags/createTag.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -11,7 +11,7 @@ from pydantic import BaseModel, Field from ...entity.tags import tagCategory -class CreateTagApiRequest(BaseModel): +class CreateTagEntityRequest(BaseModel): name: tagCategory.TagName description: str = Field(..., description='Unique name of the tag category') associatedTags: Optional[List[str]] = Field( diff --git a/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py b/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py index f9161bed3e5..a17e0ebbc95 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py +++ b/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/tags/createTagCategory.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field from ...entity.tags import tagCategory -class CreateTagCategoryRequest(BaseModel): +class CreateTagCategoryEntityRequest(BaseModel): name: tagCategory.TagName description: str = Field(..., description='Description of the tag category') categoryType: tagCategory.TagCategoryType diff --git a/ingestion/src/metadata/generated/schema/api/teams/__init__.py b/ingestion/src/metadata/generated/schema/api/teams/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/teams/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/teams/createTeam.py b/ingestion/src/metadata/generated/schema/api/teams/createTeam.py index 4042333d71a..1e5fd7901ce 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/createTeam.py +++ b/ingestion/src/metadata/generated/schema/api/teams/createTeam.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/teams/createTeam.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -12,7 +12,7 @@ from ...entity.teams import team from ...type import basic, profile -class TeamEntity(BaseModel): +class CreateTeamEntityRequest(BaseModel): name: team.TeamName displayName: Optional[str] = Field( None, diff --git a/ingestion/src/metadata/generated/schema/api/teams/createUser.py b/ingestion/src/metadata/generated/schema/api/teams/createUser.py index 4ceb1a6052a..592a20920f8 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/createUser.py +++ b/ingestion/src/metadata/generated/schema/api/teams/createUser.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/teams/createUser.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -12,7 +12,7 @@ from ...entity.teams import user from ...type import basic, profile -class RequestToCreateUserEntity(BaseModel): +class CreateUserEntityRequest(BaseModel): name: user.UserName displayName: Optional[str] = Field( None, description="Name used for display purposes. Example 'FirstName LastName'" diff --git a/ingestion/src/metadata/generated/schema/entity/__init__.py b/ingestion/src/metadata/generated/schema/entity/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/entity/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/bots.py b/ingestion/src/metadata/generated/schema/entity/bots.py index 3d0b55b34af..3b27dbc4475 100644 --- a/ingestion/src/metadata/generated/schema/entity/bots.py +++ b/ingestion/src/metadata/generated/schema/entity/bots.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/bots.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -23,6 +23,14 @@ class Bot(BaseModel): description="Name used for display purposes. Example 'FirstName LastName'.", ) description: Optional[str] = Field(None, description='Description of the bot.') + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') href: Optional[basic.Href] = Field( None, description='Link to the resource corresponding to this bot.' ) diff --git a/ingestion/src/metadata/generated/schema/entity/data/__init__.py b/ingestion/src/metadata/generated/schema/entity/data/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/data/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/data/chart.py b/ingestion/src/metadata/generated/schema/entity/data/chart.py index ad01a579b1c..5d63b68a88f 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/chart.py +++ b/ingestion/src/metadata/generated/schema/entity/data/chart.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/chart.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -43,6 +43,14 @@ class Chart(BaseModel): description: Optional[str] = Field( None, description='Description of the dashboard, what it is, and how to use it.' ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') chartType: Optional[ChartType] = None chartUrl: Optional[AnyUrl] = Field( None, description='Chart URL, pointing to its own Service URL.' diff --git a/ingestion/src/metadata/generated/schema/entity/data/dashboard.py b/ingestion/src/metadata/generated/schema/entity/data/dashboard.py index b6697dc6be0..badb1ad0f03 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/dashboard.py +++ b/ingestion/src/metadata/generated/schema/entity/data/dashboard.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/dashboard.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -29,6 +29,14 @@ class Dashboard(BaseModel): description: Optional[str] = Field( None, description='Description of the dashboard, what it is, and how to use it.' ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') dashboardUrl: Optional[AnyUrl] = Field(None, description='Dashboard URL.') charts: Optional[List[entityReference.EntityReference]] = Field( None, description='All the charts included in this Dashboard.' diff --git a/ingestion/src/metadata/generated/schema/entity/data/database.py b/ingestion/src/metadata/generated/schema/entity/data/database.py index 0f1f188d875..3af7c471ed1 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/database.py +++ b/ingestion/src/metadata/generated/schema/entity/data/database.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/database.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -26,9 +26,20 @@ class Database(BaseModel): None, description="Name that uniquely identifies a database in the format 'ServiceName.DatabaseName'.", ) + displayName: Optional[str] = Field( + None, description='Display Name that identifies this database.' + ) description: Optional[str] = Field( None, description='Description of the database instance.' ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') href: Optional[basic.Href] = Field( None, description='Link to the resource corresponding to this entity.' ) diff --git a/ingestion/src/metadata/generated/schema/entity/data/metrics.py b/ingestion/src/metadata/generated/schema/entity/data/metrics.py index 5a17f1ab702..948a713e175 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/metrics.py +++ b/ingestion/src/metadata/generated/schema/entity/data/metrics.py @@ -1,14 +1,14 @@ # generated by datamodel-codegen: # filename: schema/entity/data/metrics.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations -from typing import Optional +from typing import List, Optional from pydantic import BaseModel, Field, constr -from ...type import basic, entityReference, usageDetails +from ...type import basic, entityReference, tagLabel, usageDetails class Metrics(BaseModel): @@ -22,16 +22,30 @@ class Metrics(BaseModel): None, description="A unique name that identifies a metric in the format 'ServiceName.MetricName'.", ) + displayName: Optional[str] = Field( + None, description='Display Name that identifies this metric.' + ) description: Optional[str] = Field( None, description='Description of metrics instance, what it is, and how to use it.', ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') href: Optional[basic.Href] = Field( None, description='Link to the resource corresponding to this entity.' ) owner: Optional[entityReference.EntityReference] = Field( None, description='Owner of this metrics.' ) + tags: Optional[List[tagLabel.TagLabel]] = Field( + None, description='Tags for this chart.' + ) service: entityReference.EntityReference = Field( ..., description='Link to service where this metrics is hosted in.' ) diff --git a/ingestion/src/metadata/generated/schema/entity/data/model.py b/ingestion/src/metadata/generated/schema/entity/data/model.py index 3d07bb66ff0..61dc70f7067 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/model.py +++ b/ingestion/src/metadata/generated/schema/entity/data/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/model.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -22,12 +22,20 @@ class Model(BaseModel): displayName: Optional[str] = Field( None, description='Display Name that identifies this model.' ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') description: Optional[str] = Field( None, description='Description of the model, what it is, and how to use it.' ) - algorithm: str = Field(..., description='Algorithm used to train the model') + algorithm: str = Field(..., description='Algorithm used to train the model.') dashboard: Optional[entityReference.EntityReference] = Field( - None, description='Performance Dashboard URL to track metric evolution' + None, description='Performance Dashboard URL to track metric evolution.' ) href: Optional[basic.Href] = Field( None, description='Link to the resource corresponding to this entity.' diff --git a/ingestion/src/metadata/generated/schema/entity/data/pipeline.py b/ingestion/src/metadata/generated/schema/entity/data/pipeline.py index 56950f97be9..d015e40d199 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/pipeline.py +++ b/ingestion/src/metadata/generated/schema/entity/data/pipeline.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/pipeline.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -29,14 +29,22 @@ class Pipeline(BaseModel): description: Optional[str] = Field( None, description='Description of this Pipeline.' ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') pipelineUrl: Optional[AnyUrl] = Field( None, - description='Pipeline URL to visit/manage. This URL points to respective pipeline service UI', + description='Pipeline URL to visit/manage. This URL points to respective pipeline service UI.', ) - concurrency: Optional[int] = Field(None, description='Concurrency of the Pipeline') - pipelineLocation: Optional[str] = Field(None, description='Pipeline Code Location') + concurrency: Optional[int] = Field(None, description='Concurrency of the Pipeline.') + pipelineLocation: Optional[str] = Field(None, description='Pipeline Code Location.') startDate: Optional[basic.DateTime] = Field( - None, description='Start date of the workflow' + None, description='Start date of the workflow.' ) tasks: Optional[List[entityReference.EntityReference]] = Field( None, description='All the tasks that are part of pipeline.' diff --git a/ingestion/src/metadata/generated/schema/entity/data/report.py b/ingestion/src/metadata/generated/schema/entity/data/report.py index 5cd8b4a3549..afd86d688d7 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/report.py +++ b/ingestion/src/metadata/generated/schema/entity/data/report.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/report.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -22,9 +22,21 @@ class Report(BaseModel): None, description="A unique name that identifies a report in the format 'ServiceName.ReportName'.", ) + displayName: Optional[str] = Field( + None, + description='Display Name that identifies this report. It could be title or label from the source services.', + ) description: Optional[str] = Field( None, description='Description of this report instance.' ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') href: Optional[basic.Href] = Field( None, description='Link to the resource corresponding to this report.' ) diff --git a/ingestion/src/metadata/generated/schema/entity/data/table.py b/ingestion/src/metadata/generated/schema/entity/data/table.py index a792c2c34a0..8ffa6afdb5a 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/table.py +++ b/ingestion/src/metadata/generated/schema/entity/data/table.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/table.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -80,7 +80,7 @@ class TableConstraint(BaseModel): class ColumnName(BaseModel): __root__: constr(regex=r'^[^.]*$', min_length=1, max_length=64) = Field( ..., - description='Local name (not fully qualified name) of the column. ColumnName is `-` when the column is not named in struct dataType. For example, BigQuery supports struct with unnamed fields', + description='Local name (not fully qualified name) of the column. ColumnName is `-` when the column is not named in struct dataType. For example, BigQuery supports struct with unnamed fields.', ) @@ -219,26 +219,32 @@ class Table(BaseModel): name: TableName = Field( ..., description='Name of a table. Expected to be unique within a database.' ) - description: Optional[str] = Field(None, description='Description of a table.') - href: Optional[basic.Href] = Field(None, description='Link to this table resource.') - tableType: Optional[TableType] = None + displayName: Optional[str] = Field( + None, + description='Display Name that identifies this table. It could be title or label from the source services.', + ) fullyQualifiedName: Optional[str] = Field( None, description='Fully qualified name of a table in the form `serviceName.databaseName.tableName`.', ) + description: Optional[str] = Field(None, description='Description of a table.') + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') + href: Optional[basic.Href] = Field(None, description='Link to this table resource.') + tableType: Optional[TableType] = None columns: List[Column] = Field(..., description='Columns in this table.') tableConstraints: Optional[List[TableConstraint]] = Field( None, description='Table constraints.' ) - usageSummary: Optional[usageDetails.TypeUsedToReturnUsageDetailsOfAnEntity] = Field( - None, description='Latest usage information for this table.' - ) owner: Optional[entityReference.EntityReference] = Field( None, description='Owner of this table.' ) - followers: Optional[entityReference.EntityReferenceList] = Field( - None, description='Followers of this table.' - ) database: Optional[entityReference.EntityReference] = Field( None, description='Reference to Database that contains this table.' ) @@ -248,6 +254,12 @@ class Table(BaseModel): tags: Optional[List[tagLabel.TagLabel]] = Field( None, description='Tags for this table.' ) + usageSummary: Optional[usageDetails.TypeUsedToReturnUsageDetailsOfAnEntity] = Field( + None, description='Latest usage information for this table.' + ) + followers: Optional[entityReference.EntityReferenceList] = Field( + None, description='Followers of this table.' + ) joins: Optional[TableJoins] = Field( None, description='Details of other tables this table is frequently joined with.', diff --git a/ingestion/src/metadata/generated/schema/entity/data/task.py b/ingestion/src/metadata/generated/schema/entity/data/task.py index fb9c357e629..9abf969d50a 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/task.py +++ b/ingestion/src/metadata/generated/schema/entity/data/task.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/task.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -27,23 +27,31 @@ class Task(BaseModel): description="A unique name that identifies a pipeline in the format 'ServiceName.PipelineName.TaskName'.", ) description: Optional[str] = Field(None, description='Description of this Task.') + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') taskUrl: Optional[AnyUrl] = Field( None, - description='Task URL to visit/manage. This URL points to respective pipeline service UI', + description='Task URL to visit/manage. This URL points to respective pipeline service UI.', ) downstreamTasks: Optional[List[constr(min_length=1, max_length=64)]] = Field( None, description='All the tasks that are downstream of this task.' ) taskType: Optional[str] = Field( - None, description='Type of the Task. Usually refers to the class it implements' + None, description='Type of the Task. Usually refers to the class it implements.' ) taskSQL: Optional[str] = Field( - None, description='SQL used in the task. Can be used to determine the lineage' + None, description='SQL used in the task. Can be used to determine the lineage.' ) startDate: Optional[basic.DateTime] = Field( - None, description='Start date of the task' + None, description='Start date of the task.' ) - endDate: Optional[basic.DateTime] = Field(None, description='End date of the task') + endDate: Optional[basic.DateTime] = Field(None, description='End date of the task.') tags: Optional[List[tagLabel.TagLabel]] = Field( None, description='Tags for this Pipeline.' ) diff --git a/ingestion/src/metadata/generated/schema/entity/data/topic.py b/ingestion/src/metadata/generated/schema/entity/data/topic.py index 25e8e6bfd71..0887739c4bd 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/topic.py +++ b/ingestion/src/metadata/generated/schema/entity/data/topic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/topic.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -39,9 +39,21 @@ class Topic(BaseModel): None, description="Name that uniquely identifies a topic in the format 'messagingServiceName.topicName'.", ) + displayName: Optional[str] = Field( + None, + description='Display Name that identifies this topic. It could be title or label from the source services.', + ) description: Optional[str] = Field( None, description='Description of the topic instance.' ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') service: entityReference.EntityReference = Field( ..., description='Link to the messaging cluster/service where this topic is hosted in.', diff --git a/ingestion/src/metadata/generated/schema/entity/feed/__init__.py b/ingestion/src/metadata/generated/schema/entity/feed/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/entity/feed/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/feed/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/feed/thread.py b/ingestion/src/metadata/generated/schema/entity/feed/thread.py index 827f830a321..e58eb1c3fcd 100644 --- a/ingestion/src/metadata/generated/schema/entity/feed/thread.py +++ b/ingestion/src/metadata/generated/schema/entity/feed/thread.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/feed/thread.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/__init__.py b/ingestion/src/metadata/generated/schema/entity/services/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/services/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py b/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py index 8e1634fcabc..9c58f2207ae 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/dashboardService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -26,12 +26,23 @@ class DashboardService(BaseModel): name: constr(min_length=1, max_length=64) = Field( ..., description='Name that identifies this dashboard service.' ) + displayName: Optional[str] = Field( + None, description='Display Name that identifies this dashboard service.' + ) serviceType: DashboardServiceType = Field( ..., description='Type of dashboard service such as Looker or Superset...' ) description: Optional[str] = Field( None, description='Description of a dashboard service instance.' ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') dashboardUrl: AnyUrl = Field( ..., description='Dashboard Service URL. This will be used to make REST API calls to Dashboard Service.', diff --git a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py index c3b74b10289..b7da8a3bc75 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/databaseService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -34,6 +34,9 @@ class DatabaseService(BaseModel): name: constr(min_length=1, max_length=64) = Field( ..., description='Name that identifies this database service.' ) + displayName: Optional[str] = Field( + None, description='Display Name that identifies this database service.' + ) serviceType: DatabaseServiceType = Field( ..., description='Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres...', @@ -41,6 +44,14 @@ class DatabaseService(BaseModel): description: Optional[str] = Field( None, description='Description of a database service instance.' ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') href: basic.Href = Field( ..., description='Link to the resource corresponding to this database service.' ) diff --git a/ingestion/src/metadata/generated/schema/entity/services/messagingService.py b/ingestion/src/metadata/generated/schema/entity/services/messagingService.py index 738b72a886a..5636b5085dc 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/messagingService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/messagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/messagingService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -37,6 +37,18 @@ class MessagingService(BaseModel): description: Optional[str] = Field( None, description='Description of a messaging service instance.' ) + displayName: Optional[str] = Field( + None, + description='Display Name that identifies this messaging service. It could be title or label from the source services.', + ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') brokers: Brokers = Field( ..., description='Multiple bootstrap addresses for Kafka. Single proxy address for Pulsar.', diff --git a/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py b/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py index e93cbaba690..6859f060595 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/pipelineService.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -30,7 +30,19 @@ class PipelineService(BaseModel): description: Optional[str] = Field( None, description='Description of a pipeline service instance.' ) - pipelineUrl: AnyUrl = Field(..., description='Pipeline Service Management/UI URL') + displayName: Optional[str] = Field( + None, + description='Display Name that identifies this pipeline service. It could be title or label from the source services.', + ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') + pipelineUrl: AnyUrl = Field(..., description='Pipeline Service Management/UI URL.') ingestionSchedule: Optional[schedule.Schedule] = Field( None, description='Schedule for running metadata ingestion jobs.' ) diff --git a/ingestion/src/metadata/generated/schema/entity/tags/__init__.py b/ingestion/src/metadata/generated/schema/entity/tags/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/entity/tags/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py b/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py index bafb8bdf673..c548348c26e 100644 --- a/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py +++ b/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/tags/tagCategory.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -33,6 +33,14 @@ class Tag(BaseModel): description='Unique name of the tag of format Category.PrimaryTag.SecondaryTag.', ) description: str = Field(..., description='Unique name of the tag category.') + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') href: Optional[basic.Href] = Field( None, description='Link to the resource corresponding to the tag.' ) @@ -55,7 +63,18 @@ class TagCategory(BaseModel): extra = Extra.forbid name: TagName + displayName: Optional[str] = Field( + None, description='Display Name that identifies this tag category.' + ) description: str = Field(..., description='Description of the tag category.') + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') categoryType: TagCategoryType href: Optional[basic.Href] = Field( None, description='Link to the resource corresponding to the tag category.' diff --git a/ingestion/src/metadata/generated/schema/entity/teams/__init__.py b/ingestion/src/metadata/generated/schema/entity/teams/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/teams/team.py b/ingestion/src/metadata/generated/schema/entity/teams/team.py index d23f6ea5c60..b12693452c8 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/team.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/team.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/teams/team.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -25,6 +25,14 @@ class Team(BaseModel): None, description="Name used for display purposes. Example 'Data Science team'." ) description: Optional[str] = Field(None, description='Description of the team.') + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') href: basic.Href = Field( ..., description='Link to the resource corresponding to this entity.' ) diff --git a/ingestion/src/metadata/generated/schema/entity/teams/user.py b/ingestion/src/metadata/generated/schema/entity/teams/user.py index 5b247cba2fc..ba53dcf090d 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/user.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/user.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/teams/user.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -30,6 +30,14 @@ class User(BaseModel): None, description="Name used for display purposes. Example 'FirstName LastName'.", ) + version: Optional[basic.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') email: basic.Email = Field(..., description='Email address of the user.') href: basic.Href = Field( ..., description='Link to the resource corresponding to this entity.' diff --git a/ingestion/src/metadata/generated/schema/type/__init__.py b/ingestion/src/metadata/generated/schema/type/__init__.py index 248c6d5f83f..26e7f4e157c 100644 --- a/ingestion/src/metadata/generated/schema/type/__init__.py +++ b/ingestion/src/metadata/generated/schema/type/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 diff --git a/ingestion/src/metadata/generated/schema/type/auditLog.py b/ingestion/src/metadata/generated/schema/type/auditLog.py index cddd9dac149..ddd38acfc63 100644 --- a/ingestion/src/metadata/generated/schema/type/auditLog.py +++ b/ingestion/src/metadata/generated/schema/type/auditLog.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/auditLog.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/basic.py b/ingestion/src/metadata/generated/schema/type/basic.py index 871e9941826..0c9c20275e3 100644 --- a/ingestion/src/metadata/generated/schema/type/basic.py +++ b/ingestion/src/metadata/generated/schema/type/basic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/basic.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -8,7 +8,7 @@ from datetime import date, datetime from typing import Any, Optional from uuid import UUID -from pydantic import AnyUrl, BaseModel, EmailStr, Field, constr +from pydantic import AnyUrl, BaseModel, EmailStr, Field, confloat, constr class Basic(BaseModel): @@ -29,13 +29,6 @@ class Email(BaseModel): ) -class EntityLink(BaseModel): - __root__: constr(regex=r'^<#E/\S+/\S+>$') = Field( - ..., - description='Link to an entity or field within an entity using this format `<#E/{enties}/{entityName}/{field}/{fieldValue}`.', - ) - - class Timestamp(BaseModel): __root__: str = Field(..., description='Timestamp in unixTimeMillis.') @@ -68,6 +61,20 @@ class DateTime(BaseModel): ) +class EntityVersion(BaseModel): + __root__: confloat(ge=0.1, multiple_of=0.1) = Field( + ..., + description='Metadata version of the entity in the form `Major.Minor`. First version always starts from `0.1` when the entity is created. When the backward compatible changes are made to the entity, only the `Minor` version is incremented - example `1.0` is changed to `1.1`. When backward incompatible changes are made the `Major` version is incremented - example `1.1` to `2.0`.', + ) + + +class EntityLink(BaseModel): + __root__: constr(regex=r'^<#E/\S+/\S+>$') = Field( + ..., + description='Link to an entity or field within an entity using this format `<#E/{enties}/{entityName}/{field}/{fieldValue}`.', + ) + + class SqlQuery(BaseModel): __root__: str = Field( ..., description="SQL query statement. Example - 'select * from orders'." diff --git a/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py b/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py index 8fe6568c3d6..ffe83e5233c 100644 --- a/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py +++ b/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/collectionDescriptor.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/dailyCount.py b/ingestion/src/metadata/generated/schema/type/dailyCount.py index 61c0af8eb5b..237ad912aa2 100644 --- a/ingestion/src/metadata/generated/schema/type/dailyCount.py +++ b/ingestion/src/metadata/generated/schema/type/dailyCount.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/dailyCount.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityLineage.py b/ingestion/src/metadata/generated/schema/type/entityLineage.py index e881a59aca6..0bddcf93fe2 100644 --- a/ingestion/src/metadata/generated/schema/type/entityLineage.py +++ b/ingestion/src/metadata/generated/schema/type/entityLineage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityLineage.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -26,7 +26,7 @@ class EntityLineage(BaseModel): extra = Extra.forbid entity: entityReference.EntityReference = Field( - ..., description='Primary entity for which this lineage graph is created' + ..., description='Primary entity for which this lineage graph is created.' ) nodes: Optional[List[entityReference.EntityReference]] = None upstreamEdges: Optional[List[Edge]] = None diff --git a/ingestion/src/metadata/generated/schema/type/entityReference.py b/ingestion/src/metadata/generated/schema/type/entityReference.py index 537018c0a17..c9897fa9a4f 100644 --- a/ingestion/src/metadata/generated/schema/type/entityReference.py +++ b/ingestion/src/metadata/generated/schema/type/entityReference.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityReference.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations @@ -29,6 +29,9 @@ class EntityReference(BaseModel): description: Optional[str] = Field( None, description='Optional description of entity.' ) + displayName: Optional[str] = Field( + None, description='Display Name that identifies this entity.' + ) href: Optional[basic.Href] = Field(None, description='Link to the entity resource.') diff --git a/ingestion/src/metadata/generated/schema/type/entityUsage.py b/ingestion/src/metadata/generated/schema/type/entityUsage.py index 7030f2a3829..af9b8169c0a 100644 --- a/ingestion/src/metadata/generated/schema/type/entityUsage.py +++ b/ingestion/src/metadata/generated/schema/type/entityUsage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityUsage.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/jdbcConnection.py b/ingestion/src/metadata/generated/schema/type/jdbcConnection.py index cf8b3bf02a8..50f8958419d 100644 --- a/ingestion/src/metadata/generated/schema/type/jdbcConnection.py +++ b/ingestion/src/metadata/generated/schema/type/jdbcConnection.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/jdbcConnection.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/paging.py b/ingestion/src/metadata/generated/schema/type/paging.py index 892ea5fde59..38e35adda1a 100644 --- a/ingestion/src/metadata/generated/schema/type/paging.py +++ b/ingestion/src/metadata/generated/schema/type/paging.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/paging.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/profile.py b/ingestion/src/metadata/generated/schema/type/profile.py index 8c7878be5b7..585c30ce4da 100644 --- a/ingestion/src/metadata/generated/schema/type/profile.py +++ b/ingestion/src/metadata/generated/schema/type/profile.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/profile.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/schedule.py b/ingestion/src/metadata/generated/schema/type/schedule.py index 34b8672226b..de9cbe09b76 100644 --- a/ingestion/src/metadata/generated/schema/type/schedule.py +++ b/ingestion/src/metadata/generated/schema/type/schedule.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/schedule.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/tagLabel.py b/ingestion/src/metadata/generated/schema/type/tagLabel.py index 97d12d3df2f..45b133fd434 100644 --- a/ingestion/src/metadata/generated/schema/type/tagLabel.py +++ b/ingestion/src/metadata/generated/schema/type/tagLabel.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/tagLabel.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/usageDetails.py b/ingestion/src/metadata/generated/schema/type/usageDetails.py index 97116cb977b..fe8d91cdf60 100644 --- a/ingestion/src/metadata/generated/schema/type/usageDetails.py +++ b/ingestion/src/metadata/generated/schema/type/usageDetails.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/usageDetails.json -# timestamp: 2021-10-12T00:34:28+00:00 +# timestamp: 2021-10-21T16:10:22+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py new file mode 100644 index 00000000000..b8bcbda65af --- /dev/null +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -0,0 +1,329 @@ +import logging +from typing import Generic, List, Type, TypeVar, Union, get_args + +from pydantic import BaseModel + +from metadata.generated.schema.api.lineage.addLineage import AddLineage +from metadata.generated.schema.entity.data.chart import Chart +from metadata.generated.schema.entity.data.dashboard import Dashboard +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.metrics import Metrics +from metadata.generated.schema.entity.data.model import Model +from metadata.generated.schema.entity.data.pipeline import Pipeline +from metadata.generated.schema.entity.data.report import Report +from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.data.task import Task +from metadata.generated.schema.entity.data.topic import Topic +from metadata.generated.schema.entity.services.dashboardService import DashboardService +from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.entity.services.messagingService import MessagingService +from metadata.generated.schema.entity.services.pipelineService import PipelineService +from metadata.generated.schema.entity.teams.user import User +from metadata.ingestion.ometa.auth_provider import AuthenticationProvider +from metadata.ingestion.ometa.client import REST, ClientConfig +from metadata.ingestion.ometa.openmetadata_rest import ( + Auth0AuthenticationProvider, + GoogleAuthenticationProvider, + MetadataServerConfig, + NoOpAuthenticationProvider, + OktaAuthenticationProvider, +) + +logger = logging.getLogger(__name__) + + +# The naming convention is T for Entity Types and C for Create Types +T = TypeVar("T", bound=BaseModel) +C = TypeVar("C", bound=BaseModel) + + +class MissingEntityTypeException(Exception): + """ + We are receiving an Entity Type[T] not covered + in our suffix generation list + """ + + +class EntityList(Generic[T], BaseModel): + entities: List[T] + total: int + after: str = None + + +class OMeta(Generic[T, C]): + """ + Generic interface to the OpenMetadata API + + It is a polymorphism on all our different Entities + """ + + client: REST + _auth_provider: AuthenticationProvider + + class_root = ".".join(["metadata", "generated", "schema"]) + entity_path = "entity" + api_path = "api" + data_path = "data" + services_path = "services" + teams_path = "teams" + + def __init__(self, config: MetadataServerConfig, raw_data: bool = False): + self.config = config + if self.config.auth_provider_type == "google": + self._auth_provider: AuthenticationProvider = ( + GoogleAuthenticationProvider.create(self.config) + ) + elif self.config.auth_provider_type == "okta": + self._auth_provider: AuthenticationProvider = ( + OktaAuthenticationProvider.create(self.config) + ) + elif self.config.auth_provider_type == "auth0": + self._auth_provider: AuthenticationProvider = ( + Auth0AuthenticationProvider.create(self.config) + ) + else: + self._auth_provider: AuthenticationProvider = ( + NoOpAuthenticationProvider.create(self.config) + ) + client_config: ClientConfig = ClientConfig( + base_url=self.config.api_endpoint, + api_version=self.config.api_version, + auth_header="X-Catalog-Source", + auth_token=self._auth_provider.auth_token(), + ) + self.client = REST(client_config) + self._use_raw_data = raw_data + + def get_suffix(self, entity: Type[T]) -> str: + """ + Given an entity Type from the generated sources, + return the endpoint to run requests. + + Might be interesting to follow a more strict + and type-checked approach + """ + + # Entity Schemas + if issubclass( + entity, get_args(Union[Model, self.get_create_entity_type(Model)]) + ): + return "/models" + + if issubclass( + entity, get_args(Union[Chart, self.get_create_entity_type(Chart)]) + ): + return "/charts" + + if issubclass( + entity, get_args(Union[Dashboard, self.get_create_entity_type(Dashboard)]) + ): + return "/dashboards" + + if issubclass( + entity, get_args(Union[Database, self.get_create_entity_type(Database)]) + ): + return "/databases" + + if issubclass( + entity, get_args(Union[Pipeline, self.get_create_entity_type(Pipeline)]) + ): + return "/pipelines" + + if issubclass( + entity, get_args(Union[Table, self.get_create_entity_type(Table)]) + ): + return "/tables" + + if issubclass(entity, get_args(Union[Task, self.get_create_entity_type(Task)])): + return "/tasks" + + if issubclass( + entity, get_args(Union[Topic, self.get_create_entity_type(Topic)]) + ): + return "/topics" + + if issubclass(entity, Metrics): + return "/metrics" + + if issubclass(entity, AddLineage): + return "/lineage" + + if issubclass(entity, Report): + return "/reports" + + if issubclass(entity, get_args(Union[User, self.get_create_entity_type(User)])): + return "/users" + + # Services Schemas + if issubclass( + entity, + get_args( + Union[DatabaseService, self.get_create_entity_type(DatabaseService)] + ), + ): + return "/services/databaseServices" + + if issubclass( + entity, + get_args( + Union[DashboardService, self.get_create_entity_type(DashboardService)] + ), + ): + return "/services/dashboardServices" + + if issubclass( + entity, + get_args( + Union[MessagingService, self.get_create_entity_type(MessagingService)] + ), + ): + return "/services/messagingServices" + + if issubclass( + entity, + get_args( + Union[PipelineService, self.get_create_entity_type(PipelineService)] + ), + ): + return "/services/pipelineServices" + + raise MissingEntityTypeException( + f"Missing {entity} type when generating suffixes" + ) + + def get_module_path(self, entity: Type[T]) -> str: + """ + Based on the entity, return the module path + it is found inside generated + """ + + if "service" in entity.__name__.lower(): + return self.services_path + + if "user" in entity.__name__.lower(): + return self.teams_path + + return self.data_path + + def get_create_entity_type(self, entity: Type[T]) -> Type[C]: + """ + imports and returns the Create Type from an Entity Type T. + + We are following the expected path structure to import + on-the-fly the necessary class and pass it to the consumer + """ + file_name = f"create{entity.__name__}" + + class_path = ".".join( + [self.class_root, self.api_path, self.get_module_path(entity), file_name] + ) + + class_name = f"Create{entity.__name__}EntityRequest" + + create_class = getattr( + __import__(class_path, globals(), locals(), [class_name]), class_name + ) + return create_class + + def get_entity_from_create(self, create: Type[C]) -> Type[T]: + """ + Inversely, import the Entity type based on the create Entity class + """ + + class_name = create.__name__.replace("Create", "").replace("EntityRequest", "") + file_name = class_name.lower() + + class_path = ".".join( + [ + self.class_root, + self.entity_path, + self.get_module_path(create), + file_name.replace("service", "Service") + if "service" in create.__name__.lower() + else file_name, + ] + ) + + entity_class = getattr( + __import__(class_path, globals(), locals(), [class_name]), class_name + ) + return entity_class + + def create_or_update(self, entity: Type[T], data: T) -> Type[C]: + """ + We allow both Entity and CreateEntity for PUT + If Entity, no need to find response class mapping. + + We PUT to the endpoint and return the Entity generated result + + Here the typing is a bit more weird. We will get a type T, be it + Entity or CreateEntity, and we are always going to return Entity + """ + + is_create = "create" in entity.__name__.lower() + is_service = "service" in entity.__name__.lower() + + # Prepare the return Entity Type + if is_create: + entity_class = self.get_entity_from_create(entity) + else: + entity_class = entity + + # Prepare the request method + if is_service and is_create: + # Services can only be created via POST + method = self.client.post + else: + method = self.client.put + + resp = method(self.get_suffix(entity), data=data.json()) + return entity_class(**resp) + + def get_by_name(self, entity: Type[T], name: str) -> Type[T]: + resp = self.client.get(f"{self.get_suffix(entity)}/name/{name}") + return entity(**resp) + + def get_by_id(self, entity: Type[T], entity_id: str) -> Type[T]: + resp = self.client.get(f"{self.get_suffix(entity)}/{entity_id}") + return entity(**resp) + + def list_entities( + self, entity: Type[T], fields: str = None, after: str = None, limit: int = 1000 + ) -> EntityList[T]: + """ + Helps us paginate over the collection + """ + + suffix = self.get_suffix(entity) + + if fields is None: + resp = self.client.get(suffix) + else: + if after is not None: + resp = self.client.get( + f"{suffix}?fields={fields}&after={after}&limit={limit}" + ) + else: + resp = self.client.get(f"{suffix}?fields={fields}&limit={limit}") + + if self._use_raw_data: + return resp + else: + entities = [entity(**t) for t in resp["data"]] + total = resp["paging"]["total"] + after = resp["paging"]["after"] if "after" in resp["paging"] else None + return EntityList(entities=entities, total=total, after=after) + + def list_services(self, entity: Type[T]) -> List[EntityList[T]]: + """ + Service listing does not implement paging + """ + + resp = self.client.get(self.get_suffix(entity)) + if self._use_raw_data: + return resp + else: + return [entity(**p) for p in resp["data"]] + + def delete(self, entity: Type[T], entity_id: str) -> None: + self.client.delete(f"{self.get_suffix(entity)}/{entity_id}") diff --git a/ingestion/src/metadata/ingestion/ometa/openmetadata_rest.py b/ingestion/src/metadata/ingestion/ometa/openmetadata_rest.py index f0b014b7e4d..5313578a2ff 100644 --- a/ingestion/src/metadata/ingestion/ometa/openmetadata_rest.py +++ b/ingestion/src/metadata/ingestion/ometa/openmetadata_rest.py @@ -40,7 +40,7 @@ from metadata.generated.schema.api.data.createPipeline import ( ) from metadata.generated.schema.api.data.createTable import CreateTableEntityRequest from metadata.generated.schema.api.data.createTask import CreateTaskEntityRequest -from metadata.generated.schema.api.data.createTopic import CreateTopic +from metadata.generated.schema.api.data.createTopic import CreateTopicEntityRequest from metadata.generated.schema.api.lineage.addLineage import AddLineage from metadata.generated.schema.api.services.createDashboardService import ( CreateDashboardServiceEntityRequest, @@ -419,7 +419,9 @@ class OpenMetadataAPIClient(object): ) return MessagingService(**resp) - def create_or_update_topic(self, create_topic_request: CreateTopic) -> Topic: + def create_or_update_topic( + self, create_topic_request: CreateTopicEntityRequest + ) -> Topic: """Create or Update a Table""" resp = self.client.put("/topics", data=create_topic_request.json()) return Topic(**resp) diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest.py b/ingestion/src/metadata/ingestion/sink/metadata_rest.py index d58095b10f9..13b639b67f0 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest.py @@ -31,7 +31,7 @@ from metadata.generated.schema.api.data.createPipeline import ( ) from metadata.generated.schema.api.data.createTable import CreateTableEntityRequest from metadata.generated.schema.api.data.createTask import CreateTaskEntityRequest -from metadata.generated.schema.api.data.createTopic import CreateTopic +from metadata.generated.schema.api.data.createTopic import CreateTopicEntityRequest from metadata.generated.schema.api.lineage.addLineage import AddLineage from metadata.generated.schema.entity.data.chart import ChartType from metadata.generated.schema.entity.data.model import Model @@ -99,7 +99,7 @@ class MetadataRestSink(Sink): def write_record(self, record: Record) -> None: if isinstance(record, OMetaDatabaseAndTable): self.write_tables(record) - elif isinstance(record, CreateTopic): + elif isinstance(record, CreateTopicEntityRequest): self.write_topics(record) elif isinstance(record, Chart): self.write_charts(record) @@ -173,7 +173,7 @@ class MetadataRestSink(Sink): logger.error(err) self.status.failure(f"Table: {table_and_db.table.name.__root__}") - def write_topics(self, topic: CreateTopic) -> None: + def write_topics(self, topic: CreateTopicEntityRequest) -> None: try: created_topic = self.client.create_or_update_topic(topic) logger.info(f"Successfully ingested topic {created_topic.name.__root__}") diff --git a/ingestion/src/metadata/ingestion/source/kafka.py b/ingestion/src/metadata/ingestion/source/kafka.py index 7d66fdf11ac..5b66f97f83d 100644 --- a/ingestion/src/metadata/ingestion/source/kafka.py +++ b/ingestion/src/metadata/ingestion/source/kafka.py @@ -24,7 +24,7 @@ from confluent_kafka.schema_registry.schema_registry_client import ( ) from metadata.config.common import ConfigModel -from metadata.generated.schema.api.data.createTopic import CreateTopic +from metadata.generated.schema.api.data.createTopic import CreateTopicEntityRequest from metadata.generated.schema.entity.data.topic import SchemaType, Topic from metadata.generated.schema.entity.services.messagingService import ( MessagingServiceType, @@ -104,13 +104,13 @@ class KafkaSource(Source): def prepare(self): pass - def next_record(self) -> Iterable[CreateTopic]: + def next_record(self) -> Iterable[CreateTopicEntityRequest]: topics = self.admin_client.list_topics().topics for t in topics: if self.config.filter_pattern.included(t): logger.info("Fetching topic schema {}".format(t)) topic_schema = self._parse_topic_metadata(t) - topic = CreateTopic( + topic = CreateTopicEntityRequest( name=t, service=EntityReference( id=self.service.id, type="messagingService" diff --git a/ingestion/src/metadata/ingestion/source/sample_data.py b/ingestion/src/metadata/ingestion/source/sample_data.py index c4444b69ef7..3db8577b0f3 100644 --- a/ingestion/src/metadata/ingestion/source/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/sample_data.py @@ -27,7 +27,7 @@ from faker import Faker from pydantic import ValidationError from metadata.config.common import ConfigModel -from metadata.generated.schema.api.data.createTopic import CreateTopic +from metadata.generated.schema.api.data.createTopic import CreateTopicEntityRequest from metadata.generated.schema.api.lineage.addLineage import AddLineage from metadata.generated.schema.api.services.createDashboardService import ( CreateDashboardServiceEntityRequest, @@ -435,12 +435,12 @@ class SampleDataSource(Source): self.status.scanned("table", table_metadata.name.__root__) yield table_and_db - def ingest_topics(self) -> Iterable[CreateTopic]: + def ingest_topics(self) -> Iterable[CreateTopicEntityRequest]: for topic in self.topics["topics"]: topic["service"] = EntityReference( id=self.kafka_service.id, type="messagingService" ) - create_topic = CreateTopic(**topic) + create_topic = CreateTopicEntityRequest(**topic) self.status.scanned("topic", create_topic.name.__root__) yield create_topic diff --git a/ingestion/src/metadata/ingestion/source/sample_entity.py b/ingestion/src/metadata/ingestion/source/sample_entity.py index f159b45ffa4..9631a61ec9d 100644 --- a/ingestion/src/metadata/ingestion/source/sample_entity.py +++ b/ingestion/src/metadata/ingestion/source/sample_entity.py @@ -7,7 +7,7 @@ from typing import Iterable, List from faker import Faker -from metadata.generated.schema.api.data.createTopic import CreateTopic +from metadata.generated.schema.api.data.createTopic import CreateTopicEntityRequest from metadata.generated.schema.api.services.createDashboardService import ( CreateDashboardServiceEntityRequest, ) @@ -281,7 +281,7 @@ class SampleEntitySource(Source): ) yield dashboard - def ingest_topics(self) -> Iterable[CreateTopic]: + def ingest_topics(self) -> Iterable[CreateTopicEntityRequest]: for h in range(self.config.no_of_services): create_service = None while True: @@ -304,7 +304,7 @@ class SampleEntitySource(Source): "Ingesting service {}/{}".format(h + 1, self.config.no_of_services) ) for j in range(self.config.no_of_topics): - topic_entity = CreateTopic( + topic_entity = CreateTopicEntityRequest( name=self.table_name(), description=self.description(), partitions=self.chart_ids(), diff --git a/ingestion/tests/integration/ometa/test_ometa_database_api.py b/ingestion/tests/integration/ometa/test_ometa_database_api.py new file mode 100644 index 00000000000..e4decd4e805 --- /dev/null +++ b/ingestion/tests/integration/ometa/test_ometa_database_api.py @@ -0,0 +1,177 @@ +""" +OpenMetadata high-level API Database test +""" +import uuid +from unittest import TestCase + +from metadata.generated.schema.api.data.createDatabase import ( + CreateDatabaseEntityRequest, +) +from metadata.generated.schema.api.services.createDatabaseService import ( + CreateDatabaseServiceEntityRequest, +) +from metadata.generated.schema.api.teams.createUser import CreateUserEntityRequest +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseService, + DatabaseServiceType, +) +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.generated.schema.type.jdbcConnection import JdbcInfo +from metadata.ingestion.ometa.ometa_api import OMeta +from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig + + +class OMetaDatabaseTest(TestCase): + """ + Run this integration test with the local API available + Install the ingestion package before running the tests + """ + + service_entity_id = None + + server_config = MetadataServerConfig(api_endpoint="http://localhost:8585/api") + metadata = OMeta(server_config) + + user = metadata.create_or_update( + entity=CreateUserEntityRequest, + data=CreateUserEntityRequest(name="random-user", email="random@user.com"), + ) + owner = EntityReference(id=user.id, type="user") + + service = CreateDatabaseServiceEntityRequest( + name="test-service", + serviceType=DatabaseServiceType.MySQL, + jdbc=JdbcInfo(driverClass="jdbc", connectionUrl="jdbc://localhost"), + ) + + def setUp(self) -> None: + """ + Prepare ingredients + """ + self.service_entity = self.metadata.create_or_update( + entity=CreateDatabaseServiceEntityRequest, data=self.service + ) + + self.entity = Database( + id=uuid.uuid4(), + name="test-db", + service=EntityReference(id=self.service_entity.id, type="databaseService"), + fullyQualifiedName="test-service.test-db", + ) + self.create = CreateDatabaseEntityRequest( + name="test-db", + service=EntityReference(id=self.service_entity.id, type="databaseService"), + ) + + self.service_entity_id = str(self.service_entity.id.__root__) + + def tearDown(self) -> None: + """ + Clean up + """ + self.metadata.delete(entity=DatabaseService, entity_id=self.service_entity_id) + + def test_create(self): + """ + We can create a Database and we receive it back as Entity + """ + + res = self.metadata.create_or_update( + entity=CreateDatabaseEntityRequest, data=self.create + ) + + self.assertEqual(res.name, self.create.name) + self.assertEqual(res.service.id, self.create.service.id) + self.assertEqual(res.owner, None) + + def test_update(self): + """ + Updating it properly changes its properties + """ + + res_create = self.metadata.create_or_update( + entity=CreateDatabaseEntityRequest, data=self.create + ) + + updated = self.entity.dict(exclude_unset=True) + updated["owner"] = self.owner + updated_entity = Database(**updated) + + res = self.metadata.create_or_update(entity=Database, data=updated_entity) + + # Same ID, updated algorithm + self.assertEqual(res.service.id, updated_entity.service.id) + self.assertEqual(res_create.id, res.id) + self.assertEqual(res.owner.id, self.user.id) + + def test_get_name(self): + """ + We can fetch a Database by name and get it back as Entity + """ + + self.metadata.create_or_update(entity=Database, data=self.entity) + + res = self.metadata.get_by_name( + entity=Database, fqdn=self.entity.fullyQualifiedName + ) + self.assertEqual(res.name, self.entity.name) + + def test_get_id(self): + """ + We can fetch a Database by ID and get it back as Entity + """ + + self.metadata.create_or_update(entity=Database, data=self.entity) + + # First pick up by name + res_name = self.metadata.get_by_name( + entity=Database, fqdn=self.entity.fullyQualifiedName + ) + # Then fetch by ID + res = self.metadata.get_by_id( + entity=Database, entity_id=str(res_name.id.__root__) + ) + + self.assertEqual(res_name.id, res.id) + + def test_list(self): + """ + We can list all our Database + """ + + self.metadata.create_or_update(entity=Database, data=self.entity) + + res = self.metadata.list_entities(entity=Database) + + # Fetch our test Database. We have already inserted it, so we should find it + data = next( + iter(ent for ent in res.entities if ent.name == self.entity.name), None + ) + assert data + + def test_delete(self): + """ + We can delete a Database by ID + """ + + self.metadata.create_or_update(entity=Database, data=self.entity) + + # Find by name + res_name = self.metadata.get_by_name( + entity=Database, fqdn=self.entity.fullyQualifiedName + ) + # Then fetch by ID + res_id = self.metadata.get_by_id( + entity=Database, entity_id=str(res_name.id.__root__) + ) + + # Delete + self.metadata.delete(entity=Database, entity_id=str(res_id.id.__root__)) + + # Then we should not find it + res = self.metadata.list_entities(entity=Database) + print(res) + assert not next( + iter(ent for ent in res.entities if ent.name == self.entity.name), None + ) diff --git a/ingestion/tests/integration/ometa/test_ometa_model_api.py b/ingestion/tests/integration/ometa/test_ometa_model_api.py new file mode 100644 index 00000000000..d561e566fa1 --- /dev/null +++ b/ingestion/tests/integration/ometa/test_ometa_model_api.py @@ -0,0 +1,138 @@ +""" +OpenMetadata high-level API Model test +""" +import uuid +from unittest import TestCase + +from metadata.generated.schema.api.data.createModel import CreateModelEntityRequest +from metadata.generated.schema.api.teams.createUser import CreateUserEntityRequest +from metadata.generated.schema.entity.data.model import Model +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.ingestion.ometa.ometa_api import OMeta +from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig + + +class OMetaModelTest(TestCase): + """ + Run this integration test with the local API available + Install the ingestion package before running the tests + """ + + server_config = MetadataServerConfig(api_endpoint="http://localhost:8585/api") + metadata = OMeta(server_config) + + user = metadata.create_or_update( + entity=CreateUserEntityRequest, + data=CreateUserEntityRequest(name="random-user", email="random@user.com"), + ) + owner = EntityReference(id=user.id, type="user") + + entity = Model( + id=uuid.uuid4(), + name="test-model", + algorithm="algo", + fullyQualifiedName="test-model", + ) + create = CreateModelEntityRequest(name="test-model", algorithm="algo") + + def test_create(self): + """ + We can create a Model and we receive it back as Entity + """ + + res = self.metadata.create_or_update( + entity=CreateModelEntityRequest, data=self.create + ) + + self.assertEqual(res.name, self.create.name) + self.assertEqual(res.algorithm, self.create.algorithm) + self.assertEqual(res.owner, None) + + def test_update(self): + """ + Updating it properly changes its properties + """ + + res_create = self.metadata.create_or_update( + entity=CreateModelEntityRequest, data=self.create + ) + + updated = self.entity.dict(exclude_unset=True) + updated["owner"] = self.owner + updated_entity = Model(**updated) + + res = self.metadata.create_or_update(entity=Model, data=updated_entity) + + # Same ID, updated algorithm + self.assertEqual(res.algorithm, updated_entity.algorithm) + self.assertEqual(res_create.id, res.id) + self.assertEqual(res.owner.id, self.user.id) + + def test_get_name(self): + """ + We can fetch a model by name and get it back as Entity + """ + + self.metadata.create_or_update(entity=Model, data=self.entity) + + res = self.metadata.get_by_name( + entity=Model, fqdn=self.entity.fullyQualifiedName + ) + self.assertEqual(res.name, self.entity.name) + + def test_get_id(self): + """ + We can fetch a model by ID and get it back as Entity + """ + + self.metadata.create_or_update(entity=Model, data=self.entity) + + # First pick up by name + res_name = self.metadata.get_by_name( + entity=Model, fqdn=self.entity.fullyQualifiedName + ) + # Then fetch by ID + res = self.metadata.get_by_id(entity=Model, entity_id=str(res_name.id.__root__)) + + self.assertEqual(res_name.id, res.id) + + def test_list(self): + """ + We can list all our models + """ + + self.metadata.create_or_update(entity=Model, data=self.entity) + + res = self.metadata.list_entities(entity=Model) + + # Fetch our test model. We have already inserted it, so we should find it + data = next( + iter(ent for ent in res.entities if ent.name == self.entity.name), None + ) + assert data + + def test_delete(self): + """ + We can delete a model by ID + """ + + self.metadata.create_or_update(entity=Model, data=self.entity) + + # Find by name + res_name = self.metadata.get_by_name( + entity=Model, fqdn=self.entity.fullyQualifiedName + ) + # Then fetch by ID + res_id = self.metadata.get_by_id( + entity=Model, entity_id=str(res_name.id.__root__) + ) + + # Delete + self.metadata.delete(entity=Model, entity_id=str(res_id.id.__root__)) + + # Then we should not find it + res = self.metadata.list_entities(entity=Model) + print(res) + assert not next( + iter(ent for ent in res.entities if ent.name == self.entity.name), None + ) diff --git a/ingestion/tests/unit/test_ometa_endpoints.py b/ingestion/tests/unit/test_ometa_endpoints.py new file mode 100644 index 00000000000..fb4626fcc13 --- /dev/null +++ b/ingestion/tests/unit/test_ometa_endpoints.py @@ -0,0 +1,120 @@ +""" +OpenMetadata high-level API endpoint test +""" +from unittest import TestCase + +from metadata.generated.schema.api.data.createModel import CreateModelEntityRequest +from metadata.generated.schema.api.data.createTopic import CreateTopicEntityRequest +from metadata.generated.schema.api.services.createDatabaseService import ( + CreateDatabaseServiceEntityRequest, +) +from metadata.generated.schema.api.teams.createUser import CreateUserEntityRequest +from metadata.generated.schema.entity.data.chart import Chart +from metadata.generated.schema.entity.data.dashboard import Dashboard +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.metrics import Metrics +from metadata.generated.schema.entity.data.model import Model +from metadata.generated.schema.entity.data.pipeline import Pipeline +from metadata.generated.schema.entity.data.report import Report +from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.data.task import Task +from metadata.generated.schema.entity.data.topic import Topic +from metadata.generated.schema.entity.services.dashboardService import DashboardService +from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.entity.services.messagingService import MessagingService +from metadata.generated.schema.entity.services.pipelineService import PipelineService +from metadata.generated.schema.entity.teams.user import User +from metadata.ingestion.ometa.ometa_api import OMeta +from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig + + +class OMetaEndpointTest(TestCase): + """ + Make sure that we can infer the proper endpoints + from the generated entity classes + """ + + server_config = MetadataServerConfig(api_endpoint="http://localhost:8585/api") + metadata = OMeta(server_config) + + def test_entities_suffix(self): + """ + Pass Entities and test their suffix generation + """ + + # ML + self.assertEqual(self.metadata.get_suffix(Model), "/models") + + # Db + self.assertEqual(self.metadata.get_suffix(Database), "/databases") + self.assertEqual(self.metadata.get_suffix(Table), "/tables") + + # Dashboards + self.assertEqual(self.metadata.get_suffix(Dashboard), "/dashboards") + self.assertEqual(self.metadata.get_suffix(Chart), "/charts") + self.assertEqual(self.metadata.get_suffix(Metrics), "/metrics") + self.assertEqual(self.metadata.get_suffix(Report), "/reports") + + # Pipelines + self.assertEqual(self.metadata.get_suffix(Pipeline), "/pipelines") + self.assertEqual(self.metadata.get_suffix(Task), "/tasks") + + # Topic + self.assertEqual(self.metadata.get_suffix(Topic), "/topics") + + def test_services_suffix(self): + """ + Pass Services and test their suffix generation + """ + self.assertEqual( + self.metadata.get_suffix(DashboardService), "/services/dashboardServices" + ) + self.assertEqual( + self.metadata.get_suffix(DatabaseService), "/services/databaseServices" + ) + self.assertEqual( + self.metadata.get_suffix(MessagingService), "/services/messagingServices" + ) + self.assertEqual( + self.metadata.get_suffix(PipelineService), "/services/pipelineServices" + ) + + def test_teams_suffix(self): + """ + Pass Teams and test their suffix generation + """ + self.assertEqual(self.metadata.get_suffix(User), "/users") + + def test_get_create_entity_type(self): + """ + Validate the mapping from Entity to CreateEntity + """ + create = self.metadata.get_create_entity_type(Model) + assert issubclass(create, CreateModelEntityRequest) + + create = self.metadata.get_create_entity_type(Topic) + assert issubclass(create, CreateTopicEntityRequest) + + create = self.metadata.get_create_entity_type(DatabaseService) + assert issubclass(create, CreateDatabaseServiceEntityRequest) + + create = self.metadata.get_create_entity_type(User) + assert issubclass(create, CreateUserEntityRequest) + + def test_get_entity_from_create(self): + """ + Validate the mapping from CreateEntity to Entity + """ + entity = self.metadata.get_entity_from_create(CreateModelEntityRequest) + assert issubclass(entity, Model) + + entity = self.metadata.get_entity_from_create(CreateTopicEntityRequest) + assert issubclass(entity, Topic) + + entity = self.metadata.get_entity_from_create( + CreateDatabaseServiceEntityRequest + ) + assert issubclass(entity, DatabaseService) + + entity = self.metadata.get_entity_from_create(CreateUserEntityRequest) + assert issubclass(entity, User)