diff --git a/ingestion/tests/integration/ometa/test_ometa_objectstore_api.py b/ingestion/tests/integration/ometa/test_ometa_objectstore_api.py new file mode 100644 index 00000000000..6ae18e6c0a0 --- /dev/null +++ b/ingestion/tests/integration/ometa/test_ometa_objectstore_api.py @@ -0,0 +1,263 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +OpenMetadata high-level API Container test +""" +import uuid +from unittest import TestCase + +from metadata.generated.schema.api.data.createContainer import CreateContainerRequest +from metadata.generated.schema.api.services.createObjectStoreService import ( + CreateObjectStoreServiceRequest, +) +from metadata.generated.schema.api.teams.createUser import CreateUserRequest +from metadata.generated.schema.entity.data.container import Container +from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( + OpenMetadataConnection, +) +from metadata.generated.schema.entity.services.connections.objectstore.s3ObjectStoreConnection import ( + S3StoreConnection, +) +from metadata.generated.schema.entity.services.objectstoreService import ( + ObjectStoreConnection, + ObjectStoreService, + ObjectStoreServiceType, +) +from metadata.generated.schema.security.client.openMetadataJWTClientConfig import ( + OpenMetadataJWTClientConfig, +) +from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.ingestion.ometa.ometa_api import OpenMetadata + + +class OMetaObjectStoreTest(TestCase): + """ + Run this integration test with the local API available + Install the ingestion package before running the tests + """ + + service_entity_id = None + + server_config = OpenMetadataConnection( + hostPort="http://localhost:8585/api", + authProvider="openmetadata", + securityConfig=OpenMetadataJWTClientConfig( + jwtToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" + ), + ) + metadata = OpenMetadata(server_config) + + assert metadata.health_check() + + user = metadata.create_or_update( + data=CreateUserRequest(name="random-user", email="random@user.com"), + ) + owner = EntityReference(id=user.id, type="user") + + service = CreateObjectStoreServiceRequest( + name="test-service-object", + serviceType=ObjectStoreServiceType.S3, + connection=ObjectStoreConnection( + config=S3StoreConnection(awsConfig=AWSCredentials(awsRegion="us-east-2")) + ), + ) + service_type = "objectstoreService" + + @classmethod + def setUpClass(cls) -> None: + """ + Prepare ingredients + """ + cls.service_entity = cls.metadata.create_or_update(data=cls.service) + + cls.entity = Container( + id=uuid.uuid4(), + name="test", + service=EntityReference(id=cls.service_entity.id, type=cls.service_type), + fullyQualifiedName="test-service-object.test", + ) + + cls.create = CreateContainerRequest( + name="test", + service=cls.service_entity.fullyQualifiedName, + ) + + @classmethod + def tearDownClass(cls) -> None: + """ + Clean up + """ + + service_id = str( + cls.metadata.get_by_name( + entity=ObjectStoreService, fqn="test-service-object" + ).id.__root__ + ) + + cls.metadata.delete( + entity=ObjectStoreService, + entity_id=service_id, + recursive=True, + hard_delete=True, + ) + + def test_create(self): + """ + We can create a Container and we receive it back as Entity + """ + + res = self.metadata.create_or_update(data=self.create) + + self.assertEqual(res.name, self.entity.name) + self.assertEqual(res.service.id, self.entity.service.id) + self.assertEqual(res.owner, None) + + def test_update(self): + """ + Updating it properly changes its properties + """ + + res_create = self.metadata.create_or_update(data=self.create) + + updated = self.create.dict(exclude_unset=True) + updated["owner"] = self.owner + updated_entity = CreateContainerRequest(**updated) + + res = self.metadata.create_or_update(data=updated_entity) + + # Same ID, updated algorithm + self.assertEqual( + res.service.fullyQualifiedName, updated_entity.service.__root__ + ) + self.assertEqual(res_create.id, res.id) + self.assertEqual(res.owner.id, self.user.id) + + def test_get_name(self): + """ + We can fetch a Container by name and get it back as Entity + """ + + self.metadata.create_or_update(data=self.create) + + res = self.metadata.get_by_name( + entity=Container, fqn=self.entity.fullyQualifiedName + ) + self.assertEqual(res.name, self.entity.name) + + def test_get_id(self): + """ + We can fetch a Container by ID and get it back as Entity + """ + + self.metadata.create_or_update(data=self.create) + + # First pick up by name + res_name = self.metadata.get_by_name( + entity=Container, fqn=self.entity.fullyQualifiedName + ) + # Then fetch by ID + res = self.metadata.get_by_id(entity=Container, entity_id=res_name.id) + + self.assertEqual(res_name.id, res.id) + + def test_list(self): + """ + We can list all our Containers + """ + + self.metadata.create_or_update(data=self.create) + + res = self.metadata.list_entities(entity=Container, limit=100) + + # Fetch our test Database. We have already inserted it, so we should find it + data = next( + iter(ent for ent in res.entities if ent.name == self.entity.name), None + ) + assert data + + def test_delete(self): + """ + We can delete a Container by ID + """ + + self.metadata.create_or_update(data=self.create) + + # Find by name + res_name = self.metadata.get_by_name( + entity=Container, fqn=self.entity.fullyQualifiedName + ) + # Then fetch by ID + res_id = self.metadata.get_by_id( + entity=Container, entity_id=str(res_name.id.__root__) + ) + + # Delete + self.metadata.delete( + entity=Container, entity_id=str(res_id.id.__root__), recursive=True + ) + + # Then we should not find it + res = self.metadata.list_entities(entity=Container) + assert not next( + iter( + ent + for ent in res.entities + if ent.fullyQualifiedName == self.entity.fullyQualifiedName + ), + None, + ) + + def test_list_versions(self): + """ + test list Container entity versions + """ + self.metadata.create_or_update(data=self.create) + + # Find by name + res_name = self.metadata.get_by_name( + entity=Container, fqn=self.entity.fullyQualifiedName + ) + + res = self.metadata.get_list_entity_versions( + entity=Container, entity_id=res_name.id.__root__ + ) + assert res + + def test_get_entity_version(self): + """ + test get Container entity version + """ + self.metadata.create_or_update(data=self.create) + + # Find by name + res_name = self.metadata.get_by_name( + entity=Container, fqn=self.entity.fullyQualifiedName + ) + res = self.metadata.get_entity_version( + entity=Container, entity_id=res_name.id.__root__, version=0.1 + ) + + # check we get the correct version requested and the correct entity ID + assert res.version.__root__ == 0.1 + assert res.id == res_name.id + + def test_get_entity_ref(self): + """ + test get EntityReference + """ + res = self.metadata.create_or_update(data=self.create) + entity_ref = self.metadata.get_entity_reference( + entity=Container, fqn=res.fullyQualifiedName + ) + + assert res.id == entity_ref.id diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java index cd33b61b2c7..513d0504e18 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java @@ -109,7 +109,8 @@ public class ContainerRepository extends EntityRepository { public void prepare(Container container) throws IOException { // the objectStoreService is not fully filled in terms of props - go to the db and get it in full and re-set it ObjectStoreService objectStoreService = Entity.getEntity(container.getService(), "", Include.NON_DELETED); - container.withService(objectStoreService.getEntityReference()); + container.setService(objectStoreService.getEntityReference()); + container.setServiceType(objectStoreService.getServiceType()); if (container.getParent() != null) { Container parent = Entity.getEntity(container.getParent(), "owner", ALL); diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/container.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/container.json index 56eb39124dd..b9f09ee4489 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/container.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/container.json @@ -132,6 +132,10 @@ }, "default": null }, + "serviceType": { + "description": "Service type this table is hosted in.", + "$ref": "../services/objectstoreService.json#/definitions/objectStoreServiceType" + }, "followers": { "description": "Followers of this container.", "$ref": "../../type/entityReference.json#/definitions/entityReferenceList" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/azureObjectStoreConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/azureObjectStoreConnection.json index 50bfe6a4d1a..1e59bf738e9 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/azureObjectStoreConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/azureObjectStoreConnection.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/services/connections/objectstore/azureStoreConnection.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Azure StoreConnection", + "title": "Azure Store Connection", "description": "Azure Store Connection.", "type": "object", "javaType": "org.openmetadata.schema.services.connections.objectstore.AzureConnection", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/gcsObjectStoreConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/gcsObjectStoreConnection.json index d7176184fdd..2dfd4c89518 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/gcsObjectStoreConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/gcsObjectStoreConnection.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/services/connections/objectstore/gcsStoreConnection.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "GCS StoreConnection", + "title": "GCS Store Connection", "description": "GCS Store Connection.", "type": "object", "javaType": "org.openmetadata.schema.services.connections.objectstore.GcsConnection", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/s3ObjectStoreConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/s3ObjectStoreConnection.json index 37f44a3603f..85e86443466 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/s3ObjectStoreConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/objectstore/s3ObjectStoreConnection.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/services/connections/objectstore/s3StoreConnection.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "S3 StoreConnection", + "title": "S3 Store Connection", "description": "S3 Store Connection.", "type": "object", "javaType": "org.openmetadata.schema.services.connections.objectstore.S3Connection",