From c53c7b680d44ce6f6f386592e69224581ac076e3 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 6 Apr 2023 07:09:38 +0200 Subject: [PATCH] Rel #10828 - Update container index and fix airflow ingestion (#10873) * Update container index and fix airflow ingestion * Fix lint --------- Co-authored-by: Ayush Shah --- ingestion/src/metadata/ingestion/models/es_documents.py | 2 +- .../container_search_index_mapping.py | 2 +- .../workflows/ingestion/common.py | 8 ++++++++ .../elasticsearch/en/container_index_mapping.json | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ingestion/src/metadata/ingestion/models/es_documents.py b/ingestion/src/metadata/ingestion/models/es_documents.py index fe218311319..8df6260a121 100644 --- a/ingestion/src/metadata/ingestion/models/es_documents.py +++ b/ingestion/src/metadata/ingestion/models/es_documents.py @@ -217,7 +217,7 @@ class ContainerESDocument(BaseModel): children: Optional[List[dict]] = None prefix: Optional[str] = None numberOfObjects: Optional[int] = None - size: Optional[int] = None + size: Optional[float] = None fileFormats: Optional[List[str]] = None diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/container_search_index_mapping.py b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/container_search_index_mapping.py index 464ee5b06ff..75a3e71b5c9 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/container_search_index_mapping.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/container_search_index_mapping.py @@ -239,7 +239,7 @@ CONTAINER_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent( "type": "integer" }, "size": { - "type": "integer" + "type": "long" }, "fileFormats": { "type": "keyword" diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py index bcc834d70e3..ede4af4513c 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py @@ -28,6 +28,9 @@ from metadata.generated.schema.entity.services.databaseService import DatabaseSe from metadata.generated.schema.entity.services.messagingService import MessagingService from metadata.generated.schema.entity.services.metadataService import MetadataService from metadata.generated.schema.entity.services.mlmodelService import MlModelService +from metadata.generated.schema.entity.services.objectstoreService import ( + ObjectStoreService, +) from metadata.generated.schema.entity.services.pipelineService import PipelineService from metadata.generated.schema.tests.testSuite import TestSuite from metadata.ingestion.models.encoders import show_secrets_encoder @@ -169,6 +172,11 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource: service: MetadataService = metadata.get_by_name( entity=entity_class, fqn=ingestion_pipeline.service.name ) + elif service_type == "objectStoreService": + entity_class = ObjectStoreService + service: ObjectStoreService = metadata.get_by_name( + entity=entity_class, fqn=ingestion_pipeline.service.name + ) else: raise InvalidServiceException(f"Invalid Service Type: {service_type}") except ValidationError as original_error: diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json index bed6bbe57fa..edbae0c5286 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json @@ -222,7 +222,7 @@ "type": "integer" }, "size": { - "type": "integer" + "type": "long" }, "fileFormats": { "type": "keyword"