yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-03 03:59:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rel #10828 - Update container index and fix airflow ingestion (#10873)

Browse Source 
* Update container index and fix airflow ingestion

* Fix lint

---------

Co-authored-by: Ayush Shah <ayush@getcollate.io>
This commit is contained in:
Pere Miquel Brull 2023-04-06 07:09:38 +02:00 committed by GitHub
parent 38d4afb6cb
commit c53c7b680d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ingestion/src/metadata/ingestion/models/es_documents.py
View File

@ -217,7 +217,7 @@ class ContainerESDocument(BaseModel):
children: Optional[List[dict]] = None children: Optional[List[dict]] = None
prefix: Optional[str] = None prefix: Optional[str] = None
numberOfObjects: Optional[int] = None numberOfObjects: Optional[int] = None
size: Optional[int] = None size: Optional[float] = None
fileFormats: Optional[List[str]] = None fileFormats: Optional[List[str]] = None

2
ingestion/src/metadata/ingestion/sink/elasticsearch_mapping/container_search_index_mapping.py
View File

@ -239,7 +239,7 @@ CONTAINER_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
"type": "integer" "type": "integer"
}, },
"size": { "size": {
"type": "integer" "type": "long"
}, },
"fileFormats": { "fileFormats": {
"type": "keyword" "type": "keyword"

8
openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py
View File

@ -28,6 +28,9 @@ from metadata.generated.schema.entity.services.databaseService import DatabaseSe
from metadata.generated.schema.entity.services.messagingService import MessagingService from metadata.generated.schema.entity.services.messagingService import MessagingService
from metadata.generated.schema.entity.services.metadataService import MetadataService from metadata.generated.schema.entity.services.metadataService import MetadataService
from metadata.generated.schema.entity.services.mlmodelService import MlModelService from metadata.generated.schema.entity.services.mlmodelService import MlModelService
from metadata.generated.schema.entity.services.objectstoreService import (
ObjectStoreService,
)
from metadata.generated.schema.entity.services.pipelineService import PipelineService from metadata.generated.schema.entity.services.pipelineService import PipelineService
from metadata.generated.schema.tests.testSuite import TestSuite from metadata.generated.schema.tests.testSuite import TestSuite
from metadata.ingestion.models.encoders import show_secrets_encoder from metadata.ingestion.models.encoders import show_secrets_encoder
@ -169,6 +172,11 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource:
service: MetadataService = metadata.get_by_name( service: MetadataService = metadata.get_by_name(
entity=entity_class, fqn=ingestion_pipeline.service.name entity=entity_class, fqn=ingestion_pipeline.service.name
) )
elif service_type == "objectStoreService":
entity_class = ObjectStoreService
service: ObjectStoreService = metadata.get_by_name(
entity=entity_class, fqn=ingestion_pipeline.service.name
)
else: else:
raise InvalidServiceException(f"Invalid Service Type: {service_type}") raise InvalidServiceException(f"Invalid Service Type: {service_type}")
except ValidationError as original_error: except ValidationError as original_error:

2
openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json
View File

@ -222,7 +222,7 @@
"type": "integer" "type": "integer"
}, },
"size": { "size": {
"type": "integer" "type": "long"
}, },
"fileFormats": { "fileFormats": {
"type": "keyword" "type": "keyword"