diff --git a/docker/openmetadata-start.sh b/docker/openmetadata-start.sh index 6087907cc7f..f3bdbe487ef 100644 --- a/docker/openmetadata-start.sh +++ b/docker/openmetadata-start.sh @@ -11,8 +11,6 @@ # limitations under the License. echo "Initializing OpenMetadata Server..."; -# echo "Migrating the database to the latest version and the indexes in ElasticSearch..."; -# ./bootstrap/bootstrap_storage.sh migrate-all echo " ||||||| " echo " |||| |||| ____ " echo " |||| |||| / __ \ " diff --git a/ingestion/src/airflow_provider_openmetadata/lineage/runner.py b/ingestion/src/airflow_provider_openmetadata/lineage/runner.py index 38a607ef391..c88cdd41f75 100644 --- a/ingestion/src/airflow_provider_openmetadata/lineage/runner.py +++ b/ingestion/src/airflow_provider_openmetadata/lineage/runner.py @@ -200,7 +200,7 @@ class AirflowLineageRunner: return self.metadata.patch( entity=Pipeline, source=pipeline, - destination=pipeline.copy(update=pipeline_request.__dict__), + destination=pipeline.model_copy(update=pipeline_request.__dict__), allowed_fields=ALLOWED_COMMON_PATCH_FIELDS, restrict_update_fields=RESTRICT_UPDATE_LIST, ) diff --git a/ingestion/src/metadata/clients/domo_client.py b/ingestion/src/metadata/clients/domo_client.py index e10200b3541..948e01639a0 100644 --- a/ingestion/src/metadata/clients/domo_client.py +++ b/ingestion/src/metadata/clients/domo_client.py @@ -17,7 +17,7 @@ import traceback from dataclasses import dataclass from typing import List, Optional, Union -from pydantic import BaseModel, Extra +from pydantic import BaseModel, ConfigDict from pydomo import Domo from metadata.generated.schema.entity.services.connections.dashboard.domoDashboardConnection import ( @@ -43,8 +43,7 @@ class DomoBaseModel(BaseModel): Domo basic configurations """ - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") id: str name: str @@ -75,8 +74,7 @@ class DomoChartMetadataDetails(BaseModel): Metadata Details in chart """ - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") chartType: Optional[str] = None diff --git a/ingestion/src/metadata/config/common.py b/ingestion/src/metadata/config/common.py index c537205e6f1..f3f7df9d490 100644 --- a/ingestion/src/metadata/config/common.py +++ b/ingestion/src/metadata/config/common.py @@ -19,14 +19,13 @@ from abc import ABC, abstractmethod from typing import IO, Any, Optional import yaml -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class ConfigModel(BaseModel): """Class definition for config model""" - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") class DynamicTypedConfig(ConfigModel): diff --git a/ingestion/src/metadata/data_insight/source/metadata.py b/ingestion/src/metadata/data_insight/source/metadata.py index 15f2b86e480..2baa5553bf1 100644 --- a/ingestion/src/metadata/data_insight/source/metadata.py +++ b/ingestion/src/metadata/data_insight/source/metadata.py @@ -17,7 +17,7 @@ from datetime import datetime, timezone from types import MappingProxyType from typing import Dict, Iterable, Optional, Union, cast -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from metadata.data_insight.processor.reports.cost_analysis_report_data_processor import ( AggregatedCostAnalysisReportDataProcessor, @@ -48,9 +48,7 @@ logger = profiler_logger() class DataInsightRecord(BaseModel): """Return class for the OpenMetadata Profiler Source""" - class Config: - arbitrary_types_allowed = True - extra = "forbid" + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) data: ReportData diff --git a/ingestion/src/metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py b/ingestion/src/metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py index 05cee62fd27..3e4111dfdba 100644 --- a/ingestion/src/metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +++ b/ingestion/src/metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py @@ -175,7 +175,7 @@ class SQATestSuiteInterface(SQAInterfaceMixin, TestSuiteInterface): runtime_params = setter.get_parameters(test_case) test_case.parameterValues.append( TestCaseParameterValue( - name="runtimeParams", value=runtime_params.json() + name="runtimeParams", value=runtime_params.model_dump_json() ) ) diff --git a/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py index f80bb5e56ba..9167de56d84 100644 --- a/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py @@ -243,7 +243,7 @@ class TableDiffValidator(BaseTestValidator, SQAValidatorMixin): raw = self.get_test_case_param_value( self.test_case.parameterValues, "runtimeParams", str ) - runtime_params = TableDiffRuntimeParameters.parse_raw(raw) + runtime_params = TableDiffRuntimeParameters.model_validate_json(raw) return runtime_params def get_row_diff_test_case_result( diff --git a/ingestion/src/metadata/ingestion/api/common.py b/ingestion/src/metadata/ingestion/api/common.py index 2e43d82b098..d7b1a44ba3e 100644 --- a/ingestion/src/metadata/ingestion/api/common.py +++ b/ingestion/src/metadata/ingestion/api/common.py @@ -13,7 +13,7 @@ Common definitions for configuration management """ from typing import Any, Optional, TypeVar -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from metadata.utils.logger import ingestion_logger @@ -27,8 +27,7 @@ Entity = TypeVar("Entity", bound=BaseModel) class ConfigModel(BaseModel): - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") class DynamicTypedConfig(ConfigModel): diff --git a/ingestion/src/metadata/ingestion/api/parser.py b/ingestion/src/metadata/ingestion/api/parser.py index c37aee590dd..639c26492bf 100644 --- a/ingestion/src/metadata/ingestion/api/parser.py +++ b/ingestion/src/metadata/ingestion/api/parser.py @@ -294,7 +294,7 @@ def _unsafe_parse_config(config: dict, cls: Type[T], message: str) -> None: cls.model_validate(config) except ValidationError as err: logger.debug( - f"The supported properties for {cls.__name__} are {list(cls.__fields__.keys())}" + f"The supported properties for {cls.__name__} are {list(cls.model_fields.keys())}" ) raise err @@ -315,7 +315,7 @@ def _unsafe_parse_dbt_config(config: dict, cls: Type[T], message: str) -> None: cls.model_validate(config) except ValidationError as err: logger.debug( - f"The supported properties for {cls.__name__} are {list(cls.__fields__.keys())}" + f"The supported properties for {cls.__name__} are {list(cls.model_fields.keys())}" ) raise err diff --git a/ingestion/src/metadata/ingestion/api/topology_runner.py b/ingestion/src/metadata/ingestion/api/topology_runner.py index 4acc698c82d..280674e1476 100644 --- a/ingestion/src/metadata/ingestion/api/topology_runner.py +++ b/ingestion/src/metadata/ingestion/api/topology_runner.py @@ -328,7 +328,7 @@ class TopologyRunnerMixin(Generic[C]): """ return PatchRequest( original_entity=original_entity, - new_entity=original_entity.copy(update=create_request.__dict__), + new_entity=original_entity.model_copy(update=create_request.__dict__), override_metadata=self.source_config.overrideMetadata, ) diff --git a/ingestion/src/metadata/ingestion/models/patch_request.py b/ingestion/src/metadata/ingestion/models/patch_request.py index 43ff7ae7bb4..68dbfe23eaf 100644 --- a/ingestion/src/metadata/ingestion/models/patch_request.py +++ b/ingestion/src/metadata/ingestion/models/patch_request.py @@ -440,7 +440,7 @@ def _sort_array_entity_fields( destination_attr = destination_dict.get(model_str(source_attr.name)) if destination_attr: updated_attributes.append( - source_attr.copy(update=destination_attr.__dict__) + source_attr.model_copy(update=destination_attr.__dict__) ) # Remove the updated attribute from the destination dictionary del destination_dict[model_str(source_attr.name)] diff --git a/ingestion/src/metadata/ingestion/models/topology.py b/ingestion/src/metadata/ingestion/models/topology.py index 6e6d7614fc6..0ed771dab45 100644 --- a/ingestion/src/metadata/ingestion/models/topology.py +++ b/ingestion/src/metadata/ingestion/models/topology.py @@ -296,7 +296,7 @@ class TopologyContextManager: # If it does not exist yet, copies the Parent Context in order to have all context gathered until this point. self.contexts.setdefault( - thread_id, self.contexts[parent_thread_id].copy(deep=True) + thread_id, self.contexts[parent_thread_id].model_copy(deep=True) ) diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py index 40faacff0f6..83308982eed 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py @@ -199,7 +199,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): return None # https://docs.pydantic.dev/latest/usage/exporting_models/#modelcopy - destination = source.copy(deep=True) + destination = source.model_copy(deep=True) destination.description = Markdown(description) return self.patch(entity=entity, source=source, destination=destination) @@ -228,7 +228,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): table.tableConstraints = instance.tableConstraints - destination = table.copy(deep=True) + destination = table.model_copy(deep=True) destination.tableConstraints = constraints return self.patch(entity=Table, source=table, destination=destination) @@ -253,7 +253,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): if not source: return None - destination = source.copy(deep=True) + destination = source.model_copy(deep=True) destination.entityLink = EntityLink(entity_link) if test_case_parameter_values: @@ -291,7 +291,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): # Initialize empty tag list or the last updated tags source.tags = instance.tags or [] - destination = source.copy(deep=True) + destination = source.model_copy(deep=True) tag_fqns = {label.tagFQN.root for label in tag_labels} @@ -385,7 +385,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): # Make sure we run the patch against the last updated data from the API table.columns = instance.columns - destination = table.copy(deep=True) + destination = table.model_copy(deep=True) for column_tag in column_tags or []: update_column_tags(destination.columns, column_tag, operation) @@ -472,7 +472,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): # Make sure we run the patch against the last updated data from the API table.columns = instance.columns - destination = table.copy(deep=True) + destination = table.model_copy(deep=True) update_column_description(destination.columns, column_descriptions, force) patched_entity = self.patch(entity=Table, source=table, destination=destination) @@ -531,7 +531,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): :param life_cycle_data: Life Cycle data to add """ try: - destination = entity.copy(deep=True) + destination = entity.model_copy(deep=True) destination.lifeCycle = life_cycle return self.patch( entity=type(entity), source=entity, destination=destination @@ -546,7 +546,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase): def patch_domain(self, entity: Entity, domain: Domain) -> Optional[Entity]: """Patch domain data for an Entity""" try: - destination: Entity = entity.copy(deep=True) + destination: Entity = entity.model_copy(deep=True) destination.domain = EntityReference(id=domain.id, type="domain") return self.patch( entity=type(entity), source=entity, destination=destination diff --git a/ingestion/src/metadata/ingestion/sink/file.py b/ingestion/src/metadata/ingestion/sink/file.py index 2d862500aa3..73beaa90f9a 100644 --- a/ingestion/src/metadata/ingestion/sink/file.py +++ b/ingestion/src/metadata/ingestion/sink/file.py @@ -60,7 +60,7 @@ class FileSink(Sink): if self.wrote_something: self.file.write(",\n") - self.file.write(record.json()) + self.file.write(record.model_dump_json()) self.wrote_something = True return Either(right=get_log_name(record)) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py index 54b82d43523..3a989bf2c5d 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py @@ -205,7 +205,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC): config: WorkflowSource metadata: OpenMetadata # Big union of types we want to fetch dynamically - service_connection: DashboardConnection.__fields__["config"].annotation + service_connection: DashboardConnection.model_fields["config"].annotation topology = DashboardServiceTopology() context = TopologyContextManager(topology) @@ -608,7 +608,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC): """ patch_request = PatchRequest( original_entity=original_entity, - new_entity=original_entity.copy(update=create_request.__dict__), + new_entity=original_entity.model_copy(update=create_request.__dict__), ) if isinstance(original_entity, Dashboard): # For patch the charts need to be entity ref instead of fqn diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py index 246e56d8604..51da219b97c 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py @@ -18,7 +18,7 @@ from time import sleep from typing import List, Optional, Tuple import msal -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import ( PowerBIConnection, @@ -327,8 +327,7 @@ class PowerBiApiClient: class PowerBiClient(BaseModel): - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) api_client: PowerBiApiClient file_client: Optional[PowerBiFileClient] diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/models.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/models.py index 75790e18a58..81d99ca9f1a 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/models.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/models.py @@ -15,7 +15,7 @@ Tableau Source Model module from typing import Any, Dict, List, Optional, Union -from pydantic import BaseModel, Extra, Field, validator +from pydantic import BaseModel, ConfigDict, Field, validator from metadata.generated.schema.entity.data.chart import ChartType @@ -25,8 +25,7 @@ class TableauBaseModel(BaseModel): Tableau basic configurations """ - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") id: str name: Optional[str] = None @@ -54,8 +53,7 @@ class TableauTag(BaseModel): Aux class for Tag object of the tableau_api_lib response """ - class Config: - frozen = True + model_config = ConfigDict(frozen=True) label: str @@ -153,8 +151,7 @@ class TableauDashboard(TableauBaseModel): Aux class for Dashboard object of the tableau_api_lib response """ - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") project: Optional[TableauBaseModel] = None description: Optional[str] = None diff --git a/ingestion/src/metadata/ingestion/source/database/database_service.py b/ingestion/src/metadata/ingestion/source/database/database_service.py index da651278eb3..ebaa3e9e231 100644 --- a/ingestion/src/metadata/ingestion/source/database/database_service.py +++ b/ingestion/src/metadata/ingestion/source/database/database_service.py @@ -223,7 +223,7 @@ class DatabaseServiceSource( database_source_state: Set = set() stored_procedure_source_state: Set = set() # Big union of types we want to fetch dynamically - service_connection: DatabaseConnection.__fields__["config"].annotation + service_connection: DatabaseConnection.model_fields["config"].annotation # When processing the database, the source will update the inspector if needed inspector: Inspector diff --git a/ingestion/src/metadata/ingestion/source/database/domodatabase/models.py b/ingestion/src/metadata/ingestion/source/database/domodatabase/models.py index 9cdaa1eb9f1..ef4ce4e968d 100644 --- a/ingestion/src/metadata/ingestion/source/database/domodatabase/models.py +++ b/ingestion/src/metadata/ingestion/source/database/domodatabase/models.py @@ -15,12 +15,11 @@ Domo Database Source Model module from typing import List, Optional -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field class DomoDatabaseBaseModel(BaseModel): - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") id: str name: str diff --git a/ingestion/src/metadata/ingestion/source/database/life_cycle_query_mixin.py b/ingestion/src/metadata/ingestion/source/database/life_cycle_query_mixin.py index 546911666f5..b25692a6f14 100644 --- a/ingestion/src/metadata/ingestion/source/database/life_cycle_query_mixin.py +++ b/ingestion/src/metadata/ingestion/source/database/life_cycle_query_mixin.py @@ -17,7 +17,7 @@ from datetime import datetime from functools import lru_cache from typing import Dict, Iterable, List, Optional, Type -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from sqlalchemy.engine import Engine from metadata.generated.schema.entity.data.table import Table @@ -46,12 +46,11 @@ class LifeCycleQueryByTable(BaseModel): Query executed get life cycle """ + model_config = ConfigDict(populate_by_name=True) + table_name: str = Field(..., alias="TABLE_NAME") created_at: Optional[datetime] = Field(None, alias="CREATED_AT") - class Config: - populate_by_name = True - class LifeCycleQueryMixin: """ diff --git a/ingestion/src/metadata/ingestion/source/database/sas/metadata.py b/ingestion/src/metadata/ingestion/source/database/sas/metadata.py index 082280d9736..aadfc8c3c9f 100644 --- a/ingestion/src/metadata/ingestion/source/database/sas/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/sas/metadata.py @@ -591,7 +591,7 @@ class SasSource( ) self.metadata.client.put( path=f"{self.metadata.get_suffix(Table)}/{table_entity.id.root}/tableProfile", - data=table_profile_request.json(), + data=table_profile_request.model_dump_json(), ) except Exception as exc: diff --git a/ingestion/src/metadata/ingestion/source/messaging/kinesis/models.py b/ingestion/src/metadata/ingestion/source/messaging/kinesis/models.py index 5574a75a661..2182303f7a1 100644 --- a/ingestion/src/metadata/ingestion/source/messaging/kinesis/models.py +++ b/ingestion/src/metadata/ingestion/source/messaging/kinesis/models.py @@ -16,7 +16,7 @@ Kinesis Models from enum import Enum from typing import List, Optional -from pydantic import BaseModel, Extra +from pydantic import BaseModel, ConfigDict class KinesisEnum(Enum): @@ -66,8 +66,7 @@ class KinesisArgs(BaseModel): Model for Kinesis API Arguments """ - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") Limit: int = 100 @@ -77,8 +76,7 @@ class KinesisStreamArgs(BaseModel): Model for Kinesis Stream API Arguments """ - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") StreamName: str diff --git a/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py b/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py index 10d472c907d..faa033784a0 100644 --- a/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py +++ b/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py @@ -116,7 +116,7 @@ class MessagingServiceSource(TopologyRunnerMixin, Source, ABC): source_config: MessagingServiceMetadataPipeline config: WorkflowSource # Big union of types we want to fetch dynamically - service_connection: MessagingConnection.__fields__["config"].annotation + service_connection: MessagingConnection.model_fields["config"].annotation topology = MessagingServiceTopology() context = TopologyContextManager(topology) diff --git a/ingestion/src/metadata/ingestion/source/mlmodel/mlmodel_service.py b/ingestion/src/metadata/ingestion/source/mlmodel/mlmodel_service.py index d6d74446f68..f49baae8cf9 100644 --- a/ingestion/src/metadata/ingestion/source/mlmodel/mlmodel_service.py +++ b/ingestion/src/metadata/ingestion/source/mlmodel/mlmodel_service.py @@ -104,7 +104,7 @@ class MlModelServiceSource(TopologyRunnerMixin, Source, ABC): source_config: MlModelServiceMetadataPipeline config: WorkflowSource # Big union of types we want to fetch dynamically - service_connection: MlModelConnection.__fields__["config"].annotation + service_connection: MlModelConnection.model_fields["config"].annotation topology = MlModelServiceTopology() context = TopologyContextManager(topology) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airflow/lineage_parser.py b/ingestion/src/metadata/ingestion/source/pipeline/airflow/lineage_parser.py index 9e68749b686..5e57f421ee7 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/airflow/lineage_parser.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/airflow/lineage_parser.py @@ -73,7 +73,7 @@ from functools import singledispatch from typing import Any, DefaultDict, Dict, List, Optional, Type import attr -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from metadata.generated.schema.entity.data.table import Table from metadata.ingestion.ometa.models import T @@ -129,12 +129,11 @@ class XLets(BaseModel): Group inlets and outlets from all tasks in a DAG """ + model_config = ConfigDict(arbitrary_types_allowed=True) + inlets: List[OMEntity] outlets: List[OMEntity] - class Config: - arbitrary_types_allowed = True - def concat_dict_values( dict_1: DefaultDict[str, List[Any]], dict_2: Optional[Dict[str, List[Any]]] diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airflow/models.py b/ingestion/src/metadata/ingestion/source/pipeline/airflow/models.py index 66f1e1b74c2..6f5e14b4b80 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/airflow/models.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/airflow/models.py @@ -16,7 +16,7 @@ Tableau Source Model module from datetime import datetime from typing import Any, List, Optional -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field class AirflowBaseModel(BaseModel): @@ -24,9 +24,7 @@ class AirflowBaseModel(BaseModel): Tableau basic configurations """ - class Config: - extra = Extra.allow - arbitrary_types_allowed = True + model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) dag_id: str @@ -44,8 +42,7 @@ class AirflowTask(BaseModel): owner: Optional[str] = None # Allow picking up data from key `inlets` and `_inlets` - class Config: - populate_by_name = True + model_config = ConfigDict(populate_by_name=True) class TaskList(BaseModel): diff --git a/ingestion/src/metadata/ingestion/source/pipeline/pipeline_service.py b/ingestion/src/metadata/ingestion/source/pipeline/pipeline_service.py index bf54fdb03a0..c04c3c53c36 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/pipeline_service.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/pipeline_service.py @@ -120,7 +120,7 @@ class PipelineServiceSource(TopologyRunnerMixin, Source, ABC): source_config: PipelineServiceMetadataPipeline config: WorkflowSource # Big union of types we want to fetch dynamically - service_connection: PipelineConnection.__fields__["config"].annotation + service_connection: PipelineConnection.model_fields["config"].annotation topology = PipelineServiceTopology() context = TopologyContextManager(topology) diff --git a/ingestion/src/metadata/ingestion/source/search/search_service.py b/ingestion/src/metadata/ingestion/source/search/search_service.py index 0f64e42d28b..ce49c3245a1 100644 --- a/ingestion/src/metadata/ingestion/source/search/search_service.py +++ b/ingestion/src/metadata/ingestion/source/search/search_service.py @@ -114,7 +114,7 @@ class SearchServiceSource(TopologyRunnerMixin, Source, ABC): source_config: SearchServiceMetadataPipeline config: WorkflowSource # Big union of types we want to fetch dynamically - service_connection: SearchConnection.__fields__["config"].annotation + service_connection: SearchConnection.model_fields["config"].annotation topology = SearchServiceTopology() context = TopologyContextManager(topology) diff --git a/ingestion/src/metadata/ingestion/source/storage/storage_service.py b/ingestion/src/metadata/ingestion/source/storage/storage_service.py index 256045a06f6..838624dd9b8 100644 --- a/ingestion/src/metadata/ingestion/source/storage/storage_service.py +++ b/ingestion/src/metadata/ingestion/source/storage/storage_service.py @@ -121,7 +121,7 @@ class StorageServiceSource(TopologyRunnerMixin, Source, ABC): config: WorkflowSource metadata: OpenMetadata # Big union of types we want to fetch dynamically - service_connection: StorageConnection.__fields__["config"].annotation + service_connection: StorageConnection.model_fields["config"].annotation topology = StorageServiceTopology() context = TopologyContextManager(topology) diff --git a/ingestion/src/metadata/profiler/api/models.py b/ingestion/src/metadata/profiler/api/models.py index 73d2edaf2ab..274710778a3 100644 --- a/ingestion/src/metadata/profiler/api/models.py +++ b/ingestion/src/metadata/profiler/api/models.py @@ -17,6 +17,7 @@ multiple profilers per table and columns. """ from typing import List, Optional, Type, Union +from pydantic import ConfigDict from sqlalchemy import Column from sqlalchemy.orm import DeclarativeMeta @@ -124,10 +125,9 @@ class ProfilerResponse(ConfigModel): class ThreadPoolMetrics(ConfigModel): """A container for all metrics to be computed on the same thread.""" + model_config = ConfigDict(arbitrary_types_allowed=True) + metrics: Union[List[Union[Type[Metric], CustomMetric]], Type[Metric]] metric_type: MetricTypes column: Optional[Union[Column, SQALikeColumn]] = None table: Union[Table, DeclarativeMeta] - - class Config: - arbitrary_types_allowed = True diff --git a/ingestion/src/metadata/profiler/source/metadata.py b/ingestion/src/metadata/profiler/source/metadata.py index ad56bb252b2..b6b6ec27f1f 100644 --- a/ingestion/src/metadata/profiler/source/metadata.py +++ b/ingestion/src/metadata/profiler/source/metadata.py @@ -14,7 +14,7 @@ OpenMetadata source for the profiler import traceback from typing import Iterable, Optional, cast -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema @@ -50,9 +50,7 @@ TAGS_FIELD = ["tags"] class ProfilerSourceAndEntity(BaseModel): """Return class for the OpenMetadata Profiler Source""" - class Config: - arbitrary_types_allowed = True - extra = "forbid" + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) profiler_source: ProfilerSource entity: Table diff --git a/ingestion/src/metadata/readers/file/credentials.py b/ingestion/src/metadata/readers/file/credentials.py index 8d80288d8a9..34a454aec9d 100644 --- a/ingestion/src/metadata/readers/file/credentials.py +++ b/ingestion/src/metadata/readers/file/credentials.py @@ -26,7 +26,7 @@ def update_repository_name( Given an original set of credentials and a new repository name, return the updated credentials """ - updated = original.copy(deep=True) + updated = original.model_copy(deep=True) updated.repositoryName = RepositoryName(name) return updated diff --git a/ingestion/src/metadata/utils/ssl_registry.py b/ingestion/src/metadata/utils/ssl_registry.py index 591c2914b03..7bea9a70e3b 100644 --- a/ingestion/src/metadata/utils/ssl_registry.py +++ b/ingestion/src/metadata/utils/ssl_registry.py @@ -39,6 +39,10 @@ def ignore_ssl_init(_: Optional[SslConfig]) -> bool: @ssl_verification_registry.add(VerifySSL.validate.value) def validate_ssl_init(ssl_config: Optional[SslConfig]) -> str: + if ssl_config is None: + raise InvalidSSLVerificationException( + "You have Verify SSL but the SSL Config is missing. Make sure to inform the CA Certificate path." + ) return ssl_config.root.caCertificate.get_secret_value() diff --git a/ingestion/tests/cli_e2e/common/test_cli_db.py b/ingestion/tests/cli_e2e/common/test_cli_db.py index 0156e7023a2..a41b736d086 100644 --- a/ingestion/tests/cli_e2e/common/test_cli_db.py +++ b/ingestion/tests/cli_e2e/common/test_cli_db.py @@ -103,7 +103,7 @@ class CliCommonDB: self.get_profiler_time_partition_results() ) if expected_profiler_time_partition_results: - table_profile = profile.profile.dict() + table_profile = profile.profile.model_dump() for key in expected_profiler_time_partition_results["table_profile"]: self.assertEqual( table_profile[key], @@ -122,7 +122,7 @@ class CliCommonDB: None, ) if expected_column_profile: - column_profile = column.profile.dict() + column_profile = column.profile.model_dump() for key in expected_column_profile: # type: ignore if key == "nonParametricSkew": self.assertEqual( diff --git a/ingestion/tests/integration/data_quality/conftest.py b/ingestion/tests/integration/data_quality/conftest.py index 0e0142096cb..52db5796ff4 100644 --- a/ingestion/tests/integration/data_quality/conftest.py +++ b/ingestion/tests/integration/data_quality/conftest.py @@ -53,7 +53,7 @@ def ingest_mysql_service( "sink": {"type": "metadata-rest", "config": {}}, "workflowConfig": { "loggerLevel": LogLevels.DEBUG.value, - "openMetadataServerConfig": metadata.config.dict(), + "openMetadataServerConfig": metadata.config.model_dump(), }, } metadata_ingestion = MetadataWorkflow.create(workflow_config) diff --git a/ingestion/tests/integration/ometa/test_ometa_chart_api.py b/ingestion/tests/integration/ometa/test_ometa_chart_api.py index 258dd531196..b844891f0fa 100644 --- a/ingestion/tests/integration/ometa/test_ometa_chart_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_chart_api.py @@ -130,7 +130,7 @@ class OMetaChartTest(TestCase): res_create = self.metadata.create_or_update(data=self.create) - updated = self.create.dict(exclude_unset=True) + updated = self.create.model_dump(exclude_unset=True) updated["owner"] = self.owner updated_entity = CreateChartRequest(**updated) diff --git a/ingestion/tests/integration/ometa/test_ometa_dashboard_api.py b/ingestion/tests/integration/ometa/test_ometa_dashboard_api.py index a44a699a651..530b7561d24 100644 --- a/ingestion/tests/integration/ometa/test_ometa_dashboard_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_dashboard_api.py @@ -130,7 +130,7 @@ class OMetaDashboardTest(TestCase): res_create = self.metadata.create_or_update(data=self.create) - updated = self.create.dict(exclude_unset=True) + updated = self.create.model_dump(exclude_unset=True) updated["owner"] = self.owner updated_entity = CreateDashboardRequest(**updated) diff --git a/ingestion/tests/integration/ometa/test_ometa_database_api.py b/ingestion/tests/integration/ometa/test_ometa_database_api.py index adc4a7f1c49..5228ff470ed 100644 --- a/ingestion/tests/integration/ometa/test_ometa_database_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_database_api.py @@ -135,7 +135,7 @@ class OMetaDatabaseTest(TestCase): res_create = self.metadata.create_or_update(data=self.create) - updated = self.create.dict(exclude_unset=True) + updated = self.create.model_dump(exclude_unset=True) updated["owner"] = self.owner updated_entity = CreateDatabaseRequest(**updated) diff --git a/ingestion/tests/integration/ometa/test_ometa_mlmodel_api.py b/ingestion/tests/integration/ometa/test_ometa_mlmodel_api.py index ee92e8fc679..8af6bae5477 100644 --- a/ingestion/tests/integration/ometa/test_ometa_mlmodel_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_mlmodel_api.py @@ -162,7 +162,7 @@ class OMetaModelTest(TestCase): res_create = self.metadata.create_or_update(data=self.create) - updated = self.create.dict(exclude_unset=True) + updated = self.create.model_dump(exclude_unset=True) updated["owner"] = self.owner updated_entity = CreateMlModelRequest(**updated) diff --git a/ingestion/tests/integration/ometa/test_ometa_pipeline_api.py b/ingestion/tests/integration/ometa/test_ometa_pipeline_api.py index 6c8e0ff7f28..0f7a84793ab 100644 --- a/ingestion/tests/integration/ometa/test_ometa_pipeline_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_pipeline_api.py @@ -142,7 +142,7 @@ class OMetaPipelineTest(TestCase): res_create = self.metadata.create_or_update(data=self.create) - updated = self.create.dict(exclude_unset=True) + updated = self.create.model_dump(exclude_unset=True) updated["owner"] = self.owner updated_entity = CreatePipelineRequest(**updated) diff --git a/ingestion/tests/integration/ometa/test_ometa_role_policy_api.py b/ingestion/tests/integration/ometa/test_ometa_role_policy_api.py index c5d2751b91e..0819b8ca984 100644 --- a/ingestion/tests/integration/ometa/test_ometa_role_policy_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_role_policy_api.py @@ -243,7 +243,7 @@ class OMetaRolePolicyTest(TestCase): res_create = self.metadata.create_or_update(data=self.create_policy) - updated = self.create_policy.dict(exclude_unset=True) + updated = self.create_policy.model_dump(exclude_unset=True) updated["rules"] = [self.rule_3] updated_policy_entity = CreatePolicyRequest(**updated) @@ -463,7 +463,7 @@ class OMetaRolePolicyTest(TestCase): res_create = self.metadata.create_or_update(data=self.create_role) - updated = self.create_role.dict(exclude_unset=True) + updated = self.create_role.model_dump(exclude_unset=True) updated["policies"] = [self.role_policy_2.name] updated_entity = CreateRoleRequest(**updated) diff --git a/ingestion/tests/integration/ometa/test_ometa_storage_api.py b/ingestion/tests/integration/ometa/test_ometa_storage_api.py index b17aa907201..d9fe2473e87 100644 --- a/ingestion/tests/integration/ometa/test_ometa_storage_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_storage_api.py @@ -129,7 +129,7 @@ class OMetaObjectStoreTest(TestCase): res_create = self.metadata.create_or_update(data=self.create) - updated = self.create.dict(exclude_unset=True) + updated = self.create.model_dump(exclude_unset=True) updated["owner"] = self.owner updated_entity = CreateContainerRequest(**updated) diff --git a/ingestion/tests/integration/ometa/test_ometa_table_api.py b/ingestion/tests/integration/ometa/test_ometa_table_api.py index 0e47ba9221a..8f7c8d58045 100644 --- a/ingestion/tests/integration/ometa/test_ometa_table_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_table_api.py @@ -231,7 +231,7 @@ class OMetaTableTest(TestCase): res_create = self.metadata.create_or_update(data=self.create) - updated = self.create.dict(exclude_unset=True) + updated = self.create.model_dump(exclude_unset=True) updated["owner"] = self.owner updated_entity = CreateTableRequest(**updated) diff --git a/ingestion/tests/integration/ometa/test_ometa_topic_api.py b/ingestion/tests/integration/ometa/test_ometa_topic_api.py index 1d3fdd6958c..17ee42641b2 100644 --- a/ingestion/tests/integration/ometa/test_ometa_topic_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_topic_api.py @@ -130,7 +130,7 @@ class OMetaTopicTest(TestCase): res_create = self.metadata.create_or_update(data=self.create) - updated = self.create.dict(exclude_unset=True) + updated = self.create.model_dump(exclude_unset=True) updated["owner"] = self.owner updated_entity = CreateTopicRequest(**updated) diff --git a/ingestion/tests/integration/postgres/test_data_quality.py b/ingestion/tests/integration/postgres/test_data_quality.py index 9f5ee475f5b..34996344eac 100644 --- a/ingestion/tests/integration/postgres/test_data_quality.py +++ b/ingestion/tests/integration/postgres/test_data_quality.py @@ -133,7 +133,7 @@ def test_incompatible_column_type(ingest_metadata, metadata: OpenMetadata, db_se "entityFullyQualifiedName": f"{db_service.fullyQualifiedName.root}.dvdrental.public.customer", } }, - "serviceConnection": db_service.connection.dict(), + "serviceConnection": db_service.connection.model_dump(), }, "processor": { "type": "orm-test-runner", @@ -166,7 +166,7 @@ def test_incompatible_column_type(ingest_metadata, metadata: OpenMetadata, db_se }, "workflowConfig": { "loggerLevel": "DEBUG", - "openMetadataServerConfig": metadata.config.dict(), + "openMetadataServerConfig": metadata.config.model_dump(), }, } test_suite_procesor = TestSuiteWorkflow.create(workflow_config) diff --git a/ingestion/tests/integration/postgres/test_postgres.py b/ingestion/tests/integration/postgres/test_postgres.py index fe0d662caed..d35e4b6fa07 100644 --- a/ingestion/tests/integration/postgres/test_postgres.py +++ b/ingestion/tests/integration/postgres/test_postgres.py @@ -145,7 +145,7 @@ def test_ingest_query_log(db_service, ingest_metadata, metadata: OpenMetadata): "sink": {"type": "metadata-rest", "config": {}}, "workflowConfig": { "loggerLevel": "DEBUG", - "openMetadataServerConfig": metadata.config.dict(), + "openMetadataServerConfig": metadata.config.model_dump(), }, } metadata_ingestion = MetadataWorkflow.create(workflow_config) @@ -199,7 +199,7 @@ def run_profiler_workflow(ingest_metadata, db_service, metadata): loggerLevel=LogLevels.DEBUG, openMetadataServerConfig=metadata.config ), ) - metadata_ingestion = ProfilerWorkflow.create(workflow_config.dict()) + metadata_ingestion = ProfilerWorkflow.create(workflow_config.model_dump()) search_cache.clear() metadata_ingestion.execute() return @@ -212,7 +212,7 @@ def ingest_query_usage(ingest_metadata, db_service, metadata): "source": { "type": "postgres-usage", "serviceName": db_service.fullyQualifiedName.root, - "serviceConnection": db_service.connection.dict(), + "serviceConnection": db_service.connection.model_dump(), "sourceConfig": { "config": {"type": DatabaseUsageConfigType.DatabaseUsage.value} }, @@ -233,7 +233,7 @@ def ingest_query_usage(ingest_metadata, db_service, metadata): "sink": {"type": "metadata-rest", "config": {}}, "workflowConfig": { "loggerLevel": "DEBUG", - "openMetadataServerConfig": metadata.config.dict(), + "openMetadataServerConfig": metadata.config.model_dump(), }, } workflow = UsageWorkflow.create(workflow_config) @@ -278,7 +278,7 @@ def run_usage_workflow(db_service, metadata): "source": { "type": "postgres-usage", "serviceName": db_service.fullyQualifiedName.root, - "serviceConnection": db_service.connection.dict(), + "serviceConnection": db_service.connection.model_dump(), "sourceConfig": { "config": {"type": DatabaseUsageConfigType.DatabaseUsage.value} }, @@ -299,7 +299,7 @@ def run_usage_workflow(db_service, metadata): "sink": {"type": "metadata-rest", "config": {}}, "workflowConfig": { "loggerLevel": "DEBUG", - "openMetadataServerConfig": metadata.config.dict(), + "openMetadataServerConfig": metadata.config.model_dump(), }, } workflow = UsageWorkflow.create(workflow_config) @@ -317,7 +317,7 @@ def test_usage_delete_usage(db_service, ingest_postgres_lineage, metadata): "source": { "type": "postgres-usage", "serviceName": db_service.fullyQualifiedName.root, - "serviceConnection": db_service.connection.dict(), + "serviceConnection": db_service.connection.model_dump(), "sourceConfig": { "config": {"type": DatabaseUsageConfigType.DatabaseUsage.value} }, @@ -338,7 +338,7 @@ def test_usage_delete_usage(db_service, ingest_postgres_lineage, metadata): "sink": {"type": "metadata-rest", "config": {}}, "workflowConfig": { "loggerLevel": "DEBUG", - "openMetadataServerConfig": metadata.config.dict(), + "openMetadataServerConfig": metadata.config.model_dump(), }, } workflow = UsageWorkflow.create(workflow_config) diff --git a/ingestion/tests/integration/profiler/test_dynamodb.py b/ingestion/tests/integration/profiler/test_dynamodb.py index 56d942423e2..d6c8e6df2ea 100644 --- a/ingestion/tests/integration/profiler/test_dynamodb.py +++ b/ingestion/tests/integration/profiler/test_dynamodb.py @@ -73,7 +73,7 @@ def test_sample_data(db_service, db_fqn, metadata): }, "workflowConfig": { "loggerLevel": LogLevels.DEBUG, - "openMetadataServerConfig": metadata.config.dict(), + "openMetadataServerConfig": metadata.config.model_dump(), }, } profiler_workflow = ProfilerWorkflow.create(workflow_config) diff --git a/ingestion/tests/integration/sql_server/conftest.py b/ingestion/tests/integration/sql_server/conftest.py index 3e46d6dee3b..a2831137b65 100644 --- a/ingestion/tests/integration/sql_server/conftest.py +++ b/ingestion/tests/integration/sql_server/conftest.py @@ -136,7 +136,7 @@ def ingest_metadata(mssql_container, metadata: OpenMetadata, request): "sink": {"type": "metadata-rest", "config": {}}, "workflowConfig": { "loggerLevel": "DEBUG", - "openMetadataServerConfig": metadata.config.dict(), + "openMetadataServerConfig": metadata.config.model_dump(), }, } metadata_ingestion = MetadataWorkflow.create(workflow_config) @@ -183,7 +183,7 @@ def run_lineage_workflow( "sink": {"type": "metadata-rest", "config": {}}, "workflowConfig": { "loggerLevel": "INFO", - "openMetadataServerConfig": metadata.config.dict(), + "openMetadataServerConfig": metadata.config.model_dump(), }, } metadata_ingestion = MetadataWorkflow.create(workflow_config) diff --git a/ingestion/tests/unit/data_insight/test_entity_report_processor.py b/ingestion/tests/unit/data_insight/test_entity_report_processor.py index 935ad59f2cd..7b18b1b0c51 100644 --- a/ingestion/tests/unit/data_insight/test_entity_report_processor.py +++ b/ingestion/tests/unit/data_insight/test_entity_report_processor.py @@ -194,7 +194,7 @@ class EntityReportProcessorTest(unittest.TestCase): flat_result.timestamp = Timestamp(1695324826495) processed.append(flat_result) assert all( - k in flat_result.data.dict() + k in flat_result.data.model_dump() for k in [ "entityType", "entityTier", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 4e92f39e6b5..ed02a510efc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -448,12 +448,12 @@ public class OpenMetadataApplication extends Application