mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-06-27 04:22:05 +00:00
d8e2187980
* pydantic v2 * pydanticv2 * fix parser * fix annotated * fix model dumping * mysql ingestion * clean root models * clean root models * bump airflow * bump airflow * bump airflow * optionals * optionals * optionals * jdk * airflow migrate * fab provider * fab provider * fab provider * some more fixes * fixing tests and imports * model_dump and model_validate * model_dump and model_validate * model_dump and model_validate * union * pylint * pylint * integration tests * fix CostAnalysisReportData * integration tests * tests * missing defaults * missing defaults
84 lines
4.0 KiB
Python
84 lines
4.0 KiB
Python
# Copyright 2021 Collate
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""
|
|
This script generates the Python models from the JSON Schemas definition. Additionally, it replaces the `SecretStr`
|
|
pydantic class used for the password fields with the `CustomSecretStr` pydantic class which retrieves the secrets
|
|
from a configured secrets' manager.
|
|
"""
|
|
import datamodel_code_generator.model.pydantic
|
|
from datamodel_code_generator.imports import Import
|
|
import os
|
|
import re
|
|
|
|
|
|
datamodel_code_generator.model.pydantic.types.IMPORT_SECRET_STR = Import.from_full_path(
|
|
"metadata.ingestion.models.custom_pydantic.CustomSecretStr"
|
|
)
|
|
|
|
from datamodel_code_generator.__main__ import main
|
|
|
|
current_directory = os.getcwd()
|
|
ingestion_path = "./" if current_directory.endswith("/ingestion") else "ingestion/"
|
|
directory_root = "../" if current_directory.endswith("/ingestion") else "./"
|
|
|
|
UTF_8 = "UTF-8"
|
|
UNICODE_REGEX_REPLACEMENT_FILE_PATHS = [
|
|
f"{ingestion_path}src/metadata/generated/schema/entity/classification/tag.py",
|
|
f"{ingestion_path}src/metadata/generated/schema/entity/events/webhook.py",
|
|
f"{ingestion_path}src/metadata/generated/schema/entity/teams/user.py",
|
|
f"{ingestion_path}src/metadata/generated/schema/entity/type.py",
|
|
f"{ingestion_path}src/metadata/generated/schema/type/basic.py",
|
|
]
|
|
|
|
args = f"--custom-template-dir {directory_root}/ingestion/codegen_custom_templates --input {directory_root}openmetadata-spec/src/main/resources/json/schema --output-model-type pydantic_v2.BaseModel --use-annotated --base-class metadata.ingestion.models.custom_pydantic.BaseModel --input-file-type jsonschema --output {ingestion_path}src/metadata/generated/schema --set-default-enum-member".split(" ")
|
|
|
|
main(args)
|
|
|
|
for file_path in UNICODE_REGEX_REPLACEMENT_FILE_PATHS:
|
|
with open(file_path, "r", encoding=UTF_8) as file_:
|
|
content = file_.read()
|
|
# Python now requires to move the global flags at the very start of the expression
|
|
content = content.replace("(?U)", "(?u)")
|
|
with open(file_path, "w", encoding=UTF_8) as file_:
|
|
file_.write(content)
|
|
|
|
# Until https://github.com/koxudaxi/datamodel-code-generator/issues/1895
|
|
# TODO: This has been merged but `Union` is still not there. We'll need to validate
|
|
MISSING_IMPORTS = [f"{ingestion_path}src/metadata/generated/schema/entity/applications/app.py",]
|
|
WRITE_AFTER = "from __future__ import annotations"
|
|
|
|
for file_path in MISSING_IMPORTS:
|
|
with open(file_path, "r", encoding=UTF_8) as file_:
|
|
lines = file_.readlines()
|
|
with open(file_path, "w", encoding=UTF_8) as file_:
|
|
for line in lines:
|
|
file_.write(line)
|
|
if line.strip() == WRITE_AFTER:
|
|
file_.write("from typing import Union # custom generate import\n\n")
|
|
|
|
|
|
# unsupported rust regex pattern for pydantic v2
|
|
# https://docs.pydantic.dev/2.7/api/config/#pydantic.config.ConfigDict.regex_engine
|
|
# We'll remove validation from the client and let it fail on the server, rather than on the model generation
|
|
UNSUPPORTED_REGEX_PATTERN_FILE_PATHS = [
|
|
f"{ingestion_path}src/metadata/generated/schema/type/basic.py",
|
|
f"{ingestion_path}src/metadata/generated/schema/entity/data/searchIndex.py",
|
|
f"{ingestion_path}src/metadata/generated/schema/entity/data/table.py",
|
|
]
|
|
|
|
for file_path in UNSUPPORTED_REGEX_PATTERN_FILE_PATHS:
|
|
with open(file_path, "r", encoding=UTF_8) as file_:
|
|
content = file_.read()
|
|
content = content.replace("pattern='^((?!::).)*$',", "")
|
|
with open(file_path, "w", encoding=UTF_8) as file_:
|
|
file_.write(content)
|
|
|