diff --git a/metadata-ingestion/src/gometa/ingestion/sink/datahub_kafka.py b/metadata-ingestion/src/gometa/ingestion/sink/datahub_kafka.py index 0860d62bc8..a86776752c 100644 --- a/metadata-ingestion/src/gometa/ingestion/sink/datahub_kafka.py +++ b/metadata-ingestion/src/gometa/ingestion/sink/datahub_kafka.py @@ -10,7 +10,6 @@ from confluent_kafka import SerializingProducer from confluent_kafka.serialization import StringSerializer from confluent_kafka.schema_registry import SchemaRegistryClient from confluent_kafka.schema_registry.avro import AvroSerializer -from gometa.metadata import json_converter from gometa.metadata.schema_classes import SCHEMA_JSON_STR from gometa.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent @@ -44,32 +43,25 @@ class DatahubKafkaSink(Sink): self.config = config self.report = SinkReport() - mce_schema = MetadataChangeEvent.RECORD_SCHEMA - - producer_config = { - "bootstrap.servers": self.config.connection.bootstrap, - "schema.registry.url": self.config.connection.schema_registry_url, - **self.config.connection.producer_config, - } - schema_registry_conf = { 'url': self.config.connection.schema_registry_url, **self.config.connection.schema_registry_config, } schema_registry_client = SchemaRegistryClient(schema_registry_conf) - def convert_mce_to_dict(mce, ctx): - tuple_encoding = json_converter.with_tuple_union().to_json_object(mce) + def convert_mce_to_dict(mce: MetadataChangeEvent, ctx): + tuple_encoding = mce.to_obj(tuples=True) return tuple_encoding avro_serializer = AvroSerializer(SCHEMA_JSON_STR, schema_registry_client, to_dict=convert_mce_to_dict) - producer_conf = { + producer_config = { "bootstrap.servers": self.config.connection.bootstrap, 'key.serializer': StringSerializer('utf_8'), 'value.serializer': avro_serializer, + **self.config.connection.producer_config, } - self.producer = SerializingProducer(producer_conf) + self.producer = SerializingProducer(producer_config) @classmethod def create(cls, config_dict, ctx: PipelineContext): diff --git a/metadata-ingestion/src/gometa/ingestion/sink/datahub_rest.py b/metadata-ingestion/src/gometa/ingestion/sink/datahub_rest.py index 68419cf659..517c7faaa0 100644 --- a/metadata-ingestion/src/gometa/ingestion/sink/datahub_rest.py +++ b/metadata-ingestion/src/gometa/ingestion/sink/datahub_rest.py @@ -9,7 +9,6 @@ from requests.exceptions import HTTPError from gometa.ingestion.api.sink import Sink, WriteCallback, SinkReport from gometa.ingestion.api.common import RecordEnvelope, WorkUnit import json -from gometa.metadata import json_converter from gometa.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from gometa.metadata import ( ChartSnapshotClass, @@ -96,7 +95,7 @@ class DatahubRestSink(Sink): mce = record_envelope.record url = self.get_ingest_endpoint(mce) - raw_mce_obj = json_converter.to_json_object(mce.proposedSnapshot) + raw_mce_obj = mce.proposedSnapshot.to_obj() mce_obj = _rest_li_ify(raw_mce_obj) snapshot = {'snapshot': mce_obj} diff --git a/metadata-ingestion/src/gometa/ingestion/sink/file.py b/metadata-ingestion/src/gometa/ingestion/sink/file.py index 318c6f4590..562ea34f1e 100644 --- a/metadata-ingestion/src/gometa/ingestion/sink/file.py +++ b/metadata-ingestion/src/gometa/ingestion/sink/file.py @@ -5,7 +5,6 @@ import os import pathlib import logging import json -from gometa.metadata import json_converter from gometa.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent logger = logging.getLogger(__name__) @@ -44,7 +43,7 @@ class FileSink(Sink): def write_record_async(self, record_envelope: RecordEnvelope[MetadataChangeEvent], write_callback: WriteCallback): mce = record_envelope.record - obj = json_converter.to_json_object(mce, MetadataChangeEvent.RECORD_SCHEMA) + obj = mce.to_obj() if self.wrote_something: self.file.write(',\n') diff --git a/metadata-ingestion/src/gometa/ingestion/source/mce_file.py b/metadata-ingestion/src/gometa/ingestion/source/mce_file.py index 75a731e627..9a942a5b41 100644 --- a/metadata-ingestion/src/gometa/ingestion/source/mce_file.py +++ b/metadata-ingestion/src/gometa/ingestion/source/mce_file.py @@ -4,7 +4,6 @@ from pydantic import BaseModel from typing import Optional, Iterable from gometa.ingestion.api.source import Source, SourceReport from gometa.ingestion.source.metadata_common import MetadataWorkUnit -from gometa.metadata import json_converter from gometa.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent class MetadataFileSourceConfig(BaseModel): @@ -27,7 +26,7 @@ class MetadataFileSource(Source): mce_obj_list = [mce_obj_list] for i, obj in enumerate(mce_obj_list): - mce = json_converter.from_json_object(obj, MetadataChangeEvent.RECORD_SCHEMA) + mce: MetadataChangeEvent = MetadataChangeEvent.from_obj(obj) wu = MetadataWorkUnit(f"file://{self.config.filename}:{i}", mce) self.report.report_workunit(wu) yield wu diff --git a/metadata-ingestion/src/gometa/metadata/schema_classes.py b/metadata-ingestion/src/gometa/metadata/schema_classes.py index 8b192918e5..b95dcd4480 100644 --- a/metadata-ingestion/src/gometa/metadata/schema_classes.py +++ b/metadata-ingestion/src/gometa/metadata/schema_classes.py @@ -75,6 +75,13 @@ class KafkaAuditHeaderClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "KafkaAuditHeaderClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def time(self) -> int: @@ -213,6 +220,13 @@ class ChartInfoClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "ChartInfoClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def title(self) -> str: @@ -339,6 +353,13 @@ class ChartQueryClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "ChartQueryClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def rawQuery(self) -> str: @@ -419,6 +440,13 @@ class AuditStampClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "AuditStampClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def time(self) -> int: @@ -487,6 +515,13 @@ class ChangeAuditStampsClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "ChangeAuditStampsClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def created(self) -> "AuditStampClass": @@ -553,6 +588,13 @@ class CostClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "CostClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def costType(self) -> "CostTypeClass": @@ -609,6 +651,13 @@ class CostCostClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "CostCostClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def costId(self) -> Union[None, float]: @@ -692,6 +741,13 @@ class DeprecationClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DeprecationClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def deprecated(self) -> bool: @@ -768,6 +824,13 @@ class InstitutionalMemoryClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "InstitutionalMemoryClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def elements(self) -> List["InstitutionalMemoryMetadataClass"]: @@ -812,6 +875,13 @@ class InstitutionalMemoryMetadataClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "InstitutionalMemoryMetadataClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def url(self) -> str: @@ -899,6 +969,13 @@ class OwnerClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "OwnerClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def owner(self) -> str: @@ -965,6 +1042,13 @@ class OwnershipClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "OwnershipClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def owners(self) -> List["OwnerClass"]: @@ -1019,6 +1103,13 @@ class OwnershipSourceClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "OwnershipSourceClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def type(self) -> "OwnershipSourceTypeClass": @@ -1095,6 +1186,13 @@ class StatusClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "StatusClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def removed(self) -> bool: @@ -1135,6 +1233,13 @@ class VersionTagClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "VersionTagClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def versionTag(self) -> Union[None, str]: @@ -1182,6 +1287,13 @@ class UDFTransformerClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "UDFTransformerClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def udf(self) -> str: @@ -1234,6 +1346,13 @@ class DashboardInfoClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DashboardInfoClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def title(self) -> str: @@ -1348,6 +1467,13 @@ class DataProcessInfoClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DataProcessInfoClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def inputs(self) -> Union[None, List[str]]: @@ -1406,6 +1532,13 @@ class DatasetDeprecationClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DatasetDeprecationClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def deprecated(self) -> bool: @@ -1488,6 +1621,13 @@ class DatasetFieldMappingClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DatasetFieldMappingClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def created(self) -> "AuditStampClass": @@ -1578,6 +1718,13 @@ class DatasetPropertiesClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DatasetPropertiesClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def description(self) -> Union[None, str]: @@ -1654,6 +1801,13 @@ class DatasetUpstreamLineageClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DatasetUpstreamLineageClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def fieldMappings(self) -> List["DatasetFieldMappingClass"]: @@ -1698,6 +1852,13 @@ class UpstreamClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "UpstreamClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def auditStamp(self) -> "AuditStampClass": @@ -1762,6 +1923,13 @@ class UpstreamLineageClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "UpstreamLineageClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def upstreams(self) -> List["UpstreamClass"]: @@ -1808,6 +1976,13 @@ class CorpGroupInfoClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "CorpGroupInfoClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def email(self) -> str: @@ -1890,6 +2065,13 @@ class CorpUserEditableInfoClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "CorpUserEditableInfoClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def aboutMe(self) -> Union[None, str]: @@ -1986,6 +2168,13 @@ class CorpUserInfoClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "CorpUserInfoClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def active(self) -> bool: @@ -2148,6 +2337,13 @@ class ChartSnapshotClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "ChartSnapshotClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def urn(self) -> str: @@ -2202,6 +2398,13 @@ class CorpGroupSnapshotClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "CorpGroupSnapshotClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def urn(self) -> str: @@ -2256,6 +2459,13 @@ class CorpUserSnapshotClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "CorpUserSnapshotClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def urn(self) -> str: @@ -2310,6 +2520,13 @@ class DashboardSnapshotClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DashboardSnapshotClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def urn(self) -> str: @@ -2364,6 +2581,13 @@ class DataProcessSnapshotClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DataProcessSnapshotClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def urn(self) -> str: @@ -2418,6 +2642,13 @@ class DatasetSnapshotClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DatasetSnapshotClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def urn(self) -> str: @@ -2472,6 +2703,13 @@ class MLFeatureSnapshotClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "MLFeatureSnapshotClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def urn(self) -> str: @@ -2526,6 +2764,13 @@ class MLModelSnapshotClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "MLModelSnapshotClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def urn(self) -> str: @@ -2582,6 +2827,13 @@ class BaseDataClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "BaseDataClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def dataset(self) -> str: @@ -2650,6 +2902,13 @@ class CaveatDetailsClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "CaveatDetailsClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def needsFurtherTesting(self) -> Union[None, bool]: @@ -2720,6 +2979,13 @@ class CaveatsAndRecommendationsClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "CaveatsAndRecommendationsClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def caveats(self) -> Union[None, "CaveatDetailsClass"]: @@ -2792,6 +3058,13 @@ class EthicalConsiderationsClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "EthicalConsiderationsClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def data(self) -> Union[None, List[str]]: @@ -2880,6 +3153,13 @@ class EvaluationDataClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "EvaluationDataClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def evaluationData(self) -> List["BaseDataClass"]: @@ -2924,6 +3204,13 @@ class IntendedUseClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "IntendedUseClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def primaryUses(self) -> Union[None, List[str]]: @@ -3000,6 +3287,13 @@ class MLFeaturePropertiesClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "MLFeaturePropertiesClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def description(self) -> Union[None, str]: @@ -3066,6 +3360,13 @@ class MLModelFactorPromptsClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "MLModelFactorPromptsClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def relevantFactors(self) -> Union[None, List["MLModelFactorsClass"]]: @@ -3122,6 +3423,13 @@ class MLModelFactorsClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "MLModelFactorsClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def groups(self) -> Union[None, List[str]]: @@ -3204,6 +3512,13 @@ class MLModelPropertiesClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "MLModelPropertiesClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def description(self) -> Union[None, str]: @@ -3318,6 +3633,13 @@ class MetricsClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "MetricsClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def performanceMeasures(self) -> Union[None, List[str]]: @@ -3372,6 +3694,13 @@ class QuantitativeAnalysesClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "QuantitativeAnalysesClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def unitaryResults(self) -> Union[None, str]: @@ -3424,6 +3753,13 @@ class SourceCodeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "SourceCodeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def sourceCode(self) -> List["SourceCodeUrlClass"]: @@ -3466,6 +3802,13 @@ class SourceCodeUrlClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "SourceCodeUrlClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def type(self) -> "SourceCodeUrlTypeClass": @@ -3526,6 +3869,13 @@ class TrainingDataClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "TrainingDataClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def trainingData(self) -> List["BaseDataClass"]: @@ -3570,6 +3920,13 @@ class MetadataChangeEventClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "MetadataChangeEventClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def auditHeader(self) -> Union[None, "KafkaAuditHeaderClass"]: @@ -3634,6 +3991,13 @@ class ArrayTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "ArrayTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def nestedType(self) -> Union[None, List[str]]: @@ -3674,6 +4038,13 @@ class BinaryJsonSchemaClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "BinaryJsonSchemaClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def schema(self) -> str: @@ -3713,6 +4084,13 @@ class BooleanTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "BooleanTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + class BytesTypeClass(DictWrapper): """Bytes field type.""" @@ -3740,6 +4118,13 @@ class BytesTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "BytesTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + class DatasetFieldForeignKeyClass(DictWrapper): """For non-urn based foregin keys.""" @@ -3772,6 +4157,13 @@ class DatasetFieldForeignKeyClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "DatasetFieldForeignKeyClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def parentDataset(self) -> str: @@ -3835,6 +4227,13 @@ class EnumTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "EnumTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + class EspressoSchemaClass(DictWrapper): """Schema text of an espresso table schema.""" @@ -3865,6 +4264,13 @@ class EspressoSchemaClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "EspressoSchemaClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def documentSchema(self) -> str: @@ -3916,6 +4322,13 @@ class FixedTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "FixedTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + class ForeignKeySpecClass(DictWrapper): """Description of a foreign key in a schema.""" @@ -3944,6 +4357,13 @@ class ForeignKeySpecClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "ForeignKeySpecClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def foreignKey(self) -> Union["DatasetFieldForeignKeyClass", "UrnForeignKeyClass"]: @@ -3984,6 +4404,13 @@ class KafkaSchemaClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "KafkaSchemaClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def documentSchema(self) -> str: @@ -4026,6 +4453,13 @@ class KeyValueSchemaClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "KeyValueSchemaClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def keySchema(self) -> str: @@ -4080,6 +4514,13 @@ class MapTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "MapTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def keyType(self) -> Union[None, str]: @@ -4132,6 +4573,13 @@ class MySqlDDLClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "MySqlDDLClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def tableSchema(self) -> str: @@ -4171,6 +4619,13 @@ class NullTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "NullTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + class NumberTypeClass(DictWrapper): """Number data type: long, integer, short, etc..""" @@ -4198,6 +4653,13 @@ class NumberTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "NumberTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + class OracleDDLClass(DictWrapper): """Schema holder for oracle data definition language that describes an oracle table.""" @@ -4226,6 +4688,13 @@ class OracleDDLClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "OracleDDLClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def tableSchema(self) -> str: @@ -4266,6 +4735,13 @@ class OrcSchemaClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "OrcSchemaClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def schema(self) -> str: @@ -4306,6 +4782,13 @@ class OtherSchemaClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "OtherSchemaClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def rawSchema(self) -> str: @@ -4346,6 +4829,13 @@ class PrestoDDLClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "PrestoDDLClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def rawSchema(self) -> str: @@ -4385,6 +4875,13 @@ class RecordTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "RecordTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + class SchemaFieldClass(DictWrapper): """SchemaField to describe metadata related to dataset schema. Schema normalization rules: http://go/tms-schema""" @@ -4425,6 +4922,13 @@ class SchemaFieldClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "SchemaFieldClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def fieldPath(self) -> str: @@ -4537,6 +5041,13 @@ class SchemaFieldDataTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "SchemaFieldDataTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def type(self) -> Union["BooleanTypeClass", "FixedTypeClass", "StringTypeClass", "BytesTypeClass", "NumberTypeClass", "EnumTypeClass", "NullTypeClass", "MapTypeClass", "ArrayTypeClass", "UnionTypeClass", "RecordTypeClass"]: @@ -4601,6 +5112,13 @@ class SchemaMetadataClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "SchemaMetadataClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def schemaName(self) -> str: @@ -4784,6 +5302,13 @@ class SchemalessClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "SchemalessClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + class StringTypeClass(DictWrapper): """String field type.""" @@ -4811,6 +5336,13 @@ class StringTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "StringTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + class UnionTypeClass(DictWrapper): """Union field type.""" @@ -4839,6 +5371,13 @@ class UnionTypeClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "UnionTypeClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def nestedTypes(self) -> Union[None, List[str]]: @@ -4879,6 +5418,13 @@ class UrnForeignKeyClass(DictWrapper): getattr(self, key) setattr(self, key, value) + @classmethod + def from_obj(cls, obj: dict, tuples=False) -> "UrnForeignKeyClass": + ... + + def to_obj(self, tuples=False) -> dict: + ... + @property def currentFieldPath(self) -> str: