2023-03-02 15:02:06 -08:00
|
|
|
from dataclasses import dataclass
|
|
|
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
|
|
2023-10-10 16:08:34 +05:30
|
|
|
from datahub.metadata.schema_classes import EdgeClass, SchemaFieldDataTypeClass
|
2023-03-02 15:02:06 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class Field:
|
|
|
|
|
name: str
|
|
|
|
|
type: SchemaFieldDataTypeClass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class Dataset:
|
|
|
|
|
id: str
|
|
|
|
|
platform: str
|
|
|
|
|
properties: Optional[Dict[Any, Any]] = None
|
|
|
|
|
schema_metadata: Optional[List[Field]] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class Task:
|
|
|
|
|
name: str
|
|
|
|
|
upstream_edges: List[EdgeClass]
|
|
|
|
|
downstream_edges: List[EdgeClass]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class Pipeline:
|
|
|
|
|
platform: str
|
|
|
|
|
name: str
|
|
|
|
|
tasks: List[Task]
|