yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-01 13:23:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(ingest): remove typing workarounds (#6108)

Browse Source 
Possible now that we're on mypy 0.980.
This commit is contained in:
Harshal Sheth 2022-10-05 04:12:05 +00:00 committed by GitHub
parent 4ee3ef14ef
commit 164e47a2e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 6 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
metadata-ingestion/src/datahub/ingestion/api/committable.py
View File

@ -11,17 +11,10 @@ class CommitPolicy(Enum):
@dataclass @dataclass
class _CommittableConcrete: class Committable(ABC):
name: str name: str
commit_policy: CommitPolicy commit_policy: CommitPolicy
committed: bool committed: bool = False
# The concrete portion Committable is separated from the abstract portion due to
# https://github.com/python/mypy/issues/5374#issuecomment-568335302.
class Committable(_CommittableConcrete, ABC):
def __init__(self, name: str, commit_policy: CommitPolicy):
super(Committable, self).__init__(name, commit_policy, committed=False)
@abstractmethod @abstractmethod
def commit(self) -> None: def commit(self) -> None:
@ -34,25 +27,15 @@ StateType = TypeVar("StateType")
FilterType = TypeVar("FilterType") FilterType = TypeVar("FilterType")
class _StatefulCommittableConcrete(Generic[StateType]):
def __init__(self, state_to_commit: StateType):
self.state_to_commit: StateType = state_to_commit
class StatefulCommittable( class StatefulCommittable(
Committable, Committable,
_StatefulCommittableConcrete[StateType],
Generic[StateKeyType, StateType, FilterType], Generic[StateKeyType, StateType, FilterType],
): ):
def __init__( def __init__(
self, name: str, commit_policy: CommitPolicy, state_to_commit: StateType self, name: str, commit_policy: CommitPolicy, state_to_commit: StateType
): ):
# _ConcreteCommittable will be the first from this class. super().__init__(name=name, commit_policy=commit_policy)
super(StatefulCommittable, self).__init__( self.state_to_commit: StateType = state_to_commit
name=name, commit_policy=commit_policy
)
# _StatefulCommittableConcrete will be after _CommittableConcrete in the __mro__.
super(_CommittableConcrete, self).__init__(state_to_commit=state_to_commit)
def has_successfully_committed(self) -> bool: def has_successfully_committed(self) -> bool:
return bool(not self.state_to_commit or self.committed) return bool(not self.state_to_commit or self.committed)

7
metadata-ingestion/src/datahub/ingestion/api/common.py
View File

@ -33,14 +33,9 @@ class EndOfStream(ControlRecord):
@dataclass @dataclass
class _WorkUnitId(metaclass=ABCMeta): class WorkUnit(metaclass=ABCMeta):
id: str id: str
# For information on why the WorkUnit class is structured this way
# and is separating the dataclass portion from the abstract methods, see
# https://github.com/python/mypy/issues/5374#issuecomment-568335302.
class WorkUnit(_WorkUnitId, metaclass=ABCMeta):
@abstractmethod @abstractmethod
def get_metadata(self) -> dict: def get_metadata(self) -> dict:
pass pass

3
metadata-ingestion/src/datahub/ingestion/api/source.py
View File

@ -114,8 +114,7 @@ class Extractor(Generic[WorkUnitType, ExtractorConfig], Closeable, metaclass=ABC
pass pass
# See https://github.com/python/mypy/issues/5374 for why we suppress this mypy error. @dataclass
@dataclass # type: ignore[misc]
class Source(Closeable, metaclass=ABCMeta): class Source(Closeable, metaclass=ABCMeta):
ctx: PipelineContext ctx: PipelineContext