yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-15 19:03:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): remove alphabet field from allow/deny config (#5629)

Browse Source
This commit is contained in:
Harshal Sheth 2022-08-16 03:55:17 +00:00 committed by GitHub
parent dc08bedd6e
commit dfd0d1581b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
metadata-ingestion/src/datahub/configuration/common.py
View File

@ -1,7 +1,7 @@
import re import re
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from enum import Enum from enum import Enum
from typing import IO, Any, Dict, List, Optional, Pattern, cast from typing import IO, Any, ClassVar, Dict, List, Optional, Pattern, cast
from pydantic import BaseModel, Extra, validator from pydantic import BaseModel, Extra, validator
from pydantic.fields import Field from pydantic.fields import Field
@ -121,6 +121,12 @@ class OauthConfiguration(ConfigModel):
class AllowDenyPattern(ConfigModel): class AllowDenyPattern(ConfigModel):
"""A class to store allow deny regexes""" """A class to store allow deny regexes"""
# This regex is used to check if a given rule is a regex expression or a literal.
# Note that this is not a perfect check. For example, the '.' character should
# be considered a regex special character, but it's used frequently in literal
# patterns and hence we allow it anyway.
IS_SIMPLE_REGEX: ClassVar = re.compile(r"^[A-Za-z0-9 _.-]+$")
allow: List[str] = Field( allow: List[str] = Field(
default=[".*"], default=[".*"],
description="List of regex patterns to include in ingestion", description="List of regex patterns to include in ingestion",
@ -133,13 +139,6 @@ class AllowDenyPattern(ConfigModel):
default=True, default=True,
description="Whether to ignore case sensitivity during pattern matching.", description="Whether to ignore case sensitivity during pattern matching.",
) # Name comparisons should default to ignoring case ) # Name comparisons should default to ignoring case
alphabet: str = Field(
default="[A-Za-z0-9 _.-]", description="Allowed alphabets pattern"
)
@property
def alphabet_pattern(self) -> Pattern:
return re.compile(f"^{self.alphabet}+$")
@property @property
def regex_flags(self) -> int: def regex_flags(self) -> int:
@ -167,7 +166,7 @@ class AllowDenyPattern(ConfigModel):
much more efficient in some cases. much more efficient in some cases.
""" """
return all( return all(
self.alphabet_pattern.match(allow_pattern) for allow_pattern in self.allow self.IS_SIMPLE_REGEX.match(allow_pattern) for allow_pattern in self.allow
) )
def get_allowed_list(self) -> List[str]: def get_allowed_list(self) -> List[str]: