yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-02 13:53:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): add pydantic version upper bound (#5775)

Browse Source
This commit is contained in:
Harshal Sheth 2022-08-30 18:07:23 +00:00 committed by GitHub
parent 47aa32a8a5
commit 138846acca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/setup.py
View File

@ -224,7 +224,7 @@ plugins: Dict[str, Set[str]] = {
| bigquery_common | bigquery_common
| {"sqlalchemy-bigquery>=1.4.1", "sqllineage==1.3.5", "sqlparse"}, | {"sqlalchemy-bigquery>=1.4.1", "sqllineage==1.3.5", "sqlparse"},
"bigquery-usage": bigquery_common | usage_common | {"cachetools"}, "bigquery-usage": bigquery_common | usage_common | {"cachetools"},
"bigquery-beta": bigquery_common| {"sql_metadata"}, "bigquery-beta": bigquery_common | {"sql_metadata"},
"clickhouse": sql_common | {"clickhouse-sqlalchemy==0.1.8"}, "clickhouse": sql_common | {"clickhouse-sqlalchemy==0.1.8"},
"clickhouse-usage": sql_common "clickhouse-usage": sql_common
| usage_common | usage_common
@ -350,7 +350,8 @@ base_dev_requirements = {
"mypy>=0.950", "mypy>=0.950",
# pydantic 1.8.2 is incompatible with mypy 0.910. # pydantic 1.8.2 is incompatible with mypy 0.910.
# See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910. # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.
"pydantic>=1.9.0", # Restricting top version to <1.10 until we can fix our types.
"pydantic >=1.9.0, <1.10",
"pytest>=6.2.2", "pytest>=6.2.2",
"pytest-asyncio>=0.16.0", "pytest-asyncio>=0.16.0",
"pytest-cov>=2.8.1", "pytest-cov>=2.8.1",

2
metadata-ingestion/src/datahub/configuration/common.py
View File

@ -105,7 +105,7 @@ class OauthConfiguration(ConfigModel):
scopes: Optional[List[str]] = Field( scopes: Optional[List[str]] = Field(
description="scopes required to connect to snowflake" description="scopes required to connect to snowflake"
) )
use_certificate: Optional[str] = Field( use_certificate: Optional[bool] = Field(
description="Do you want to use certificate and private key to authenticate using oauth", description="Do you want to use certificate and private key to authenticate using oauth",
default=False, default=False,
) )

5
metadata-ingestion/src/datahub/ingestion/source/azure/azure_common.py
View File

@ -27,18 +27,23 @@ class AdlsSourceConfig(ConfigModel):
) )
account_key: Optional[str] = Field( account_key: Optional[str] = Field(
description="Azure storage account access key that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**", description="Azure storage account access key that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**",
default=None,
) )
sas_token: Optional[str] = Field( sas_token: Optional[str] = Field(
description="Azure storage account Shared Access Signature (SAS) token that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**", description="Azure storage account Shared Access Signature (SAS) token that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**",
default=None,
) )
client_secret: Optional[str] = Field( client_secret: Optional[str] = Field(
description="Azure client secret that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**", description="Azure client secret that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**",
default=None,
) )
client_id: Optional[str] = Field( client_id: Optional[str] = Field(
description="Azure client (Application) ID required when a `client_secret` is used as a credential.", description="Azure client (Application) ID required when a `client_secret` is used as a credential.",
default=None,
) )
tenant_id: Optional[str] = Field( tenant_id: Optional[str] = Field(
description="Azure tenant (Directory) ID required when a `client_secret` is used as a credential.", description="Azure tenant (Directory) ID required when a `client_secret` is used as a credential.",
default=None,
) )
def get_abfss_url(self, folder_path: str = "") -> str: def get_abfss_url(self, folder_path: str = "") -> str:

2
metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
View File

@ -332,7 +332,7 @@ class SnowflakeConfig(BaseSnowflakeConfig, SQLAlchemyConfig):
self.oauth_config.authority_url, self.oauth_config.authority_url,
self.oauth_config.provider, self.oauth_config.provider,
) )
if self.oauth_config.use_certificate is True: if self.oauth_config.use_certificate:
response = generator.get_token_with_certificate( response = generator.get_token_with_certificate(
private_key_content=str(self.oauth_config.encoded_oauth_public_key), private_key_content=str(self.oauth_config.encoded_oauth_public_key),
public_key_content=str(self.oauth_config.encoded_oauth_private_key), public_key_content=str(self.oauth_config.encoded_oauth_private_key),