fix(ingest): add pydantic version upper bound (#5775)

metadata-ingestion/setup.py

@@ -350,7 +350,8 @@ base_dev_requirements = {
     "mypy>=0.950",
     # pydantic 1.8.2 is incompatible with mypy 0.910.
     # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.
-    "pydantic>=1.9.0",
+    # Restricting top version to <1.10 until we can fix our types.
+    "pydantic >=1.9.0, <1.10",
     "pytest>=6.2.2",
     "pytest-asyncio>=0.16.0",
     "pytest-cov>=2.8.1",

metadata-ingestion/src/datahub/configuration/common.py

@@ -105,7 +105,7 @@ class OauthConfiguration(ConfigModel):
     scopes: Optional[List[str]] = Field(
         description="scopes required to connect to snowflake"
     )
-    use_certificate: Optional[str] = Field(
+    use_certificate: Optional[bool] = Field(
         description="Do you want to use certificate and private key to authenticate using oauth",
         default=False,
     )

metadata-ingestion/src/datahub/ingestion/source/azure/azure_common.py

@@ -27,18 +27,23 @@ class AdlsSourceConfig(ConfigModel):
     )
     account_key: Optional[str] = Field(
         description="Azure storage account access key that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**",
+        default=None,
     )
     sas_token: Optional[str] = Field(
         description="Azure storage account Shared Access Signature (SAS) token that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**",
+        default=None,
     )
     client_secret: Optional[str] = Field(
         description="Azure client secret that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**",
+        default=None,
     )
     client_id: Optional[str] = Field(
         description="Azure client (Application) ID required when a `client_secret` is used as a credential.",
+        default=None,
     )
     tenant_id: Optional[str] = Field(
         description="Azure tenant (Directory) ID required when a `client_secret` is used as a credential.",
+        default=None,
     )
 
     def get_abfss_url(self, folder_path: str = "") -> str:

metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py

@@ -332,7 +332,7 @@ class SnowflakeConfig(BaseSnowflakeConfig, SQLAlchemyConfig):
             self.oauth_config.authority_url,
             self.oauth_config.provider,
         )
-        if self.oauth_config.use_certificate is True:
+        if self.oauth_config.use_certificate:
             response = generator.get_token_with_certificate(
                 private_key_content=str(self.oauth_config.encoded_oauth_public_key),
                 public_key_content=str(self.oauth_config.encoded_oauth_private_key),