Add config file env var loading tests (#8799)

* Add env var load test

* Add env var load test

ingestion/tests/unit/metadata/cli/test_openmetadata_imports_migration.py

@@ -12,8 +12,8 @@
 """
 test import migration cli script
 """
-
 import os
+from pathlib import Path
 from unittest import TestCase
 
 from metadata.cli.openmetadata_imports_migration import (
@@ -25,21 +25,21 @@ class TestOpenmetadataImportsMigration(TestCase):
     """Test class for the cli scrip test"""
 
     store = dict()
-    path_to_ress_dir = os.path.join(os.path.dirname(__file__), "resources")
+    resources_path = Path(__file__).parent.absolute() / "resources"
 
     @classmethod
     def setUpClass(cls) -> None:
-        for root, _, filenames in os.walk(cls.path_to_ress_dir):
+        for root, _, filenames in os.walk(cls.resources_path):
             for filename in filenames:
                 with open(os.path.join(root, filename), "r", encoding="utf-8") as fle:
                     cls.store[os.path.join(root, filename)] = fle.read()
 
     def test_run_openmetadata_imports_migration(self):
         """test the run openmetadata function"""
-        run_openmetadata_imports_migration(self.path_to_ress_dir, True)
+        run_openmetadata_imports_migration(self.resources_path, True)
         failures = []
 
-        for root, _, filenames in os.walk(self.path_to_ress_dir):
+        for root, _, filenames in os.walk(self.resources_path):
             for filename in filenames:
                 if os.path.splitext(filename)[1] == ".py":
                     with open(

ingestion/tests/unit/metadata/common/resources/bigquery.yaml

@@ -0,0 +1,28 @@
+---
+source:
+  type: bigquery
+  serviceName: bigquery_${PROJECT_ID}
+  serviceConnection:
+    config:
+      type: BigQuery
+      credentials:
+        gcsConfig: "${KEY_PATH}"
+  sourceConfig:
+    config:
+      type: DatabaseMetadata
+      markDeletedTables: true
+      includeTables: true
+      includeViews: true
+      databaseFilterPattern:
+        includes:
+          - '${PROJECT_ID}'
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG
+  openMetadataServerConfig:
+    hostPort: "http://localhost:8585/api"
+    authProvider: "openmetadata"
+    securityConfig:
+      jwtToken: ${JWT_TOKEN}

ingestion/tests/unit/metadata/common/test_ingest_file_load.py

@@ -0,0 +1,59 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Validate how we are reading ingestion configs
+"""
+import os
+from pathlib import Path
+from unittest import TestCase, mock
+
+from metadata.config.common import load_config_file
+
+
+class TestIngestionFileLoad(TestCase):
+    """Test class for the ingestion loading test"""
+
+    resources_path = Path(__file__).parent.absolute() / "resources"
+
+    @mock.patch.dict(
+        os.environ,
+        {
+            "PROJECT_ID": "my-project-id",
+            "KEY_PATH": "/random/path",
+            "JWT_TOKEN": "jwt-token",
+        },
+        clear=True,
+    )
+    def test_load_env_vars(self):
+        """Check that we are picking up the right env vars with different quoting marks"""
+        config_file = self.resources_path / "bigquery.yaml"
+        config_dict = load_config_file(config_file)
+
+        self.assertEqual(config_dict["source"]["serviceName"], "bigquery_my-project-id")
+        self.assertEqual(
+            config_dict["source"]["sourceConfig"]["config"]["databaseFilterPattern"][
+                "includes"
+            ][0],
+            "my-project-id",
+        )
+        self.assertEqual(
+            config_dict["source"]["serviceConnection"]["config"]["credentials"][
+                "gcsConfig"
+            ],
+            "/random/path",
+        )
+        self.assertEqual(
+            config_dict["workflowConfig"]["openMetadataServerConfig"]["securityConfig"][
+                "jwtToken"
+            ],
+            "jwt-token",
+        )