Fix oracle bulk view definitions and comments & improve metadata version (#10620)

* Add more info to client version

* lower oracle views and comments

ingestion/src/metadata/__version__.py

@@ -28,7 +28,22 @@ class VersionParsingException(Exception):
     """
 
 
-def get_version_from_string(raw_version: str) -> str:
+def get_client_version_from_string(raw_version: str) -> str:
+    """
+    Given a raw version string, such as `0.10.1.dev0` or
+    `0.11.0-SNAPSHOT`, we should extract the major.minor.patch
+    :param raw_version: raw string with version info
+    :return: Clean version string
+    """
+    try:
+        return re.match(r"\d+.\d+.\d+.\d+", raw_version).group(0)
+    except AttributeError as err:
+        raise VersionParsingException(
+            f"Can't extract client version from {raw_version}: {err}"
+        )
+
+
+def get_server_version_from_string(raw_version: str) -> str:
     """
     Given a raw version string, such as `0.10.1.dev0` or
     `0.11.0-SNAPSHOT`, we should extract the major.minor.patch
@@ -39,7 +54,7 @@ def get_version_from_string(raw_version: str) -> str:
         return re.match(r"\d+.\d+.\d+", raw_version).group(0)
     except AttributeError as err:
         raise VersionParsingException(
-            f"Can't extract version from {raw_version}: {err}"
+            f"Can't extract server version from {raw_version}: {err}"
         )
 
 
@@ -49,7 +64,7 @@ def get_client_version() -> str:
     :return: client version
     """
     raw_version = version("openmetadata-ingestion")
-    return get_version_from_string(raw_version)
+    return get_client_version_from_string(raw_version)
 
 
 def get_metadata_version() -> str:

ingestion/src/metadata/ingestion/ometa/mixins/server_mixin.py

@@ -13,7 +13,7 @@ Mixin class containing Server and client specific methods
 
 To be used by OpenMetadata class
 """
-from metadata.__version__ import get_client_version, get_version_from_string
+from metadata.__version__ import get_client_version, get_server_version_from_string
 from metadata.ingestion.ometa.client import REST
 from metadata.utils.logger import ometa_logger
 
@@ -53,7 +53,7 @@ class OMetaServerMixin:
                 "Cannot Find Version at api/v1/system/version."
                 + " If running the server in DEV mode locally, make sure to `mvn clean install`."
             )
-        return get_version_from_string(raw_version)
+        return get_server_version_from_string(raw_version)
 
     def validate_versions(self) -> None:
         """
@@ -65,7 +65,9 @@ class OMetaServerMixin:
         server_version = self.get_server_version()
         client_version = get_client_version()
 
-        if server_version != client_version:
+        # Server version will be 0.13.2, vs 0.13.2.X from the client.
+        # If the server version is contained in the client version, then we're good to go
+        if server_version not in client_version:
             raise VersionMismatchException(
                 f"Server version is {server_version} vs. Client version {client_version}. Both should match."
             )

ingestion/src/metadata/ingestion/source/database/oracle/metadata.py

@@ -37,26 +37,39 @@ from metadata.utils.sqlalchemy_utils import (
 
 @reflection.cache
 def get_table_comment(
-    self, connection, table_name, schema=None, resolve_synonyms=False, dblink="", **kw
+    self,
+    connection,
+    table_name: str,
+    schema: str = None,
+    resolve_synonyms=False,
+    dblink="",
+    **kw,
 ):  # pylint: disable=unused-argument
     return get_table_comment_wrapper(
         self,
         connection,
-        table_name=table_name,
-        schema=schema,
+        table_name=table_name.lower(),
+        schema=schema.lower() if schema else None,
         query=ORACLE_ALL_TABLE_COMMENTS,
     )
 
 
 @reflection.cache
 def get_view_definition(
-    self, connection, view_name, schema=None, resolve_synonyms=False, dblink="", **kw
+    self,
+    connection,
+    view_name: str,
+    schema: str = None,
+    resolve_synonyms=False,
+    dblink="",
+    **kw,
 ):  # pylint: disable=unused-argument
+
     return get_view_definition_wrapper(
         self,
         connection,
-        table_name=view_name,
-        schema=schema,
+        table_name=view_name.lower(),
+        schema=schema.lower() if schema else None,
         query=ORACLE_ALL_VIEW_DEFINITIONS,
     )
 

ingestion/src/metadata/ingestion/source/database/oracle/queries.py

@@ -15,8 +15,8 @@ SQL Queries used during ingestion
 ORACLE_ALL_TABLE_COMMENTS = """
 SELECT 
 	comments table_comment,
-	table_name,
-	owner "schema" 	
+	LOWER(table_name) "table_name",
+	LOWER(owner) "schema" 	
 FROM all_tab_comments
 where comments is not null and owner not in ('SYSTEM', 'SYS')
 """
@@ -24,8 +24,8 @@ where comments is not null and owner not in ('SYSTEM', 'SYS')
 
 ORACLE_ALL_VIEW_DEFINITIONS = """
 SELECT 
-	view_name, 
-	owner "schema",
+	LOWER(view_name) "view_name", 
+	LOWER(owner) "schema",
 	text view_def 
 FROM all_views 
 where text is not null and owner not in ('SYSTEM', 'SYS')

ingestion/tests/unit/test_version.py

@@ -14,7 +14,10 @@ Validate Server Mixin version methods
 
 from unittest import TestCase
 
-from metadata.__version__ import get_version_from_string
+from metadata.__version__ import (
+    get_client_version_from_string,
+    get_server_version_from_string,
+)
 
 
 class OMetaVersionTest(TestCase):
@@ -26,8 +29,26 @@ class OMetaVersionTest(TestCase):
         """
         We should be able to parse regular version responses
         """
-        self.assertEqual("0.11.0", get_version_from_string("0.11.0.dev0"))
-        self.assertEqual("0.11.0", get_version_from_string("0.11.0"))
-        self.assertEqual("1111.11.111", get_version_from_string("1111.11.111"))
-        self.assertEqual("1111.11.111", get_version_from_string("1111.11.111-SNAPSHOT"))
-        self.assertEqual("0.11.1", get_version_from_string("0.11.1.0.0.1.patch"))
+        self.assertEqual("0.11.0", get_server_version_from_string("0.11.0.dev0"))
+        self.assertEqual("0.11.0", get_server_version_from_string("0.11.0"))
+        self.assertEqual("1111.11.111", get_server_version_from_string("1111.11.111"))
+        self.assertEqual(
+            "1111.11.111", get_server_version_from_string("1111.11.111-SNAPSHOT")
+        )
+        self.assertEqual("0.11.1", get_server_version_from_string("0.11.1.0.0.1.patch"))
+
+    def test_get_client_version_from_string(self):
+        """
+        We should be able to parse regular version responses
+        """
+        self.assertEqual("0.13.2.5", get_client_version_from_string("0.13.2.5.dev0"))
+        self.assertEqual("0.11.0.1", get_client_version_from_string("0.11.0.1"))
+        self.assertEqual(
+            "1111.11.111.1", get_client_version_from_string("1111.11.111.1")
+        )
+        self.assertEqual(
+            "1111.11.111.2", get_client_version_from_string("1111.11.111.2-SNAPSHOT")
+        )
+        self.assertEqual(
+            "0.11.1.0", get_client_version_from_string("0.11.1.0.0.1.patch")
+        )