From cd6ece1cf4a026f7737fc5743d1bd13566f0d7d6 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 16 Mar 2023 14:11:22 +0100 Subject: [PATCH] Fix oracle bulk view definitions and comments & improve metadata version (#10620) * Add more info to client version * lower oracle views and comments --- ingestion/src/metadata/__version__.py | 21 ++++++++++-- .../ingestion/ometa/mixins/server_mixin.py | 8 +++-- .../source/database/oracle/metadata.py | 25 ++++++++++---- .../source/database/oracle/queries.py | 8 ++--- ingestion/tests/unit/test_version.py | 33 +++++++++++++++---- 5 files changed, 73 insertions(+), 22 deletions(-) diff --git a/ingestion/src/metadata/__version__.py b/ingestion/src/metadata/__version__.py index 9081d0b168a..c81446723b1 100644 --- a/ingestion/src/metadata/__version__.py +++ b/ingestion/src/metadata/__version__.py @@ -28,7 +28,22 @@ class VersionParsingException(Exception): """ -def get_version_from_string(raw_version: str) -> str: +def get_client_version_from_string(raw_version: str) -> str: + """ + Given a raw version string, such as `0.10.1.dev0` or + `0.11.0-SNAPSHOT`, we should extract the major.minor.patch + :param raw_version: raw string with version info + :return: Clean version string + """ + try: + return re.match(r"\d+.\d+.\d+.\d+", raw_version).group(0) + except AttributeError as err: + raise VersionParsingException( + f"Can't extract client version from {raw_version}: {err}" + ) + + +def get_server_version_from_string(raw_version: str) -> str: """ Given a raw version string, such as `0.10.1.dev0` or `0.11.0-SNAPSHOT`, we should extract the major.minor.patch @@ -39,7 +54,7 @@ def get_version_from_string(raw_version: str) -> str: return re.match(r"\d+.\d+.\d+", raw_version).group(0) except AttributeError as err: raise VersionParsingException( - f"Can't extract version from {raw_version}: {err}" + f"Can't extract server version from {raw_version}: {err}" ) @@ -49,7 +64,7 @@ def get_client_version() -> str: :return: client version """ raw_version = version("openmetadata-ingestion") - return get_version_from_string(raw_version) + return get_client_version_from_string(raw_version) def get_metadata_version() -> str: diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/server_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/server_mixin.py index 283532d718e..c1a1ab3abd4 100755 --- a/ingestion/src/metadata/ingestion/ometa/mixins/server_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/server_mixin.py @@ -13,7 +13,7 @@ Mixin class containing Server and client specific methods To be used by OpenMetadata class """ -from metadata.__version__ import get_client_version, get_version_from_string +from metadata.__version__ import get_client_version, get_server_version_from_string from metadata.ingestion.ometa.client import REST from metadata.utils.logger import ometa_logger @@ -53,7 +53,7 @@ class OMetaServerMixin: "Cannot Find Version at api/v1/system/version." + " If running the server in DEV mode locally, make sure to `mvn clean install`." ) - return get_version_from_string(raw_version) + return get_server_version_from_string(raw_version) def validate_versions(self) -> None: """ @@ -65,7 +65,9 @@ class OMetaServerMixin: server_version = self.get_server_version() client_version = get_client_version() - if server_version != client_version: + # Server version will be 0.13.2, vs 0.13.2.X from the client. + # If the server version is contained in the client version, then we're good to go + if server_version not in client_version: raise VersionMismatchException( f"Server version is {server_version} vs. Client version {client_version}. Both should match." ) diff --git a/ingestion/src/metadata/ingestion/source/database/oracle/metadata.py b/ingestion/src/metadata/ingestion/source/database/oracle/metadata.py index f98ec59d418..407a3c53d37 100644 --- a/ingestion/src/metadata/ingestion/source/database/oracle/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/oracle/metadata.py @@ -37,26 +37,39 @@ from metadata.utils.sqlalchemy_utils import ( @reflection.cache def get_table_comment( - self, connection, table_name, schema=None, resolve_synonyms=False, dblink="", **kw + self, + connection, + table_name: str, + schema: str = None, + resolve_synonyms=False, + dblink="", + **kw, ): # pylint: disable=unused-argument return get_table_comment_wrapper( self, connection, - table_name=table_name, - schema=schema, + table_name=table_name.lower(), + schema=schema.lower() if schema else None, query=ORACLE_ALL_TABLE_COMMENTS, ) @reflection.cache def get_view_definition( - self, connection, view_name, schema=None, resolve_synonyms=False, dblink="", **kw + self, + connection, + view_name: str, + schema: str = None, + resolve_synonyms=False, + dblink="", + **kw, ): # pylint: disable=unused-argument + return get_view_definition_wrapper( self, connection, - table_name=view_name, - schema=schema, + table_name=view_name.lower(), + schema=schema.lower() if schema else None, query=ORACLE_ALL_VIEW_DEFINITIONS, ) diff --git a/ingestion/src/metadata/ingestion/source/database/oracle/queries.py b/ingestion/src/metadata/ingestion/source/database/oracle/queries.py index 093bb44b94b..2a5a767a9e9 100644 --- a/ingestion/src/metadata/ingestion/source/database/oracle/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/oracle/queries.py @@ -15,8 +15,8 @@ SQL Queries used during ingestion ORACLE_ALL_TABLE_COMMENTS = """ SELECT comments table_comment, - table_name, - owner "schema" + LOWER(table_name) "table_name", + LOWER(owner) "schema" FROM all_tab_comments where comments is not null and owner not in ('SYSTEM', 'SYS') """ @@ -24,8 +24,8 @@ where comments is not null and owner not in ('SYSTEM', 'SYS') ORACLE_ALL_VIEW_DEFINITIONS = """ SELECT - view_name, - owner "schema", + LOWER(view_name) "view_name", + LOWER(owner) "schema", text view_def FROM all_views where text is not null and owner not in ('SYSTEM', 'SYS') diff --git a/ingestion/tests/unit/test_version.py b/ingestion/tests/unit/test_version.py index d4dd05e34ba..cea527e4da6 100644 --- a/ingestion/tests/unit/test_version.py +++ b/ingestion/tests/unit/test_version.py @@ -14,7 +14,10 @@ Validate Server Mixin version methods from unittest import TestCase -from metadata.__version__ import get_version_from_string +from metadata.__version__ import ( + get_client_version_from_string, + get_server_version_from_string, +) class OMetaVersionTest(TestCase): @@ -26,8 +29,26 @@ class OMetaVersionTest(TestCase): """ We should be able to parse regular version responses """ - self.assertEqual("0.11.0", get_version_from_string("0.11.0.dev0")) - self.assertEqual("0.11.0", get_version_from_string("0.11.0")) - self.assertEqual("1111.11.111", get_version_from_string("1111.11.111")) - self.assertEqual("1111.11.111", get_version_from_string("1111.11.111-SNAPSHOT")) - self.assertEqual("0.11.1", get_version_from_string("0.11.1.0.0.1.patch")) + self.assertEqual("0.11.0", get_server_version_from_string("0.11.0.dev0")) + self.assertEqual("0.11.0", get_server_version_from_string("0.11.0")) + self.assertEqual("1111.11.111", get_server_version_from_string("1111.11.111")) + self.assertEqual( + "1111.11.111", get_server_version_from_string("1111.11.111-SNAPSHOT") + ) + self.assertEqual("0.11.1", get_server_version_from_string("0.11.1.0.0.1.patch")) + + def test_get_client_version_from_string(self): + """ + We should be able to parse regular version responses + """ + self.assertEqual("0.13.2.5", get_client_version_from_string("0.13.2.5.dev0")) + self.assertEqual("0.11.0.1", get_client_version_from_string("0.11.0.1")) + self.assertEqual( + "1111.11.111.1", get_client_version_from_string("1111.11.111.1") + ) + self.assertEqual( + "1111.11.111.2", get_client_version_from_string("1111.11.111.2-SNAPSHOT") + ) + self.assertEqual( + "0.11.1.0", get_client_version_from_string("0.11.1.0.0.1.patch") + )