diff --git a/ingestion/setup.py b/ingestion/setup.py index 966bf70fcad..929f8ccb3b2 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -57,6 +57,7 @@ VERSIONS = { "giturlparse": "giturlparse", "validators": "validators~=0.22.0", "teradata": "teradatasqlalchemy>=20.0.0.0", + "cockroach": "sqlalchemy-cockroachdb~=2.0", "cassandra": "cassandra-driver>=3.28.0", } @@ -233,6 +234,10 @@ plugins: Dict[str, Set[str]] = { "glue": {VERSIONS["boto3"]}, "great-expectations": {VERSIONS["great-expectations"]}, "greenplum": {*COMMONS["postgres"]}, + "cockroach": { + VERSIONS["cockroach"], + "psycopg2-binary", + }, "hive": { *COMMONS["hive"], "thrift>=0.13,<1", @@ -380,6 +385,7 @@ test = { VERSIONS["avro"], # Sample Data VERSIONS["grpc-tools"], VERSIONS["neo4j"], + VERSIONS["cockroach"], "testcontainers==3.7.1;python_version<'3.9'", "testcontainers~=4.8.0;python_version>='3.9'", "minio==7.2.5", diff --git a/ingestion/src/metadata/examples/workflows/cockroach.yaml b/ingestion/src/metadata/examples/workflows/cockroach.yaml new file mode 100644 index 00000000000..e9e77b46b06 --- /dev/null +++ b/ingestion/src/metadata/examples/workflows/cockroach.yaml @@ -0,0 +1,24 @@ +source: + type: cockroach + serviceName: local_cockroach + serviceConnection: + config: + type: Cockroach + username: username + authType: + password: password + hostPort: localhost:26257 + database: database + sourceConfig: + config: + type: DatabaseMetadata +sink: + type: metadata-rest + config: {} +workflowConfig: +# loggerLevel: INFO # DEBUG, INFO, WARN or ERROR + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" diff --git a/ingestion/src/metadata/ingestion/source/database/cockroach/__init__.py b/ingestion/src/metadata/ingestion/source/database/cockroach/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/src/metadata/ingestion/source/database/cockroach/connection.py b/ingestion/src/metadata/ingestion/source/database/cockroach/connection.py new file mode 100644 index 00000000000..a73f1389ffe --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/database/cockroach/connection.py @@ -0,0 +1,71 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Source connection handler +""" +from typing import Optional + +from sqlalchemy.engine import Engine + +from metadata.generated.schema.entity.automations.workflow import ( + Workflow as AutomationWorkflow, +) +from metadata.generated.schema.entity.services.connections.database.cockroachConnection import ( + CockroachConnection, +) +from metadata.generated.schema.entity.services.connections.testConnectionResult import ( + TestConnectionResult, +) +from metadata.ingestion.connections.builders import ( + create_generic_db_connection, + get_connection_args_common, + get_connection_url_common, +) +from metadata.ingestion.connections.test_connections import test_connection_db_common +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.database.cockroach.queries import COCKROACH_GET_DATABASE +from metadata.utils.constants import THREE_MIN + + +def get_connection(connection: CockroachConnection) -> Engine: + """ + Create connection + """ + return create_generic_db_connection( + connection=connection, + get_connection_url_fn=get_connection_url_common, + get_connection_args_fn=get_connection_args_common, + ) + + +def test_connection( + metadata: OpenMetadata, + engine: Engine, + service_connection: CockroachConnection, + automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, +) -> TestConnectionResult: + """ + Test connection. This can be executed either as part + of a metadata workflow or during an Automation Workflow + """ + queries = { + "GetDatabases": COCKROACH_GET_DATABASE, + } + return test_connection_db_common( + metadata=metadata, + engine=engine, + service_connection=service_connection, + automation_workflow=automation_workflow, + queries=queries, + timeout_seconds=timeout_seconds, + ) diff --git a/ingestion/src/metadata/ingestion/source/database/cockroach/metadata.py b/ingestion/src/metadata/ingestion/source/database/cockroach/metadata.py new file mode 100644 index 00000000000..aa677f58860 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/database/cockroach/metadata.py @@ -0,0 +1,204 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Postgres source module +""" +import traceback +from collections import namedtuple +from typing import Iterable, Optional + +from sqlalchemy import String as SqlAlchemyString +from sqlalchemy import sql +from sqlalchemy.dialects.postgresql.base import ischema_names + +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.table import TableType +from metadata.generated.schema.entity.services.connections.database.cockroachConnection import ( + CockroachConnection, +) +from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( + OpenMetadataConnection, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + Source as WorkflowSource, +) +from metadata.ingestion.api.steps import InvalidSourceException +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.database.cockroach.queries import ( + COCKROACH_GET_DB_NAMES, + COCKROACH_GET_TABLE_NAMES, + COCKROACH_GET_VIEW_NAMES, + COCKROACH_SCHEMA_COMMENTS, +) +from metadata.ingestion.source.database.column_type_parser import create_sqlalchemy_type +from metadata.ingestion.source.database.common_db_source import ( + CommonDbSourceService, + TableNameAndType, +) +from metadata.ingestion.source.database.multi_db_source import MultiDBSource +from metadata.utils import fqn +from metadata.utils.filters import filter_by_database +from metadata.utils.importer import import_side_effects +from metadata.utils.logger import ingestion_logger + +import_side_effects( + "metadata.ingestion.source.database.postgres.converter_orm", + "metadata.ingestion.source.database.postgres.metrics", +) + +TableKey = namedtuple("TableKey", ["schema", "table_name"]) + +logger = ingestion_logger() + +RELKIND_MAP = { + "r": TableType.Regular, + "p": TableType.Partitioned, + "f": TableType.Foreign, + "v": TableType.View, +} + +GEOMETRY = create_sqlalchemy_type("GEOMETRY") +POINT = create_sqlalchemy_type("POINT") +POLYGON = create_sqlalchemy_type("POLYGON") + +ischema_names.update( + { + "geometry": GEOMETRY, + "point": POINT, + "polygon": POLYGON, + "box": create_sqlalchemy_type("BOX"), + "bpchar": SqlAlchemyString, + "circle": create_sqlalchemy_type("CIRCLE"), + "line": create_sqlalchemy_type("LINE"), + "lseg": create_sqlalchemy_type("LSEG"), + "path": create_sqlalchemy_type("PATH"), + "pg_lsn": create_sqlalchemy_type("PG_LSN"), + "pg_snapshot": create_sqlalchemy_type("PG_SNAPSHOT"), + "tsquery": create_sqlalchemy_type("TSQUERY"), + "txid_snapshot": create_sqlalchemy_type("TXID_SNAPSHOT"), + "xid": SqlAlchemyString, + "xml": create_sqlalchemy_type("XML"), + } +) + + +class CockroachSource(CommonDbSourceService, MultiDBSource): + """ + Implements the necessary methods to extract + Database metadata from Cockroach Source + """ + + def __init__(self, config: WorkflowSource, metadata: OpenMetadata): + super().__init__(config, metadata) + self.schema_desc_map = {} + + @classmethod + def create( + cls, + config_dict, + metadata: OpenMetadataConnection, + pipeline_name: Optional[str] = None, + ): + config: WorkflowSource = WorkflowSource.model_validate(config_dict) + connection: CockroachConnection = config.serviceConnection.root.config + if not isinstance(connection, CockroachConnection): + raise InvalidSourceException( + f"Expected CockroachConnection, but got {connection}" + ) + return cls(config, metadata) + + def set_schema_description_map(self) -> None: + self.schema_desc_map.clear() + results = self.engine.execute(COCKROACH_SCHEMA_COMMENTS).all() + for row in results: + self.schema_desc_map[(row.database_name, row.schema_name)] = row.comment + + def get_schema_description(self, schema_name: str) -> Optional[str]: + """ + Method to fetch the schema description + """ + return self.schema_desc_map.get((self.context.get().database, schema_name)) + + def query_table_names_and_types( + self, schema_name: str + ) -> Iterable[TableNameAndType]: + """ + Overwrite the inspector implementation to handle partitioned + and foreign types + """ + result = self.connection.execute( + sql.text(COCKROACH_GET_TABLE_NAMES), + {"schema": schema_name}, + ) + return [ + TableNameAndType( + name=name, type_=RELKIND_MAP.get(relkind, TableType.Regular) + ) + for name, relkind in result + ] + + def query_view_names_and_types( + self, schema_name: str + ) -> Iterable[TableNameAndType]: + result = self.connection.execute( + sql.text(COCKROACH_GET_VIEW_NAMES), + {"schema": schema_name}, + ) + return [ + TableNameAndType( + name=name, type_=RELKIND_MAP.get(relkind, TableType.Regular) + ) + for name, relkind in result + ] + + def get_configured_database(self) -> Optional[str]: + if not self.service_connection.ingestAllDatabases: + return self.service_connection.database + return None + + def get_database_names_raw(self) -> Iterable[str]: + yield from self._execute_database_query(COCKROACH_GET_DB_NAMES) + + def get_database_names(self) -> Iterable[str]: + if not self.config.serviceConnection.root.config.ingestAllDatabases: + configured_db = self.config.serviceConnection.root.config.database + self.set_inspector(database_name=configured_db) + self.set_schema_description_map() + yield configured_db + else: + for new_database in self.get_database_names_raw(): + database_fqn = fqn.build( + self.metadata, + entity_type=Database, + service_name=self.context.get().database_service, + database_name=new_database, + ) + + if filter_by_database( + self.source_config.databaseFilterPattern, + ( + database_fqn + if self.source_config.useFqnForFiltering + else new_database + ), + ): + self.status.filter(database_fqn, "Database Filtered Out") + continue + + try: + self.set_inspector(database_name=new_database) + self.set_schema_description_map() + yield new_database + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.error( + f"Error trying to connect to database {new_database}: {exc}" + ) diff --git a/ingestion/src/metadata/ingestion/source/database/cockroach/queries.py b/ingestion/src/metadata/ingestion/source/database/cockroach/queries.py new file mode 100644 index 00000000000..154174da828 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/database/cockroach/queries.py @@ -0,0 +1,72 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +SQL Queries used during ingestion +""" + +from sqlalchemy import text + +COCKROACH_GET_TABLE_NAMES = """ + SELECT + c.relname AS table_name, + c.relkind AS relkind + FROM + pg_class c + JOIN + pg_namespace n ON n.oid = c.relnamespace + WHERE + n.nspname = :schema + AND c.relkind IN ('r', 'p', 'f') + ORDER BY + c.relname +""" + +COCKROACH_GET_VIEW_NAMES = """ + SELECT + c.relname AS table_name, + c.relkind AS relkind + FROM + pg_class c + JOIN + pg_namespace n ON n.oid = c.relnamespace + WHERE + n.nspname = :schema + AND c.relkind IN ('v') + ORDER BY + c.relname +""" + + +COCKROACH_SCHEMA_COMMENTS = """ + SELECT + current_database() AS database_name, + n.nspname AS schema_name, + d.description AS comment +FROM + pg_namespace n +LEFT JOIN + pg_description d +ON + n.oid = d.objoid +WHERE + d.objsubid = 0; +""" + + +COCKROACH_GET_DATABASE = text( + """ + select datname FROM pg_catalog.pg_database +""" +) + +COCKROACH_GET_DB_NAMES = """ + select datname from pg_catalog.pg_database +""" diff --git a/ingestion/src/metadata/ingestion/source/database/cockroach/service_spec.py b/ingestion/src/metadata/ingestion/source/database/cockroach/service_spec.py new file mode 100644 index 00000000000..abd2a45dede --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/database/cockroach/service_spec.py @@ -0,0 +1,4 @@ +from metadata.ingestion.source.database.cockroach.metadata import CockroachSource +from metadata.utils.service_spec.default import DefaultDatabaseSpec + +ServiceSpec = DefaultDatabaseSpec(metadata_source_class=CockroachSource) diff --git a/ingestion/tests/integration/cockroach/__init__.py b/ingestion/tests/integration/cockroach/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/integration/cockroach/conftest.py b/ingestion/tests/integration/cockroach/conftest.py new file mode 100644 index 00000000000..9b6898d0298 --- /dev/null +++ b/ingestion/tests/integration/cockroach/conftest.py @@ -0,0 +1,64 @@ +import os +import textwrap + +import pytest +from sqlalchemy import create_engine + +from _openmetadata_testutils.helpers.docker import try_bind +from metadata.generated.schema.api.services.createDatabaseService import ( + CreateDatabaseServiceRequest, +) +from metadata.generated.schema.entity.services.connections.database.cockroachConnection import ( + CockroachConnection, +) +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseServiceType, +) + + +@pytest.fixture(scope="module") +def cockroach_container(tmp_path_factory): + """ + Start a Cockroach container. + """ + from testcontainers.cockroachdb import CockroachDBContainer + + container = CockroachDBContainer(image="cockroachdb/cockroach:v23.1.0") + + with ( + try_bind(container, 26257, None) if not os.getenv("CI") else container + ) as container: + engine = create_engine(container.get_connection_url()) + engine.execute( + textwrap.dedent( + """ + CREATE TABLE user_profiles ( + user_id UUID PRIMARY KEY, + first_name TEXT, + last_name TEXT, + email TEXT, + signup_date TIMESTAMP, + is_active BOOLEAN + ); + """ + ) + ) + + yield container + + +@pytest.fixture(scope="module") +def create_service_request(cockroach_container, tmp_path_factory): + return CreateDatabaseServiceRequest( + name="docker_test_" + tmp_path_factory.mktemp("cockroach").name, + serviceType=DatabaseServiceType.Cockroach, + connection=DatabaseConnection( + config=CockroachConnection( + username=cockroach_container.username, + authType={"password": cockroach_container.password}, + hostPort=f"localhost:{cockroach_container.get_exposed_port(26257)}", + database=cockroach_container.dbname, + ) + ), + ) diff --git a/ingestion/tests/integration/cockroach/test_metadata.py b/ingestion/tests/integration/cockroach/test_metadata.py new file mode 100644 index 00000000000..61c25a4a75b --- /dev/null +++ b/ingestion/tests/integration/cockroach/test_metadata.py @@ -0,0 +1,76 @@ +import sys + +import pytest +from sqlalchemy import create_engine + +from metadata.generated.schema.entity.data.table import Constraint, Table +from metadata.workflow.metadata import MetadataWorkflow + +if not sys.version_info >= (3, 9): + pytest.skip("requires python 3.9+", allow_module_level=True) + + +@pytest.fixture(scope="module") +def prepare_cockroach(cockroach_container): + engine = create_engine(cockroach_container.get_connection_url()) + + sql = [ + """ + CREATE TABLE test_table ( + id INTEGER NOT NULL DEFAULT unique_rowid(), + name VARCHAR(100), + age INTEGER, + PRIMARY KEY (id) + ); + """, + """ + INSERT INTO test_table (name, age) VALUES + ('John Doe', 25), + ('Jane Smith', 30); + """, + ] + for stmt in sql: + engine.execute(stmt) + + +@pytest.mark.parametrize( + "table_fqn,expected_columns", + [ + [ + "{service}.roach.public.test_table", + { + "id": {"type": "int", "nullable": False}, + "name": {"type": "varchar", "nullable": True}, + "age": {"type": "int", "nullable": True}, + }, + ] + ], + ids=lambda x: x.split(".")[-1] if isinstance(x, str) else "", +) +def test_ingest_metadata( + patch_passwords_for_db_services, + run_workflow, + ingestion_config, + metadata, + table_fqn, + expected_columns, + db_service, + prepare_cockroach, +): + run_workflow(MetadataWorkflow, ingestion_config) + + table = metadata.get_by_name( + entity=Table, fqn=table_fqn.format(service=db_service.fullyQualifiedName.root) + ) + assert table + assert table.fullyQualifiedName.root.split(".")[-1] == "test_table" + + for name, properties in expected_columns.items(): + column = next((col for col in table.columns if col.name.root == name), None) + assert column is not None + assert column.dataType.name.lower() == properties["type"] + assert ( + column.constraint == Constraint.PRIMARY_KEY + if name == "id" + else Constraint.NULL + ) diff --git a/ingestion/tests/integration/kafka/__init__.py b/ingestion/tests/integration/kafka/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/unit/topology/database/test_athena.py b/ingestion/tests/unit/topology/database/test_athena.py index 04447362619..5ef1a49dc7b 100644 --- a/ingestion/tests/unit/topology/database/test_athena.py +++ b/ingestion/tests/unit/topology/database/test_athena.py @@ -269,9 +269,7 @@ mock_athena_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "athena"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_bigquery.py b/ingestion/tests/unit/topology/database/test_bigquery.py index b5b317efb33..2bce2514a04 100644 --- a/ingestion/tests/unit/topology/database/test_bigquery.py +++ b/ingestion/tests/unit/topology/database/test_bigquery.py @@ -87,7 +87,7 @@ mock_bq_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": {"jwtToken": "jwt"}, + "securityConfig": {"jwtToken": "bigquery"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_bigtable.py b/ingestion/tests/unit/topology/database/test_bigtable.py index 2df8446f8ba..de676fad540 100644 --- a/ingestion/tests/unit/topology/database/test_bigtable.py +++ b/ingestion/tests/unit/topology/database/test_bigtable.py @@ -86,9 +86,7 @@ mock_bigtable_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "bigtable"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_cassandra.py b/ingestion/tests/unit/topology/database/test_cassandra.py index 2fd157d077a..21edaf77e4c 100644 --- a/ingestion/tests/unit/topology/database/test_cassandra.py +++ b/ingestion/tests/unit/topology/database/test_cassandra.py @@ -66,9 +66,7 @@ mock_cassandra_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "cassandra"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_cockroach.py b/ingestion/tests/unit/topology/database/test_cockroach.py new file mode 100644 index 00000000000..2f9ee884a60 --- /dev/null +++ b/ingestion/tests/unit/topology/database/test_cockroach.py @@ -0,0 +1,263 @@ +""" +Test Cockroach using the topology +""" +import types +from unittest import TestCase +from unittest.mock import patch + +from sqlalchemy.types import VARCHAR + +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema +from metadata.generated.schema.entity.data.table import Column, Constraint, DataType +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseService, + DatabaseServiceType, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, +) +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.ingestion.source.database.cockroach.metadata import ( + GEOMETRY, + POINT, + POLYGON, + CockroachSource, +) + +mock_cockroach_config = { + "source": { + "type": "cockroach", + "serviceName": "local_cockroach1", + "serviceConnection": { + "config": { + "type": "Cockroach", + "username": "username", + "authType": { + "password": "password", + }, + "hostPort": "localhost:26257", + "database": "cockroach", + } + }, + "sourceConfig": { + "config": { + "type": "DatabaseMetadata", + } + }, + }, + "sink": { + "type": "metadata-rest", + "config": {}, + }, + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": {"jwtToken": "cockroach"}, + } + }, +} + +MOCK_DATABASE_SERVICE = DatabaseService( + id="85811038-099a-11ed-861d-0242ac120002", + name="cockroach_source", + connection=DatabaseConnection(), + serviceType=DatabaseServiceType.Cockroach, +) + +MOCK_DATABASE = Database( + id="2aaa012e-099a-11ed-861d-0242ac120002", + name="default", + fullyQualifiedName="cockroach_source.default", + displayName="default", + description="", + service=EntityReference( + id="85811038-099a-11ed-861d-0242ac120002", + type="databaseService", + ), +) + +MOCK_DATABASE_SCHEMA = DatabaseSchema( + id="2aaa012e-099a-11ed-861d-0242ac120056", + name="default", + fullyQualifiedName="cockroach_source.default.default", + displayName="default", + description="", + database=EntityReference( + id="2aaa012e-099a-11ed-861d-0242ac120002", + type="database", + ), + service=EntityReference( + id="2aaa012e-099a-11ed-861d-0242ac120002", + type="database", + ), +) + + +MOCK_COLUMN_VALUE = [ + { + "name": "username", + "type": VARCHAR(), + "nullable": True, + "default": None, + "autoincrement": False, + "system_data_type": "varchar(50)", + "comment": None, + }, + { + "name": "geom_c", + "type": GEOMETRY(), + "nullable": True, + "default": None, + "autoincrement": False, + "system_data_type": "geometry", + "comment": None, + }, + { + "name": "point_c", + "type": POINT(), + "nullable": True, + "default": None, + "autoincrement": False, + "system_data_type": "point", + "comment": None, + }, + { + "name": "polygon_c", + "type": POLYGON(), + "nullable": True, + "default": None, + "autoincrement": False, + "comment": None, + "system_data_type": "polygon", + }, +] + + +EXPECTED_COLUMN_VALUE = [ + Column( + name="username", + displayName=None, + dataType=DataType.VARCHAR, + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="varchar(50)", + description=None, + fullyQualifiedName=None, + tags=None, + constraint=Constraint.NULL, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + Column( + name="geom_c", + displayName=None, + dataType=DataType.GEOMETRY, + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="geometry", + description=None, + fullyQualifiedName=None, + tags=None, + constraint=Constraint.NULL, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + Column( + name="point_c", + displayName=None, + dataType=DataType.GEOMETRY, + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="point", + description=None, + fullyQualifiedName=None, + tags=None, + constraint=Constraint.NULL, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + Column( + name="polygon_c", + displayName=None, + dataType=DataType.GEOMETRY, + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="polygon", + description=None, + fullyQualifiedName=None, + tags=None, + constraint=Constraint.NULL, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), +] + + +class cockroachUnitTest(TestCase): + @patch( + "metadata.ingestion.source.database.common_db_source.CommonDbSourceService.test_connection" + ) + def __init__(self, methodName, test_connection) -> None: + super().__init__(methodName) + test_connection.return_value = False + self.config = OpenMetadataWorkflowConfig.model_validate(mock_cockroach_config) + self.cockroach_source = CockroachSource.create( + mock_cockroach_config["source"], + self.config.workflowConfig.openMetadataServerConfig, + ) + + self.cockroach_source.context.get().__dict__[ + "database_service" + ] = MOCK_DATABASE_SERVICE.name.root + self.cockroach_source.context.get().__dict__[ + "database" + ] = MOCK_DATABASE.name.root + self.cockroach_source.context.get().__dict__[ + "database_schema" + ] = MOCK_DATABASE_SCHEMA.name.root + + def test_datatype(self): + inspector = types.SimpleNamespace() + inspector.get_columns = ( + lambda table_name, schema_name, db_name: MOCK_COLUMN_VALUE + ) + inspector.get_pk_constraint = lambda table_name, schema_name: [] + inspector.get_unique_constraints = lambda table_name, schema_name: [] + inspector.get_foreign_keys = lambda table_name, schema_name: [] + + result, _, _ = self.cockroach_source.get_columns_and_constraints( + "public", "user", "cockroach", inspector + ) + for i, _ in enumerate(EXPECTED_COLUMN_VALUE): + self.assertEqual(result[i], EXPECTED_COLUMN_VALUE[i]) + + @patch("sqlalchemy.engine.base.Engine") + @patch( + "metadata.ingestion.source.database.common_db_source.CommonDbSourceService.connection" + ) + def test_close_connection(self, engine, connection): + connection.return_value = True + self.cockroach_source.close() diff --git a/ingestion/tests/unit/topology/database/test_couchbase.py b/ingestion/tests/unit/topology/database/test_couchbase.py index bfc8c0395bf..2ecf94aea3c 100644 --- a/ingestion/tests/unit/topology/database/test_couchbase.py +++ b/ingestion/tests/unit/topology/database/test_couchbase.py @@ -53,9 +53,7 @@ mock_couch_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "couchbase"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_databricks.py b/ingestion/tests/unit/topology/database/test_databricks.py index 6f1d9525943..8fb88155795 100644 --- a/ingestion/tests/unit/topology/database/test_databricks.py +++ b/ingestion/tests/unit/topology/database/test_databricks.py @@ -63,9 +63,7 @@ mock_databricks_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "databricks"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_datalake.py b/ingestion/tests/unit/topology/database/test_datalake.py index 60233805634..0a318cf479a 100644 --- a/ingestion/tests/unit/topology/database/test_datalake.py +++ b/ingestion/tests/unit/topology/database/test_datalake.py @@ -73,9 +73,7 @@ mock_datalake_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "datalake"}, } }, } @@ -583,9 +581,7 @@ mock_datalake_gcs_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "datalake"}, }, }, } diff --git a/ingestion/tests/unit/topology/database/test_domodatabase.py b/ingestion/tests/unit/topology/database/test_domodatabase.py index a0676237c70..64fa54ebfc4 100644 --- a/ingestion/tests/unit/topology/database/test_domodatabase.py +++ b/ingestion/tests/unit/topology/database/test_domodatabase.py @@ -94,14 +94,7 @@ mock_domodatabase_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc" - "iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE" - "2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB" - "iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN" - "r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u" - "d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "domodatabase"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_doris.py b/ingestion/tests/unit/topology/database/test_doris.py index be8aad50065..a384219ca6a 100644 --- a/ingestion/tests/unit/topology/database/test_doris.py +++ b/ingestion/tests/unit/topology/database/test_doris.py @@ -52,9 +52,7 @@ mock_doris_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "doris"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_glue.py b/ingestion/tests/unit/topology/database/test_glue.py index d9da11f3ebb..03ce287f08e 100644 --- a/ingestion/tests/unit/topology/database/test_glue.py +++ b/ingestion/tests/unit/topology/database/test_glue.py @@ -62,9 +62,7 @@ mock_glue_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "glue"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_greenplum.py b/ingestion/tests/unit/topology/database/test_greenplum.py index 40f45db4403..e5674487ffb 100644 --- a/ingestion/tests/unit/topology/database/test_greenplum.py +++ b/ingestion/tests/unit/topology/database/test_greenplum.py @@ -50,9 +50,7 @@ mock_greenplum_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "greenplum"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_hive.py b/ingestion/tests/unit/topology/database/test_hive.py index 45628bf8e89..50312890107 100644 --- a/ingestion/tests/unit/topology/database/test_hive.py +++ b/ingestion/tests/unit/topology/database/test_hive.py @@ -63,14 +63,7 @@ mock_hive_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc" - "iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE" - "2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB" - "iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN" - "r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u" - "d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "hive"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_iceberg.py b/ingestion/tests/unit/topology/database/test_iceberg.py index 8123a80758f..556b65a144f 100644 --- a/ingestion/tests/unit/topology/database/test_iceberg.py +++ b/ingestion/tests/unit/topology/database/test_iceberg.py @@ -390,7 +390,7 @@ MOCK_HIVE_CONFIG = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": {"jwtToken": "token"}, + "securityConfig": {"jwtToken": "iceberg"}, } }, } @@ -415,7 +415,7 @@ MOCK_REST_CONFIG = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": {"jwtToken": "token"}, + "securityConfig": {"jwtToken": "iceberg"}, } }, } @@ -447,7 +447,7 @@ MOCK_GLUE_CONFIG = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": {"jwtToken": "token"}, + "securityConfig": {"jwtToken": "iceberg"}, } }, } @@ -481,7 +481,7 @@ MOCK_DYNAMO_CONFIG = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": {"jwtToken": "token"}, + "securityConfig": {"jwtToken": "iceberg"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_mongodb.py b/ingestion/tests/unit/topology/database/test_mongodb.py index 522fc0b3f7c..2a505e5cdd7 100644 --- a/ingestion/tests/unit/topology/database/test_mongodb.py +++ b/ingestion/tests/unit/topology/database/test_mongodb.py @@ -66,9 +66,7 @@ mock_mongo_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "mongodb"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_mssql.py b/ingestion/tests/unit/topology/database/test_mssql.py index 32994476645..0ac7516bec7 100644 --- a/ingestion/tests/unit/topology/database/test_mssql.py +++ b/ingestion/tests/unit/topology/database/test_mssql.py @@ -65,14 +65,7 @@ mock_mssql_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc" - "iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE" - "2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB" - "iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN" - "r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u" - "d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "mssql"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_mysql.py b/ingestion/tests/unit/topology/database/test_mysql.py index d69a2f59577..6b1c115229a 100644 --- a/ingestion/tests/unit/topology/database/test_mysql.py +++ b/ingestion/tests/unit/topology/database/test_mysql.py @@ -51,9 +51,7 @@ mock_mysql_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "mysql"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_oracle.py b/ingestion/tests/unit/topology/database/test_oracle.py index cbbdcdfddd7..60796af2f14 100644 --- a/ingestion/tests/unit/topology/database/test_oracle.py +++ b/ingestion/tests/unit/topology/database/test_oracle.py @@ -65,14 +65,7 @@ mock_oracle_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc" - "iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE" - "2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB" - "iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN" - "r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u" - "d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "oracle"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_postgres.py b/ingestion/tests/unit/topology/database/test_postgres.py index 86da8904347..362aa178c49 100644 --- a/ingestion/tests/unit/topology/database/test_postgres.py +++ b/ingestion/tests/unit/topology/database/test_postgres.py @@ -73,9 +73,7 @@ mock_postgres_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "postgres"}, } }, } @@ -110,9 +108,7 @@ mock_postgres_usage_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "postgres"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_redshift.py b/ingestion/tests/unit/topology/database/test_redshift.py index 04095e6d2ac..4b8989fbda3 100644 --- a/ingestion/tests/unit/topology/database/test_redshift.py +++ b/ingestion/tests/unit/topology/database/test_redshift.py @@ -45,9 +45,7 @@ mock_redshift_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "redshift"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_salesforce.py b/ingestion/tests/unit/topology/database/test_salesforce.py index 8c2ceb780f3..6102d4ff377 100644 --- a/ingestion/tests/unit/topology/database/test_salesforce.py +++ b/ingestion/tests/unit/topology/database/test_salesforce.py @@ -63,9 +63,7 @@ mock_salesforce_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "salesforce"}, } }, } diff --git a/ingestion/tests/unit/topology/database/test_snowflake.py b/ingestion/tests/unit/topology/database/test_snowflake.py index 14fde4908c4..c681ea95b86 100644 --- a/ingestion/tests/unit/topology/database/test_snowflake.py +++ b/ingestion/tests/unit/topology/database/test_snowflake.py @@ -47,9 +47,7 @@ SNOWFLAKE_CONFIGURATION = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "snowflake"}, } }, "ingestionPipelineFQN": "snowflake.mock_pipeline", diff --git a/ingestion/tests/unit/topology/database/test_unity_catalog.py b/ingestion/tests/unit/topology/database/test_unity_catalog.py index 620fb916fac..05403b101d0 100644 --- a/ingestion/tests/unit/topology/database/test_unity_catalog.py +++ b/ingestion/tests/unit/topology/database/test_unity_catalog.py @@ -87,9 +87,7 @@ mock_unitycatalog_config = { "openMetadataServerConfig": { "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", - "securityConfig": { - "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" - }, + "securityConfig": {"jwtToken": "unity_catalog"}, } }, } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index 3a9fedee940..a48ebd67c47 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -27,6 +27,7 @@ import org.openmetadata.schema.services.connections.dashboard.TableauConnection; import org.openmetadata.schema.services.connections.database.BigQueryConnection; import org.openmetadata.schema.services.connections.database.BigTableConnection; import org.openmetadata.schema.services.connections.database.CassandraConnection; +import org.openmetadata.schema.services.connections.database.CockroachConnection; import org.openmetadata.schema.services.connections.database.DatalakeConnection; import org.openmetadata.schema.services.connections.database.DeltaLakeConnection; import org.openmetadata.schema.services.connections.database.GreenplumConnection; @@ -85,7 +86,9 @@ public final class ClassConverterFactory { TestServiceConnectionRequest.class, new TestServiceConnectionRequestClassConverter()), Map.entry(TrinoConnection.class, new TrinoConnectionClassConverter()), - Map.entry(Workflow.class, new WorkflowClassConverter())); + Map.entry(Workflow.class, new WorkflowClassConverter()), + Map.entry(CockroachConnection.class, new CockroachConnectionClassConverter())); + Map.entry(Workflow.class, new WorkflowClassConverter()); Map.entry(CassandraConnection.class, new CassandraConnectionClassConverter()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/CockroachConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/CockroachConnectionClassConverter.java new file mode 100644 index 00000000000..67b545a541f --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/CockroachConnectionClassConverter.java @@ -0,0 +1,34 @@ +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.security.ssl.ValidateSSLClientConfig; +import org.openmetadata.schema.services.connections.database.CockroachConnection; +import org.openmetadata.schema.services.connections.database.common.basicAuth; +import org.openmetadata.service.util.JsonUtils; + +/** + * Converter class to get an `CockroachConnection` object. + */ +public class CockroachConnectionClassConverter extends ClassConverter { + + private static final List> SSL_SOURCE_CLASS = List.of(ValidateSSLClientConfig.class); + private static final List> CONFIG_SOURCE_CLASSES = List.of(basicAuth.class); + + public CockroachConnectionClassConverter() { + super(CockroachConnection.class); + } + + @Override + public Object convert(Object object) { + CockroachConnection cockroachConnection = + (CockroachConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvert(cockroachConnection.getAuthType(), CONFIG_SOURCE_CLASSES) + .ifPresent(cockroachConnection::setAuthType); + + tryToConvert(cockroachConnection.getSslConfig(), SSL_SOURCE_CLASS) + .ifPresent(cockroachConnection::setSslConfig); + + return cockroachConnection; + } +} diff --git a/openmetadata-service/src/main/resources/json/data/testConnections/database/cockroach.json b/openmetadata-service/src/main/resources/json/data/testConnections/database/cockroach.json new file mode 100644 index 00000000000..7a212cd7766 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/testConnections/database/cockroach.json @@ -0,0 +1,39 @@ +{ + "name": "Cockroach", + "displayName": "Cockroach Test Connection", + "description": "This Test Connection validates the access against the database and basic metadata extraction of schemas and tables.", + "steps": [ + { + "name": "CheckAccess", + "description": "Validate that we can properly reach the database and authenticate with the given credentials.", + "errorMessage": "Failed to connect to Cockroach, please validate the credentials", + "mandatory": true + }, + { + "name": "GetDatabases", + "description": "List all the databases available to the user.", + "errorMessage": "Failed to fetch databases, please validate if the user has enough privilege to fetch databases.", + "mandatory": true + }, + { + "name": "GetSchemas", + "description": "List all the schemas available to the user.", + "errorMessage": "Failed to fetch schemas, please validate if the user has enough privilege to fetch schemas.", + "mandatory": true + }, + { + "name": "GetTables", + "description": "From a given schema, list the tables belonging to that schema. If no schema is specified, we'll list the tables of a random schema.", + "errorMessage": "Failed to fetch tables, please validate if the user has enough privilege to fetch tables.", + "mandatory": true + }, + { + "name": "GetViews", + "description": "From a given schema, list the views belonging to that schema. If no schema is specified, we'll list the tables of a random schema.", + "errorMessage": "Failed to fetch views, please validate if the user has enough privilege to fetch views.", + "mandatory": false + } + ] + } + + \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/cockroachConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/cockroachConnection.json new file mode 100644 index 00000000000..0d123bd80d2 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/cockroachConnection.json @@ -0,0 +1,93 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/database/cockroachConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CockroachConnection", + "description": "Cockroach Database Connection Config", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.database.CockroachConnection", + "definitions": { + "cockroachType": { + "description": "Service type.", + "type": "string", + "enum": ["Cockroach"], + "default": "Cockroach" + }, + "cockroachScheme": { + "description": "SQLAlchemy driver scheme options.", + "type": "string", + "enum": ["cockroachdb+psycopg2"], + "default": "cockroachdb+psycopg2" + } + }, + "properties": { + "type": { + "title": "Service Type", + "description": "Service Type", + "$ref": "#/definitions/cockroachType", + "default": "Cockroach" + }, + "scheme": { + "title": "Connection Scheme", + "description": "SQLAlchemy driver scheme options.", + "$ref": "#/definitions/cockroachScheme", + "default": "cockroachdb+psycopg2" + }, + "username": { + "title": "Username", + "description": "Username to connect to Cockroach. This user should have privileges to read all the metadata in Cockroach.", + "type": "string" + }, + "authType": { + "title": "Auth Configuration Type", + "description": "Choose Auth Config Type.", + "oneOf": [ + { + "$ref": "./common/basicAuth.json" + } + ] + }, + "hostPort": { + "title": "Host and Port", + "description": "Host and port of the Cockrooach service.", + "type": "string" + }, + "database": { + "title": "Database Name", + "description": "Optional name to give to the database in OpenMetadata. If left blank, we will use default as the database name.", + "type": "string" + }, + "ingestAllDatabases": { + "title": "Ingest All Databases", + "description": "Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of this.", + "type": "boolean", + "default": false + }, + "databaseSchema": { + "title": "Database Schema", + "description": "Database Schema of the data source. This is optional parameter, if you would like to restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion attempts to scan all the schemas.", + "type": "string" + }, + "sslConfig": { + "title": "SSL", + "description": "SSL Configuration details.", + "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" + }, + "sslMode": { + "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslMode" + }, + "connectionOptions": { + "title": "Connection Options", + "$ref": "../connectionBasicType.json#/definitions/connectionOptions" + }, + "connectionArguments": { + "title": "Connection Arguments", + "$ref": "../connectionBasicType.json#/definitions/connectionArguments" + }, + "supportsMetadataExtraction": { + "title": "Supports Metadata Extraction", + "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" + } + }, + "additionalProperties": false, + "required": ["hostPort", "username"] +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json index 031ed496d99..1de1f882adc 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json @@ -59,7 +59,8 @@ "Teradata", "SapErp", "Synapse", - "Exasol" + "Exasol", + "Cockroach" ], "javaEnums": [ { @@ -196,6 +197,9 @@ }, { "name": "Exasol" + }, + { + "name": "Cockroach" } ] }, @@ -337,6 +341,9 @@ }, { "$ref": "./connections/database/exasolConnection.json" + }, + { + "$ref": "./connections/database/cockroachConnection.json" } ] } diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Cockroach.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Cockroach.md new file mode 100644 index 00000000000..141baf8078f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Cockroach.md @@ -0,0 +1,73 @@ +# Cockroach + +In this section, we provide guides and references to use the Cockroach connector. + +## Requirements + +### Profiler & Data Quality + +Executing the profiler Workflow or data quality tests, will require the user to have `SELECT` permission on the tables/schemas where the profiler/tests will be executed. More information on the profiler workflow setup can be found [here](https://docs.open-metadata.org/how-to-guides/data-quality-observability/profiler/workflow) and data quality tests [here](https://docs.open-metadata.org/connectors/ingestion/workflows/data-quality). + +You can find further information on the Cockroach connector in the [docs](https://docs.open-metadata.org/connectors/database/cockroach). + +## Connection Details + +$$section +### Connection Scheme $(id="scheme") + +SQLAlchemy driver scheme options. +$$ + +$$section +### Username $(id="username") + +Username to connect to Cockroach. This user should have privileges to read all the metadata in Cockroach. +$$ + + +$$section +### Auth Config $(id="authType") +There is one auth config: +- Basic Auth. + +User can authenticate the Cockroach Instance with auth type as `Basic Authentication` i.e. Password +$$ + +## Basic Auth +$$section +### Password $(id="password") + +Password to connect to Cockroach. +$$ + +$$section +### Host and Port $(id="hostPort") + +This parameter specifies the host and port of the Cockroach instance. This should be specified as a string in the format `hostname:port`. For example, you might set the hostPort parameter to `localhost:26257`. + +If you are running the OpenMetadata ingestion in a docker and your services are hosted on the `localhost`, then use `host.docker.internal:26257` as the value. +$$ + +$$section +### Database $(id="database") + +Initial Cockroach database to connect to. If you want to ingest all databases, set `ingestAllDatabases` to true. +$$ + +$$section +### Ingest All Databases $(id="ingestAllDatabases") + +If ticked, the workflow will be able to ingest all database in the cluster. If not ticked, the workflow will only ingest tables from the database set above. +$$ + +$$section +### Connection Arguments $(id="connectionArguments") + +Additional connection arguments such as security or protocol configs that can be sent to service during connection. +$$ + +$$section +### Connection Options $(id="connectionOptions") + +Additional connection options to build the URL that can be sent to service during the connection. +$$ diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/img/service-icon-cockroach.png b/openmetadata-ui/src/main/resources/ui/src/assets/img/service-icon-cockroach.png new file mode 100644 index 00000000000..44e3e7753cb Binary files /dev/null and b/openmetadata-ui/src/main/resources/ui/src/assets/img/service-icon-cockroach.png differ diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts index f9c205eaa3b..61d0576397e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts @@ -24,6 +24,7 @@ import azuresql from '../assets/img/service-icon-azuresql.png'; import bigtable from '../assets/img/service-icon-bigtable.png'; import cassandra from '../assets/img/service-icon-cassandra.png'; import clickhouse from '../assets/img/service-icon-clickhouse.png'; +import cockroach from '../assets/img/service-icon-cockroach.png'; import couchbase from '../assets/img/service-icon-couchbase.svg'; import dagster from '../assets/img/service-icon-dagster.png'; import databrick from '../assets/img/service-icon-databrick.png'; @@ -202,6 +203,7 @@ export const ICEBERGE = iceberge; export const TERADATA = teradata; export const FLINK = flink; export const REST_SERVICE = restService; +export const COCKROACH = cockroach; export const excludedService = [ MlModelServiceType.Sklearn, MetadataServiceType.MetadataES, @@ -399,6 +401,7 @@ export const BETA_SERVICES = [ DatabaseServiceType.Cassandra, MetadataServiceType.AlationSink, DatabaseServiceType.Synapse, + DatabaseServiceType.Cockroach, ]; export const TEST_CONNECTION_INITIAL_MESSAGE = i18n.t( diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createSearchIndex.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createSearchIndex.ts index fa6aa223d85..a6c8e1a7a87 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createSearchIndex.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createSearchIndex.ts @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * Create a SearchIndex entity request */ export interface CreateSearchIndex { diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/database.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/database.ts index ac3036eec9b..5dc38de7d52 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/database.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/database.ts @@ -511,6 +511,7 @@ export enum DatabaseServiceType { BigTable = "BigTable", Cassandra = "Cassandra", Clickhouse = "Clickhouse", + Cockroach = "Cockroach", Couchbase = "Couchbase", CustomDatabase = "CustomDatabase", Databricks = "Databricks", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/searchIndex.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/searchIndex.ts index 92d3e1642d3..3bb6fea8cd0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/searchIndex.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/searchIndex.ts @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * A `SearchIndex` is a index mapping definition in ElasticSearch or OpenSearch */ export interface SearchIndex { diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/cockroachConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/cockroachConnection.ts new file mode 100644 index 00000000000..f01830961c9 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/cockroachConnection.ts @@ -0,0 +1,138 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /** + * Cockroach Database Connection Config + */ +export interface CockroachConnection { + /** + * Choose Auth Config Type. + */ + authType?: AuthConfigurationType; + connectionArguments?: { [key: string]: any }; + connectionOptions?: { [key: string]: string }; + /** + * Optional name to give to the database in OpenMetadata. If left blank, we will use default + * as the database name. + */ + database?: string; + /** + * Database Schema of the data source. This is optional parameter, if you would like to + * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion + * attempts to scan all the schemas. + */ + databaseSchema?: string; + /** + * Host and port of the Cockrooach service. + */ + hostPort: string; + /** + * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of + * this. + */ + ingestAllDatabases?: boolean; + /** + * SQLAlchemy driver scheme options. + */ + scheme?: CockroachScheme; + /** + * SSL Configuration details. + */ + sslConfig?: Config; + sslMode?: SSLMode; + supportsMetadataExtraction?: boolean; + /** + * Service Type + */ + type?: CockroachType; + /** + * Username to connect to Cockroach. This user should have privileges to read all the + * metadata in Cockroach. + */ + username: string; +} + +/** + * Choose Auth Config Type. + * + * Common Database Connection Config + */ +export interface AuthConfigurationType { + /** + * Password to connect to source. + */ + password?: string; +} + +/** + * SQLAlchemy driver scheme options. + */ +export enum CockroachScheme { + CockroachdbPsycopg2 = "cockroachdb+psycopg2", +} + +/** + * SSL Configuration details. + * + * Client SSL configuration + * + * OpenMetadata Client configured to validate SSL certificates. + */ +export interface Config { + /** + * The CA certificate used for SSL validation. + */ + caCertificate?: string; + /** + * The SSL certificate used for client authentication. + */ + sslCertificate?: string; + /** + * The private key associated with the SSL certificate. + */ + sslKey?: string; +} + +/** + * SSL Mode to connect to database. + */ +export enum SSLMode { + Allow = "allow", + Disable = "disable", + Prefer = "prefer", + Require = "require", + VerifyCA = "verify-ca", + VerifyFull = "verify-full", +} + +/** + * Service Type + * + * Service type. + */ +export enum CockroachType { + Cockroach = "Cockroach", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/db2Connection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/db2Connection.ts index a30308439b5..b29f802914d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/db2Connection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/db2Connection.ts @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * Db2 Connection Config */ export interface Db2Connection { @@ -26,6 +24,14 @@ export interface Db2Connection { * Host and port of the DB2 service. */ hostPort: string; + /** + * License to connect to DB2. + */ + license?: string; + /** + * License file name to connect to DB2. + */ + licenseFileName?: string; /** * Password to connect to DB2. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts index 36d674af285..f5711a26c70 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts @@ -10,8 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - /** * This schema defines the `Database Service` is a service such as MySQL, BigQuery, * Redshift, Postgres, or Snowflake. Alternative terms such as Database Cluster, Database @@ -22,7 +20,7 @@ export interface DatabaseService { * Change that lead to this version of the entity. */ changeDescription?: ChangeDescription; - connection?: DatabaseConnection; + connection?: DatabaseConnection; /** * List of data products this entity is part of. */ @@ -230,13 +228,15 @@ export interface DatabaseConnection { * Synapse Database Connection Config * * Exasol Database Connection Config + * + * Cockroach Database Connection Config */ export interface ConfigClass { /** * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. */ connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; + connectionOptions?: { [key: string]: string }; /** * GCP Credentials */ @@ -295,8 +295,10 @@ export interface ConfigClass { * Host and Port of the SAP ERP instance. * * Host and port of the Azure Synapse service. + * + * Host and port of the Cockrooach service. */ - hostPort?: string; + hostPort?: string; sampleDataStorageConfig?: SampleDataStorageConfig; /** * SQLAlchemy driver scheme options. @@ -305,18 +307,18 @@ export interface ConfigClass { * * Couchbase driver scheme options. */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; + scheme?: ConfigScheme; + supportsDatabase?: boolean; + supportsDataDiff?: boolean; supportsDBTExtraction?: boolean; /** * Supports Lineage Extraction. */ - supportsLineageExtraction?: boolean; + supportsLineageExtraction?: boolean; supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; + supportsProfiler?: boolean; + supportsQueryComment?: boolean; + supportsSystemProfile?: boolean; /** * Supports Usage Extraction. */ @@ -341,7 +343,7 @@ export interface ConfigClass { * multi-regions are not yet in GA. */ usageLocation?: string; - awsConfig?: AWSCredentials; + awsConfig?: AWSCredentials; /** * Optional name to give to the database in OpenMetadata. If left blank, we will use default * as the database name. @@ -378,6 +380,9 @@ export interface ConfigClass { * * Initial Redshift database to connect to. If you want to ingest all databases, set * ingestAllDatabases to true. + * + * Optional name to give to the database in OpenMetadata. If left blank, we will use default + * as the database name. */ database?: string; /** @@ -443,8 +448,6 @@ export interface ConfigClass { * * Password to connect to MongoDB. * - * Password to connect to Cassandra. - * * Password to connect to Couchbase. * * Password to connect to Doris. @@ -542,6 +545,9 @@ export interface ConfigClass { * * Username to connect to Exasol. This user should have privileges to read all the metadata * in Exasol. + * + * Username to connect to Cockroach. This user should have privileges to read all the + * metadata in Cockroach. */ username?: string; /** @@ -604,7 +610,7 @@ export interface ConfigClass { /** * License file name to connect to DB2. */ - licenseFileName?: string; + licenseFileName?: string; supportsViewLineageExtraction?: boolean; /** * Available sources to fetch the metadata. @@ -664,7 +670,7 @@ export interface ConfigClass { * Custom OpenMetadata Classification name for Postgres policy tags. */ classificationName?: string; - sslMode?: SSLMode; + sslMode?: SSLMode; /** * Protocol ( Connection Argument ) to connect to Presto. */ @@ -779,10 +785,6 @@ export interface ConfigClass { * Choose between Database connection or HDB User Store connection. */ connection?: SAPHanaConnection; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; /** * Couchbase connection Bucket options. */ @@ -846,7 +848,7 @@ export interface ConfigClass { * Pagination limit used while querying the SAP ERP API for fetching the entities */ paginationLimit?: number; - verifySSL?: VerifySSL; + verifySSL?: VerifySSL; /** * Client SSL/TLS settings. */ @@ -887,18 +889,24 @@ export enum AuthMechanismEnum { * IAM Auth Database Connection Config * * Azure Database Connection Config + * + * Configuration for connecting to DataStax Astra DB in the cloud. */ export interface AuthConfigurationType { /** * Password to connect to source. */ - password?: string; - awsConfig?: AWSCredentials; + password?: string; + awsConfig?: AWSCredentials; azureConfig?: AzureCredentials; /** * JWT to connect to source. */ jwt?: string; + /** + * Configuration for connecting to DataStax Astra DB in the cloud. + */ + cloudConfig?: DataStaxAstraDBConfiguration; } /** @@ -977,6 +985,30 @@ export interface AzureCredentials { vaultName?: string; } +/** + * Configuration for connecting to DataStax Astra DB in the cloud. + */ +export interface DataStaxAstraDBConfiguration { + /** + * Timeout in seconds for establishing new connections to Cassandra. + */ + connectTimeout?: number; + /** + * Timeout in seconds for individual Cassandra requests. + */ + requestTimeout?: number; + /** + * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax + * Astra DB. + */ + secureConnectBundle?: string; + /** + * The Astra DB application token used for authentication. + */ + token?: string; + [property: string]: any; +} + /** * Database Authentication types not requiring config. */ @@ -1078,7 +1110,7 @@ export interface Connection { /** * Berarer token to use for the 'Authorization' header. */ - token?: string; + token?: string; awsConfig?: AWSCredentials; /** * DynamoDB table name. @@ -1214,30 +1246,6 @@ export interface SSLCertificatesByPath { privateKeyPath?: string; } -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - /** * Available sources to fetch the metadata. * @@ -1273,7 +1281,7 @@ export interface TaLakeConfigurationSource { /** * Prefix of the data source. */ - prefix?: string; + prefix?: string; securityConfig?: SecurityConfigClass; } @@ -1318,7 +1326,7 @@ export interface ConnectionClass { * Local path for the local file with metastore data. E.g., /tmp/metastore.db */ metastoreFilePath?: string; - securityConfig?: AWSCredentials; + securityConfig?: AWSCredentials; } /** @@ -1586,9 +1594,9 @@ export interface HiveMetastoreConnectionDetails { /** * Custom OpenMetadata Classification name for Postgres policy tags. */ - classificationName?: string; + classificationName?: string; connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; + connectionOptions?: { [key: string]: string }; /** * Database of the data source. This is optional parameter, if you would like to restrict * the metadata reading to a single database. When left blank, OpenMetadata Ingestion @@ -1605,7 +1613,7 @@ export interface HiveMetastoreConnectionDetails { * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of * this. */ - ingestAllDatabases?: boolean; + ingestAllDatabases?: boolean; sampleDataStorageConfig?: SampleDataStorageConfig; /** * SQLAlchemy driver scheme options. @@ -1614,16 +1622,16 @@ export interface HiveMetastoreConnectionDetails { /** * SSL Configuration details. */ - sslConfig?: Config; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; + sslConfig?: Config; + sslMode?: SSLMode; + supportsDatabase?: boolean; + supportsDataDiff?: boolean; + supportsDBTExtraction?: boolean; + supportsLineageExtraction?: boolean; supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; + supportsProfiler?: boolean; + supportsQueryComment?: boolean; + supportsUsageExtraction?: boolean; /** * Service Type */ @@ -1662,8 +1670,8 @@ export interface HiveMetastoreConnectionDetailsAuthConfigurationType { /** * Password to connect to source. */ - password?: string; - awsConfig?: AWSCredentials; + password?: string; + awsConfig?: AWSCredentials; azureConfig?: AzureCredentials; } @@ -1694,7 +1702,7 @@ export interface DataStorageConfig { /** * Prefix of the data source. */ - prefix?: string; + prefix?: string; storageConfig?: AwsCredentials; [property: string]: any; } @@ -1833,6 +1841,7 @@ export enum ConfigScheme { Bigquery = "bigquery", ClickhouseHTTP = "clickhouse+http", ClickhouseNative = "clickhouse+native", + CockroachdbPsycopg2 = "cockroachdb+psycopg2", Couchbase = "couchbase", DatabricksConnector = "databricks+connector", Db2IBMDB = "db2+ibm_db", @@ -1901,6 +1910,7 @@ export enum ConfigType { BigTable = "BigTable", Cassandra = "Cassandra", Clickhouse = "Clickhouse", + Cockroach = "Cockroach", Couchbase = "Couchbase", CustomDatabase = "CustomDatabase", Databricks = "Databricks", @@ -2017,6 +2027,7 @@ export enum DatabaseServiceType { BigTable = "BigTable", Cassandra = "Cassandra", Clickhouse = "Clickhouse", + Cockroach = "Cockroach", Couchbase = "Couchbase", CustomDatabase = "CustomDatabase", Databricks = "Databricks", @@ -2094,7 +2105,7 @@ export interface TagLabel { * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the * entity must confirm the suggested labels before it is marked as 'Confirmed'. */ - state: State; + state: State; style?: Style; tagFQN: string; } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceAutoClassificationPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceAutoClassificationPipeline.ts index abef5ad0458..668eeec75c4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceAutoClassificationPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceAutoClassificationPipeline.ts @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * DatabaseService AutoClassification & Auto Classification Pipeline Configuration. */ export interface DatabaseServiceAutoClassificationPipeline { @@ -40,17 +38,10 @@ export interface DatabaseServiceAutoClassificationPipeline { * Optional configuration to turn off fetching metadata for views. */ includeViews?: boolean; - /** - * Percentage of data or no. of rows used to compute the profiler metrics and run data - * quality tests - */ - profileSample?: number; - profileSampleType?: ProfileSampleType; /** * Number of sample rows to ingest when 'Generate Sample Data' is enabled */ - sampleDataCount?: number; - samplingMethodType?: SamplingMethodType; + sampleDataCount?: number; /** * Regex to only fetch tables or databases that matches the pattern. */ @@ -98,22 +89,6 @@ export interface FilterPattern { includes?: string[]; } -/** - * Type of Profile Sample (percentage or rows) - */ -export enum ProfileSampleType { - Percentage = "PERCENTAGE", - Rows = "ROWS", -} - -/** - * Type of Sampling Method (BERNOULLI or SYSTEM) - */ -export enum SamplingMethodType { - Bernoulli = "BERNOULLI", - System = "SYSTEM", -} - /** * Pipeline type * diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceProfilerPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceProfilerPipeline.ts index afa3c8ba93f..16bb422468f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceProfilerPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceProfilerPipeline.ts @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * DatabaseService Profiler Pipeline Configuration. */ export interface DatabaseServiceProfilerPipeline { @@ -47,12 +45,8 @@ export interface DatabaseServiceProfilerPipeline { * Percentage of data or no. of rows used to compute the profiler metrics and run data * quality tests */ - profileSample?: number; - profileSampleType?: ProfileSampleType; - /** - * Number of sample rows to ingest when 'Generate Sample Data' is enabled - */ - sampleDataCount?: number; + profileSample?: number; + profileSampleType?: ProfileSampleType; samplingMethodType?: SamplingMethodType; /** * Regex to only fetch tables or databases that matches the pattern. diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceQueryLineagePipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceQueryLineagePipeline.ts index a3724f17d15..eb9fa3aff5d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceQueryLineagePipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceQueryLineagePipeline.ts @@ -10,12 +10,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * DatabaseService Query Lineage Pipeline Configuration. */ export interface DatabaseServiceQueryLineagePipeline { + /** + * Set 'Cross Database Service Names' to process lineage with the database. + */ + crossDatabaseServiceNames?: string[]; /** * Regex to only fetch databases that matches the pattern. */ @@ -33,6 +35,11 @@ export interface DatabaseServiceQueryLineagePipeline { * Configuration to set the timeout for parsing the query in seconds. */ parsingTimeoutLimit?: number; + /** + * Set the 'Process Cross Database Lineage' toggle to control whether to process table + * lineage across different databases. + */ + processCrossDatabaseLineage?: boolean; /** * Set the 'Process Query Lineage' toggle to control whether to process query lineage. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/type/entityLineage.ts b/openmetadata-ui/src/main/resources/ui/src/generated/type/entityLineage.ts index b81b9118e64..33ddf673345 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/type/entityLineage.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/type/entityLineage.ts @@ -19,8 +19,8 @@ export interface EntityLineage { /** * Primary entity for which this lineage graph is created. */ - entity: EntityReference; - nodes?: EntityReference[]; + entity: EntityReference; + nodes?: EntityReference[]; upstreamEdges?: Edge[]; } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/type/entityRelationship.ts b/openmetadata-ui/src/main/resources/ui/src/generated/type/entityRelationship.ts index 76827eed041..daefb0cf3b1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/type/entityRelationship.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/type/entityRelationship.ts @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * This schema defines the EntityRelationship type used for establishing relationship * between two entities. EntityRelationship is used for capturing relationships from one * entity to another. For example, a database contains tables. @@ -81,6 +79,7 @@ export enum RelationshipType { ParentOf = "parentOf", ReactedTo = "reactedTo", RelatedTo = "relatedTo", + RelatesTo = "relatesTo", RepliedTo = "repliedTo", Reviews = "reviews", TestedBy = "testedBy", diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts index 0af61010900..76c34674a0f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts @@ -20,6 +20,7 @@ import bigQueryConnection from '../jsons/connectionSchemas/connections/database/ import bigTableConnection from '../jsons/connectionSchemas/connections/database/bigTableConnection.json'; import cassandraConnection from '../jsons/connectionSchemas/connections/database/cassandraConnection.json'; import clickhouseConnection from '../jsons/connectionSchemas/connections/database/clickhouseConnection.json'; +import cockroachConnection from '../jsons/connectionSchemas/connections/database/cockroachConnection.json'; import couchbaseConnection from '../jsons/connectionSchemas/connections/database/couchbaseConnection.json'; import customDatabaseConnection from '../jsons/connectionSchemas/connections/database/customDatabaseConnection.json'; import databricksConnection from '../jsons/connectionSchemas/connections/database/databricksConnection.json'; @@ -87,6 +88,11 @@ export const getDatabaseConfig = (type: DatabaseServiceType) => { break; } + case DatabaseServiceType.Cockroach: { + schema = cockroachConnection; + + break; + } case DatabaseServiceType.Databricks: { schema = databricksConnection; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts index 92d448077d8..13be10061d6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts @@ -26,6 +26,7 @@ import { BIGTABLE, CASSANDRA, CLICKHOUSE, + COCKROACH, COUCHBASE, CUSTOM_SEARCH_DEFAULT, CUSTOM_STORAGE_DEFAULT, @@ -356,6 +357,9 @@ class ServiceUtilClassBase { case this.DatabaseServiceTypeSmallCase.Couchbase: return COUCHBASE; + case this.DatabaseServiceTypeSmallCase.Cockroach: + return COCKROACH; + case this.DatabaseServiceTypeSmallCase.Greenplum: return GREENPLUM; @@ -675,6 +679,8 @@ class ServiceUtilClassBase { return 'ElasticSearch'; case this.SearchServiceTypeSmallCase.CustomSearch: return 'Custom Search'; + case this.DatabaseServiceTypeSmallCase.Cockroach: + return 'Cockroach'; default: return capitalize(serviceType);