yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-03 12:53:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature : Cockroach db connector (#18961)

Browse Source
This commit is contained in:
Akash Verma 2025-01-02 13:07:55 +05:30 committed by GitHub
parent ceeba2ad75
commit 39dcb5baef
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
55 changed files with 1324 additions and 213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ingestion/setup.py
View File

@ -57,6 +57,7 @@ VERSIONS = {
"giturlparse": "giturlparse",
"validators": "validators~=0.22.0",
"teradata": "teradatasqlalchemy>=20.0.0.0",
"cockroach": "sqlalchemy-cockroachdb~=2.0",
"cassandra": "cassandra-driver>=3.28.0",
}
@ -233,6 +234,10 @@ plugins: Dict[str, Set[str]] = {
"glue": {VERSIONS["boto3"]},
"great-expectations": {VERSIONS["great-expectations"]},
"greenplum": {*COMMONS["postgres"]},
"cockroach": {
VERSIONS["cockroach"],
"psycopg2-binary",
},
"hive": {
*COMMONS["hive"],
"thrift>=0.13,<1",
@ -380,6 +385,7 @@ test = {
VERSIONS["avro"], # Sample Data
VERSIONS["grpc-tools"],
VERSIONS["neo4j"],
VERSIONS["cockroach"],
"testcontainers==3.7.1;python_version<'3.9'",
"testcontainers~=4.8.0;python_version>='3.9'",
"minio==7.2.5",

24
ingestion/src/metadata/examples/workflows/cockroach.yaml Normal file
View File

@ -0,0 +1,24 @@
source:
type: cockroach
serviceName: local_cockroach
serviceConnection:
config:
type: Cockroach
username: username
authType:
password: password
hostPort: localhost:26257
database: database
sourceConfig:
config:
type: DatabaseMetadata
sink:
type: metadata-rest
config: {}
workflowConfig:
# loggerLevel: INFO # DEBUG, INFO, WARN or ERROR
openMetadataServerConfig:
hostPort: http://localhost:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

0
ingestion/src/metadata/ingestion/source/database/cockroach/__init__.py Normal file
View File

71
ingestion/src/metadata/ingestion/source/database/cockroach/connection.py Normal file
View File

@ -0,0 +1,71 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Source connection handler
"""
from typing import Optional
from sqlalchemy.engine import Engine
from metadata.generated.schema.entity.automations.workflow import (
Workflow as AutomationWorkflow,
)
from metadata.generated.schema.entity.services.connections.database.cockroachConnection import (
CockroachConnection,
)
from metadata.generated.schema.entity.services.connections.testConnectionResult import (
TestConnectionResult,
)
from metadata.ingestion.connections.builders import (
create_generic_db_connection,
get_connection_args_common,
get_connection_url_common,
)
from metadata.ingestion.connections.test_connections import test_connection_db_common
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.database.cockroach.queries import COCKROACH_GET_DATABASE
from metadata.utils.constants import THREE_MIN
def get_connection(connection: CockroachConnection) -> Engine:
"""
Create connection
"""
return create_generic_db_connection(
connection=connection,
get_connection_url_fn=get_connection_url_common,
get_connection_args_fn=get_connection_args_common,
)
def test_connection(
metadata: OpenMetadata,
engine: Engine,
service_connection: CockroachConnection,
automation_workflow: Optional[AutomationWorkflow] = None,
timeout_seconds: Optional[int] = THREE_MIN,
) -> TestConnectionResult:
"""
Test connection. This can be executed either as part
of a metadata workflow or during an Automation Workflow
"""
queries = {
"GetDatabases": COCKROACH_GET_DATABASE,
}
return test_connection_db_common(
metadata=metadata,
engine=engine,
service_connection=service_connection,
automation_workflow=automation_workflow,
queries=queries,
timeout_seconds=timeout_seconds,
)

204
ingestion/src/metadata/ingestion/source/database/cockroach/metadata.py Normal file
View File

@ -0,0 +1,204 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Postgres source module
"""
import traceback
from collections import namedtuple
from typing import Iterable, Optional
from sqlalchemy import String as SqlAlchemyString
from sqlalchemy import sql
from sqlalchemy.dialects.postgresql.base import ischema_names
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.table import TableType
from metadata.generated.schema.entity.services.connections.database.cockroachConnection import (
CockroachConnection,
)
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,
)
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.ingestion.api.steps import InvalidSourceException
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.database.cockroach.queries import (
COCKROACH_GET_DB_NAMES,
COCKROACH_GET_TABLE_NAMES,
COCKROACH_GET_VIEW_NAMES,
COCKROACH_SCHEMA_COMMENTS,
)
from metadata.ingestion.source.database.column_type_parser import create_sqlalchemy_type
from metadata.ingestion.source.database.common_db_source import (
CommonDbSourceService,
TableNameAndType,
)
from metadata.ingestion.source.database.multi_db_source import MultiDBSource
from metadata.utils import fqn
from metadata.utils.filters import filter_by_database
from metadata.utils.importer import import_side_effects
from metadata.utils.logger import ingestion_logger
import_side_effects(
"metadata.ingestion.source.database.postgres.converter_orm",
"metadata.ingestion.source.database.postgres.metrics",
)
TableKey = namedtuple("TableKey", ["schema", "table_name"])
logger = ingestion_logger()
RELKIND_MAP = {
"r": TableType.Regular,
"p": TableType.Partitioned,
"f": TableType.Foreign,
"v": TableType.View,
}
GEOMETRY = create_sqlalchemy_type("GEOMETRY")
POINT = create_sqlalchemy_type("POINT")
POLYGON = create_sqlalchemy_type("POLYGON")
ischema_names.update(
{
"geometry": GEOMETRY,
"point": POINT,
"polygon": POLYGON,
"box": create_sqlalchemy_type("BOX"),
"bpchar": SqlAlchemyString,
"circle": create_sqlalchemy_type("CIRCLE"),
"line": create_sqlalchemy_type("LINE"),
"lseg": create_sqlalchemy_type("LSEG"),
"path": create_sqlalchemy_type("PATH"),
"pg_lsn": create_sqlalchemy_type("PG_LSN"),
"pg_snapshot": create_sqlalchemy_type("PG_SNAPSHOT"),
"tsquery": create_sqlalchemy_type("TSQUERY"),
"txid_snapshot": create_sqlalchemy_type("TXID_SNAPSHOT"),
"xid": SqlAlchemyString,
"xml": create_sqlalchemy_type("XML"),
}
)
class CockroachSource(CommonDbSourceService, MultiDBSource):
"""
Implements the necessary methods to extract
Database metadata from Cockroach Source
"""
def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
super().__init__(config, metadata)
self.schema_desc_map = {}
@classmethod
def create(
cls,
config_dict,
metadata: OpenMetadataConnection,
pipeline_name: Optional[str] = None,
):
config: WorkflowSource = WorkflowSource.model_validate(config_dict)
connection: CockroachConnection = config.serviceConnection.root.config
if not isinstance(connection, CockroachConnection):
raise InvalidSourceException(
f"Expected CockroachConnection, but got {connection}"
)
return cls(config, metadata)
def set_schema_description_map(self) -> None:
self.schema_desc_map.clear()
results = self.engine.execute(COCKROACH_SCHEMA_COMMENTS).all()
for row in results:
self.schema_desc_map[(row.database_name, row.schema_name)] = row.comment
def get_schema_description(self, schema_name: str) -> Optional[str]:
"""
Method to fetch the schema description
"""
return self.schema_desc_map.get((self.context.get().database, schema_name))
def query_table_names_and_types(
self, schema_name: str
) -> Iterable[TableNameAndType]:
"""
Overwrite the inspector implementation to handle partitioned
and foreign types
"""
result = self.connection.execute(
sql.text(COCKROACH_GET_TABLE_NAMES),
{"schema": schema_name},
)
return [
TableNameAndType(
name=name, type_=RELKIND_MAP.get(relkind, TableType.Regular)
)
for name, relkind in result
]
def query_view_names_and_types(
self, schema_name: str
) -> Iterable[TableNameAndType]:
result = self.connection.execute(
sql.text(COCKROACH_GET_VIEW_NAMES),
{"schema": schema_name},
)
return [
TableNameAndType(
name=name, type_=RELKIND_MAP.get(relkind, TableType.Regular)
)
for name, relkind in result
]
def get_configured_database(self) -> Optional[str]:
if not self.service_connection.ingestAllDatabases:
return self.service_connection.database
return None
def get_database_names_raw(self) -> Iterable[str]:
yield from self._execute_database_query(COCKROACH_GET_DB_NAMES)
def get_database_names(self) -> Iterable[str]:
if not self.config.serviceConnection.root.config.ingestAllDatabases:
configured_db = self.config.serviceConnection.root.config.database
self.set_inspector(database_name=configured_db)
self.set_schema_description_map()
yield configured_db
else:
for new_database in self.get_database_names_raw():
database_fqn = fqn.build(
self.metadata,
entity_type=Database,
service_name=self.context.get().database_service,
database_name=new_database,
)
if filter_by_database(
self.source_config.databaseFilterPattern,
(
database_fqn
if self.source_config.useFqnForFiltering
else new_database
),
):
self.status.filter(database_fqn, "Database Filtered Out")
continue
try:
self.set_inspector(database_name=new_database)
self.set_schema_description_map()
yield new_database
except Exception as exc:
logger.debug(traceback.format_exc())
logger.error(
f"Error trying to connect to database {new_database}: {exc}"
)

72
ingestion/src/metadata/ingestion/source/database/cockroach/queries.py Normal file
View File

@ -0,0 +1,72 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
SQL Queries used during ingestion
"""
from sqlalchemy import text
COCKROACH_GET_TABLE_NAMES = """
SELECT
c.relname AS table_name,
c.relkind AS relkind
FROM
pg_class c
JOIN
pg_namespace n ON n.oid = c.relnamespace
WHERE
n.nspname = :schema
AND c.relkind IN ('r', 'p', 'f')
ORDER BY
c.relname
"""
COCKROACH_GET_VIEW_NAMES = """
SELECT
c.relname AS table_name,
c.relkind AS relkind
FROM
pg_class c
JOIN
pg_namespace n ON n.oid = c.relnamespace
WHERE
n.nspname = :schema
AND c.relkind IN ('v')
ORDER BY
c.relname
"""
COCKROACH_SCHEMA_COMMENTS = """
SELECT
current_database() AS database_name,
n.nspname AS schema_name,
d.description AS comment
FROM
pg_namespace n
LEFT JOIN
pg_description d
ON
n.oid = d.objoid
WHERE
d.objsubid = 0;
"""
COCKROACH_GET_DATABASE = text(
"""
select datname FROM pg_catalog.pg_database
"""
)
COCKROACH_GET_DB_NAMES = """
select datname from pg_catalog.pg_database
"""

4
ingestion/src/metadata/ingestion/source/database/cockroach/service_spec.py Normal file
View File

@ -0,0 +1,4 @@
from metadata.ingestion.source.database.cockroach.metadata import CockroachSource
from metadata.utils.service_spec.default import DefaultDatabaseSpec
ServiceSpec = DefaultDatabaseSpec(metadata_source_class=CockroachSource)

0
ingestion/tests/integration/cockroach/__init__.py Normal file
View File

64
ingestion/tests/integration/cockroach/conftest.py Normal file
View File

@ -0,0 +1,64 @@
import os
import textwrap
import pytest
from sqlalchemy import create_engine
from _openmetadata_testutils.helpers.docker import try_bind
from metadata.generated.schema.api.services.createDatabaseService import (
CreateDatabaseServiceRequest,
)
from metadata.generated.schema.entity.services.connections.database.cockroachConnection import (
CockroachConnection,
)
from metadata.generated.schema.entity.services.databaseService import (
DatabaseConnection,
DatabaseServiceType,
)
@pytest.fixture(scope="module")
def cockroach_container(tmp_path_factory):
"""
Start a Cockroach container.
"""
from testcontainers.cockroachdb import CockroachDBContainer
container = CockroachDBContainer(image="cockroachdb/cockroach:v23.1.0")
with (
try_bind(container, 26257, None) if not os.getenv("CI") else container
) as container:
engine = create_engine(container.get_connection_url())
engine.execute(
textwrap.dedent(
"""
CREATE TABLE user_profiles (
user_id UUID PRIMARY KEY,
first_name TEXT,
last_name TEXT,
email TEXT,
signup_date TIMESTAMP,
is_active BOOLEAN
);
"""
)
)
yield container
@pytest.fixture(scope="module")
def create_service_request(cockroach_container, tmp_path_factory):
return CreateDatabaseServiceRequest(
name="docker_test_" + tmp_path_factory.mktemp("cockroach").name,
serviceType=DatabaseServiceType.Cockroach,
connection=DatabaseConnection(
config=CockroachConnection(
username=cockroach_container.username,
authType={"password": cockroach_container.password},
hostPort=f"localhost:{cockroach_container.get_exposed_port(26257)}",
database=cockroach_container.dbname,
)
),
)

76
ingestion/tests/integration/cockroach/test_metadata.py Normal file
View File

@ -0,0 +1,76 @@
import sys
import pytest
from sqlalchemy import create_engine
from metadata.generated.schema.entity.data.table import Constraint, Table
from metadata.workflow.metadata import MetadataWorkflow
if not sys.version_info >= (3, 9):
pytest.skip("requires python 3.9+", allow_module_level=True)
@pytest.fixture(scope="module")
def prepare_cockroach(cockroach_container):
engine = create_engine(cockroach_container.get_connection_url())
sql = [
"""
CREATE TABLE test_table (
id INTEGER NOT NULL DEFAULT unique_rowid(),
name VARCHAR(100),
age INTEGER,
PRIMARY KEY (id)
);
""",
"""
INSERT INTO test_table (name, age) VALUES
('John Doe', 25),
('Jane Smith', 30);
""",
]
for stmt in sql:
engine.execute(stmt)
@pytest.mark.parametrize(
"table_fqn,expected_columns",
[
[
"{service}.roach.public.test_table",
{
"id": {"type": "int", "nullable": False},
"name": {"type": "varchar", "nullable": True},
"age": {"type": "int", "nullable": True},
},
]
],
ids=lambda x: x.split(".")[-1] if isinstance(x, str) else "",
)
def test_ingest_metadata(
patch_passwords_for_db_services,
run_workflow,
ingestion_config,
metadata,
table_fqn,
expected_columns,
db_service,
prepare_cockroach,
):
run_workflow(MetadataWorkflow, ingestion_config)
table = metadata.get_by_name(
entity=Table, fqn=table_fqn.format(service=db_service.fullyQualifiedName.root)
)
assert table
assert table.fullyQualifiedName.root.split(".")[-1] == "test_table"
for name, properties in expected_columns.items():
column = next((col for col in table.columns if col.name.root == name), None)
assert column is not None
assert column.dataType.name.lower() == properties["type"]
assert (
column.constraint == Constraint.PRIMARY_KEY
if name == "id"
else Constraint.NULL
)

0
ingestion/tests/integration/kafka/__init__.py Normal file
View File

4
ingestion/tests/unit/topology/database/test_athena.py
View File

@ -269,9 +269,7 @@ mock_athena_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "athena"},
}
},
}

2
ingestion/tests/unit/topology/database/test_bigquery.py
View File

@ -87,7 +87,7 @@ mock_bq_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {"jwtToken": "jwt"},
"securityConfig": {"jwtToken": "bigquery"},
}
},
}

4
ingestion/tests/unit/topology/database/test_bigtable.py
View File

@ -86,9 +86,7 @@ mock_bigtable_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "bigtable"},
}
},
}

4
ingestion/tests/unit/topology/database/test_cassandra.py
View File

@ -66,9 +66,7 @@ mock_cassandra_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "cassandra"},
}
},
}

263
ingestion/tests/unit/topology/database/test_cockroach.py Normal file
View File

@ -0,0 +1,263 @@
"""
Test Cockroach using the topology
"""
import types
from unittest import TestCase
from unittest.mock import patch
from sqlalchemy.types import VARCHAR
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.table import Column, Constraint, DataType
from metadata.generated.schema.entity.services.databaseService import (
DatabaseConnection,
DatabaseService,
DatabaseServiceType,
)
from metadata.generated.schema.metadataIngestion.workflow import (
OpenMetadataWorkflowConfig,
)
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.source.database.cockroach.metadata import (
GEOMETRY,
POINT,
POLYGON,
CockroachSource,
)
mock_cockroach_config = {
"source": {
"type": "cockroach",
"serviceName": "local_cockroach1",
"serviceConnection": {
"config": {
"type": "Cockroach",
"username": "username",
"authType": {
"password": "password",
},
"hostPort": "localhost:26257",
"database": "cockroach",
}
},
"sourceConfig": {
"config": {
"type": "DatabaseMetadata",
}
},
},
"sink": {
"type": "metadata-rest",
"config": {},
},
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {"jwtToken": "cockroach"},
}
},
}
MOCK_DATABASE_SERVICE = DatabaseService(
id="85811038-099a-11ed-861d-0242ac120002",
name="cockroach_source",
connection=DatabaseConnection(),
serviceType=DatabaseServiceType.Cockroach,
)
MOCK_DATABASE = Database(
id="2aaa012e-099a-11ed-861d-0242ac120002",
name="default",
fullyQualifiedName="cockroach_source.default",
displayName="default",
description="",
service=EntityReference(
id="85811038-099a-11ed-861d-0242ac120002",
type="databaseService",
),
)
MOCK_DATABASE_SCHEMA = DatabaseSchema(
id="2aaa012e-099a-11ed-861d-0242ac120056",
name="default",
fullyQualifiedName="cockroach_source.default.default",
displayName="default",
description="",
database=EntityReference(
id="2aaa012e-099a-11ed-861d-0242ac120002",
type="database",
),
service=EntityReference(
id="2aaa012e-099a-11ed-861d-0242ac120002",
type="database",
),
)
MOCK_COLUMN_VALUE = [
{
"name": "username",
"type": VARCHAR(),
"nullable": True,
"default": None,
"autoincrement": False,
"system_data_type": "varchar(50)",
"comment": None,
},
{
"name": "geom_c",
"type": GEOMETRY(),
"nullable": True,
"default": None,
"autoincrement": False,
"system_data_type": "geometry",
"comment": None,
},
{
"name": "point_c",
"type": POINT(),
"nullable": True,
"default": None,
"autoincrement": False,
"system_data_type": "point",
"comment": None,
},
{
"name": "polygon_c",
"type": POLYGON(),
"nullable": True,
"default": None,
"autoincrement": False,
"comment": None,
"system_data_type": "polygon",
},
]
EXPECTED_COLUMN_VALUE = [
Column(
name="username",
displayName=None,
dataType=DataType.VARCHAR,
arrayDataType=None,
dataLength=1,
precision=None,
scale=None,
dataTypeDisplay="varchar(50)",
description=None,
fullyQualifiedName=None,
tags=None,
constraint=Constraint.NULL,
ordinalPosition=None,
jsonSchema=None,
children=None,
customMetrics=None,
profile=None,
),
Column(
name="geom_c",
displayName=None,
dataType=DataType.GEOMETRY,
arrayDataType=None,
dataLength=1,
precision=None,
scale=None,
dataTypeDisplay="geometry",
description=None,
fullyQualifiedName=None,
tags=None,
constraint=Constraint.NULL,
ordinalPosition=None,
jsonSchema=None,
children=None,
customMetrics=None,
profile=None,
),
Column(
name="point_c",
displayName=None,
dataType=DataType.GEOMETRY,
arrayDataType=None,
dataLength=1,
precision=None,
scale=None,
dataTypeDisplay="point",
description=None,
fullyQualifiedName=None,
tags=None,
constraint=Constraint.NULL,
ordinalPosition=None,
jsonSchema=None,
children=None,
customMetrics=None,
profile=None,
),
Column(
name="polygon_c",
displayName=None,
dataType=DataType.GEOMETRY,
arrayDataType=None,
dataLength=1,
precision=None,
scale=None,
dataTypeDisplay="polygon",
description=None,
fullyQualifiedName=None,
tags=None,
constraint=Constraint.NULL,
ordinalPosition=None,
jsonSchema=None,
children=None,
customMetrics=None,
profile=None,
),
]
class cockroachUnitTest(TestCase):
@patch(
"metadata.ingestion.source.database.common_db_source.CommonDbSourceService.test_connection"
)
def __init__(self, methodName, test_connection) -> None:
super().__init__(methodName)
test_connection.return_value = False
self.config = OpenMetadataWorkflowConfig.model_validate(mock_cockroach_config)
self.cockroach_source = CockroachSource.create(
mock_cockroach_config["source"],
self.config.workflowConfig.openMetadataServerConfig,
)
self.cockroach_source.context.get().__dict__[
"database_service"
] = MOCK_DATABASE_SERVICE.name.root
self.cockroach_source.context.get().__dict__[
"database"
] = MOCK_DATABASE.name.root
self.cockroach_source.context.get().__dict__[
"database_schema"
] = MOCK_DATABASE_SCHEMA.name.root
def test_datatype(self):
inspector = types.SimpleNamespace()
inspector.get_columns = (
lambda table_name, schema_name, db_name: MOCK_COLUMN_VALUE
)
inspector.get_pk_constraint = lambda table_name, schema_name: []
inspector.get_unique_constraints = lambda table_name, schema_name: []
inspector.get_foreign_keys = lambda table_name, schema_name: []
result, _, _ = self.cockroach_source.get_columns_and_constraints(
"public", "user", "cockroach", inspector
)
for i, _ in enumerate(EXPECTED_COLUMN_VALUE):
self.assertEqual(result[i], EXPECTED_COLUMN_VALUE[i])
@patch("sqlalchemy.engine.base.Engine")
@patch(
"metadata.ingestion.source.database.common_db_source.CommonDbSourceService.connection"
)
def test_close_connection(self, engine, connection):
connection.return_value = True
self.cockroach_source.close()

4
ingestion/tests/unit/topology/database/test_couchbase.py
View File

@ -53,9 +53,7 @@ mock_couch_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "couchbase"},
}
},
}

4
ingestion/tests/unit/topology/database/test_databricks.py
View File

@ -63,9 +63,7 @@ mock_databricks_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "databricks"},
}
},
}

8
ingestion/tests/unit/topology/database/test_datalake.py
View File

@ -73,9 +73,7 @@ mock_datalake_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "datalake"},
}
},
}
@ -583,9 +581,7 @@ mock_datalake_gcs_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "datalake"},
},
},
}

9
ingestion/tests/unit/topology/database/test_domodatabase.py
View File

@ -94,14 +94,7 @@ mock_domodatabase_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc"
"iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE"
"2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB"
"iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN"
"r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u"
"d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "domodatabase"},
}
},
}

4
ingestion/tests/unit/topology/database/test_doris.py
View File

@ -52,9 +52,7 @@ mock_doris_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "doris"},
}
},
}

4
ingestion/tests/unit/topology/database/test_glue.py
View File

@ -62,9 +62,7 @@ mock_glue_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "glue"},
}
},
}

4
ingestion/tests/unit/topology/database/test_greenplum.py
View File

@ -50,9 +50,7 @@ mock_greenplum_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "greenplum"},
}
},
}

9
ingestion/tests/unit/topology/database/test_hive.py
View File

@ -63,14 +63,7 @@ mock_hive_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc"
"iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE"
"2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB"
"iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN"
"r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u"
"d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "hive"},
}
},
}

8
ingestion/tests/unit/topology/database/test_iceberg.py
View File

@ -390,7 +390,7 @@ MOCK_HIVE_CONFIG = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {"jwtToken": "token"},
"securityConfig": {"jwtToken": "iceberg"},
}
},
}
@ -415,7 +415,7 @@ MOCK_REST_CONFIG = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {"jwtToken": "token"},
"securityConfig": {"jwtToken": "iceberg"},
}
},
}
@ -447,7 +447,7 @@ MOCK_GLUE_CONFIG = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {"jwtToken": "token"},
"securityConfig": {"jwtToken": "iceberg"},
}
},
}
@ -481,7 +481,7 @@ MOCK_DYNAMO_CONFIG = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {"jwtToken": "token"},
"securityConfig": {"jwtToken": "iceberg"},
}
},
}

4
ingestion/tests/unit/topology/database/test_mongodb.py
View File

@ -66,9 +66,7 @@ mock_mongo_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "mongodb"},
}
},
}

9
ingestion/tests/unit/topology/database/test_mssql.py
View File

@ -65,14 +65,7 @@ mock_mssql_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc"
"iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE"
"2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB"
"iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN"
"r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u"
"d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "mssql"},
}
},
}

4
ingestion/tests/unit/topology/database/test_mysql.py
View File

@ -51,9 +51,7 @@ mock_mysql_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "mysql"},
}
},
}

9
ingestion/tests/unit/topology/database/test_oracle.py
View File

@ -65,14 +65,7 @@ mock_oracle_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc"
"iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE"
"2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB"
"iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN"
"r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u"
"d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "oracle"},
}
},
}

8
ingestion/tests/unit/topology/database/test_postgres.py
View File

@ -73,9 +73,7 @@ mock_postgres_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "postgres"},
}
},
}
@ -110,9 +108,7 @@ mock_postgres_usage_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "postgres"},
}
},
}

4
ingestion/tests/unit/topology/database/test_redshift.py
View File

@ -45,9 +45,7 @@ mock_redshift_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "redshift"},
}
},
}

4
ingestion/tests/unit/topology/database/test_salesforce.py
View File

@ -63,9 +63,7 @@ mock_salesforce_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "salesforce"},
}
},
}

4
ingestion/tests/unit/topology/database/test_snowflake.py
View File

@ -47,9 +47,7 @@ SNOWFLAKE_CONFIGURATION = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "snowflake"},
}
},
"ingestionPipelineFQN": "snowflake.mock_pipeline",

4
ingestion/tests/unit/topology/database/test_unity_catalog.py
View File

@ -87,9 +87,7 @@ mock_unitycatalog_config = {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
"securityConfig": {"jwtToken": "unity_catalog"},
}
},
}

5
openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java
View File

@ -27,6 +27,7 @@ import org.openmetadata.schema.services.connections.dashboard.TableauConnection;
import org.openmetadata.schema.services.connections.database.BigQueryConnection;
import org.openmetadata.schema.services.connections.database.BigTableConnection;
import org.openmetadata.schema.services.connections.database.CassandraConnection;
import org.openmetadata.schema.services.connections.database.CockroachConnection;
import org.openmetadata.schema.services.connections.database.DatalakeConnection;
import org.openmetadata.schema.services.connections.database.DeltaLakeConnection;
import org.openmetadata.schema.services.connections.database.GreenplumConnection;
@ -85,7 +86,9 @@ public final class ClassConverterFactory {
TestServiceConnectionRequest.class,
new TestServiceConnectionRequestClassConverter()),
Map.entry(TrinoConnection.class, new TrinoConnectionClassConverter()),
Map.entry(Workflow.class, new WorkflowClassConverter()));
Map.entry(Workflow.class, new WorkflowClassConverter()),
Map.entry(CockroachConnection.class, new CockroachConnectionClassConverter()));
Map.entry(Workflow.class, new WorkflowClassConverter());
Map.entry(CassandraConnection.class, new CassandraConnectionClassConverter());
}

34
openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/CockroachConnectionClassConverter.java Normal file
View File

@ -0,0 +1,34 @@
package org.openmetadata.service.secrets.converter;
import java.util.List;
import org.openmetadata.schema.security.ssl.ValidateSSLClientConfig;
import org.openmetadata.schema.services.connections.database.CockroachConnection;
import org.openmetadata.schema.services.connections.database.common.basicAuth;
import org.openmetadata.service.util.JsonUtils;
/**
* Converter class to get an `CockroachConnection` object.
*/
public class CockroachConnectionClassConverter extends ClassConverter {
private static final List<Class<?>> SSL_SOURCE_CLASS = List.of(ValidateSSLClientConfig.class);
private static final List<Class<?>> CONFIG_SOURCE_CLASSES = List.of(basicAuth.class);
public CockroachConnectionClassConverter() {
super(CockroachConnection.class);
}
@Override
public Object convert(Object object) {
CockroachConnection cockroachConnection =
(CockroachConnection) JsonUtils.convertValue(object, this.clazz);
tryToConvert(cockroachConnection.getAuthType(), CONFIG_SOURCE_CLASSES)
.ifPresent(cockroachConnection::setAuthType);
tryToConvert(cockroachConnection.getSslConfig(), SSL_SOURCE_CLASS)
.ifPresent(cockroachConnection::setSslConfig);
return cockroachConnection;
}
}

39
openmetadata-service/src/main/resources/json/data/testConnections/database/cockroach.json Normal file
View File

@ -0,0 +1,39 @@
{
"name": "Cockroach",
"displayName": "Cockroach Test Connection",
"description": "This Test Connection validates the access against the database and basic metadata extraction of schemas and tables.",
"steps": [
{
"name": "CheckAccess",
"description": "Validate that we can properly reach the database and authenticate with the given credentials.",
"errorMessage": "Failed to connect to Cockroach, please validate the credentials",
"mandatory": true
},
{
"name": "GetDatabases",
"description": "List all the databases available to the user.",
"errorMessage": "Failed to fetch databases, please validate if the user has enough privilege to fetch databases.",
"mandatory": true
},
{
"name": "GetSchemas",
"description": "List all the schemas available to the user.",
"errorMessage": "Failed to fetch schemas, please validate if the user has enough privilege to fetch schemas.",
"mandatory": true
},
{
"name": "GetTables",
"description": "From a given schema, list the tables belonging to that schema. If no schema is specified, we'll list the tables of a random schema.",
"errorMessage": "Failed to fetch tables, please validate if the user has enough privilege to fetch tables.",
"mandatory": true
},
{
"name": "GetViews",
"description": "From a given schema, list the views belonging to that schema. If no schema is specified, we'll list the tables of a random schema.",
"errorMessage": "Failed to fetch views, please validate if the user has enough privilege to fetch views.",
"mandatory": false
}
]
}

93
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/cockroachConnection.json Normal file
View File

@ -0,0 +1,93 @@
{
"$id": "https://open-metadata.org/schema/entity/services/connections/database/cockroachConnection.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CockroachConnection",
"description": "Cockroach Database Connection Config",
"type": "object",
"javaType": "org.openmetadata.schema.services.connections.database.CockroachConnection",
"definitions": {
"cockroachType": {
"description": "Service type.",
"type": "string",
"enum": ["Cockroach"],
"default": "Cockroach"
},
"cockroachScheme": {
"description": "SQLAlchemy driver scheme options.",
"type": "string",
"enum": ["cockroachdb+psycopg2"],
"default": "cockroachdb+psycopg2"
}
},
"properties": {
"type": {
"title": "Service Type",
"description": "Service Type",
"$ref": "#/definitions/cockroachType",
"default": "Cockroach"
},
"scheme": {
"title": "Connection Scheme",
"description": "SQLAlchemy driver scheme options.",
"$ref": "#/definitions/cockroachScheme",
"default": "cockroachdb+psycopg2"
},
"username": {
"title": "Username",
"description": "Username to connect to Cockroach. This user should have privileges to read all the metadata in Cockroach.",
"type": "string"
},
"authType": {
"title": "Auth Configuration Type",
"description": "Choose Auth Config Type.",
"oneOf": [
{
"$ref": "./common/basicAuth.json"
}
]
},
"hostPort": {
"title": "Host and Port",
"description": "Host and port of the Cockrooach service.",
"type": "string"
},
"database": {
"title": "Database Name",
"description": "Optional name to give to the database in OpenMetadata. If left blank, we will use default as the database name.",
"type": "string"
},
"ingestAllDatabases": {
"title": "Ingest All Databases",
"description": "Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of this.",
"type": "boolean",
"default": false
},
"databaseSchema": {
"title": "Database Schema",
"description": "Database Schema of the data source. This is optional parameter, if you would like to restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion attempts to scan all the schemas.",
"type": "string"
},
"sslConfig": {
"title": "SSL",
"description": "SSL Configuration details.",
"$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig"
},
"sslMode": {
"$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslMode"
},
"connectionOptions": {
"title": "Connection Options",
"$ref": "../connectionBasicType.json#/definitions/connectionOptions"
},
"connectionArguments": {
"title": "Connection Arguments",
"$ref": "../connectionBasicType.json#/definitions/connectionArguments"
},
"supportsMetadataExtraction": {
"title": "Supports Metadata Extraction",
"$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
}
},
"additionalProperties": false,
"required": ["hostPort", "username"]
}

9
openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json
View File

@ -59,7 +59,8 @@
"Teradata",
"SapErp",
"Synapse",
"Exasol"
"Exasol",
"Cockroach"
],
"javaEnums": [
{
@ -196,6 +197,9 @@
},
{
"name": "Exasol"
},
{
"name": "Cockroach"
}
]
},
@ -337,6 +341,9 @@
},
{
"$ref": "./connections/database/exasolConnection.json"
},
{
"$ref": "./connections/database/cockroachConnection.json"
}
]
}

73
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Cockroach.md Normal file
View File

@ -0,0 +1,73 @@
# Cockroach
In this section, we provide guides and references to use the Cockroach connector.
## Requirements
### Profiler & Data Quality
Executing the profiler Workflow or data quality tests, will require the user to have `SELECT` permission on the tables/schemas where the profiler/tests will be executed. More information on the profiler workflow setup can be found [here](https://docs.open-metadata.org/how-to-guides/data-quality-observability/profiler/workflow) and data quality tests [here](https://docs.open-metadata.org/connectors/ingestion/workflows/data-quality).
You can find further information on the Cockroach connector in the [docs](https://docs.open-metadata.org/connectors/database/cockroach).
## Connection Details
$$section
### Connection Scheme $(id="scheme")
SQLAlchemy driver scheme options.
$$
$$section
### Username $(id="username")
Username to connect to Cockroach. This user should have privileges to read all the metadata in Cockroach.
$$
$$section
### Auth Config $(id="authType")
There is one auth config:
- Basic Auth.
User can authenticate the Cockroach Instance with auth type as `Basic Authentication` i.e. Password
$$
## Basic Auth
$$section
### Password $(id="password")
Password to connect to Cockroach.
$$
$$section
### Host and Port $(id="hostPort")
This parameter specifies the host and port of the Cockroach instance. This should be specified as a string in the format `hostname:port`. For example, you might set the hostPort parameter to `localhost:26257`.
If you are running the OpenMetadata ingestion in a docker and your services are hosted on the `localhost`, then use `host.docker.internal:26257` as the value.
$$
$$section
### Database $(id="database")
Initial Cockroach database to connect to. If you want to ingest all databases, set `ingestAllDatabases` to true.
$$
$$section
### Ingest All Databases $(id="ingestAllDatabases")
If ticked, the workflow will be able to ingest all database in the cluster. If not ticked, the workflow will only ingest tables from the database set above.
$$
$$section
### Connection Arguments $(id="connectionArguments")
Additional connection arguments such as security or protocol configs that can be sent to service during connection.
$$
$$section
### Connection Options $(id="connectionOptions")
Additional connection options to build the URL that can be sent to service during the connection.
$$

BIN
openmetadata-ui/src/main/resources/ui/src/assets/img/service-icon-cockroach.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 798 KiB

3
openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts
View File

@ -24,6 +24,7 @@ import azuresql from '../assets/img/service-icon-azuresql.png';
import bigtable from '../assets/img/service-icon-bigtable.png';
import cassandra from '../assets/img/service-icon-cassandra.png';
import clickhouse from '../assets/img/service-icon-clickhouse.png';
import cockroach from '../assets/img/service-icon-cockroach.png';
import couchbase from '../assets/img/service-icon-couchbase.svg';
import dagster from '../assets/img/service-icon-dagster.png';
import databrick from '../assets/img/service-icon-databrick.png';
@ -202,6 +203,7 @@ export const ICEBERGE = iceberge;
export const TERADATA = teradata;
export const FLINK = flink;
export const REST_SERVICE = restService;
export const COCKROACH = cockroach;
export const excludedService = [
MlModelServiceType.Sklearn,
MetadataServiceType.MetadataES,
@ -399,6 +401,7 @@ export const BETA_SERVICES = [
DatabaseServiceType.Cassandra,
MetadataServiceType.AlationSink,
DatabaseServiceType.Synapse,
DatabaseServiceType.Cockroach,
];
export const TEST_CONNECTION_INITIAL_MESSAGE = i18n.t(

4
openmetadata-ui/src/main/resources/ui/src/generated/api/data/createSearchIndex.ts
View File

@ -10,9 +10,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
/**
* Create a SearchIndex entity request
*/
export interface CreateSearchIndex {

1
openmetadata-ui/src/main/resources/ui/src/generated/entity/data/database.ts
View File

@ -511,6 +511,7 @@ export enum DatabaseServiceType {
BigTable = "BigTable",
Cassandra = "Cassandra",
Clickhouse = "Clickhouse",
Cockroach = "Cockroach",
Couchbase = "Couchbase",
CustomDatabase = "CustomDatabase",
Databricks = "Databricks",

4
openmetadata-ui/src/main/resources/ui/src/generated/entity/data/searchIndex.ts
View File

@ -10,9 +10,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
/**
* A `SearchIndex` is a index mapping definition in ElasticSearch or OpenSearch
*/
export interface SearchIndex {

138
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/cockroachConnection.ts Normal file
View File

@ -0,0 +1,138 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* eslint-disable @typescript-eslint/no-explicit-any */
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Cockroach Database Connection Config
*/
export interface CockroachConnection {
/**
* Choose Auth Config Type.
*/
authType?: AuthConfigurationType;
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Optional name to give to the database in OpenMetadata. If left blank, we will use default
* as the database name.
*/
database?: string;
/**
* Database Schema of the data source. This is optional parameter, if you would like to
* restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Host and port of the Cockrooach service.
*/
hostPort: string;
/**
* Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of
* this.
*/
ingestAllDatabases?: boolean;
/**
* SQLAlchemy driver scheme options.
*/
scheme?: CockroachScheme;
/**
* SSL Configuration details.
*/
sslConfig?: Config;
sslMode?: SSLMode;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: CockroachType;
/**
* Username to connect to Cockroach. This user should have privileges to read all the
* metadata in Cockroach.
*/
username: string;
}
/**
* Choose Auth Config Type.
*
* Common Database Connection Config
*/
export interface AuthConfigurationType {
/**
* Password to connect to source.
*/
password?: string;
}
/**
* SQLAlchemy driver scheme options.
*/
export enum CockroachScheme {
CockroachdbPsycopg2 = "cockroachdb+psycopg2",
}
/**
* SSL Configuration details.
*
* Client SSL configuration
*
* OpenMetadata Client configured to validate SSL certificates.
*/
export interface Config {
/**
* The CA certificate used for SSL validation.
*/
caCertificate?: string;
/**
* The SSL certificate used for client authentication.
*/
sslCertificate?: string;
/**
* The private key associated with the SSL certificate.
*/
sslKey?: string;
}
/**
* SSL Mode to connect to database.
*/
export enum SSLMode {
Allow = "allow",
Disable = "disable",
Prefer = "prefer",
Require = "require",
VerifyCA = "verify-ca",
VerifyFull = "verify-full",
}
/**
* Service Type
*
* Service type.
*/
export enum CockroachType {
Cockroach = "Cockroach",
}

12
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/db2Connection.ts
View File

@ -10,9 +10,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
/**
* Db2 Connection Config
*/
export interface Db2Connection {
@ -26,6 +24,14 @@ export interface Db2Connection {
* Host and port of the DB2 service.
*/
hostPort: string;
/**
* License to connect to DB2.
*/
license?: string;
/**
* License file name to connect to DB2.
*/
licenseFileName?: string;
/**
* Password to connect to DB2.
*/

75
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts
View File

@ -10,8 +10,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* This schema defines the `Database Service` is a service such as MySQL, BigQuery,
* Redshift, Postgres, or Snowflake. Alternative terms such as Database Cluster, Database
@ -230,6 +228,8 @@ export interface DatabaseConnection {
* Synapse Database Connection Config
*
* Exasol Database Connection Config
*
* Cockroach Database Connection Config
*/
export interface ConfigClass {
/**
@ -295,6 +295,8 @@ export interface ConfigClass {
* Host and Port of the SAP ERP instance.
*
* Host and port of the Azure Synapse service.
*
* Host and port of the Cockrooach service.
*/
hostPort?: string;
sampleDataStorageConfig?: SampleDataStorageConfig;
@ -378,6 +380,9 @@ export interface ConfigClass {
*
* Initial Redshift database to connect to. If you want to ingest all databases, set
* ingestAllDatabases to true.
*
* Optional name to give to the database in OpenMetadata. If left blank, we will use default
* as the database name.
*/
database?: string;
/**
@ -443,8 +448,6 @@ export interface ConfigClass {
*
* Password to connect to MongoDB.
*
* Password to connect to Cassandra.
*
* Password to connect to Couchbase.
*
* Password to connect to Doris.
@ -542,6 +545,9 @@ export interface ConfigClass {
*
* Username to connect to Exasol. This user should have privileges to read all the metadata
* in Exasol.
*
* Username to connect to Cockroach. This user should have privileges to read all the
* metadata in Cockroach.
*/
username?: string;
/**
@ -779,10 +785,6 @@ export interface ConfigClass {
* Choose between Database connection or HDB User Store connection.
*/
connection?: SAPHanaConnection;
/**
* Configuration for connecting to DataStax Astra DB in the cloud.
*/
cloudConfig?: DataStaxAstraDBConfiguration;
/**
* Couchbase connection Bucket options.
*/
@ -887,6 +889,8 @@ export enum AuthMechanismEnum {
* IAM Auth Database Connection Config
*
* Azure Database Connection Config
*
* Configuration for connecting to DataStax Astra DB in the cloud.
*/
export interface AuthConfigurationType {
/**
@ -899,6 +903,10 @@ export interface AuthConfigurationType {
* JWT to connect to source.
*/
jwt?: string;
/**
* Configuration for connecting to DataStax Astra DB in the cloud.
*/
cloudConfig?: DataStaxAstraDBConfiguration;
}
/**
@ -977,6 +985,30 @@ export interface AzureCredentials {
vaultName?: string;
}
/**
* Configuration for connecting to DataStax Astra DB in the cloud.
*/
export interface DataStaxAstraDBConfiguration {
/**
* Timeout in seconds for establishing new connections to Cassandra.
*/
connectTimeout?: number;
/**
* Timeout in seconds for individual Cassandra requests.
*/
requestTimeout?: number;
/**
* File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax
* Astra DB.
*/
secureConnectBundle?: string;
/**
* The Astra DB application token used for authentication.
*/
token?: string;
[property: string]: any;
}
/**
* Database Authentication types not requiring config.
*/
@ -1214,30 +1246,6 @@ export interface SSLCertificatesByPath {
privateKeyPath?: string;
}
/**
* Configuration for connecting to DataStax Astra DB in the cloud.
*/
export interface DataStaxAstraDBConfiguration {
/**
* Timeout in seconds for establishing new connections to Cassandra.
*/
connectTimeout?: number;
/**
* Timeout in seconds for individual Cassandra requests.
*/
requestTimeout?: number;
/**
* File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax
* Astra DB.
*/
secureConnectBundle?: string;
/**
* The Astra DB application token used for authentication.
*/
token?: string;
[property: string]: any;
}
/**
* Available sources to fetch the metadata.
*
@ -1833,6 +1841,7 @@ export enum ConfigScheme {
Bigquery = "bigquery",
ClickhouseHTTP = "clickhouse+http",
ClickhouseNative = "clickhouse+native",
CockroachdbPsycopg2 = "cockroachdb+psycopg2",
Couchbase = "couchbase",
DatabricksConnector = "databricks+connector",
Db2IBMDB = "db2+ibm_db",
@ -1901,6 +1910,7 @@ export enum ConfigType {
BigTable = "BigTable",
Cassandra = "Cassandra",
Clickhouse = "Clickhouse",
Cockroach = "Cockroach",
Couchbase = "Couchbase",
CustomDatabase = "CustomDatabase",
Databricks = "Databricks",
@ -2017,6 +2027,7 @@ export enum DatabaseServiceType {
BigTable = "BigTable",
Cassandra = "Cassandra",
Clickhouse = "Clickhouse",
Cockroach = "Cockroach",
Couchbase = "Couchbase",
CustomDatabase = "CustomDatabase",
Databricks = "Databricks",

27
openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceAutoClassificationPipeline.ts
View File

@ -10,9 +10,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
/**
* DatabaseService AutoClassification & Auto Classification Pipeline Configuration.
*/
export interface DatabaseServiceAutoClassificationPipeline {
@ -40,17 +38,10 @@ export interface DatabaseServiceAutoClassificationPipeline {
* Optional configuration to turn off fetching metadata for views.
*/
includeViews?: boolean;
/**
* Percentage of data or no. of rows used to compute the profiler metrics and run data
* quality tests
*/
profileSample?: number;
profileSampleType?: ProfileSampleType;
/**
* Number of sample rows to ingest when 'Generate Sample Data' is enabled
*/
sampleDataCount?: number;
samplingMethodType?: SamplingMethodType;
/**
* Regex to only fetch tables or databases that matches the pattern.
*/
@ -98,22 +89,6 @@ export interface FilterPattern {
includes?: string[];
}
/**
* Type of Profile Sample (percentage or rows)
*/
export enum ProfileSampleType {
Percentage = "PERCENTAGE",
Rows = "ROWS",
}
/**
* Type of Sampling Method (BERNOULLI or SYSTEM)
*/
export enum SamplingMethodType {
Bernoulli = "BERNOULLI",
System = "SYSTEM",
}
/**
* Pipeline type
*

8
openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceProfilerPipeline.ts
View File

@ -10,9 +10,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
/**
* DatabaseService Profiler Pipeline Configuration.
*/
export interface DatabaseServiceProfilerPipeline {
@ -49,10 +47,6 @@ export interface DatabaseServiceProfilerPipeline {
*/
profileSample?: number;
profileSampleType?: ProfileSampleType;
/**
* Number of sample rows to ingest when 'Generate Sample Data' is enabled
*/
sampleDataCount?: number;
samplingMethodType?: SamplingMethodType;
/**
* Regex to only fetch tables or databases that matches the pattern.

13
openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceQueryLineagePipeline.ts
View File

@ -10,12 +10,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
/**
* DatabaseService Query Lineage Pipeline Configuration.
*/
export interface DatabaseServiceQueryLineagePipeline {
/**
* Set 'Cross Database Service Names' to process lineage with the database.
*/
crossDatabaseServiceNames?: string[];
/**
* Regex to only fetch databases that matches the pattern.
*/
@ -33,6 +35,11 @@ export interface DatabaseServiceQueryLineagePipeline {
* Configuration to set the timeout for parsing the query in seconds.
*/
parsingTimeoutLimit?: number;
/**
* Set the 'Process Cross Database Lineage' toggle to control whether to process table
* lineage across different databases.
*/
processCrossDatabaseLineage?: boolean;
/**
* Set the 'Process Query Lineage' toggle to control whether to process query lineage.
*/

5
openmetadata-ui/src/main/resources/ui/src/generated/type/entityRelationship.ts
View File

@ -10,9 +10,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
/**
* This schema defines the EntityRelationship type used for establishing relationship
* between two entities. EntityRelationship is used for capturing relationships from one
* entity to another. For example, a database contains tables.
@ -81,6 +79,7 @@ export enum RelationshipType {
ParentOf = "parentOf",
ReactedTo = "reactedTo",
RelatedTo = "relatedTo",
RelatesTo = "relatesTo",
RepliedTo = "repliedTo",
Reviews = "reviews",
TestedBy = "testedBy",

6
openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts
View File

@ -20,6 +20,7 @@ import bigQueryConnection from '../jsons/connectionSchemas/connections/database/
import bigTableConnection from '../jsons/connectionSchemas/connections/database/bigTableConnection.json';
import cassandraConnection from '../jsons/connectionSchemas/connections/database/cassandraConnection.json';
import clickhouseConnection from '../jsons/connectionSchemas/connections/database/clickhouseConnection.json';
import cockroachConnection from '../jsons/connectionSchemas/connections/database/cockroachConnection.json';
import couchbaseConnection from '../jsons/connectionSchemas/connections/database/couchbaseConnection.json';
import customDatabaseConnection from '../jsons/connectionSchemas/connections/database/customDatabaseConnection.json';
import databricksConnection from '../jsons/connectionSchemas/connections/database/databricksConnection.json';
@ -87,6 +88,11 @@ export const getDatabaseConfig = (type: DatabaseServiceType) => {
break;
}
case DatabaseServiceType.Cockroach: {
schema = cockroachConnection;
break;
}
case DatabaseServiceType.Databricks: {
schema = databricksConnection;

6
openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts
View File

@ -26,6 +26,7 @@ import {
BIGTABLE,
CASSANDRA,
CLICKHOUSE,
COCKROACH,
COUCHBASE,
CUSTOM_SEARCH_DEFAULT,
CUSTOM_STORAGE_DEFAULT,
@ -356,6 +357,9 @@ class ServiceUtilClassBase {
case this.DatabaseServiceTypeSmallCase.Couchbase:
return COUCHBASE;
case this.DatabaseServiceTypeSmallCase.Cockroach:
return COCKROACH;
case this.DatabaseServiceTypeSmallCase.Greenplum:
return GREENPLUM;
@ -675,6 +679,8 @@ class ServiceUtilClassBase {
return 'ElasticSearch';
case this.SearchServiceTypeSmallCase.CustomSearch:
return 'Custom Search';
case this.DatabaseServiceTypeSmallCase.Cockroach:
return 'Cockroach';
default:
return capitalize(serviceType);