mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-07-27 03:10:04 +00:00
* use db for mysql/mariadb * format
This commit is contained in:
parent
6b7a9fe76c
commit
603d61eaa2
@ -24,6 +24,12 @@ from metadata.generated.schema.entity.data.table import Table
|
|||||||
from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
|
from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
|
||||||
DatabricksConnection,
|
DatabricksConnection,
|
||||||
)
|
)
|
||||||
|
from metadata.generated.schema.entity.services.connections.database.mariaDBConnection import (
|
||||||
|
MariaDBConnection,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
|
||||||
|
MysqlConnection,
|
||||||
|
)
|
||||||
from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import (
|
from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import (
|
||||||
SnowflakeType,
|
SnowflakeType,
|
||||||
)
|
)
|
||||||
@ -79,22 +85,27 @@ class SQAInterfaceMixin(Root):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def set_catalog(self, session) -> None:
|
def set_catalog(self, session) -> None:
|
||||||
"""Set catalog for the session. Right now only databricks and unity catalog requires it
|
"""Set the catalog or database for the session.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
session (Session): sqa session object
|
session (Session): sqa session object
|
||||||
"""
|
"""
|
||||||
if not isinstance(
|
if isinstance(
|
||||||
self.service_connection_config,
|
self.service_connection_config,
|
||||||
(UnityCatalogConnection, DatabricksConnection),
|
(UnityCatalogConnection, DatabricksConnection),
|
||||||
):
|
):
|
||||||
return
|
session.get_bind().execute(
|
||||||
bind = session.get_bind()
|
|
||||||
bind.execute(
|
|
||||||
"USE CATALOG %(catalog)s;",
|
"USE CATALOG %(catalog)s;",
|
||||||
{"catalog": self.service_connection_config.catalog},
|
{"catalog": self.service_connection_config.catalog},
|
||||||
).first()
|
).first()
|
||||||
|
|
||||||
|
if isinstance(
|
||||||
|
self.service_connection_config, (MysqlConnection, MariaDBConnection)
|
||||||
|
):
|
||||||
|
session.get_bind().execute(
|
||||||
|
f"USE {self.table_entity.databaseSchema.name};",
|
||||||
|
)
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
"""close session"""
|
"""close session"""
|
||||||
self.session.close()
|
self.session.close()
|
||||||
|
@ -297,7 +297,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|||||||
column: the column to compute the metrics against
|
column: the column to compute the metrics against
|
||||||
metrics: list of metrics to compute
|
metrics: list of metrics to compute
|
||||||
Returns:
|
Returns:
|
||||||
dictionnary of results
|
dictionary of results
|
||||||
"""
|
"""
|
||||||
|
|
||||||
if not metrics:
|
if not metrics:
|
||||||
|
@ -88,7 +88,7 @@ class FirstQuartile(StaticMetric, PercentilMixin):
|
|||||||
except MemoryError:
|
except MemoryError:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Unable to compute Median for {self.col.name} due to memory constraints."
|
f"Unable to compute Median for {self.col.name} due to memory constraints."
|
||||||
f"We recommend using a smaller sample size or partitionning."
|
f"We recommend using a smaller sample size or partitioning."
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
# check if nan
|
# check if nan
|
||||||
|
@ -87,7 +87,7 @@ class Median(StaticMetric, PercentilMixin):
|
|||||||
except MemoryError:
|
except MemoryError:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Unable to compute Median for {self.col.name} due to memory constraints."
|
f"Unable to compute Median for {self.col.name} due to memory constraints."
|
||||||
f"We recommend using a smaller sample size or partitionning."
|
f"We recommend using a smaller sample size or partitioning."
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
|
@ -88,7 +88,7 @@ class ThirdQuartile(StaticMetric, PercentilMixin):
|
|||||||
except MemoryError:
|
except MemoryError:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Unable to compute Median for {self.col.name} due to memory constraints."
|
f"Unable to compute Median for {self.col.name} due to memory constraints."
|
||||||
f"We recommend using a smaller sample size or partitionning."
|
f"We recommend using a smaller sample size or partitioning."
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
# check if nan
|
# check if nan
|
||||||
|
@ -51,7 +51,9 @@ class AzureSQLSampler(SQASampler):
|
|||||||
|
|
||||||
def get_sample_query(self, *, column=None) -> Query:
|
def get_sample_query(self, *, column=None) -> Query:
|
||||||
"""get query for sample data"""
|
"""get query for sample data"""
|
||||||
rnd = self._base_sample_query(column).cte(f"{self.get_sampler_table_name()}_rnd")
|
rnd = self._base_sample_query(column).cte(
|
||||||
|
f"{self.get_sampler_table_name()}_rnd"
|
||||||
|
)
|
||||||
query = self.client.query(rnd)
|
query = self.client.query(rnd)
|
||||||
return query.cte(f"{self.get_sampler_table_name()}_sample")
|
return query.cte(f"{self.get_sampler_table_name()}_sample")
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user