diff --git a/ingestion/src/metadata/examples/workflows/saperp.yaml b/ingestion/src/metadata/examples/workflows/saperp.yaml index f24f95954b8..481ff57333b 100644 --- a/ingestion/src/metadata/examples/workflows/saperp.yaml +++ b/ingestion/src/metadata/examples/workflows/saperp.yaml @@ -8,7 +8,7 @@ source: apiKey: api_key databaseName: databaseName databaseSchema: databaseSchema - paginationLimit: 10 + paginationLimit: 100 sourceConfig: config: type: DatabaseMetadata diff --git a/ingestion/src/metadata/ingestion/source/database/saperp/client.py b/ingestion/src/metadata/ingestion/source/database/saperp/client.py index 608e85bd027..4779fbe354d 100644 --- a/ingestion/src/metadata/ingestion/source/database/saperp/client.py +++ b/ingestion/src/metadata/ingestion/source/database/saperp/client.py @@ -59,7 +59,7 @@ class SapErpClient: api_version="v1", allow_redirects=True, retry_codes=[500, 504], - retry_wait=5, + retry_wait=2, verify=get_verify_ssl(config.sslConfig), ) self.client = REST(client_config) diff --git a/ingestion/src/metadata/ingestion/source/database/saperp/metadata.py b/ingestion/src/metadata/ingestion/source/database/saperp/metadata.py index 2750b35129d..8605e3a0abf 100644 --- a/ingestion/src/metadata/ingestion/source/database/saperp/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/saperp/metadata.py @@ -12,7 +12,7 @@ SAP ERP source module """ import traceback -from typing import Iterable, List, Optional +from typing import Iterable, List, Optional, Tuple from metadata.generated.schema.api.data.createTable import CreateTableRequest from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema @@ -127,23 +127,20 @@ class SaperpSource(CommonDbSourceService): ) def _check_col_length( # pylint: disable=arguments-differ - self, datatype: str, col_length: Optional[str] - ) -> Optional[int]: + self, datatype: str, col_length: Optional[str], col_decimals: Optional[str] + ) -> Tuple[Optional[int], Optional[int]]: """ return the column length for the dataLength attribute """ try: - if datatype and datatype.upper() in { - "CHAR", - "VARCHAR", - "BINARY", - "VARBINARY", - }: - return int(col_length) if col_length else None + return ( + int(col_length) if col_length else None, + int(col_decimals) if col_decimals else None, + ) except Exception as exc: logger.debug(traceback.format_exc()) logger.warning(f"Failed to fetch column length: {exc}") - return None + return None, None def _get_table_constraints( self, columns: Optional[List[Column]] @@ -190,14 +187,21 @@ class SaperpSource(CommonDbSourceService): return Constraint.NOT_NULL if column.notnull == "X" else Constraint.NULL def _get_display_datatype( # pylint: disable=arguments-differ - self, column_type: str, col_data_length: Optional[int] + self, + column_type: str, + col_data_length: Optional[int], + decimals: Optional[int], + sap_column_type: Optional[str], ) -> str: """ Method to get the display datatype """ + column_type_name = sap_column_type if sap_column_type else column_type + if col_data_length and decimals: + return f"{column_type_name}({str(col_data_length)},{str(decimals)})" if col_data_length: - return f"{column_type}({str(col_data_length)})" - return column_type + return f"{column_type_name}({str(col_data_length)})" + return column_type_name def get_columns_and_constraints( # pylint: disable=arguments-differ self, table_name: str @@ -211,8 +215,10 @@ class SaperpSource(CommonDbSourceService): for sap_column in sap_columns or []: try: column_type = ColumnTypeParser.get_column_type(sap_column.datatype) - col_data_length = self._check_col_length( - datatype=column_type, col_length=sap_column.leng + col_data_length, col_decimal_length = self._check_col_length( + datatype=column_type, + col_length=sap_column.leng, + col_decimals=sap_column.decimals, ) column_name = ( f"{sap_column.fieldname}({sap_column.precfield})" @@ -226,7 +232,10 @@ class SaperpSource(CommonDbSourceService): f"Unknown type {repr(sap_column.datatype)}: {sap_column.fieldname}" ) data_type_display = self._get_display_datatype( - column_type, col_data_length + column_type, + col_data_length, + col_decimal_length, + sap_column.datatype, ) col_data_length = 1 if col_data_length is None else col_data_length om_column = Column( @@ -251,6 +260,9 @@ class SaperpSource(CommonDbSourceService): ) if column_type == DataType.ARRAY.value: om_column.arrayDataType = DataType.UNKNOWN + if col_data_length and col_decimal_length: + om_column.precision = col_data_length + om_column.scale = col_decimal_length om_columns.append(om_column) except Exception as exc: logger.debug(traceback.format_exc()) diff --git a/ingestion/src/metadata/ingestion/source/database/saperp/models.py b/ingestion/src/metadata/ingestion/source/database/saperp/models.py index 3ca0a23f251..8475ac82621 100644 --- a/ingestion/src/metadata/ingestion/source/database/saperp/models.py +++ b/ingestion/src/metadata/ingestion/source/database/saperp/models.py @@ -45,6 +45,7 @@ class SapErpColumn(BaseModel): i_ddtext: Optional[str] = None dd_text: Optional[str] = None leng: Optional[str] = None + decimals: Optional[str] = None class SapErpTableList(BaseModel): diff --git a/ingestion/tests/unit/topology/database/test_saperp.py b/ingestion/tests/unit/topology/database/test_saperp.py index 39fabdb7a2d..c9701af4a1f 100644 --- a/ingestion/tests/unit/topology/database/test_saperp.py +++ b/ingestion/tests/unit/topology/database/test_saperp.py @@ -152,10 +152,10 @@ EXPECTED_TABLES_AND_COLUMNS = [ displayName="Client", dataType="INT", arrayDataType=None, - dataLength=1, + dataLength=3, precision=None, scale=None, - dataTypeDisplay="INT", + dataTypeDisplay="CLNT(3)", description=Markdown(root="Client"), fullyQualifiedName=None, tags=None,