From 66a89ffe616e627fcf1e4a540bc599c78bc83d39 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 10 Sep 2023 18:46:28 +0200 Subject: [PATCH] Fix e2e profiler status (#13127) --- ingestion/src/metadata/profiler/api/models.py | 4 ++++ .../src/metadata/profiler/source/metadata.py | 4 ++++ ingestion/tests/cli_e2e/base/test_cli.py | 16 ++++++---------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ingestion/src/metadata/profiler/api/models.py b/ingestion/src/metadata/profiler/api/models.py index 6250fdb4d18..af5e389dd8e 100644 --- a/ingestion/src/metadata/profiler/api/models.py +++ b/ingestion/src/metadata/profiler/api/models.py @@ -87,3 +87,7 @@ class ProfilerResponse(ConfigModel): profile: CreateTableProfileRequest sample_data: Optional[TableData] = None column_tags: Optional[List[PatchColumnTagResponse]] = None + + def __str__(self): + """Return the table name being processed""" + return f"Table [{self.table.name.__root__}]" diff --git a/ingestion/src/metadata/profiler/source/metadata.py b/ingestion/src/metadata/profiler/source/metadata.py index edb4655b384..3780759150b 100644 --- a/ingestion/src/metadata/profiler/source/metadata.py +++ b/ingestion/src/metadata/profiler/source/metadata.py @@ -52,6 +52,10 @@ class ProfilerSourceAndEntity(BaseModel): profiler_source: ProfilerSource entity: Table + def __str__(self): + """Return the information of the table being profiler""" + return f"Table [{self.entity.name.__root__}]" + class OpenMetadataSource(Source): """ diff --git a/ingestion/tests/cli_e2e/base/test_cli.py b/ingestion/tests/cli_e2e/base/test_cli.py index c69cef81419..4324297e741 100644 --- a/ingestion/tests/cli_e2e/base/test_cli.py +++ b/ingestion/tests/cli_e2e/base/test_cli.py @@ -91,9 +91,9 @@ class CliBase(ABC): output_clean = re.sub(" +", " ", output_clean) output_clean_ansi = re.compile(r"\x1b[^m]*m") output_clean = output_clean_ansi.sub(" ", output_clean) - regex = r"Source Status:%(log)s(.*?)%(log)sSink Status: .*" % REGEX_AUX - output_clean = re.findall(regex, output_clean.strip()) - return Status.parse_obj(literal_eval(output_clean[0].strip())) + regex = r"Source Status:%(log)s(.*?)%(log)s.* Status: .*" % REGEX_AUX + output_clean_regex = re.findall(regex, output_clean.strip()) + return Status.parse_obj(literal_eval(output_clean_regex[0].strip())) @staticmethod def extract_sink_status(output) -> Status: @@ -101,13 +101,9 @@ class CliBase(ABC): output_clean = re.sub(" +", " ", output_clean) output_clean_ansi = re.compile(r"\x1b[^m]*m") output_clean = output_clean_ansi.sub("", output_clean) - if re.match(".* Processor Status: .*", output_clean): - regex = r"Sink Status:%(log)s(.*?)%(log)sProcessor Status: .*" % REGEX_AUX - output_clean = re.findall(regex, output_clean.strip())[0].strip() - else: - regex = r".*Sink Status:%(log)s(.*?)%(log)sWorkflow.*Summary.*" % REGEX_AUX - output_clean = re.findall(regex, output_clean.strip())[0].strip() - return Status.parse_obj(literal_eval(output_clean)) + regex = r".*Sink Status:%(log)s(.*?)%(log)sWorkflow.*Summary.*" % REGEX_AUX + output_clean_regex = re.findall(regex, output_clean.strip())[0].strip() + return Status.parse_obj(literal_eval(output_clean_regex)) @staticmethod def build_yaml(config_yaml: dict, test_type: E2EType, extra_args: dict):