OpenMetadata/ingestion/tests/integration/workflow/test_workflow.py

#  Copyright 2021 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import importlib
import pathlib
import uuid
from unittest import TestCase

from metadata.config.common import ConfigurationError, load_config_file
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
    OpenMetadataConnection,
)
from metadata.generated.schema.entity.services.databaseService import DatabaseService
from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
    IngestionPipeline,
)
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
    StepSummary,
)
from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
    OpenMetadataJWTClientConfig,
)
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.workflow.metadata import MetadataWorkflow


class WorkflowTest(TestCase):
    """
    Validate workflow methods
    """

    server_config = OpenMetadataConnection(
        hostPort="http://localhost:8585/api",
        authProvider="openmetadata",
        securityConfig=OpenMetadataJWTClientConfig(
            jwtToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
        ),
    )
    metadata = OpenMetadata(server_config)

    assert metadata.health_check()

    def delete_service(self):
        service_id = str(
            self.metadata.get_by_name(
                entity=DatabaseService, fqn="local_mysql_test"
            ).id.root
        )

        self.metadata.delete(
            entity=DatabaseService,
            entity_id=service_id,
            recursive=True,
            hard_delete=True,
        )

    def test_get_200(self):
        key = "metadata.ingestion.sink.metadata_rest.MetadataRestSink"
        if key.find(".") >= 0:
            module_name, class_name = key.rsplit(".", 1)
            my_class = getattr(importlib.import_module(module_name), class_name)
            self.assertEqual((my_class is not None), True)

    def test_get_4xx(self):
        key = "metadata.ingestion.sink.MYSQL.mysqlSINK"
        try:
            if key.find(".") >= 0:
                module_name, class_name = key.rsplit(".", 1)
                getattr(importlib.import_module(module_name), class_name)
        except ModuleNotFoundError:
            self.assertRaises(ModuleNotFoundError)

    def test_title_typeClassFetch(self):
        is_file = True
        file_type = "query-parser"
        if is_file:
            replace = file_type.replace("-", "_")
        else:
            replace = "".join(
                [i.title() for i in file_type.replace("-", "_").split("_")]
            )
        self.assertEqual(replace, "query_parser")

    def test_title_typeClassFetch_4xx(self):
        is_file = False
        file_type = "query-parser"
        if is_file:
            replace = file_type.replace("-", "_")
        else:
            replace = "".join(
                [i.title() for i in file_type.replace("-", "_").split("_")]
            )
        self.assertEqual(replace, "QueryParser")

    def test_execute_200(self):
        current_dir = pathlib.Path(__file__).resolve().parent
        config_file = current_dir.joinpath("mysql_test.yaml")
        workflow_config = load_config_file(config_file)
        workflow = MetadataWorkflow.create(workflow_config)
        workflow.execute()
        workflow.stop()

        # Service is created
        self.assertIsNotNone(
            self.metadata.get_by_name(entity=DatabaseService, fqn="local_mysql_test")
        )

        # The service has an ingestion pipeline (since it has the ingestionPipelineFQN inside and the runId)
        self.assertIsNotNone(
            self.metadata.get_by_name(
                entity=IngestionPipeline, fqn=workflow_config["ingestionPipelineFQN"]
            )
        )

        # The pipeline has the right status
        pipeline_status = self.metadata.get_pipeline_status(
            workflow_config["ingestionPipelineFQN"], workflow_config["pipelineRunId"]
        )

        # We have status for the source and sink
        self.assertEqual(len(pipeline_status.status.root), 2)
        self.assertTrue(isinstance(pipeline_status.status.root[0], StepSummary))

        # Rerunning with a different Run ID still generates the correct status
        new_run_id = str(uuid.uuid4())
        workflow_config["pipelineRunId"] = new_run_id

        workflow = MetadataWorkflow.create(workflow_config)
        workflow.execute()
        workflow.stop()

        pipeline_status = self.metadata.get_pipeline_status(
            workflow_config["ingestionPipelineFQN"], new_run_id
        )

        # We have status for the source and sink
        self.assertEqual(len(pipeline_status.status.root), 2)
        self.assertTrue(isinstance(pipeline_status.status.root[0], StepSummary))

        self.delete_service()

    def test_execute_4xx(self):
        config_file = pathlib.Path("/tmp/mysql_test123")
        try:
            load_config_file(config_file)
        except ConfigurationError:
            self.assertRaises(ConfigurationError)

    def test_fail_no_service_connection_and_overwrite(self):
        current_dir = pathlib.Path(__file__).resolve().parent
        config_file = current_dir.joinpath("mysql_test.yaml")
        workflow_config = load_config_file(config_file)

        del workflow_config["source"]["serviceConnection"]
        workflow_config["workflowConfig"]["openMetadataServerConfig"][
            "forceEntityOverwriting"
        ] = True

        with self.assertRaises(AttributeError):
            MetadataWorkflow.create(workflow_config)