mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-31 18:48:35 +00:00
d8e2187980
* pydantic v2 * pydanticv2 * fix parser * fix annotated * fix model dumping * mysql ingestion * clean root models * clean root models * bump airflow * bump airflow * bump airflow * optionals * optionals * optionals * jdk * airflow migrate * fab provider * fab provider * fab provider * some more fixes * fixing tests and imports * model_dump and model_validate * model_dump and model_validate * model_dump and model_validate * union * pylint * pylint * integration tests * fix CostAnalysisReportData * integration tests * tests * missing defaults * missing defaults
69 lines
2.6 KiB
Python
69 lines
2.6 KiB
Python
# Copyright 2021 Collate
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""
|
|
You can run this DAG from the default OM installation
|
|
"""
|
|
from datetime import datetime
|
|
|
|
from airflow import models
|
|
from airflow.providers.docker.operators.docker import DockerOperator
|
|
|
|
from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
|
|
PipelineType,
|
|
)
|
|
|
|
config = """
|
|
source:
|
|
type: mysql
|
|
serviceName: local_mysql
|
|
serviceConnection:
|
|
config:
|
|
type: Mysql
|
|
username: openmetadata_user
|
|
password: openmetadata_password
|
|
hostPort: localhost:3306
|
|
databaseSchema: openmetadata_db
|
|
connectionOptions: {}
|
|
connectionArguments: {}
|
|
sourceConfig:
|
|
config:
|
|
type: DatabaseMetadata
|
|
sink:
|
|
type: metadata-rest
|
|
config: {}
|
|
workflowConfig:
|
|
openMetadataServerConfig:
|
|
hostPort: http://localhost:8585/api
|
|
authProvider: openmetadata
|
|
securityConfig:
|
|
jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
|
|
"""
|
|
|
|
|
|
with models.DAG(
|
|
"ingestion-docker-operator",
|
|
schedule_interval="@once",
|
|
start_date=datetime(2021, 1, 1),
|
|
catchup=False,
|
|
tags=["OpenMetadata"],
|
|
) as dag:
|
|
DockerOperator(
|
|
command="python main.py",
|
|
image="openmetadata/ingestion-base:local",
|
|
environment={"config": config, "pipelineType": PipelineType.metadata.value},
|
|
docker_url="unix://var/run/docker.sock", # To allow to start Docker. Needs chmod 666 permissions
|
|
tty=True,
|
|
auto_remove="success",
|
|
network_mode="host", # To reach the OM server
|
|
task_id="ingest",
|
|
dag=dag,
|
|
)
|