mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-11-16 10:53:31 +00:00
Fix 14084: Removed Airflow Mssql backend connection (#14420)
* Removed Airflow Mssql backend conn * added breaking changes docs
This commit is contained in:
Onkar Ravgan
GitHub
parent
cce059d98f
commit
7d032977d7
@ -32,6 +32,12 @@ set json = JSON_INSERT(
|
||||
)
|
||||
where name = 'DataInsightsApplication';
|
||||
|
||||
-- Remove mssql connection from airflow db
|
||||
UPDATE pipeline_service_entity pse
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.connection')
|
||||
WHERE serviceType = 'Airflow'
|
||||
AND JSON_EXTRACT(json, '$.connection.config.connection.type') = 'Mssql';
|
||||
|
||||
-- Rename NOOP Secret Manager to DB
|
||||
update metadata_service_entity
|
||||
set json = JSON_REPLACE(json, '$.connection.config.secretsManagerProvider', 'db')
|
||||
|
||||
@ -39,6 +39,16 @@ SET json = jsonb_set(
|
||||
)
|
||||
where name = 'DataInsightsApplication';
|
||||
|
||||
-- Remove mssql connection from airflow db
|
||||
UPDATE pipeline_service_entity pse
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{connection, config}',
|
||||
json->'connection'->'config' #- '{connection}'
|
||||
)
|
||||
WHERE serviceType = 'Airflow'
|
||||
AND json #>> '{connection,config,connection,type}' = 'Mssql';
|
||||
|
||||
-- Rename NOOP Secret Manager to DB
|
||||
update metadata_service_entity
|
||||
set json = jsonb_set(
|
||||
|
||||
@ -21,9 +21,6 @@ from sqlalchemy.engine import Engine
|
||||
from metadata.generated.schema.entity.automations.workflow import (
|
||||
Workflow as AutomationWorkflow,
|
||||
)
|
||||
from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
|
||||
MssqlConnection,
|
||||
)
|
||||
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
|
||||
MysqlConnection,
|
||||
)
|
||||
@ -81,15 +78,6 @@ def _(airflow_connection: PostgresConnection) -> Engine:
|
||||
return get_postgres_connection(airflow_connection)
|
||||
|
||||
|
||||
@_get_connection.register
|
||||
def _(airflow_connection: MssqlConnection) -> Engine:
|
||||
from metadata.ingestion.source.database.mssql.connection import (
|
||||
get_connection as get_mssql_connection,
|
||||
)
|
||||
|
||||
return get_mssql_connection(airflow_connection)
|
||||
|
||||
|
||||
@_get_connection.register
|
||||
def _(airflow_connection: SQLiteConnection) -> Engine:
|
||||
from metadata.ingestion.source.database.sqlite.connection import (
|
||||
|
||||
@ -44,7 +44,7 @@ In terms of `connection` we support the following selections:
|
||||
|
||||
- `backend`: Should not be used from the UI. This is only applicable when ingesting Airflow metadata locally
|
||||
by running the ingestion from a DAG. It will use the current Airflow SQLAlchemy connection to extract the data.
|
||||
- `MySQL`, `Postgres`, `MSSQL` and `SQLite`: Pass the required credentials to reach out each of these services. We
|
||||
- `MySQL`, `Postgres`, and `SQLite`: Pass the required credentials to reach out each of these services. We
|
||||
will create a connection to the pointed database and read Airflow data from there.
|
||||
|
||||
{% /extraContent %}
|
||||
|
||||
@ -72,7 +72,7 @@ In terms of `connection` we support the following selections:
|
||||
|
||||
- `backend`: Should not be used from the UI. This is only applicable when ingesting Airflow metadata locally by running
|
||||
the ingestion from a DAG. It will use the current Airflow SQLAlchemy connection to extract the data.
|
||||
- `MySQL`, `Postgres`, `MSSQL` and `SQLite`: Pass the required credentials to reach out each of these services. We will
|
||||
- `MySQL`, `Postgres`, and `SQLite`: Pass the required credentials to reach out each of these services. We will
|
||||
create a connection to the pointed database and read Airflow data from there.
|
||||
|
||||
**hostPort**: URL to the Airflow instance.
|
||||
@ -98,7 +98,7 @@ In terms of `connection` we support the following selections:
|
||||
|
||||
- `backend`: Should not be used from the UI. This is only applicable when ingesting Airflow metadata locally by running
|
||||
the ingestion from a DAG. It will use the current Airflow SQLAlchemy connection to extract the data.
|
||||
- `MySQL`, `Postgres`, `MSSQL` and `SQLite`: Pass the required credentials to reach out each of these services. We will
|
||||
- `MySQL`, `Postgres`, and `SQLite`: Pass the required credentials to reach out each of these services. We will
|
||||
create a connection to the pointed database and read Airflow data from there.
|
||||
|
||||
{% /codeInfo %}
|
||||
@ -128,7 +128,7 @@ source:
|
||||
numberOfStatus: 10
|
||||
```
|
||||
```yaml {% srNumber=6 %}
|
||||
# Connection needs to be one of Mysql, Postgres, Mssql or Sqlite
|
||||
# Connection needs to be one of Mysql, Postgres or Sqlite
|
||||
connection:
|
||||
type: Mysql
|
||||
username: airflow_user
|
||||
@ -142,13 +142,6 @@ source:
|
||||
# database: airflow_db
|
||||
# hostPort: localhost:3306
|
||||
# #
|
||||
# type: Mssql
|
||||
# username: airflow_user
|
||||
# password: airflow_pass
|
||||
# database: airflow_db
|
||||
# hostPort: localhost:3306
|
||||
# uriString: http://... (optional)
|
||||
# #
|
||||
# type: Sqlite
|
||||
# username: airflow_user
|
||||
# password: airflow_pass
|
||||
|
||||
@ -0,0 +1,34 @@
|
||||
---
|
||||
title: Upgrade 1.2.x to 1.3.x
|
||||
slug: /deployment/upgrade/versions/120-to-130
|
||||
---
|
||||
|
||||
# Upgrade from 1.2.x to 1.3.x
|
||||
|
||||
Upgrading from 1.2.x to 1.3.x can be done directly on your instances. This page will list few general details you should take into consideration when running the upgrade.
|
||||
|
||||
## Deprecation Notice
|
||||
|
||||
|
||||
## Breaking Changes for 1.3.x Stable Release
|
||||
|
||||
|
||||
## Service Connection Changes
|
||||
|
||||
### Airflow Connection
|
||||
|
||||
Removed the `MSSQL` connection option from airflow backend database. This is due to the option being experimental and will be deprecated by the Airflow team. For more information refer to the [link](https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html#choosing-database-backend).
|
||||
|
||||
If you are using airflow with `MSSQL` backend, we recommend switching it to the supported backends e.g `MYSQL` or `POSTGRES`.
|
||||
|
||||
#### Removed below MSSQL config option from Airflow
|
||||
|
||||
```yaml
|
||||
...
|
||||
connection:
|
||||
type: Mssql
|
||||
username: user
|
||||
password: pass
|
||||
hostPort: localhost:1433
|
||||
database: dev
|
||||
```
|
||||
@ -185,6 +185,8 @@ site_menu:
|
||||
url: /deployment/upgrade/versions
|
||||
- category: Deployment / Upgrade OpenMetadata / Version Changes / 1.0.x to 1.1.x
|
||||
url: /deployment/upgrade/versions/100-to-110
|
||||
- category: Deployment / Upgrade OpenMetadata / Version Changes / 1.2.x to 1.3.x
|
||||
url: /deployment/upgrade/versions/120-to-130
|
||||
|
||||
- category: Deployment / Backup & Restore Metadata
|
||||
url: /deployment/backup-restore-metadata
|
||||
|
||||
@ -45,9 +45,6 @@
|
||||
{
|
||||
"$ref": "../database/postgresConnection.json"
|
||||
},
|
||||
{
|
||||
"$ref": "../database/mssqlConnection.json"
|
||||
},
|
||||
{
|
||||
"$ref": "../database/sqliteConnection.json"
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user