diff --git a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql index e75a2cda010..c740727616b 100644 --- a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql @@ -32,6 +32,12 @@ set json = JSON_INSERT( ) where name = 'DataInsightsApplication'; +-- Remove mssql connection from airflow db +UPDATE pipeline_service_entity pse +SET json = JSON_REMOVE(json, '$.connection.config.connection') +WHERE serviceType = 'Airflow' +AND JSON_EXTRACT(json, '$.connection.config.connection.type') = 'Mssql'; + -- Rename NOOP Secret Manager to DB update metadata_service_entity set json = JSON_REPLACE(json, '$.connection.config.secretsManagerProvider', 'db') diff --git a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql index f41724585c6..5d2490c4861 100644 --- a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql @@ -39,6 +39,16 @@ SET json = jsonb_set( ) where name = 'DataInsightsApplication'; +-- Remove mssql connection from airflow db +UPDATE pipeline_service_entity pse +SET json = jsonb_set( + json, + '{connection, config}', + json->'connection'->'config' #- '{connection}' +) +WHERE serviceType = 'Airflow' +AND json #>> '{connection,config,connection,type}' = 'Mssql'; + -- Rename NOOP Secret Manager to DB update metadata_service_entity set json = jsonb_set( diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py index 2fc5610cfad..705030cb9de 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py @@ -21,9 +21,6 @@ from sqlalchemy.engine import Engine from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) -from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( - MssqlConnection, -) from metadata.generated.schema.entity.services.connections.database.mysqlConnection import ( MysqlConnection, ) @@ -81,15 +78,6 @@ def _(airflow_connection: PostgresConnection) -> Engine: return get_postgres_connection(airflow_connection) -@_get_connection.register -def _(airflow_connection: MssqlConnection) -> Engine: - from metadata.ingestion.source.database.mssql.connection import ( - get_connection as get_mssql_connection, - ) - - return get_mssql_connection(airflow_connection) - - @_get_connection.register def _(airflow_connection: SQLiteConnection) -> Engine: from metadata.ingestion.source.database.sqlite.connection import ( diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/pipeline/airflow/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/pipeline/airflow/index.md index d3fc7365250..da5267782aa 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/pipeline/airflow/index.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/pipeline/airflow/index.md @@ -44,7 +44,7 @@ In terms of `connection` we support the following selections: - `backend`: Should not be used from the UI. This is only applicable when ingesting Airflow metadata locally by running the ingestion from a DAG. It will use the current Airflow SQLAlchemy connection to extract the data. -- `MySQL`, `Postgres`, `MSSQL` and `SQLite`: Pass the required credentials to reach out each of these services. We +- `MySQL`, `Postgres`, and `SQLite`: Pass the required credentials to reach out each of these services. We will create a connection to the pointed database and read Airflow data from there. {% /extraContent %} diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/pipeline/airflow/yaml.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/pipeline/airflow/yaml.md index 966f0b5bb0b..10aa0b203e3 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/pipeline/airflow/yaml.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/pipeline/airflow/yaml.md @@ -72,7 +72,7 @@ In terms of `connection` we support the following selections: - `backend`: Should not be used from the UI. This is only applicable when ingesting Airflow metadata locally by running the ingestion from a DAG. It will use the current Airflow SQLAlchemy connection to extract the data. -- `MySQL`, `Postgres`, `MSSQL` and `SQLite`: Pass the required credentials to reach out each of these services. We will +- `MySQL`, `Postgres`, and `SQLite`: Pass the required credentials to reach out each of these services. We will create a connection to the pointed database and read Airflow data from there. **hostPort**: URL to the Airflow instance. @@ -98,7 +98,7 @@ In terms of `connection` we support the following selections: - `backend`: Should not be used from the UI. This is only applicable when ingesting Airflow metadata locally by running the ingestion from a DAG. It will use the current Airflow SQLAlchemy connection to extract the data. -- `MySQL`, `Postgres`, `MSSQL` and `SQLite`: Pass the required credentials to reach out each of these services. We will +- `MySQL`, `Postgres`, and `SQLite`: Pass the required credentials to reach out each of these services. We will create a connection to the pointed database and read Airflow data from there. {% /codeInfo %} @@ -128,7 +128,7 @@ source: numberOfStatus: 10 ``` ```yaml {% srNumber=6 %} - # Connection needs to be one of Mysql, Postgres, Mssql or Sqlite + # Connection needs to be one of Mysql, Postgres or Sqlite connection: type: Mysql username: airflow_user @@ -142,13 +142,6 @@ source: # database: airflow_db # hostPort: localhost:3306 # # - # type: Mssql - # username: airflow_user - # password: airflow_pass - # database: airflow_db - # hostPort: localhost:3306 - # uriString: http://... (optional) - # # # type: Sqlite # username: airflow_user # password: airflow_pass diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/deployment/upgrade/versions/120-to-130.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/deployment/upgrade/versions/120-to-130.md new file mode 100644 index 00000000000..c0fa15ca80e --- /dev/null +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/deployment/upgrade/versions/120-to-130.md @@ -0,0 +1,34 @@ +--- +title: Upgrade 1.2.x to 1.3.x +slug: /deployment/upgrade/versions/120-to-130 +--- + +# Upgrade from 1.2.x to 1.3.x + +Upgrading from 1.2.x to 1.3.x can be done directly on your instances. This page will list few general details you should take into consideration when running the upgrade. + +## Deprecation Notice + + +## Breaking Changes for 1.3.x Stable Release + + +## Service Connection Changes + +### Airflow Connection + +Removed the `MSSQL` connection option from airflow backend database. This is due to the option being experimental and will be deprecated by the Airflow team. For more information refer to the [link](https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html#choosing-database-backend). + +If you are using airflow with `MSSQL` backend, we recommend switching it to the supported backends e.g `MYSQL` or `POSTGRES`. + +#### Removed below MSSQL config option from Airflow + +```yaml +... + connection: + type: Mssql + username: user + password: pass + hostPort: localhost:1433 + database: dev +``` \ No newline at end of file diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md index b615b75747a..a29c23c7506 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md @@ -185,6 +185,8 @@ site_menu: url: /deployment/upgrade/versions - category: Deployment / Upgrade OpenMetadata / Version Changes / 1.0.x to 1.1.x url: /deployment/upgrade/versions/100-to-110 + - category: Deployment / Upgrade OpenMetadata / Version Changes / 1.2.x to 1.3.x + url: /deployment/upgrade/versions/120-to-130 - category: Deployment / Backup & Restore Metadata url: /deployment/backup-restore-metadata diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/airflowConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/airflowConnection.json index 1c25a69960e..c3756c3131e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/airflowConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/airflowConnection.json @@ -45,9 +45,6 @@ { "$ref": "../database/postgresConnection.json" }, - { - "$ref": "../database/mssqlConnection.json" - }, { "$ref": "../database/sqliteConnection.json" }