yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-07 13:07:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #13053: Remove Connection URI config MongoDB (#14584)

Browse Source 
* Fix #13053: Remove Connection URI cofig MongoDB

* pyformat & test fixes
This commit is contained in:
Mayur Singal 2024-01-06 00:21:12 +05:30 committed by GitHub
parent c8f7fc486f
commit a789fc86d6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 195 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ingestion/src/metadata/examples/workflows/mongodb.yaml
View File

@ -5,8 +5,10 @@ source:
config:
type: MongoDB
databaseName: custom_database_name
connectionDetails:
connectionURI: mongodb+srv://user:pass@localhost:27017
scheme: mongodb+srv
username: username
password: password
hostPort: localhost:27017
sourceConfig:
config:
type: DatabaseMetadata

6
ingestion/src/metadata/ingestion/source/database/mongodb/connection.py
View File

@ -24,7 +24,6 @@ from metadata.generated.schema.entity.automations.workflow import (
Workflow as AutomationWorkflow,
)
from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import (
MongoConnectionString,
MongoDBConnection,
)
from metadata.ingestion.connections.builders import get_connection_url_common
@ -36,10 +35,7 @@ def get_connection(connection: MongoDBConnection):
"""
Create connection
"""
if isinstance(connection.connectionDetails, MongoConnectionString):
mongo_url = connection.connectionDetails.connectionURI
else:
mongo_url = get_connection_url_common(connection.connectionDetails)
mongo_url = get_connection_url_common(connection)
return MongoClient(mongo_url)

6
ingestion/tests/unit/topology/database/test_mongodb.py
View File

@ -47,9 +47,9 @@ mock_mongo_config = {
"serviceConnection": {
"config": {
"type": "MongoDB",
"connectionDetails": {
"connectionURI": "mongodb://ulixius:dummy_password@localhost:27017",
},
"username": "ulixius",
"password": "dummy_password",
"hostPort": "localhost:27017",
},
},
"sourceConfig": {

2
openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mongodb/index.md
View File

@ -61,8 +61,6 @@ To fetch the metadata from MongoDB to OpenMetadata, the MongoDB user must have a
#### Connection Details
- **MongoDB Connection Details**: Choose between MongoDB Connection String and MongoDB Connection Values to authenticate with your mongodb cluster.
- **Connection URI**: MongoDB connection string is a concise string of parameters used to establish a connection between an OpenMetadata and a MongoDB database. For ex. `mongodb://username:password@mongodb0.example.com:27017`.
- **Username**: Username to connect to Mongodb. This user must have access to perform `find` operation on collection and `listCollection` operations on database available in MongoDB.
- **Password**: Password to connect to MongoDB.
- **Host Port**: The hostPort parameter specifies the host and port of the MongoDB. This should be specified as a string in the format `hostname:port`. E.g., `localhost:27017`.

20
openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mongodb/yaml.md
View File

@ -94,12 +94,6 @@ This is a sample config for MongoDB:
{% /codeInfo %}
{% codeInfo srNumber=5 %}
**connectionURI**: MongoDB connection string is a concise string of parameters used to establish a connection between an OpenMetadata and a MongoDB database. For ex. `mongodb://username:password@mongodb0.example.com:27017`.
{% /codeInfo %}
{% codeInfo srNumber=6 %}
**databaseName**: Optional name to give to the database in OpenMetadata. If left blank, we will use default as the database name.
@ -132,23 +126,19 @@ source:
serviceConnection:
config:
type: MongoDB
connectionDetails:
```
```yaml {% srNumber=1 %}
username: username
username: username
```
```yaml {% srNumber=2 %}
password: password
password: password
```
```yaml {% srNumber=3 %}
hostPort: localhost:27017
```
```yaml {% srNumber=5 %}
# connectionURI: mongodb://username:password@mongodb0.example.com:27017
hostPort: localhost:27017
```
```yaml {% srNumber=7 %}
# connectionOptions:
# key: value
# connectionOptions:
# key: value
```
```yaml {% srNumber=6 %}
database: custom_database_name

33
openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v130/Migration.java Normal file
View File

@ -0,0 +1,33 @@
package org.openmetadata.service.migration.mysql.v130;
import static org.openmetadata.service.migration.utils.v130.MigrationUtil.migrateMongoDBConnStr;
import lombok.SneakyThrows;
import org.jdbi.v3.core.Handle;
import org.openmetadata.service.migration.api.MigrationProcessImpl;
import org.openmetadata.service.migration.utils.MigrationFile;
public class Migration extends MigrationProcessImpl {
private Handle handle;
public Migration(MigrationFile migrationFile) {
super(migrationFile);
}
@Override
public void initialize(Handle handle) {
super.initialize(handle);
this.handle = handle;
}
@Override
@SneakyThrows
public void runDataMigration() {
String updateSqlQuery =
"UPDATE dbservice_entity de SET json = :json "
+ "WHERE serviceType = 'MongoDB' "
+ "AND id = :id";
migrateMongoDBConnStr(handle, updateSqlQuery);
}
}

33
openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v130/Migration.java Normal file
View File

@ -0,0 +1,33 @@
package org.openmetadata.service.migration.postgres.v130;
import static org.openmetadata.service.migration.utils.v130.MigrationUtil.migrateMongoDBConnStr;
import lombok.SneakyThrows;
import org.jdbi.v3.core.Handle;
import org.openmetadata.service.migration.api.MigrationProcessImpl;
import org.openmetadata.service.migration.utils.MigrationFile;
public class Migration extends MigrationProcessImpl {
private Handle handle;
public Migration(MigrationFile migrationFile) {
super(migrationFile);
}
@Override
public void initialize(Handle handle) {
super.initialize(handle);
this.handle = handle;
}
@Override
@SneakyThrows
public void runDataMigration() {
String updateSqlQuery =
"UPDATE dbservice_entity de SET json = :json::jsonb "
+ "WHERE serviceType = 'MongoDB' "
+ "AND id = :id";
migrateMongoDBConnStr(handle, updateSqlQuery);
}
}

82
openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v130/MigrationUtil.java Normal file
View File

@ -0,0 +1,82 @@
package org.openmetadata.service.migration.utils.v130;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import org.jdbi.v3.core.Handle;
import org.openmetadata.schema.entity.services.DatabaseService;
import org.openmetadata.service.util.JsonUtils;
public class MigrationUtil {
private static final String GET_MONGO_DB_SERVICES =
"SELECT id, json from " + "dbservice_entity de WHERE " + "serviceType = 'MongoDB'";
private MigrationUtil() {
/* Cannot create object util class*/
}
private static Map extractConnectionURIDetails(String connectionString) {
Map connectionDetailsMap = new LinkedHashMap();
try {
// Parse the MongoDB connection string
URI uri = new URI(connectionString);
// Extract components
String username = uri.getUserInfo().split(":")[0];
String password = uri.getUserInfo().split(":")[1];
String host = uri.getHost();
String scheme = uri.getScheme();
int port = uri.getPort();
String query = uri.getQuery();
Map queryMap = new HashMap<>();
if (query != null) {
String[] queryParams = query.split("&");
System.out.println("Query Parameters:");
for (String param : queryParams) {
queryMap.put(param.split("=")[0], param.split("=")[1]);
}
}
// populate connection details map the extracted components
connectionDetailsMap.put("username", username);
connectionDetailsMap.put("password", password);
connectionDetailsMap.put("hostPort", host + ":" + port);
connectionDetailsMap.put("scheme", scheme);
connectionDetailsMap.put("connectionOptions", queryMap);
} catch (URISyntaxException e) {
e.printStackTrace();
}
return connectionDetailsMap;
}
public static void migrateMongoDBConnStr(Handle handle, String updateSqlQuery) {
handle
.createQuery(GET_MONGO_DB_SERVICES)
.mapToMap()
.forEach(
row -> {
DatabaseService mongoService =
JsonUtils.readValue(row.get("json").toString(), DatabaseService.class);
String id = row.get("id").toString();
Map mongoDBConnection = (LinkedHashMap) mongoService.getConnection().getConfig();
Map connDetails = (LinkedHashMap) mongoDBConnection.get("connectionDetails");
Map finalConnectionDetails;
if (connDetails.get("connectionURI") != null) {
String connectionURI = connDetails.get("connectionURI").toString();
finalConnectionDetails = extractConnectionURIDetails(connectionURI);
} else {
finalConnectionDetails = connDetails;
}
mongoDBConnection.putAll(finalConnectionDetails);
mongoDBConnection.remove("connectionDetails");
String json = JsonUtils.pojoToJson(mongoService);
handle.createUpdate(updateSqlQuery).bind("json", json).bind("id", id).execute();
});
}
}

49
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDB/mongoDBValues.json
View File

@ -1,49 +0,0 @@
{
"$id": "https://open-metadata.org/schema/entity/services/connections/database/datalake/azureConfig.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Mongo Connection Values",
"description": "Azure Datalake Storage will ingest files in container",
"type": "object",
"javaType": "org.openmetadata.schema.services.connections.database.mongo.MongoConnectionValues",
"definitions": {
"mongoDBScheme": {
"description": "Mongo connection scheme options.",
"type": "string",
"enum": [
"mongodb",
"mongodb+srv"
],
"default": "mongodb"
}
},
"properties": {
"scheme": {
"title": "Connection Scheme",
"description": "Mongo connection scheme options.",
"$ref": "#/definitions/mongoDBScheme",
"default": "mongodb"
},
"username": {
"title": "Username",
"description": "Username to connect to MongoDB. This user should have privileges to read all the metadata in MongoDB.",
"type": "string"
},
"password": {
"title": "Password",
"description": "Password to connect to MongoDB.",
"type": "string",
"format": "password"
},
"hostPort": {
"title": "Host and Port",
"description": "Host and port of the MongoDB service.",
"type": "string"
},
"connectionOptions": {
"title": "Connection Options",
"$ref": "../../connectionBasicType.json#/definitions/connectionOptions"
}
},
"additionalProperties": false,
"required": ["hostPort"]
}

56
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json
View File

@ -14,17 +14,14 @@
],
"default": "MongoDB"
},
"MongoConnectionString": {
"type": "object",
"javaType": "org.openmetadata.schema.services.connections.database.mongo.MongoConnectionString",
"title": "Mongo Connection String",
"properties": {
"connectionURI": {
"title": "Connection URI",
"description": "Connection URI to connect to your MongoDB cluster",
"type": "string"
}
}
"mongoDBScheme": {
"description": "Mongo connection scheme options.",
"type": "string",
"enum": [
"mongodb",
"mongodb+srv"
],
"default": "mongodb"
}
},
@ -35,17 +32,31 @@
"$ref": "#/definitions/mongoDBType",
"default": "MongoDB"
},
"connectionDetails": {
"title": "MongoDB Connection Details",
"description": "MongoDB Connection Details.",
"oneOf": [
{
"$ref": "mongoDB/mongoDBValues.json"
},
{
"$ref": "#/definitions/MongoConnectionString"
}
]
"scheme": {
"title": "Connection Scheme",
"description": "Mongo connection scheme options.",
"$ref": "#/definitions/mongoDBScheme",
"default": "mongodb"
},
"username": {
"title": "Username",
"description": "Username to connect to MongoDB. This user should have privileges to read all the metadata in MongoDB.",
"type": "string"
},
"password": {
"title": "Password",
"description": "Password to connect to MongoDB.",
"type": "string",
"format": "password"
},
"hostPort": {
"title": "Host and Port",
"description": "Host and port of the MongoDB service.",
"type": "string"
},
"connectionOptions": {
"title": "Connection Options",
"$ref": "../connectionBasicType.json#/definitions/connectionOptions"
},
"databaseName": {
"title": "Database Name",
@ -57,5 +68,6 @@
"$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
}
},
"required": ["hostPort"],
"additionalProperties": false
}

12
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/MongoDB.md
View File

@ -8,18 +8,6 @@ You can find further information on the Hive connector in the [here](https://doc
## Connection Details
$$section
### MongoDB Connection Details $(id="connectionDetails")
Choose between MongoDB Connection String and MongoDB Connection Values to authenticate with your mongodb cluster.
$$
$$section
### Connection URI $(id="connectionURI")
MongoDB connection string is a concise string of parameters used to establish a connection between an OpenMetadata and a MongoDB database. For ex. `mongodb://username:password@mongodb0.example.com:27017`
$$
$$section
### Username $(id="username")
Username to connect to Mongodb. This user must have access to perform `find` operation on collection and `listCollection` operations on database available in MongoDB.