yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-08 23:14:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix#6615: Fix failing test still in CI (#7725)

Browse Source 
* Fix failing test still some tests are failing

* Fix bug in service entity repository

* Fix all tests except Ingestion Pipeline integration tests

* Fix Ingestion Pipeline integration tests

* Fix failing Python tests
This commit is contained in:
Nahuel 2022-09-26 20:39:42 +02:00 committed by GitHub
parent f5f7e8a5c1
commit df909703eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 142 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ingestion/tests/unit/metadata/utils/secrets/test_aws_based_secrets_manager.py
View File

@ -87,7 +87,7 @@ class AWSBasedSecretsManager(object):
aws_manager.add_auth_provider_security_config(actual_om_connection)
self.assert_client_called_once(
aws_manager, "/openmetadata/auth-provider/google"
aws_manager, "/openmetadata/bot/ingestion-bot"
)
self.assertEqual(
self.auth_provider_config, actual_om_connection.securityConfig
@ -105,7 +105,7 @@ class AWSBasedSecretsManager(object):
with self.assertRaises(ValueError) as value_error:
aws_manager.add_auth_provider_security_config(self.om_connection)
self.assertTrue(
"/openmetadata/auth-provider/google" in str(value_error.exception)
"/openmetadata/bot/ingestion-bot" in str(value_error.exception)
)
@patch("metadata.clients.aws_client.AWSClient.get_client")

1
ingestion/tests/unit/metadata/utils/secrets/test_secrets_manager_factory.py
View File

@ -69,6 +69,7 @@ class TestSecretsManagerFactory(TestCase):
secret_manager_providers = [
secret_manager_provider
for secret_manager_provider in SecretsManagerProvider
if secret_manager_provider is not SecretsManagerProvider.in_memory
]
for secret_manager_provider in secret_manager_providers:
open_metadata_connection: OpenMetadataConnection = OpenMetadataConnection(

5
ingestion/tests/unit/profiler/test_profiler_partitions.py
View File

@ -52,7 +52,10 @@ mock_bigquery_config = {
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "no-auth",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
}
},
}

5
ingestion/tests/unit/test_handle_partitions.py
View File

@ -48,7 +48,10 @@ mock_bigquery_config = {
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "no-auth",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
}
},
}

6
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java
View File

@ -20,6 +20,7 @@ import java.io.IOException;
import org.json.JSONObject;
import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig;
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType;
import org.openmetadata.schema.metadataIngestion.DatabaseServiceMetadataPipeline;
import org.openmetadata.schema.metadataIngestion.LogLevels;
import org.openmetadata.schema.services.connections.metadata.OpenMetadataServerConnection;
@ -85,7 +86,8 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
if (secretsManager.isLocal()) {
secretsManager.encryptOrDecryptDbtConfigSource(ingestionPipeline, service, true);
store(ingestionPipeline.getId(), ingestionPipeline, update);
} else {
} else if (service.getType().equals(Entity.DATABASE_SERVICE)
&& ingestionPipeline.getPipelineType().equals(PipelineType.METADATA)) {
// otherwise, nullify the config since it will be kept outside OM
DatabaseServiceMetadataPipeline databaseServiceMetadataPipeline =
JsonUtils.convertValue(
@ -98,6 +100,8 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
databaseServiceMetadataPipeline.setDbtConfigSource(dbtConfigSource);
ingestionPipeline.getSourceConfig().setConfig(databaseServiceMetadataPipeline);
secretsManager.encryptOrDecryptDbtConfigSource(ingestionPipeline, service, true);
} else {
store(ingestionPipeline.getId(), ingestionPipeline, update);
}
// Restore the relationships

1
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java
View File

@ -97,6 +97,7 @@ public abstract class ServiceEntityRepository<
} else {
// otherwise, nullify the config since it will be kept outside OM
Object connectionConfig = service.getConnection().getConfig();
secretsManager.validateServiceConnection(connectionConfig, service.getServiceType().value(), serviceType);
service.getConnection().setConfig(null);
store(service.getId(), service, update);
// save connection in the secret manager after storing the service

42
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java
View File

@ -81,6 +81,7 @@ import org.openmetadata.schema.auth.TokenRefreshRequest;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.teams.authn.BasicAuthMechanism;
import org.openmetadata.schema.teams.authn.GenerateTokenRequest;
import org.openmetadata.schema.teams.authn.JWTAuthMechanism;
import org.openmetadata.schema.teams.authn.JWTTokenExpiry;
import org.openmetadata.schema.teams.authn.SSOAuthMechanism;
@ -212,6 +213,8 @@ public class UserResource extends EntityResource<User, UserRepository> {
@DefaultValue("non-deleted")
Include include)
throws IOException {
// remove USER_PROTECTED_FIELDS from fieldsParam
fieldsParam = fieldsParam != null ? fieldsParam.replaceAll("," + USER_PROTECTED_FIELDS, "") : null;
ListFilter filter = new ListFilter(include).addQueryParam("team", teamParam);
if (isAdmin != null) {
filter.addQueryParam("isAdmin", String.valueOf(isAdmin));
@ -276,6 +279,8 @@ public class UserResource extends EntityResource<User, UserRepository> {
@DefaultValue("non-deleted")
Include include)
throws IOException {
// remove USER_PROTECTED_FIELDS from fieldsParam
fieldsParam = fieldsParam != null ? fieldsParam.replaceAll("," + USER_PROTECTED_FIELDS, "") : null;
return decryptOrNullify(securityContext, getInternal(uriInfo, securityContext, id, fieldsParam, include));
}
@ -310,6 +315,8 @@ public class UserResource extends EntityResource<User, UserRepository> {
@DefaultValue("non-deleted")
Include include)
throws IOException {
// remove USER_PROTECTED_FIELDS from fieldsParam
fieldsParam = fieldsParam != null ? fieldsParam.replaceAll("," + USER_PROTECTED_FIELDS, "") : null;
return decryptOrNullify(securityContext, getByNameInternal(uriInfo, securityContext, name, fieldsParam, include));
}
@ -459,6 +466,41 @@ public class UserResource extends EntityResource<User, UserRepository> {
return response.toResponse();
}
@PUT
@Path("/generateToken/{id}")
@Operation(
operationId = "generateJWTTokenForBotUser",
summary = "Generate JWT Token for a Bot User",
tags = "users",
description = "Generate JWT Token for a Bot User.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The user ",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = JWTTokenExpiry.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response generateToken(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") UUID id,
@Valid GenerateTokenRequest generateTokenRequest)
throws IOException {
User user = dao.get(uriInfo, id, Fields.EMPTY_FIELDS);
authorizeGenerateJWT(user);
authorizer.authorizeAdmin(securityContext, false);
JWTAuthMechanism jwtAuthMechanism =
jwtTokenGenerator.generateJWTToken(user, generateTokenRequest.getJWTTokenExpiry());
AuthenticationMechanism authenticationMechanism =
new AuthenticationMechanism().withConfig(jwtAuthMechanism).withAuthType(AuthenticationMechanism.AuthType.JWT);
user.setAuthenticationMechanism(authenticationMechanism);
User updatedUser = dao.createOrUpdate(uriInfo, user).getEntity();
jwtAuthMechanism =
JsonUtils.convertValue(updatedUser.getAuthenticationMechanism().getConfig(), JWTAuthMechanism.class);
return Response.status(Response.Status.OK).entity(jwtAuthMechanism).build();
}
@PUT
@Path("/revokeToken/{id}")
@Operation(

11
openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java
View File

@ -26,6 +26,7 @@ import org.openmetadata.schema.metadataIngestion.DatabaseServiceMetadataPipeline
import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.InvalidServiceConnectionException;
import org.openmetadata.service.exception.SecretsManagerException;
import org.openmetadata.service.util.JsonUtils;
@ -106,4 +107,14 @@ public abstract class SecretsManager {
public abstract Object encryptOrDecryptIngestionBotCredentials(
String botName, Object securityConfig, boolean encrypt);
public void validateServiceConnection(Object connectionConfig, String connectionType, ServiceType serviceType) {
try {
Class<?> clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType));
JsonUtils.readValue(JsonUtils.pojoToJson(connectionConfig), clazz);
} catch (Exception exception) {
throw InvalidServiceConnectionException.byMessage(
connectionType, String.format("Failed to construct connection instance of %s", connectionType));
}
}
}

3
openmetadata-service/src/main/java/org/openmetadata/service/secrets/ThirdPartySecretsManager.java
View File

@ -38,6 +38,7 @@ public abstract class ThirdPartySecretsManager extends SecretsManager {
String secretName = buildSecretId("service", serviceType.value(), connectionType, connectionName);
try {
if (encrypt) {
validateServiceConnection(connectionConfig, connectionType, serviceType);
String connectionConfigJson = JsonUtils.pojoToJson(connectionConfig);
if (connectionConfigJson != null) {
upsertSecret(secretName, connectionConfigJson);
@ -47,7 +48,7 @@ public abstract class ThirdPartySecretsManager extends SecretsManager {
Class<?> clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType));
return JsonUtils.readValue(getSecret(secretName), clazz);
}
} catch (ClassNotFoundException ex) {
} catch (ClassNotFoundException | InvalidServiceConnectionException ex) {
throw InvalidServiceConnectionException.byMessage(
connectionType, String.format("Failed to construct connection instance of %s", connectionType));
} catch (Exception e) {

6
openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataServerConnectionBuilder.java
View File

@ -45,8 +45,11 @@ public class OpenMetadataServerConnectionBuilder {
OpenMetadataApplicationConfig openMetadataApplicationConfig,
CollectionDAO collectionDAO) {
this.secretsManager = secretsManager;
// TODO: https://github.com/open-metadata/OpenMetadata/issues/7712
authProvider =
OpenMetadataServerConnection.AuthProvider.fromValue(
"basic".equals(openMetadataApplicationConfig.getAuthenticationConfiguration().getProvider())
? OpenMetadataServerConnection.AuthProvider.OPENMETADATA
: OpenMetadataServerConnection.AuthProvider.fromValue(
openMetadataApplicationConfig.getAuthenticationConfiguration().getProvider());
if (!OpenMetadataServerConnection.AuthProvider.NO_AUTH.equals(authProvider)) {
@ -63,6 +66,7 @@ public class OpenMetadataServerConnectionBuilder {
openMetadataURL = airflowConfiguration.getMetadataApiEndpoint();
clusterName = openMetadataApplicationConfig.getClusterName();
secretsManagerProvider = secretsManager.getSecretsManagerProvider();
verifySSL = OpenMetadataServerConnection.VerifySSL.fromValue(airflowConfiguration.getVerifySSL());
airflowSSLConfig =
getAirflowSSLConfig(
OpenMetadataServerConnection.VerifySSL.fromValue(airflowConfiguration.getVerifySSL()),

2
openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/IngestionPipelineRepositoryTest.java → openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/IngestionPipelineRepositoryUnitTest.java
View File

@ -39,7 +39,7 @@ import org.openmetadata.service.Entity;
import org.openmetadata.service.secrets.SecretsManager;
@ExtendWith(MockitoExtension.class)
public class IngestionPipelineRepositoryTest {
public class IngestionPipelineRepositoryUnitTest {
@Mock protected CollectionDAO collectionDAO;

11
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DashboardServiceResourceTest.java
View File

@ -18,6 +18,7 @@ import static javax.ws.rs.core.Response.Status.OK;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
@ -194,11 +195,7 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
if (fieldName.equals("connection")) {
DashboardConnection expectedDashboardConnection = (DashboardConnection) expected;
DashboardConnection actualDashboardConnection = JsonUtils.readValue((String) actual, DashboardConnection.class);
actualDashboardConnection.setConfig(
JsonUtils.convertValue(actualDashboardConnection.getConfig(), SupersetConnection.class));
assertEquals(expectedDashboardConnection, actualDashboardConnection);
assertTrue(((String) actual).contains("-encrypted-value"));
} else {
super.assertCommonFieldChange(fieldName, expected, actual);
}
@ -221,11 +218,11 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
}
assertEquals(expectedSupersetConnection.getHostPort(), actualSupersetConnection.getHostPort());
assertEquals(expectedSupersetConnection.getProvider(), actualSupersetConnection.getProvider());
if (ADMIN_AUTH_HEADERS.equals(authHeaders) || BOT_AUTH_HEADERS.equals(authHeaders)) {
if (ADMIN_AUTH_HEADERS.equals(authHeaders)) {
assertEquals(expectedSupersetConnection.getUsername(), actualSupersetConnection.getUsername());
assertEquals(expectedSupersetConnection.getPassword(), actualSupersetConnection.getPassword());
assertEquals(expectedSupersetConnection.getProvider(), actualSupersetConnection.getProvider());
} else {
} else if (BOT_AUTH_HEADERS.equals(authHeaders)) {
assertNull(actualSupersetConnection.getUsername());
assertNull(actualSupersetConnection.getPassword());
}

7
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DatabaseServiceResourceTest.java
View File

@ -17,6 +17,7 @@ import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static javax.ws.rs.core.Response.Status.OK;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
@ -289,11 +290,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
Schedule actualSchedule = JsonUtils.readValue((String) actual, Schedule.class);
assertEquals(expectedSchedule, actualSchedule);
} else if (fieldName.equals("connection")) {
DatabaseConnection expectedConnection = (DatabaseConnection) expected;
DatabaseConnection actualConnection = JsonUtils.readValue((String) actual, DatabaseConnection.class);
actualConnection.setConfig(JsonUtils.convertValue(actualConnection.getConfig(), SnowflakeConnection.class));
// TODO remove this hardcoding
validateDatabaseConnection(expectedConnection, actualConnection, DatabaseServiceType.Snowflake);
assertTrue(((String) actual).contains("-encrypted-value"));
} else {
super.assertCommonFieldChange(fieldName, expected, actual);
}

6
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/MessagingServiceResourceTest.java
View File

@ -16,6 +16,7 @@ package org.openmetadata.service.resources.services;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static javax.ws.rs.core.Response.Status.OK;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
@ -219,10 +220,7 @@ public class MessagingServiceResourceTest extends EntityResourceTest<MessagingSe
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
if ("connection".equals(fieldName)) {
MessagingConnection expectedConnection = (MessagingConnection) expected;
MessagingConnection actualConnection = JsonUtils.readValue((String) actual, MessagingConnection.class);
actualConnection.setConfig(JsonUtils.convertValue(actualConnection.getConfig(), KafkaConnection.class));
assertEquals(expectedConnection, actualConnection);
assertTrue(((String) actual).contains("-encrypted-value"));
} else {
super.assertCommonFieldChange(fieldName, expected, actual);
}

7
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/MlModelServiceResourceTest.java
View File

@ -16,6 +16,7 @@ package org.openmetadata.service.resources.services;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static javax.ws.rs.core.Response.Status.OK;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
@ -167,11 +168,7 @@ public class MlModelServiceResourceTest extends EntityResourceTest<MlModelServic
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
if (fieldName.equals("connection")) {
MlModelConnection expectedMlModelConnection = (MlModelConnection) expected;
MlModelConnection actualMlModelConnection = JsonUtils.readValue((String) actual, MlModelConnection.class);
actualMlModelConnection.setConfig(
JsonUtils.convertValue(actualMlModelConnection.getConfig(), MlflowConnection.class));
assertEquals(expectedMlModelConnection, actualMlModelConnection);
assertTrue(((String) actual).contains("-encrypted-value"));
} else {
super.assertCommonFieldChange(fieldName, expected, actual);
}

11
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/PipelineServiceResourceTest.java
View File

@ -18,6 +18,7 @@ import static javax.ws.rs.core.Response.Status.OK;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.service.resources.services.DatabaseServiceResourceTest.validateMysqlConnection;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
@ -254,11 +255,7 @@ public class PipelineServiceResourceTest extends EntityResourceTest<PipelineServ
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
if (fieldName.equals("connection")) {
PipelineConnection expectedConnection = (PipelineConnection) expected;
PipelineConnection actualConnection = JsonUtils.readValue((String) actual, PipelineConnection.class);
actualConnection.setConfig(JsonUtils.convertValue(actualConnection.getConfig(), AirflowConnection.class));
// TODO remove this hardcoding
validatePipelineConnection(expectedConnection, actualConnection, PipelineServiceType.Airflow, null);
assertTrue(((String) actual).contains("-encrypted-value"));
} else {
super.assertCommonFieldChange(fieldName, expected, actual);
}
@ -275,13 +272,13 @@ public class PipelineServiceResourceTest extends EntityResourceTest<PipelineServ
MysqlConnection expectedMysqlConnection = (MysqlConnection) expectedDatabaseConnection.getConfig();
// Use the database service tests utilities for the comparison
// only admin can see all connection parameters
if (ADMIN_AUTH_HEADERS.equals(authHeaders) || BOT_AUTH_HEADERS.equals(authHeaders)) {
if (ADMIN_AUTH_HEADERS.equals(authHeaders)) {
DatabaseConnection actualDatabaseConnection =
JsonUtils.convertValue(actualAirflowConnection.getConnection(), DatabaseConnection.class);
MysqlConnection actualMysqlConnection =
JsonUtils.convertValue(actualDatabaseConnection.getConfig(), MysqlConnection.class);
validateMysqlConnection(expectedMysqlConnection, actualMysqlConnection);
} else {
} else if (BOT_AUTH_HEADERS.equals(authHeaders)) {
assertNotNull(actualAirflowConnection);
assertNotNull(actualAirflowConnection.getHostPort());
assertNull(actualAirflowConnection.getConnection());

6
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResourceTest.java
View File

@ -62,11 +62,9 @@ import org.openmetadata.schema.services.connections.database.ConnectionArguments
import org.openmetadata.schema.services.connections.database.ConnectionOptions;
import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.services.DatabaseServiceResourceTest;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.FullyQualifiedName;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.TestUtils;
@ -486,8 +484,10 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
assertEquals(expectedScheduleInterval, ingestion.getAirflowConfig().getScheduleInterval());
// Update and connector orgs and options to database connection
DatabaseServiceResourceTest databaseServiceResourceTest = new DatabaseServiceResourceTest();
DatabaseService databaseService =
Entity.getEntity(ingestionPipeline.getService(), Fields.EMPTY_FIELDS, Include.ALL);
databaseServiceResourceTest.getEntity(ingestionPipeline.getService().getId(), "connection", ADMIN_AUTH_HEADERS);
DatabaseConnection databaseConnection = databaseService.getConnection();
Map<String, String> advConfig = new HashMap<>();
advConfig.put("hive.execution.engine", "tez");

37
openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java
View File

@ -73,12 +73,15 @@ import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.TestMethodOrder;
import org.openmetadata.schema.api.teams.CreateUser;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.entity.teams.Role;
import org.openmetadata.schema.entity.teams.Team;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.security.client.GoogleSSOClientConfig;
import org.openmetadata.schema.teams.authn.GenerateTokenRequest;
import org.openmetadata.schema.teams.authn.JWTAuthMechanism;
import org.openmetadata.schema.teams.authn.JWTTokenExpiry;
import org.openmetadata.schema.teams.authn.SSOAuthMechanism;
import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.ImageList;
@ -178,7 +181,15 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
create = createRequest(test, 3).withProfile(PROFILE);
createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
create = createRequest(test, 5).withDisplayName("displayName").withProfile(PROFILE).withIsBot(true);
create =
createRequest(test, 5)
.withDisplayName("displayName")
.withProfile(PROFILE)
.withIsBot(true)
.withAuthenticationMechanism(
new AuthenticationMechanism()
.withAuthType(AuthenticationMechanism.AuthType.JWT)
.withConfig(new JWTAuthMechanism().withJWTTokenExpiry(JWTTokenExpiry.Unlimited)));
createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
create = createRequest(test, 6).withDisplayName("displayName").withProfile(PROFILE).withIsAdmin(true);
@ -361,11 +372,11 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
int initialBotCount = bots.getPaging().getTotal();
// Create 3 bot users
CreateUser create = createRequest(test, 0).withIsBot(true);
CreateUser create = createBotUserRequest(test, 0);
User bot0 = createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
create = createRequest(test, 1).withIsBot(true);
create = createBotUserRequest(test, 1);
User bot1 = createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
create = createRequest(test, 2).withIsBot(true);
create = createBotUserRequest(test, 2);
User bot2 = createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
Predicate<User> isBot0 = u -> u.getId().equals(bot0.getId());
@ -391,6 +402,15 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
assertEquals(initialUserCount - initialBotCount, users.getPaging().getTotal());
}
private CreateUser createBotUserRequest(TestInfo test, int index) {
return createRequest(test, index)
.withIsBot(true)
.withAuthenticationMechanism(
new AuthenticationMechanism()
.withAuthType(AuthenticationMechanism.AuthType.JWT)
.withConfig(new JWTAuthMechanism().withJWTTokenExpiry(JWTTokenExpiry.Unlimited)));
}
@Test
void get_listUsersWithTeamsPagination(TestInfo test) throws IOException {
TeamResourceTest teamResourceTest = new TeamResourceTest();
@ -634,7 +654,14 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
.withName("ingestion-bot-jwt")
.withDisplayName("ingestion-bot-jwt")
.withEmail("ingestion-bot-jwt@email.com")
.withIsBot(true),
.withIsBot(true)
.withAuthenticationMechanism(
new AuthenticationMechanism()
.withAuthType(AuthenticationMechanism.AuthType.SSO)
.withConfig(
new SSOAuthMechanism()
.withSsoServiceType(SSOAuthMechanism.SsoServiceType.GOOGLE)
.withAuthConfig(new GoogleSSOClientConfig().withSecretKey("/path/to/secret.json")))),
authHeaders("ingestion-bot-jwt@email.com"));
TestUtils.put(
getResource(String.format("users/generateToken/%s", user.getId())),

15
openmetadata-service/src/test/java/org/openmetadata/service/resources/util/UtilResourceTest.java
View File

@ -26,6 +26,9 @@ import org.openmetadata.schema.api.teams.CreateTeam;
import org.openmetadata.schema.api.teams.CreateUser;
import org.openmetadata.schema.api.tests.CreateTestSuite;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.security.client.GoogleSSOClientConfig;
import org.openmetadata.schema.teams.authn.SSOAuthMechanism;
import org.openmetadata.schema.util.EntitiesCount;
import org.openmetadata.schema.util.ServicesCount;
import org.openmetadata.service.OpenMetadataApplicationTest;
@ -192,7 +195,17 @@ public class UtilResourceTest extends OpenMetadataApplicationTest {
// Create a bot user.
UserResourceTest userResourceTest = new UserResourceTest();
CreateUser createUser = userResourceTest.createRequest(test).withIsBot(true);
CreateUser createUser =
userResourceTest
.createRequest(test)
.withIsBot(true)
.withAuthenticationMechanism(
new AuthenticationMechanism()
.withAuthType(AuthenticationMechanism.AuthType.SSO)
.withConfig(
new SSOAuthMechanism()
.withSsoServiceType(SSOAuthMechanism.SsoServiceType.GOOGLE)
.withAuthConfig(new GoogleSSOClientConfig().withSecretKey("/fake/path/secret.json"))));
userResourceTest.createEntity(createUser, ADMIN_AUTH_HEADERS);
int afterUserCount = getEntitiesCount().getUserCount();