yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-01 11:09:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

10041 Part1 - Refactor and clean up System APIs (#10042)

Browse Source 
* 10041 Part1 - Refactor and clean up System APIs

* Update the config resource endpoint url
This commit is contained in:
Suresh Srinivas 2023-02-03 16:25:40 -08:00 committed by GitHub
parent 2ad9a7eb44
commit 1990ac56d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
43 changed files with 230 additions and 564 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
conf/openmetadata.yaml
View File

@ -145,7 +145,7 @@ authenticationConfiguration:
provider: ${AUTHENTICATION_PROVIDER:-basic}
# This will only be valid when provider type specified is customOidc
providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]}
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
clientId: ${AUTHENTICATION_CLIENT_ID:-""}
callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}

2
docker/local-metadata/docker-compose-postgres.yml
View File

@ -75,7 +75,7 @@ services:
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}

2
docker/local-metadata/docker-compose.yml
View File

@ -75,7 +75,7 @@ services:
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}

2
docker/metadata/docker-compose-postgres.yml
View File

@ -68,7 +68,7 @@ services:
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}

2
docker/metadata/docker-compose.yml
View File

@ -66,7 +66,7 @@ services:
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}

8
ingestion/src/metadata/ingestion/ometa/mixins/server_mixin.py
View File

@ -43,13 +43,15 @@ class OMetaServerMixin:
def get_server_version(self) -> str:
"""
Run endpoint /version to check server version
Run endpoint /system/version to check server version
:return: Server version
"""
try:
raw_version = self.client.get("/version")["version"]
raw_version = self.client.get("/system/version")["version"]
except KeyError:
raise VersionNotFoundException("Cannot Find Version at api/v1/version")
raise VersionNotFoundException(
"Cannot Find Version at api/v1/system/version"
)
return get_version_from_string(raw_version)
def validate_versions(self) -> None:

2
ingestion/src/metadata/ingestion/ometa/ometa_api.py
View File

@ -736,7 +736,7 @@ class OpenMetadata(
"""
Run version api call. Return `true` if response is not None
"""
raw_version = self.client.get("/version")["version"]
raw_version = self.client.get("/system/version")["version"]
return raw_version is not None
def close(self):

4
openmetadata-clients/openmetadata-java-client/src/main/java/org/openmetadata/client/gateway/OpenMetadata.java
View File

@ -22,7 +22,7 @@ import feign.okhttp.OkHttpClient;
import feign.slf4j.Slf4jLogger;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.client.ApiClient;
import org.openmetadata.client.api.CatalogApi;
import org.openmetadata.client.api.SystemApi;
import org.openmetadata.client.interceptors.CustomRequestInterceptor;
import org.openmetadata.client.security.factory.AuthenticationProviderFactory;
import org.openmetadata.schema.api.OpenMetadataServerVersion;
@ -97,7 +97,7 @@ public class OpenMetadata {
}
public String[] getServerVersion() {
CatalogApi api = apiClient.buildClient(CatalogApi.class);
SystemApi api = apiClient.buildClient(SystemApi.class);
org.openmetadata.client.model.OpenMetadataServerVersion serverVersion = api.getCatalogVersion();
return VersionUtils.getVersionFromString(serverVersion.getVersion());
}

2
openmetadata-docs/content/deployment/kubernetes/values.md
View File

@ -15,7 +15,7 @@ This page list all the supported helm values for OpenMetadata Helm Charts.
| Key | Type | Default |
| :---------- | :---------- | :---------- |
| global.authentication.provider | string | `basic` |
| global.authentication.publicKeys | list | `[http://openmetadata:8585/api/v1/config/jwks]` |
| global.authentication.publicKeys | list | `[http://openmetadata:8585/api/v1/system/config/jwks]` |
| global.authentication.authority | string | `https://accounts.google.com` |
| global.authentication.clientId | string | `Empty String` |
| global.authentication.callbackUrl | string | `Empty String` |

4
openmetadata-docs/content/deployment/security/basic-auth/index.md
View File

@ -35,7 +35,7 @@ The following configuration controls the auth mechanism for OpenMetadata. Update
```yaml
authenticationConfiguration:
provider: ${AUTHENTICATION_PROVIDER:-basic}
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]}
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
```
@ -43,7 +43,7 @@ authenticationConfiguration:
For the Basic auth we need to set:
- `provider`: basic
- `publicKeyUrls`: {http|https}://{your_domain}:{port}}/api/v1/config/jwks
- `publicKeyUrls`: {http|https}://{your_domain}:{port}}/api/v1/system/config/jwks
- `authority`: {your_domain}
- `enableSelfSignup`: This flag indicates if users can come and signup by themselves on the OM

4
openmetadata-docs/content/deployment/security/enable-jwt-tokens.md
View File

@ -78,11 +78,11 @@ authenticationConfiguration:
jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
```
add `http://{your domain}:8585/api/v1/config/jwks` to `publicKeyUrls`. You should append to the existing configuration such that
add `http://{your domain}:8585/api/v1/system/config/jwks` to `publicKeyUrls`. You should append to the existing configuration such that
your SSO and JWTToken auth verification will work.
```yaml
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[{your SSO public keys URL}, http://{your domain}:8585/api/v1/config/jwks]}
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[{your SSO public keys URL}, http://{your domain}:8585/api/v1/system/config/jwks]}
```
Once you configure the above settings, restart OpenMetadata server .

2
openmetadata-docs/content/deployment/security/enable-ssl/ssl-troubleshooting.md
View File

@ -10,7 +10,7 @@ In this section we comment common issues that the user can face when enabling SS
## Bot using JWT as authentication mechanism
After enabling SSL on the OM server, we have to update also the public keys URL for the validation of the JWT tokens by
updating to the secured URL: `https://{server_domain}:{port}/api/v1/config/jwks`.
updating to the secured URL: `https://{server_domain}:{port}/api/v1/system/config/jwks`.
In case we are using a self-signed certificate, it will fail with the error below:

8
openmetadata-docs/content/deployment/security/jwt-troubleshooting.md
View File

@ -4,17 +4,17 @@ slug: /deployment/security/jwt-troubleshooting
---
# JWT Troubleshooting
Add the `{domain}:{port}/config/jwks` in the list of publicKeys
Add the `{domain}:{port}/api/v1/sytem/config/jwks` in the list of publicKeys
```yaml
authentication:
provider: "google"
publicKeys:
- "https://www.googleapis.com/oauth2/v3/certs"
- "http://localhost:8585/api/v1/config/jwks" (your domain and port)
- "http://localhost:8585/api/v1/system/config/jwks" (your domain and port)
```
This config with `"http://localhost:8585/api/v1/config/jwks"` is the default behavior. If you are configuring and expecting a JWT token to work, configuring with that extra URL is required.
This config with `"http://localhost:8585/api/v1/system/config/jwks"` is the default behavior. If you are configuring and expecting a JWT token to work, configuring with that extra URL is required.
JWT Tokens are issued by private certificates.
@ -23,7 +23,7 @@ We need public keys to decrypt it and get that token's user name, expiry time, e
In OpenMetadata users can enable SSO for users to login and use JWT tokens issued by OpenMetadata for bots
The way OpenMetadata issues a JWT Token is using this [config](https://github.com/open-metadata/OpenMetadata/blob/main/conf/openmetadata.yaml#L155). It uses the `rsapublicKeyFilePath` file to generate a token.
When the ingestion workflow uses this token, we use `rsapublicKeyPath` to decrypt it. The way we do this is using the response from this endpoint `http://localhost:8585/api/v1/config/jwks`.
When the ingestion workflow uses this token, we use `rsapublicKeyPath` to decrypt it. The way we do this is using the response from this endpoint `http://localhost:8585/api/v1/system/config/jwks`.
## Get JWT token from UI.

4
openmetadata-docs/content/deployment/security/ldap/index.md
View File

@ -19,7 +19,7 @@ The following configuration controls the auth mechanism for OpenMetadata. Update
```yaml
authenticationConfiguration:
provider: ${AUTHENTICATION_PROVIDER:-ldap}
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]}
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-false}
ldapConfiguration:
@ -43,7 +43,7 @@ For the LDAP auth we need to set:
OpenMetadata Specific Configuration :
- `provider`: ldap
- `publicKeyUrls`: {http|https}://{your_domain}:{port}}/api/v1/config/jwks
- `publicKeyUrls`: {http|https}://{your_domain}:{port}}/api/v1/system/config/jwks
- `authority`: {your_domain}
- `enableSelfSignup`: This has to be false for Ldap.

8
openmetadata-docs/content/how-to-guides/feature-configurations/bots.md
View File

@ -35,7 +35,7 @@ Otherwise, a JWT Token will be generated to be the default authentication mechan
**2. JWT Token auth mechanism**
If you decide to configure a JWT Token for the authentication mechanism ensure that you have also the value `http://localhost:8585/api/v1/config/jwks`
If you decide to configure a JWT Token for the authentication mechanism ensure that you have also the value `http://localhost:8585/api/v1/system/config/jwks`
in your `publicKeyUrls` list:
- For **bare metal** configuration:
@ -45,13 +45,13 @@ authenticationConfiguration:
provider: "google"
publicKeyUrls:
- "https://www.googleapis.com/oauth2/v3/certs"
- "http://localhost:8585/api/v1/config/jwks"
- "http://localhost:8585/api/v1/system/config/jwks"
```
- For **docker** configuration, the value to be updated is `AUTHENTICATION_PUBLIC_KEYS`:
```bash
AUTHENTICATION_PUBLIC_KEYS=[https://www.googleapis.com/oauth2/v3/certs, http://localhost:8585/api/v1/config/jwks]
AUTHENTICATION_PUBLIC_KEYS=[https://www.googleapis.com/oauth2/v3/certs, http://localhost:8585/api/v1/system/config/jwks]
```
- In the case of **kubernetes**, you have to update `publicKeys` values:
@ -61,7 +61,7 @@ global:
authentication:
publicKeys:
- "https://www.googleapis.com/oauth2/v3/certs"
- "http://localhost:8585/api/v1/config/jwks"
- "http://localhost:8585/api/v1/system/config/jwks"
```
**3. Redeploying ingestion pipelines**

49
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -250,10 +250,7 @@ public interface CollectionDAO {
DataInsightChartDAO dataInsightChartDAO();
@CreateSqlObject
UtilDAO utilDAO();
@CreateSqlObject
SettingsDAO getSettingsDAO();
SystemDAO systemDAO();
@CreateSqlObject
TokenDAO getTokenDAO();
@ -3175,7 +3172,7 @@ public interface CollectionDAO {
}
}
interface UtilDAO {
interface SystemDAO {
@ConnectionAwareSqlQuery(
value =
"SELECT (SELECT COUNT(*) FROM table_entity <cond>) as tableCount, "
@ -3221,6 +3218,26 @@ public interface CollectionDAO {
+ "(SELECT COUNT(*) FROM mlmodel_service_entity <cond>) as mlModelServiceCount")
@RegisterRowMapper(ServicesCountRowMapper.class)
ServicesCount getAggregatedServicesCount(@Define("cond") String cond) throws StatementException;
@SqlQuery("SELECT configType,json FROM openmetadata_settings")
@RegisterRowMapper(SettingsRowMapper.class)
List<Settings> getAllConfig() throws StatementException;
@SqlQuery("SELECT configType, json FROM openmetadata_settings WHERE configType = :configType")
@RegisterRowMapper(SettingsRowMapper.class)
Settings getConfigWithKey(@Bind("configType") String configType) throws StatementException;
@ConnectionAwareSqlUpdate(
value =
"INSERT into openmetadata_settings (configType, json)"
+ "VALUES (:configType, :json) ON DUPLICATE KEY UPDATE json = :json",
connectionType = MYSQL)
@ConnectionAwareSqlUpdate(
value =
"INSERT into openmetadata_settings (configType, json)"
+ "VALUES (:configType, :json :: jsonb) ON CONFLICT (configType) DO UPDATE SET json = EXCLUDED.json",
connectionType = POSTGRES)
void insertSettings(@Bind("configType") String configType, @Bind("json") String json);
}
class SettingsRowMapper implements RowMapper<Settings> {
@ -3252,28 +3269,6 @@ public interface CollectionDAO {
}
}
interface SettingsDAO {
@SqlQuery("SELECT configType,json FROM openmetadata_settings")
@RegisterRowMapper(SettingsRowMapper.class)
List<Settings> getAllConfig() throws StatementException;
@SqlQuery("SELECT configType, json FROM openmetadata_settings WHERE configType = :configType")
@RegisterRowMapper(SettingsRowMapper.class)
Settings getConfigWithKey(@Bind("configType") String configType) throws StatementException;
@ConnectionAwareSqlUpdate(
value =
"INSERT into openmetadata_settings (configType, json)"
+ "VALUES (:configType, :json) ON DUPLICATE KEY UPDATE json = :json",
connectionType = MYSQL)
@ConnectionAwareSqlUpdate(
value =
"INSERT into openmetadata_settings (configType, json)"
+ "VALUES (:configType, :json :: jsonb) ON CONFLICT (configType) DO UPDATE SET json = EXCLUDED.json",
connectionType = POSTGRES)
void insertSettings(@Bind("configType") String configType, @Bind("json") String json);
}
class TokenRowMapper implements RowMapper<TokenInterface> {
@Override
public TokenInterface map(ResultSet rs, StatementContext ctx) throws SQLException {

37
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SettingsRepository.java → openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java
View File

@ -1,16 +1,3 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.jdbi3;
import java.util.List;
@ -19,24 +6,35 @@ import javax.json.JsonValue;
import javax.ws.rs.core.Response;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.settings.Settings;
import org.openmetadata.schema.util.EntitiesCount;
import org.openmetadata.schema.util.ServicesCount;
import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.RestUtil;
import org.openmetadata.service.util.ResultList;
@Slf4j
public class SettingsRepository {
public class SystemRepository {
private static final String FAILED_TO_UPDATE_SETTINGS = "Failed to Update Settings";
public static final String INTERNAL_SERVER_ERROR_WITH_REASON = "Internal Server Error. Reason :";
private final CollectionDAO dao;
private final SystemDAO dao;
public SettingsRepository(CollectionDAO dao) {
public SystemRepository(SystemDAO dao) {
this.dao = dao;
}
public EntitiesCount getAllEntitiesCount(ListFilter filter) {
return dao.getAggregatedEntitiesCount(filter.getCondition());
}
public ServicesCount getAllServicesCount(ListFilter filter) {
return dao.getAggregatedServicesCount(filter.getCondition());
}
public ResultList<Settings> listAllConfigs() {
List<Settings> settingsList = null;
try {
settingsList = dao.getSettingsDAO().getAllConfig();
settingsList = dao.getAllConfig();
} catch (Exception ex) {
LOG.error("Error while trying fetch all Settings " + ex.getMessage());
}
@ -49,7 +47,7 @@ public class SettingsRepository {
public Settings getConfigWithKey(String key) {
try {
return dao.getSettingsDAO().getConfigWithKey(key);
return dao.getConfigWithKey(key);
} catch (Exception ex) {
LOG.error("Error while trying fetch Settings " + ex.getMessage());
}
@ -97,8 +95,7 @@ public class SettingsRepository {
public void updateSetting(Settings setting) {
try {
dao.getSettingsDAO()
.insertSettings(setting.getConfigType().toString(), JsonUtils.pojoToJson(setting.getConfigValue()));
dao.insertSettings(setting.getConfigType().toString(), JsonUtils.pojoToJson(setting.getConfigValue()));
} catch (Exception ex) {
throw new RuntimeException(ex);
}

20
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UtilRepository.java
View File

@ -1,20 +0,0 @@
package org.openmetadata.service.jdbi3;
import org.openmetadata.schema.util.EntitiesCount;
import org.openmetadata.schema.util.ServicesCount;
public class UtilRepository {
private final CollectionDAO.UtilDAO dao;
public UtilRepository(CollectionDAO.UtilDAO dao) {
this.dao = dao;
}
public EntitiesCount getAllEntitiesCount(ListFilter filter) {
return dao.getAggregatedEntitiesCount(filter.getCondition());
}
public ServicesCount getAllServicesCount(ListFilter filter) {
return dao.getAggregatedServicesCount(filter.getCondition());
}
}

77
openmetadata-service/src/main/java/org/openmetadata/service/resources/CatalogResource.java
View File

@ -1,77 +0,0 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.resources;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.util.Arrays;
import java.util.List;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriInfo;
import org.openmetadata.schema.type.CollectionDescriptor;
import org.openmetadata.service.util.ResultList;
@Path("/v1")
@Api(value = "All collections in the catalog application", tags = "All collections in the Catalog")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "root")
public class CatalogResource {
public static class CollectionList extends ResultList<CollectionDescriptor> {
@SuppressWarnings("unused")
public CollectionList() {
/* Required for serde */
}
public CollectionList(List<CollectionDescriptor> data) {
super(data);
}
}
private static CollectionList collectionList;
public static CollectionList getCollectionList(UriInfo uriInfo) {
if (collectionList == null) {
CollectionDescriptor[] collections = CollectionRegistry.getInstance().getCollectionForPath("/v1", uriInfo);
collectionList = new CollectionList(Arrays.asList(collections));
}
return collectionList;
}
@GET
@Operation(
operationId = "listCollections",
summary = "List all collections",
tags = "catalog",
description =
"List all the collections supported by OpenMetadata. This list provides all the collections "
+ "and resource REST endpoints.",
responses = {
@ApiResponse(
responseCode = "200",
description = "All collections",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = CollectionList.class)))
})
public CollectionList getCollections(@Context UriInfo uriInfo) {
return getCollectionList(uriInfo);
}
}

51
openmetadata-service/src/main/java/org/openmetadata/service/resources/CollectionRegistry.java
View File

@ -16,7 +16,6 @@ package org.openmetadata.service.resources;
import com.google.common.annotations.VisibleForTesting;
import io.dropwizard.setup.Environment;
import io.swagger.annotations.Api;
import java.io.File;
import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@ -31,7 +30,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import javax.ws.rs.Path;
import javax.ws.rs.core.UriInfo;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@ -43,7 +41,6 @@ import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.auth.AuthenticatorHandler;
import org.openmetadata.service.util.RestUtil;
import org.reflections.Reflections;
import org.reflections.scanners.MethodAnnotationsScanner;
import org.reflections.util.ClasspathHelper;
@ -92,17 +89,6 @@ public final class CollectionRegistry {
loadConditionFunctions();
}
/** For a collection at {@code collectionPath} returns JSON document that describes it and it's children */
public CollectionDescriptor[] getCollectionForPath(String collectionPath, UriInfo uriInfo) {
CollectionDetails parent = collectionMap.get(collectionPath);
CollectionDescriptor[] children = parent.getChildCollections();
for (CollectionDescriptor child : children) {
URI href = child.getCollection().getHref();
child.getCollection().setHref(RestUtil.getHref(uriInfo, href.getPath()));
}
return children;
}
public Map<String, CollectionDetails> getCollectionMap() {
return Collections.unmodifiableMap(collectionMap);
}
@ -122,26 +108,11 @@ public final class CollectionRegistry {
}
}
}
// Now add collections to their parents
// Example add to /v1 collections services databases etc.
for (CollectionDetails details : collectionMap.values()) {
CollectionInfo collectionInfo = details.cd.getCollection();
if (collectionInfo.getName().equals("root")) {
// Collection root does not have any parent
continue;
}
String parent = new File(collectionInfo.getHref().getPath()).getParent();
CollectionDetails parentCollection = collectionMap.get(parent);
if (parentCollection != null) {
collectionMap.get(parent).addChildCollection(details);
}
}
}
/**
* Resource such as Policy provide a set of functions for authoring SpEL based conditions. The registry loads all
* those conditions and makes it available listing them.
* those conditions and makes it available for listing them over API to author expressions in Rules.
*/
private void loadConditionFunctions() {
Reflections reflections =
@ -233,7 +204,7 @@ public final class CollectionRegistry {
return new CollectionDetails(cd, cl.getCanonicalName(), order);
}
/** Compile a list of REST collection based on Resource classes marked with {@code Collection} annotation */
/** Compile a list of REST collections based on Resource classes marked with {@code Collection} annotation */
private static List<CollectionDetails> getCollections() {
Reflections reflections = new Reflections("org.openmetadata.service.resources");
@ -295,29 +266,15 @@ public final class CollectionRegistry {
}
public static class CollectionDetails {
private final int order;
@Getter private final String resourceClass;
private final CollectionDescriptor cd;
private final List<CollectionDescriptor> childCollections = new ArrayList<>();
@Getter @Setter private Object resource;
private final CollectionDescriptor cd;
private final int order;
CollectionDetails(CollectionDescriptor cd, String resourceClass, int order) {
this.cd = cd;
this.resourceClass = resourceClass;
this.order = order;
}
public void addChildCollection(CollectionDetails child) {
CollectionInfo collectionInfo = child.cd.getCollection();
LOG.info(
"Adding child collection {} to parent collection {}", collectionInfo.getName(), cd.getCollection().getName());
childCollections.add(child.cd);
}
public CollectionDescriptor[] getChildCollections() {
return childCollections.toArray(new CollectionDescriptor[0]);
}
}
}

6
openmetadata-service/src/main/java/org/openmetadata/service/resources/alerts/AlertResource.java
View File

@ -73,7 +73,7 @@ import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.resources.policies.PolicyResource;
import org.openmetadata.service.resources.settings.SettingsResource;
import org.openmetadata.service.resources.system.SystemResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.JsonUtils;
@ -315,7 +315,7 @@ public class AlertResource extends EntityResource<Alert, AlertRepository> {
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = SettingsResource.SettingsList.class)))
schema = @Schema(implementation = SystemResource.SettingsList.class)))
})
public List<TriggerConfig> getAlertBootstrapFilters(
@Context UriInfo uriInfo, @Context SecurityContext securityContext) {
@ -351,7 +351,7 @@ public class AlertResource extends EntityResource<Alert, AlertRepository> {
@Operation(
operationId = "validateCondition",
summary = "Validate a given condition",
tags = "policies",
tags = "alerts",
description = "Validate a given condition expression used in filtering rules.",
responses = {
@ApiResponse(responseCode = "204", description = "No value is returned"),

63
openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ServiceResource.java
View File

@ -1,63 +0,0 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.resources.services;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.util.Arrays;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriInfo;
import org.openmetadata.schema.type.CollectionDescriptor;
import org.openmetadata.service.resources.CatalogResource.CollectionList;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.CollectionRegistry;
@Path("/v1/services")
@Api(value = "Services collection", tags = "Services collection")
@Produces(MediaType.APPLICATION_JSON)
@Collection(name = "services")
public class ServiceResource {
private static CollectionList serviceList;
public static CollectionList getServiceList(UriInfo uriInfo) {
if (serviceList == null) {
CollectionDescriptor[] services = CollectionRegistry.getInstance().getCollectionForPath("/v1/services", uriInfo);
serviceList = new CollectionList(Arrays.asList(services));
}
return serviceList;
}
@GET
@Operation(
operationId = "listServiceCollection",
summary = "List service collections",
tags = "services",
description = "Get a list of resources under service collection.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of serviceCollections",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = CollectionList.class)))
})
public CollectionList getCollections(@Context UriInfo uriInfo) {
return getServiceList(uriInfo);
}
}

8
openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java
View File

@ -22,7 +22,7 @@ import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.settings.Settings;
import org.openmetadata.schema.settings.SettingsType;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.SettingsRepository;
import org.openmetadata.service.jdbi3.SystemRepository;
import org.openmetadata.service.util.JsonUtils;
@Slf4j
@ -30,14 +30,14 @@ public class SettingsCache {
private static final SettingsCache INSTANCE = new SettingsCache();
private static volatile boolean INITIALIZED = false;
protected static LoadingCache<String, Settings> SETTINGS_CACHE;
protected static SettingsRepository SETTINGS_REPOSITORY;
protected static SystemRepository systemRepository;
// Expected to be called only once from the DefaultAuthorizer
public static void initialize(CollectionDAO dao) {
if (!INITIALIZED) {
SETTINGS_CACHE =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new SettingsLoader());
SETTINGS_REPOSITORY = new SettingsRepository(dao);
systemRepository = new SystemRepository(dao.systemDAO());
INITIALIZED = true;
}
}
@ -71,7 +71,7 @@ public class SettingsCache {
static class SettingsLoader extends CacheLoader<String, Settings> {
@Override
public Settings load(@CheckForNull String settingsName) {
Settings setting = SETTINGS_REPOSITORY.getConfigWithKey(settingsName);
Settings setting = systemRepository.getConfigWithKey(settingsName);
LOG.info("Loaded Setting {}", setting.getConfigType());
return setting;
}

18
openmetadata-service/src/main/java/org/openmetadata/service/resources/config/ConfigResource.java → openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java
View File

@ -11,7 +11,7 @@
* limitations under the License.
*/
package org.openmetadata.service.resources.config;
package org.openmetadata.service.resources.system;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
@ -32,8 +32,8 @@ import org.openmetadata.service.sandbox.SandboxConfiguration;
import org.openmetadata.service.security.jwt.JWKSResponse;
import org.openmetadata.service.security.jwt.JWTTokenGenerator;
@Path("/v1/config")
@Api(value = "Get configuration")
@Path("/v1/system/config")
@Api(value = "System configuration APIs")
@Produces(MediaType.APPLICATION_JSON)
@Collection(name = "config")
public class ConfigResource {
@ -53,7 +53,7 @@ public class ConfigResource {
@Operation(
operationId = "getAuthConfiguration",
summary = "Get auth configuration",
tags = "config",
tags = "system",
responses = {
@ApiResponse(
responseCode = "200",
@ -76,7 +76,7 @@ public class ConfigResource {
@Operation(
operationId = "getAuthorizerConfig",
summary = "Get authorizer configuration",
tags = "config",
tags = "system",
responses = {
@ApiResponse(
responseCode = "200",
@ -99,7 +99,7 @@ public class ConfigResource {
@Operation(
operationId = "getSandboxConfiguration",
summary = "Get sandbox mode",
tags = "config",
tags = "system",
responses = {
@ApiResponse(
responseCode = "200",
@ -120,7 +120,7 @@ public class ConfigResource {
@Operation(
operationId = "getSlackChatConfiguration",
summary = "Get Slack Chat Configuration",
tags = "config",
tags = "system",
responses = {
@ApiResponse(
responseCode = "200",
@ -143,7 +143,7 @@ public class ConfigResource {
@Operation(
operationId = "getAirflowConfiguration",
summary = "Get airflow configuration",
tags = "config",
tags = "system",
responses = {
@ApiResponse(
responseCode = "200",
@ -167,7 +167,7 @@ public class ConfigResource {
@Operation(
operationId = "getJWKSResponse",
summary = "Get JWKS public key",
tags = "config",
tags = "system",
responses = {
@ApiResponse(
responseCode = "200",

127
openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsResource.java → openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java
View File

@ -1,17 +1,4 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.resources.settings;
package org.openmetadata.service.resources.system;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
@ -28,12 +15,14 @@ import java.util.Objects;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.PATCH;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
@ -42,29 +31,36 @@ import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.settings.Settings;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.util.EntitiesCount;
import org.openmetadata.schema.util.ServicesCount;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.SettingsRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.SystemRepository;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList;
/**
* Resource for managing OpenMetadata settings that an admin can change. Example - using APIs here, the conversation
* thread notification can be changed to include only events that an organization uses.
*/
@Path("/v1/settings")
@Api(value = "Settings Collection", tags = "Settings collection")
@Path("/v1/system")
@Api(value = "Util collection", tags = "Util collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "settings")
@Collection(name = "system")
@Slf4j
public class SettingsResource {
private final SettingsRepository settingsRepository;
public class SystemResource {
public static final String COLLECTION_PATH = "/v1/util";
private final SystemRepository systemRepository;
private final Authorizer authorizer;
public SystemResource(CollectionDAO dao, Authorizer authorizer) {
Objects.requireNonNull(dao, "SystemRepository must not be null");
this.systemRepository = new SystemRepository(dao.systemDAO());
this.authorizer = authorizer;
}
@SuppressWarnings("unused") // Method used for reflection
public void initialize(OpenMetadataApplicationConfig config) throws IOException {
initSettings();
@ -83,10 +79,10 @@ public class SettingsResource {
settings.forEach(
(setting) -> {
try {
Settings storedSettings = settingsRepository.getConfigWithKey(setting.getConfigType().toString());
Settings storedSettings = systemRepository.getConfigWithKey(setting.getConfigType().toString());
if (storedSettings == null) {
// Only in case a config doesn't exist in DB we insert it
settingsRepository.createNewSetting(setting);
systemRepository.createNewSetting(setting);
}
} catch (Exception ex) {
LOG.debug("Fetching from DB failed ", ex);
@ -104,18 +100,12 @@ public class SettingsResource {
}
}
public SettingsResource(CollectionDAO dao, Authorizer authorizer) {
Objects.requireNonNull(dao, "SettingsRepository must not be null");
this.settingsRepository = new SettingsRepository(dao);
SettingsCache.initialize(dao);
this.authorizer = authorizer;
}
@GET
@Path("/settings")
@Operation(
operationId = "listSettings",
summary = "List All Settings",
tags = "settings",
tags = "system",
description = "Get a List of all OpenMetadata Settings",
responses = {
@ApiResponse(
@ -125,15 +115,15 @@ public class SettingsResource {
})
public ResultList<Settings> list(@Context UriInfo uriInfo, @Context SecurityContext securityContext) {
authorizer.authorizeAdmin(securityContext);
return settingsRepository.listAllConfigs();
return systemRepository.listAllConfigs();
}
@GET
@Path("/{name}")
@Path("/settings/{name}")
@Operation(
operationId = "getSetting",
summary = "Get a Setting",
tags = "settings",
tags = "system",
description = "Get a OpenMetadata Settings",
responses = {
@ApiResponse(
@ -147,14 +137,15 @@ public class SettingsResource {
@Parameter(description = "Name of the setting", schema = @Schema(type = "string")) @PathParam("name")
String name) {
authorizer.authorizeAdmin(securityContext);
return settingsRepository.getConfigWithKey(name);
return systemRepository.getConfigWithKey(name);
}
@PUT
@Path("/settings")
@Operation(
operationId = "createOrUpdate",
summary = "Update Setting",
tags = "settings",
tags = "system",
description = "Update Existing Settings",
responses = {
@ApiResponse(
@ -165,15 +156,15 @@ public class SettingsResource {
public Response createOrUpdateSetting(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid Settings settingName) {
authorizer.authorizeAdmin(securityContext);
return settingsRepository.createOrUpdate(settingName);
return systemRepository.createOrUpdate(settingName);
}
@PATCH
@Path("/{settingName}")
@Path("/settings/{settingName}")
@Operation(
operationId = "patchSetting",
summary = "Patch a Setting",
tags = "settings",
tags = "system",
description = "Update an existing Setting using JsonPatch.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
@ -192,6 +183,56 @@ public class SettingsResource {
}))
JsonPatch patch) {
authorizer.authorizeAdmin(securityContext);
return settingsRepository.patchSetting(settingName, patch);
return systemRepository.patchSetting(settingName, patch);
}
@GET
@Path("/entities/count")
@Operation(
operationId = "listEntitiesCount",
summary = "List All Entities Counts",
tags = "system",
description = "Get a List of all Entities Count",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of Entities Count",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntitiesCount.class)))
})
public EntitiesCount listEntitiesCount(
@Context UriInfo uriInfo,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include) {
ListFilter filter = new ListFilter(include);
return systemRepository.getAllEntitiesCount(filter);
}
@GET
@Path("/services/count")
@Operation(
operationId = "listServicesCount",
summary = "List All Services Counts",
tags = "system",
description = "Get a List of all Entities Count",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of Services Count",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = ServicesCount.class)))
})
public ServicesCount listServicesCount(
@Context UriInfo uriInfo,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include) {
ListFilter filter = new ListFilter(include);
return systemRepository.getAllServicesCount(filter);
}
}

1
openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java
View File

@ -151,6 +151,7 @@ public class TagResource extends EntityResource<Tag, TagRepository> {
}
}
@Override
public void initialize(OpenMetadataApplicationConfig config) throws IOException {
// TODO: Once we have migrated to the version above 0.13.1, then this can be removed
migrateTags();

95
openmetadata-service/src/main/java/org/openmetadata/service/resources/util/UtilResource.java
View File

@ -1,95 +0,0 @@
package org.openmetadata.service.resources.util;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.util.Objects;
import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.util.EntitiesCount;
import org.openmetadata.schema.util.ServicesCount;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.UtilRepository;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.security.Authorizer;
@Path("/v1/util")
@Api(value = "Util collection", tags = "Util collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "util")
@Slf4j
public class UtilResource {
public static final String COLLECTION_PATH = "/v1/util";
private final UtilRepository utilRepository;
private final Authorizer authorizer;
public UtilResource(CollectionDAO dao, Authorizer authorizer) {
Objects.requireNonNull(dao, "UtilRepository must not be null");
this.utilRepository = new UtilRepository(dao.utilDAO());
this.authorizer = authorizer;
}
@GET
@Path("/entities/count")
@Operation(
operationId = "listEntitiesCount",
summary = "List All Entities Counts",
tags = "util",
description = "Get a List of all Entities Count",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of Entities Count",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntitiesCount.class)))
})
public EntitiesCount listEntitiesCount(
@Context UriInfo uriInfo,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include) {
ListFilter filter = new ListFilter(include);
return utilRepository.getAllEntitiesCount(filter);
}
@GET
@Path("/services/count")
@Operation(
operationId = "listServicesCount",
summary = "List All Services Counts",
tags = "util",
description = "Get a List of all Entities Count",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of Services Count",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = ServicesCount.class)))
})
public ServicesCount listServicesCount(
@Context UriInfo uriInfo,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include) {
ListFilter filter = new ListFilter(include);
return utilRepository.getAllServicesCount(filter);
}
}

6
openmetadata-service/src/main/java/org/openmetadata/service/resources/version/VersionResource.java
View File

@ -27,8 +27,8 @@ import org.openmetadata.service.OpenMetadataApplication;
import org.openmetadata.service.resources.Collection;
@Slf4j
@Path("/v1/version")
@Api(value = "Catalog version", tags = "Catalog version related operations")
@Path("/v1/system/version")
@Api(value = "System software version", tags = "System version related operations")
@Produces(MediaType.APPLICATION_JSON)
@Collection(name = "version")
public class VersionResource {
@ -55,7 +55,7 @@ public class VersionResource {
@Operation(
operationId = "getCatalogVersion",
summary = "Get version of metadata service",
tags = "catalog",
tags = "system",
description = "Get the build version of OpenMetadata service and build timestamp.")
public OpenMetadataServerVersion getCatalogVersion() {
return OPEN_METADATA_SERVER_VERSION;

4
openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java
View File

@ -62,9 +62,9 @@ public class JwtFilter implements ContainerRequestFilter {
private String providerType;
public static final List<String> EXCLUDED_ENDPOINTS =
List.of(
"v1/config",
"v1/system/config",
"v1/users/signup",
"v1/version",
"v1/system/version",
"v1/users/registrationConfirmation",
"v1/users/resendRegistrationToken",
"v1/users/generatePasswordResetLink",

2
openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java
View File

@ -118,6 +118,6 @@ public abstract class OpenMetadataApplicationTest {
}
public static WebTarget getConfigResource(String resource) {
return getClient().target(format("http://localhost:%s/api/v1/config/%s", APP.getLocalPort(), resource));
return getClient().target(format("http://localhost:%s/api/v1/system/config/%s", APP.getLocalPort(), resource));
}
}

37
openmetadata-service/src/test/java/org/openmetadata/service/resources/CollectionRegistryTest.java
View File

@ -1,37 +0,0 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.resources;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.openmetadata.schema.type.CollectionDescriptor;
class CollectionRegistryTest {
@Test
void testCollections() {
CollectionRegistry.getInstance();
Map<String, CollectionRegistry.CollectionDetails> descriptors = CollectionRegistry.getInstance().getCollectionMap();
String path = "/v1";
CollectionRegistry.CollectionDetails parent = descriptors.get(path);
CollectionDescriptor[] children = parent.getChildCollections();
assertNotEquals(0, children.length);
path = "/v1/services";
parent = descriptors.get(path);
children = parent.getChildCollections();
assertNotEquals(0, children.length);
}
}

10
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -410,16 +410,6 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
.withOwner(reduceEntityReference(owner));
}
public final K createPutRequest(String name, String description, String displayName, EntityReference owner) {
if (!supportsEmptyDescription && description == null) {
throw new IllegalArgumentException("Entity " + entityType + " does not support empty description");
}
return createPutRequest(name)
.withDescription(description)
.withDisplayName(displayName)
.withOwner(reduceEntityReference(owner));
}
public abstract K createRequest(String name);
public K createPutRequest(String name) {

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/config/ConfigResourceTest.java → openmetadata-service/src/test/java/org/openmetadata/service/resources/system/ConfigResourceTest.java
View File

@ -11,7 +11,7 @@
* limitations under the License.
*/
package org.openmetadata.service.resources.config;
package org.openmetadata.service.resources.system;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;

87
openmetadata-service/src/test/java/org/openmetadata/service/resources/util/UtilResourceTest.java → openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java
View File

@ -1,4 +1,4 @@
package org.openmetadata.service.resources.util;
package org.openmetadata.service.resources.system;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
@ -49,7 +49,7 @@ import org.openmetadata.service.util.TestUtils;
@Slf4j
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class UtilResourceTest extends OpenMetadataApplicationTest {
public class SystemResourceTest extends OpenMetadataApplicationTest {
@BeforeAll
public static void setup(TestInfo test) throws IOException, URISyntaxException {
@ -57,28 +57,10 @@ public class UtilResourceTest extends OpenMetadataApplicationTest {
entityResourceTest.setup(test);
}
public static EntitiesCount getEntitiesCount() throws HttpResponseException {
WebTarget target = getResource("util/entities/count");
return TestUtils.get(target, EntitiesCount.class, ADMIN_AUTH_HEADERS);
}
public static ServicesCount getServicesCount() throws HttpResponseException {
WebTarget target = getResource("util/services/count");
return TestUtils.get(target, ServicesCount.class, ADMIN_AUTH_HEADERS);
}
@Test
public void entitiesCount(TestInfo test) throws HttpResponseException {
// Get count before adding entities
int beforeTableCount = getEntitiesCount().getTableCount();
int beforeDashboardCount = getEntitiesCount().getDashboardCount();
int beforePipelineCount = getEntitiesCount().getPipelineCount();
int beforeTopicCount = getEntitiesCount().getTopicCount();
int beforeServiceCount = getEntitiesCount().getServicesCount();
int beforeUserCount = getEntitiesCount().getUserCount();
int beforeTeamCount = getEntitiesCount().getTeamCount();
int beforeTestSuiteCount = getEntitiesCount().getTestSuiteCount();
EntitiesCount beforeCount = getEntitiesCount();
// Create Table
TableResourceTest tableResourceTest = new TableResourceTest();
@ -120,36 +102,22 @@ public class UtilResourceTest extends OpenMetadataApplicationTest {
CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(test);
testSuiteResourceTest.createEntity(createTestSuite, ADMIN_AUTH_HEADERS);
// Get count after adding entities
int afterTableCount = getEntitiesCount().getTableCount();
int afterDashboardCount = getEntitiesCount().getDashboardCount();
int afterPipelineCount = getEntitiesCount().getPipelineCount();
int afterTopicCount = getEntitiesCount().getTopicCount();
int afterServiceCount = getEntitiesCount().getServicesCount();
int afterUserCount = getEntitiesCount().getUserCount();
int afterTeamCount = getEntitiesCount().getTeamCount();
int afterTestSuiteCount = getEntitiesCount().getTestSuiteCount();
int actualCount = 1;
Assertions.assertEquals(afterDashboardCount - beforeDashboardCount, actualCount);
Assertions.assertEquals(afterPipelineCount - beforePipelineCount, actualCount);
Assertions.assertEquals(afterServiceCount - beforeServiceCount, actualCount);
Assertions.assertEquals(afterUserCount - beforeUserCount, actualCount);
Assertions.assertEquals(afterTableCount - beforeTableCount, actualCount);
Assertions.assertEquals(afterTeamCount - beforeTeamCount, actualCount);
Assertions.assertEquals(afterTopicCount - beforeTopicCount, actualCount);
Assertions.assertEquals(afterTestSuiteCount - beforeTestSuiteCount, actualCount);
// Ensure counts of entities is increased by 1
EntitiesCount afterCount = getEntitiesCount();
Assertions.assertEquals(beforeCount.getDashboardCount() + 1, afterCount.getDashboardCount());
Assertions.assertEquals(beforeCount.getPipelineCount() + 1, afterCount.getPipelineCount());
Assertions.assertEquals(beforeCount.getServicesCount() + 1, afterCount.getServicesCount());
Assertions.assertEquals(beforeCount.getUserCount() + 1, afterCount.getUserCount());
Assertions.assertEquals(beforeCount.getTableCount() + 1, afterCount.getTableCount());
Assertions.assertEquals(beforeCount.getTeamCount() + 1, afterCount.getTeamCount());
Assertions.assertEquals(beforeCount.getTopicCount() + 1, afterCount.getTopicCount());
Assertions.assertEquals(beforeCount.getTestSuiteCount() + 1, afterCount.getTestSuiteCount());
}
@Test
public void servicesCount(TestInfo test) throws HttpResponseException {
// Get count before adding services
int beforeMessagingServiceCount = getServicesCount().getMessagingServiceCount();
int beforeDashboardServiceCount = getServicesCount().getDashboardServiceCount();
int beforePipelineServiceCount = getServicesCount().getPipelineServiceCount();
int beforeMlModelServiceCount = getServicesCount().getMlModelServiceCount();
ServicesCount beforeCount = getServicesCount();
// Create Database Service
DatabaseServiceResourceTest databaseServiceResourceTest = new DatabaseServiceResourceTest();
@ -176,17 +144,12 @@ public class UtilResourceTest extends OpenMetadataApplicationTest {
CreateMlModelService createMlModelService = mlModelServiceResourceTest.createRequest(test);
mlModelServiceResourceTest.createEntity(createMlModelService, ADMIN_AUTH_HEADERS);
// Get count after creating services
int afterMessagingServiceCount = getServicesCount().getMessagingServiceCount();
int afterDashboardServiceCount = getServicesCount().getDashboardServiceCount();
int afterPipelineServiceCount = getServicesCount().getPipelineServiceCount();
int afterMlModelServiceCount = getServicesCount().getMlModelServiceCount();
int actualCount = 1;
Assertions.assertEquals(afterMessagingServiceCount - beforeMessagingServiceCount, actualCount);
Assertions.assertEquals(afterDashboardServiceCount - beforeDashboardServiceCount, actualCount);
Assertions.assertEquals(afterPipelineServiceCount - beforePipelineServiceCount, actualCount);
Assertions.assertEquals(afterMlModelServiceCount - beforeMlModelServiceCount, actualCount);
// Get count after creating services and ensure it increased by 1
ServicesCount afterCount = getServicesCount();
Assertions.assertEquals(beforeCount.getMessagingServiceCount() + 1, afterCount.getMessagingServiceCount());
Assertions.assertEquals(beforeCount.getDashboardServiceCount() + 1, afterCount.getDashboardServiceCount());
Assertions.assertEquals(beforeCount.getPipelineServiceCount() + 1, afterCount.getPipelineServiceCount());
Assertions.assertEquals(beforeCount.getMlModelServiceCount() + 1, afterCount.getMlModelServiceCount());
}
@Test
@ -213,4 +176,14 @@ public class UtilResourceTest extends OpenMetadataApplicationTest {
// The bot user count should not be considered.
Assertions.assertEquals(beforeUserCount, afterUserCount);
}
private static EntitiesCount getEntitiesCount() throws HttpResponseException {
WebTarget target = getResource("system/entities/count");
return TestUtils.get(target, EntitiesCount.class, ADMIN_AUTH_HEADERS);
}
private static ServicesCount getServicesCount() throws HttpResponseException {
WebTarget target = getResource("system/services/count");
return TestUtils.get(target, ServicesCount.class, ADMIN_AUTH_HEADERS);
}
}

4
openmetadata-ui/src/main/resources/ui/cypress/common/common.js
View File

@ -337,7 +337,7 @@ export const editOwnerforCreatedService = (
'waitForIngestion'
);
interceptURL('GET', '/api/v1/config/airflow', 'airflow');
interceptURL('GET', '/api/v1/system/config/airflow', 'airflow');
// click on created service
cy.get(`[data-testid="service-name-${service_Name}"]`)
.should('exist')
@ -985,7 +985,7 @@ export const updateDescriptionForIngestedTables = (
`/api/v1/services/ingestionPipelines?fields=owner,pipelineStatuses&service=${serviceName}`,
'getSelectedService'
);
interceptURL('GET', '/api/v1/config/airflow', 'airflow');
interceptURL('GET', '/api/v1/system/config/airflow', 'airflow');
// click on created service
cy.get(`[data-testid="service-name-${serviceName}"]`)

2
openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js
View File

@ -114,7 +114,7 @@ describe('Postgres Ingestion', () => {
verifyResponseStatusCode('@getServices', 200);
cy.intercept('/api/v1/services/ingestionPipelines?*').as('ingestionData');
interceptURL('GET', '/api/v1/config/airflow', 'airflow');
interceptURL('GET', '/api/v1/system/config/airflow', 'airflow');
cy.get(`[data-testid="service-name-${serviceName}"]`)
.should('exist')
.click();

2
openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/redshiftWithDBT.spec.js
View File

@ -110,7 +110,7 @@ describe('RedShift Ingestion', () => {
verifyResponseStatusCode('@getServices', 200);
cy.intercept('/api/v1/services/ingestionPipelines?*').as('ingestionData');
interceptURL('GET', '/api/v1/config/airflow', 'airflow');
interceptURL('GET', '/api/v1/system/config/airflow', 'airflow');
cy.get(`[data-testid="service-name-${REDSHIFT.serviceName}"]`)
.should('exist')
.click();

2
openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/DataConsumerRole.spec.js
View File

@ -117,7 +117,7 @@ const ID = {
describe('DataConsumer Edit policy should work properly', () => {
it('Create a new account and assign Data consumer role to the user', () => {
interceptURL('GET', 'api/v1/config/auth', 'getLoginPage');
interceptURL('GET', 'api/v1/system/config/auth', 'getLoginPage');
cy.visit('/');
verifyResponseStatusCode('@getLoginPage', 200);
// Click on create account button

2
openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js
View File

@ -103,7 +103,7 @@ describe('Data Quality and Profiler should work properly', () => {
.should('be.visible')
.click();
cy.intercept('/api/v1/services/ingestionPipelines?*').as('ingestionData');
interceptURL('GET', '/api/v1/config/airflow', 'airflow');
interceptURL('GET', '/api/v1/system/config/airflow', 'airflow');
cy.get(`[data-testid="service-name-${serviceName}"]`)
.should('exist')
.click();

4
openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Login.spec.js
View File

@ -29,7 +29,7 @@ const invalidPassword = 'testUsers@123';
describe('Login flow should work properly', () => {
it('Signup and Login with signed up credentials', () => {
interceptURL('GET', 'api/v1/config/auth', 'getLoginPage');
interceptURL('GET', 'api/v1/system/config/auth', 'getLoginPage');
cy.visit('/');
verifyResponseStatusCode('@getLoginPage', 200);
// Click on create account button
@ -101,7 +101,7 @@ describe('Login flow should work properly', () => {
});
it('Forgot password and login with new password', () => {
interceptURL('GET', 'api/v1/config/auth', 'getLoginPage');
interceptURL('GET', 'api/v1/system/config/auth', 'getLoginPage');
cy.visit('/');
verifyResponseStatusCode('@getLoginPage', 200);
// Click on Forgot button

4
openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Service.spec.js
View File

@ -32,7 +32,7 @@ describe('Services page should work properly', () => {
});
it('Update service description', () => {
interceptURL('GET', '/api/v1/config/airflow', 'getService');
interceptURL('GET', '/api/v1/system/config/airflow', 'getService');
cy.get(`[data-testid="service-name-${service.name}"]`)
.should('be.visible')
.click();
@ -52,7 +52,7 @@ describe('Services page should work properly', () => {
});
it('Update owner and check description', () => {
interceptURL('GET', '/api/v1/config/airflow', 'getService');
interceptURL('GET', '/api/v1/system/config/airflow', 'getService');
cy.get(`[data-testid="service-name-${service.name}"]`)
.should('be.visible')
.click();

2
openmetadata-ui/src/main/resources/ui/cypress/support/commands.js
View File

@ -78,7 +78,7 @@ Cypress.Commands.add('loginByGoogleApi', () => {
});
Cypress.Commands.add('goToHomePage', () => {
interceptURL('GET', '/api/v1/util/entities/count', 'count');
interceptURL('GET', '/api/v1/system/entities/count', 'count');
interceptURL('GET', '/api/v1/feed*', 'feed');
interceptURL('GET', '/api/v1/users/name/*?fields=*', 'userProfile');
cy.get('[data-testid="whats-new-dialog"]')

16
openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts
View File

@ -66,7 +66,7 @@ export const getOwnershipCount = (
export const fetchAuthenticationConfig = async () => {
const response = await APIClient.get<AuthenticationConfiguration>(
'/config/auth'
'/system/config/auth'
);
return response.data;
@ -74,7 +74,7 @@ export const fetchAuthenticationConfig = async () => {
export const fetchAuthorizerConfig = async () => {
const response = await APIClient.get<AuthorizerConfiguration>(
'/config/authorizer'
'/system/config/authorizer'
);
return response.data;
@ -82,18 +82,20 @@ export const fetchAuthorizerConfig = async () => {
export const fetchSandboxConfig = async () => {
const response = await APIClient.get<{ sandboxModeEnabled: boolean }>(
'/config/sandbox'
'/system/config/sandbox'
);
return response.data;
};
export const fetchSlackConfig = (): Promise<AxiosResponse> => {
return APIClient.get('/config/slackChat');
return APIClient.get('/system/config/slackChat');
};
export const fetchAirflowConfig = async () => {
const response = await APIClient.get<AirflowConfiguration>('/config/airflow');
const response = await APIClient.get<AirflowConfiguration>(
'/system/config/airflow'
);
return response.data;
};
@ -128,7 +130,7 @@ export const getSuggestions = <T extends SearchIndex>(
};
export const getVersion = async () => {
const response = await APIClient.get<{ version: string }>('/version');
const response = await APIClient.get<{ version: string }>('/system/version');
return response.data;
};
@ -292,7 +294,7 @@ export const getEntityCount = async (
};
export const getAllEntityCount = async () => {
const response = await APIClient.get<EntitiesCount>('/util/entities/count');
const response = await APIClient.get<EntitiesCount>('/system/entities/count');
return response.data;
};