yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-25 17:04:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Usage resource migrated to jdbi3

Browse Source
This commit is contained in:
sureshms 2021-10-21 19:24:28 -07:00
parent b84220a6a8
commit a98e8c6afb
7 changed files with 134 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MetricsDAO3.java
View File

@ -1,10 +1,7 @@
package org.openmetadata.catalog.jdbi3;
import org.jdbi.v3.sqlobject.statement.SqlQuery;
import org.openmetadata.catalog.entity.data.Metrics;
import org.openmetadata.catalog.entity.data.Table;
import org.skife.jdbi.v2.sqlobject.Bind;
import org.skife.jdbi.v2.sqlobject.SqlQuery;
import org.skife.jdbi.v2.sqlobject.SqlUpdate;
import java.util.List;

13
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepositoryHelper.java
View File

@ -118,7 +118,7 @@ public class TableRepositoryHelper implements EntityRepository<Table> {
table.setJoins(fields.contains("joins") ? getJoins(table) : null);
table.setSampleData(fields.contains("sampleData") ? getSampleData(table) : null);
table.setViewDefinition(fields.contains("viewDefinition") ? table.getViewDefinition() : null);
table.setTableProfile(fields.contains("tableProfile") ? getTableProfile(table): null);
table.setTableProfile(fields.contains("tableProfile") ? getTableProfile(table) : null);
return table;
}
@ -204,7 +204,8 @@ public class TableRepositoryHelper implements EntityRepository<Table> {
@Transaction
public Status addFollower(String tableId, String userId) throws IOException {
repo3.tableDAO().findEntityById(tableId);
return EntityUtil.addFollower(repo3.relationshipDAO(), repo3.userDAO(), tableId, Entity.TABLE, userId, Entity.USER) ?
return EntityUtil.addFollower(repo3.relationshipDAO(), repo3.userDAO(), tableId, Entity.TABLE, userId,
Entity.USER) ?
Status.CREATED : Status.OK;
}
@ -263,7 +264,7 @@ public class TableRepositoryHelper implements EntityRepository<Table> {
}
}
//validate all the columns
for (ColumnProfile columnProfile: tableProfile.getColumnProfile()) {
for (ColumnProfile columnProfile : tableProfile.getColumnProfile()) {
validateColumn(table, columnProfile.getName());
}
storedMapTableProfiles.put(tableProfile.getProfileDate(), tableProfile);
@ -448,7 +449,8 @@ public class TableRepositoryHelper implements EntityRepository<Table> {
}
private EntityReference getOwner(Table table) throws IOException {
return table == null ? null : EntityUtil.populateOwner(table.getId(), repo3.relationshipDAO(), repo3.userDAO(), repo3.teamDAO());
return table == null ? null : EntityUtil.populateOwner(table.getId(), repo3.relationshipDAO(), repo3.userDAO(),
repo3.teamDAO());
}
private List<EntityReference> getFollowers(Table table) throws IOException {
@ -629,7 +631,8 @@ public class TableRepositoryHelper implements EntityRepository<Table> {
}
private List<TableProfile> getTableProfile(Table table) throws IOException {
List<TableProfile> tableProfiles = JsonUtils.readObjects(repo3.entityExtensionDAO().getExtension(table.getId().toString(),
List<TableProfile> tableProfiles =
JsonUtils.readObjects(repo3.entityExtensionDAO().getExtension(table.getId().toString(),
"table.tableProfile"),
TableProfile.class);
if (tableProfiles != null) {

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UsageDAO.java
View File

@ -1,6 +1,6 @@
package org.openmetadata.catalog.jdbi3;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDetailsMapper;
import org.openmetadata.catalog.jdbi3.UsageRepositoryHelper.UsageDetailsMapper;
import org.openmetadata.catalog.type.UsageDetails;
import org.skife.jdbi.v2.sqlobject.Bind;
import org.skife.jdbi.v2.sqlobject.SqlQuery;

57
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UsageRepository3.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import org.jdbi.v3.sqlobject.CreateSqlObject;
public interface UsageRepository3 {
@CreateSqlObject
UsageDAO3 usageDAO();
@CreateSqlObject
TableDAO3 tableDAO();
@CreateSqlObject
DatabaseDAO3 databaseDAO();
@CreateSqlObject
MetricsDAO3 metricsDAO();
@CreateSqlObject
DashboardDAO3 dashboardDAO();
@CreateSqlObject
ReportDAO3 reportDAO();
@CreateSqlObject
TopicDAO3 topicDAO();
@CreateSqlObject
ChartDAO3 chartDAO();
@CreateSqlObject
TaskDAO3 taskDAO();
@CreateSqlObject
PipelineDAO3 pipelineDAO();
@CreateSqlObject
ModelDAO3 modelDAO();
@CreateSqlObject
EntityRelationshipDAO3 relationshipDAO();
}

78
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UsageRepository.java → catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UsageRepositoryHelper.java
View File

@ -17,14 +17,13 @@
package org.openmetadata.catalog.jdbi3;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.type.DailyCount;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.EntityUsage;
import org.openmetadata.catalog.type.UsageDetails;
import org.openmetadata.catalog.type.DailyCount;
import org.openmetadata.catalog.type.UsageStats;
import org.openmetadata.catalog.util.EntityUtil;
import org.skife.jdbi.v2.StatementContext;
import org.skife.jdbi.v2.sqlobject.CreateSqlObject;
import org.skife.jdbi.v2.sqlobject.Transaction;
import org.skife.jdbi.v2.tweak.ResultSetMapper;
import org.slf4j.Logger;
@ -38,93 +37,62 @@ import java.util.UUID;
import static org.openmetadata.catalog.util.EntityUtil.getEntityReference;
public abstract class UsageRepository {
private static final Logger LOG = LoggerFactory.getLogger(UsageRepository.class);
@CreateSqlObject
abstract UsageDAO usageDAO();
@CreateSqlObject
abstract TableDAO tableDAO();
@CreateSqlObject
abstract DatabaseDAO databaseDAO();
@CreateSqlObject
abstract MetricsDAO metricsDAO();
@CreateSqlObject
abstract DashboardDAO dashboardDAO();
@CreateSqlObject
abstract ReportDAO reportDAO();
@CreateSqlObject
abstract TopicDAO topicDAO();
@CreateSqlObject
abstract ChartDAO chartDAO();
@CreateSqlObject
abstract TaskDAO taskDAO();
@CreateSqlObject
abstract PipelineDAO pipelineDAO();
@CreateSqlObject
abstract ModelDAO modelDAO();
@CreateSqlObject
abstract EntityRelationshipDAO relationshipDAO();
public class UsageRepositoryHelper {
private static final Logger LOG = LoggerFactory.getLogger(UsageRepositoryHelper.class);
public UsageRepositoryHelper(UsageRepository3 repo3) { this.repo3 = repo3; }
private final UsageRepository3 repo3;
@Transaction
public EntityUsage get(String entityType, String id, String date, int days) throws IOException {
EntityReference ref = getEntityReference(entityType, UUID.fromString(id), tableDAO(), databaseDAO(),
metricsDAO(), dashboardDAO(), reportDAO(), topicDAO(), chartDAO(), taskDAO(), modelDAO(), pipelineDAO());
List<UsageDetails> usageDetails = usageDAO().getUsageById(id, date, days - 1);
EntityReference ref = getEntityReference(entityType, UUID.fromString(id), repo3.tableDAO(), repo3.databaseDAO(),
repo3.metricsDAO(), repo3.dashboardDAO(), repo3.reportDAO(), repo3.topicDAO(), repo3.chartDAO(),
repo3.taskDAO(), repo3.modelDAO(), repo3.pipelineDAO());
List<UsageDetails> usageDetails = repo3.usageDAO().getUsageById(id, date, days - 1);
return new EntityUsage().withUsage(usageDetails).withEntity(ref);
}
@Transaction
public EntityUsage getByName(String entityType, String fqn, String date, int days) throws IOException {
EntityReference ref = EntityUtil.getEntityReferenceByName(entityType, fqn, tableDAO(), databaseDAO(),
metricsDAO(), reportDAO(), topicDAO(), chartDAO(), dashboardDAO(), taskDAO(), modelDAO(), pipelineDAO());
List<UsageDetails> usageDetails = usageDAO().getUsageById(ref.getId().toString(), date, days - 1);
EntityReference ref = EntityUtil.getEntityReferenceByName(entityType, fqn, repo3.tableDAO(), repo3.databaseDAO(),
repo3.metricsDAO(), repo3.reportDAO(), repo3.topicDAO(), repo3.chartDAO(), repo3.dashboardDAO(),
repo3.taskDAO(), repo3.modelDAO(), repo3.pipelineDAO());
List<UsageDetails> usageDetails = repo3.usageDAO().getUsageById(ref.getId().toString(), date, days - 1);
return new EntityUsage().withUsage(usageDetails).withEntity(ref);
}
@Transaction
public void create(String entityType, String id, DailyCount usage) throws IOException {
// Validate data entity for which usage is being collected
getEntityReference(entityType, UUID.fromString(id), tableDAO(), databaseDAO(), metricsDAO(),
dashboardDAO(), reportDAO(), topicDAO(), chartDAO(), taskDAO(), modelDAO(), pipelineDAO());
getEntityReference(entityType, UUID.fromString(id), repo3.tableDAO(), repo3.databaseDAO(), repo3.metricsDAO(),
repo3.dashboardDAO(), repo3.reportDAO(), repo3.topicDAO(), repo3.chartDAO(), repo3.taskDAO(),
repo3.modelDAO(), repo3.pipelineDAO());
addUsage(entityType, id, usage);
}
@Transaction
public void createByName(String entityType, String fullyQualifiedName, DailyCount usage) throws IOException {
EntityReference ref = EntityUtil.getEntityReferenceByName(entityType, fullyQualifiedName, tableDAO(),
databaseDAO(), metricsDAO(), reportDAO(), topicDAO(), chartDAO(), dashboardDAO(),
taskDAO(), modelDAO(), pipelineDAO());
EntityReference ref = EntityUtil.getEntityReferenceByName(entityType, fullyQualifiedName, repo3.tableDAO(),
repo3.databaseDAO(), repo3.metricsDAO(), repo3.reportDAO(), repo3.topicDAO(), repo3.chartDAO(),
repo3.dashboardDAO(), repo3.taskDAO(), repo3.modelDAO(), repo3.pipelineDAO());
addUsage(entityType, ref.getId().toString(), usage);
LOG.info("Usage successfully posted by name");
}
@Transaction
public void computePercentile(String entityType, String date) {
usageDAO().computePercentile(entityType, date);
repo3.usageDAO().computePercentile(entityType, date);
}
private void addUsage(String entityType, String entityId, DailyCount usage) {
// Insert usage record
usageDAO().insert(usage.getDate(), entityId, entityType, usage.getCount());
repo3.usageDAO().insert(usage.getDate(), entityId, entityType, usage.getCount());
// If table usage was reported, add the usage count to database
if (entityType.equalsIgnoreCase(Entity.TABLE)) {
List<String> databaseIds = relationshipDAO().findFrom(entityId, Relationship.CONTAINS.ordinal(), Entity.DATABASE);
usageDAO().insertOrUpdateCount(usage.getDate(), databaseIds.get(0), Entity.DATABASE, usage.getCount());
List<String> databaseIds = repo3.relationshipDAO().findFrom(entityId, Relationship.CONTAINS.ordinal(), Entity.DATABASE);
repo3.usageDAO().insertOrUpdateCount(usage.getDate(), databaseIds.get(0), Entity.DATABASE, usage.getCount());
}
}

10
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/usage/UsageResource.java
View File

@ -17,8 +17,8 @@
package org.openmetadata.catalog.resources.usage;
import com.google.inject.Inject;
import org.openmetadata.catalog.jdbi3.UsageRepositoryHelper;
import org.openmetadata.catalog.resources.teams.UserResource;
import org.openmetadata.catalog.jdbi3.UsageRepository;
import org.openmetadata.catalog.resources.Collection;
import org.openmetadata.catalog.type.EntityUsage;
import org.openmetadata.catalog.type.DailyCount;
@ -54,14 +54,14 @@ import java.util.Objects;
@Api(value = "Usage resource", tags = "Usage resource")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "usage", repositoryClass = "org.openmetadata.catalog.jdbi3.UsageRepository")
//@Collection(name = "usage", repositoryClass = "org.openmetadata.catalog.jdbi3.UsageRepositoryHelper")
public class UsageResource {
private static final Logger LOG = LoggerFactory.getLogger(UserResource.class);
private final UsageRepository dao;
private final UsageRepositoryHelper dao;
@Inject
public UsageResource(UsageRepository dao, CatalogAuthorizer authorizer) {
Objects.requireNonNull(dao, "UsageRepository must not be null");
public UsageResource(UsageRepositoryHelper dao, CatalogAuthorizer authorizer) {
Objects.requireNonNull(dao, "UsageRepositoryHelper must not be null");
this.dao = dao;
}

64
catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java
View File

@ -348,19 +348,6 @@ public final class EntityUtil {
setOwner(dao, ownedEntityId, ownedEntityType, newOwner);
}
public static List<EntityReference> getEntityReference(List<EntityReference> list, TableDAO tableDAO,
DatabaseDAO databaseDAO, MetricsDAO metricsDAO,
DashboardDAO dashboardDAO, ReportDAO reportDAO,
TopicDAO topicDAO, ChartDAO chartDAO,
TaskDAO taskDAO, ModelDAO modelDAO,
PipelineDAO pipelineDAO) throws IOException {
for (EntityReference ref : list) {
getEntityReference(ref, tableDAO, databaseDAO, metricsDAO, dashboardDAO, reportDAO, topicDAO, chartDAO,
taskDAO, modelDAO, pipelineDAO);
}
return list;
}
public static List<EntityReference> getEntityReference(List<EntityReference> list, TableDAO3 tableDAO,
DatabaseDAO3 databaseDAO, MetricsDAO3 metricsDAO,
DashboardDAO3 dashboardDAO, ReportDAO3 reportDAO,
@ -375,47 +362,6 @@ public final class EntityUtil {
}
public static EntityReference getEntityReference(EntityReference ref, TableDAO tableDAO, DatabaseDAO databaseDAO,
MetricsDAO metricsDAO, DashboardDAO dashboardDAO,
ReportDAO reportDAO, TopicDAO topicDAO, ChartDAO chartDAO,
TaskDAO taskDAO, ModelDAO modelDAO, PipelineDAO pipelineDAO)
throws IOException {
// Note href to entity reference is not added here
String entity = ref.getType();
String id = ref.getId().toString();
if (entity.equalsIgnoreCase(Entity.TABLE)) {
Table instance = EntityUtil.validate(id, tableDAO.findById(id), Table.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
} else if (entity.equalsIgnoreCase(Entity.DATABASE)) {
Database instance = EntityUtil.validate(id, databaseDAO.findById(id), Database.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
} else if (entity.equalsIgnoreCase(Entity.METRICS)) {
Metrics instance = EntityUtil.validate(id, metricsDAO.findById(id), Metrics.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
} else if (entity.equalsIgnoreCase(Entity.DASHBOARD)) {
Dashboard instance = EntityUtil.validate(id, dashboardDAO.findById(id), Dashboard.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
} else if (entity.equalsIgnoreCase(Entity.REPORT)) {
Report instance = EntityUtil.validate(id, reportDAO.findById(id), Report.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
} else if (entity.equalsIgnoreCase(Entity.TOPIC)) {
Topic instance = EntityUtil.validate(id, topicDAO.findById(id), Topic.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
} else if (entity.equalsIgnoreCase(Entity.CHART)) {
Chart instance = EntityUtil.validate(id, chartDAO.findById(id), Chart.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
} else if (entity.equalsIgnoreCase(Entity.TASK)) {
Task instance = EntityUtil.validate(id, taskDAO.findById(id), Task.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
} else if (entity.equalsIgnoreCase(Entity.PIPELINE)) {
Pipeline instance = EntityUtil.validate(id, pipelineDAO.findById(id), Pipeline.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
} else if (entity.equalsIgnoreCase(Entity.MODEL)) {
Model instance = EntityUtil.validate(id, modelDAO.findById(id), Model.class);
return ref.withDescription(instance.getDescription()).withName(instance.getFullyQualifiedName());
}
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(entity));
}
public static EntityReference getEntityReference3(EntityReference ref, TableDAO3 tableDAO, DatabaseDAO3 databaseDAO,
MetricsDAO3 metricsDAO, DashboardDAO3 dashboardDAO,
@ -459,16 +405,6 @@ public final class EntityUtil {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(entity));
}
public static EntityReference getEntityReference(String entity, UUID id, TableDAO tableDAO, DatabaseDAO databaseDAO,
MetricsDAO metricsDAO, DashboardDAO dashboardDAO,
ReportDAO reportDAO, TopicDAO topicDAO, ChartDAO chartDAO,
TaskDAO taskDAO, ModelDAO modelDAO, PipelineDAO pipelineDAO)
throws IOException {
EntityReference ref = new EntityReference().withId(id).withType(entity);
return getEntityReference(ref, tableDAO, databaseDAO, metricsDAO, dashboardDAO,
reportDAO, topicDAO, chartDAO, taskDAO, modelDAO, pipelineDAO);
}
public static EntityReference getEntityReference(String entity, UUID id, TableDAO3 tableDAO,
DatabaseDAO3 databaseDAO,
MetricsDAO3 metricsDAO, DashboardDAO3 dashboardDAO,