yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-03 04:10:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(frontend): Remove unused files 1/4 (#3014)

Browse Source
This commit is contained in:
John Joyce 2021-08-09 09:48:05 -07:00 committed by GitHub
parent 9ee091f518
commit 383bde18f5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 163 additions and 2845 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
datahub-dao/build.gradle
View File

@ -1,15 +0,0 @@
apply plugin: 'java'
dependencies {
compile project(':gms:client')
compile project(':metadata-utils')
compile externalDependency.elasticSearchRest
compile externalDependency.hibernateCore
compile externalDependency.guava
compileOnly externalDependency.lombok
annotationProcessor externalDependency.lombok
testCompile externalDependency.mockito
}

11
datahub-dao/src/main/java/com/linkedin/datahub/common/exceptions/MissingFieldException.java
View File

@ -1,11 +0,0 @@
package com.linkedin.datahub.common.exceptions;
/**
* An exception to be thrown when certain required field is missing.
*/
public class MissingFieldException extends RuntimeException {
public MissingFieldException(String message) {
super(message);
}
}

117
datahub-dao/src/main/java/com/linkedin/datahub/dao/DaoFactory.java
View File

@ -1,117 +0,0 @@
package com.linkedin.datahub.dao;
import com.linkedin.datahub.dao.table.DataPlatformsDao;
import com.linkedin.datahub.dao.table.DatasetOwnerDao;
import com.linkedin.datahub.dao.table.DatasetsDao;
import com.linkedin.datahub.dao.table.GmsDao;
import com.linkedin.datahub.dao.table.LineageDao;
import com.linkedin.datahub.dao.view.BrowseDAO;
import com.linkedin.datahub.dao.view.CorpUserViewDao;
import com.linkedin.datahub.dao.view.DatasetViewDao;
import com.linkedin.datahub.dao.view.DocumentSearchDao;
import com.linkedin.datahub.dao.view.OwnerViewDao;
import com.linkedin.util.Configuration;
public class DaoFactory {
private static final String GMS_HOST_ENV_VAR = "DATAHUB_GMS_HOST";
private static final String GMS_PORT_ENV_VAR = "DATAHUB_GMS_PORT";
private static final String GMS_USE_SSL_ENV_VAR = "DATAHUB_GMS_USE_SSL";
private static final String GMS_SSL_PROTOCOL_VAR = "DATAHUB_GMS_SSL_PROTOCOL";
private static GmsDao _gmsDao;
private static DocumentSearchDao datasetDocumentSearchDao;
private static DocumentSearchDao corpUserDocumentSearchDao;
private static CorpUserViewDao corpUserViewDao;
private static BrowseDAO datasetBrowseDao;
private static OwnerViewDao ownerViewDao;
private static DatasetViewDao datasetViewDao;
private static DatasetOwnerDao datasetOwnerDao;
private static DatasetsDao datasetsDao;
private static LineageDao lineageDao;
private static DataPlatformsDao dataPlatformsDao;
private DaoFactory() {
}
private static GmsDao getGmsDao() {
if (_gmsDao == null) {
_gmsDao = new GmsDao(Configuration.getEnvironmentVariable(GMS_HOST_ENV_VAR),
Integer.parseInt(Configuration.getEnvironmentVariable(GMS_PORT_ENV_VAR)),
Boolean.parseBoolean(Configuration.getEnvironmentVariable(GMS_USE_SSL_ENV_VAR, "False")),
Configuration.getEnvironmentVariable(GMS_SSL_PROTOCOL_VAR));
}
return _gmsDao;
}
public static DocumentSearchDao getDatasetDocumentSearchDao() {
if (datasetDocumentSearchDao == null) {
datasetDocumentSearchDao = new DocumentSearchDao<>(getGmsDao().get_datasets());
}
return datasetDocumentSearchDao;
}
public static DocumentSearchDao getCorpUserDocumentSearchDao() {
if (corpUserDocumentSearchDao == null) {
corpUserDocumentSearchDao = new DocumentSearchDao<>(getGmsDao().get_corpUsers());
}
return corpUserDocumentSearchDao;
}
public static BrowseDAO getDatasetBrowseDAO() {
if (datasetBrowseDao == null) {
datasetBrowseDao = new BrowseDAO<>(getGmsDao().get_datasets());
}
return datasetBrowseDao;
}
public static CorpUserViewDao getCorpUserViewDao() {
if (corpUserViewDao == null) {
corpUserViewDao = new CorpUserViewDao(getGmsDao().get_corpUsers());
}
return corpUserViewDao;
}
public static OwnerViewDao getOwnerViewDao() {
if (ownerViewDao == null) {
ownerViewDao = new OwnerViewDao(getGmsDao().get_ownerships(), getGmsDao().get_corpUsers());
}
return ownerViewDao;
}
public static DatasetViewDao getDatasetViewDao() {
if (datasetViewDao == null) {
datasetViewDao = new DatasetViewDao(getGmsDao().get_datasets(), getGmsDao().get_deprecations(),
getGmsDao().get_institutionalMemory(), getGmsDao().get_schemas());
}
return datasetViewDao;
}
public static DatasetOwnerDao getDatasetOwnerDao() {
if (datasetOwnerDao == null) {
datasetOwnerDao = new DatasetOwnerDao(getGmsDao().get_ownerships());
}
return datasetOwnerDao;
}
public static DatasetsDao getDatasetsDao() {
if (datasetsDao == null) {
datasetsDao = new DatasetsDao(getGmsDao().get_ownerships());
}
return datasetsDao;
}
public static LineageDao getLineageDao() {
if (lineageDao == null) {
lineageDao = new LineageDao(getGmsDao().get_lineages(), getGmsDao().get_datasets());
}
return lineageDao;
}
public static DataPlatformsDao getDataPlatformsDao() {
if (dataPlatformsDao == null) {
dataPlatformsDao = new DataPlatformsDao(getGmsDao().get_dataPlatforms());
}
return dataPlatformsDao;
}
}

29
datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DataPlatformsDao.java
View File

@ -1,29 +0,0 @@
package com.linkedin.datahub.dao.table;
import com.linkedin.dataPlatforms.DataPlatform;
import com.linkedin.dataplatform.client.DataPlatforms;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
public class DataPlatformsDao {
private final DataPlatforms _dataPlatforms;
public DataPlatformsDao(@Nonnull DataPlatforms dataPlatforms) {
_dataPlatforms = dataPlatforms;
}
/**
* Get all data platforms
*/
public List<Map<String, Object>> getAllPlatforms() throws Exception {
return _dataPlatforms.getAllPlatforms()
.stream()
.filter(DataPlatform::hasDataPlatformInfo)
.map(platform -> platform.getDataPlatformInfo().data())
.collect(Collectors.toList());
}
}

30
datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DatasetOwnerDao.java
View File

@ -1,30 +0,0 @@
package com.linkedin.datahub.dao.table;
import com.linkedin.common.OwnerArray;
import com.linkedin.common.Ownership;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.datahub.models.view.DatasetOwner;
import com.linkedin.dataset.client.Ownerships;
import java.util.List;
import javax.annotation.Nonnull;
import static com.linkedin.datahub.util.DatasetUtil.*;
import static com.linkedin.datahub.util.OwnerUtil.*;
public class DatasetOwnerDao {
private final Ownerships _ownerships;
public DatasetOwnerDao(@Nonnull Ownerships ownerships) {
this._ownerships = ownerships;
}
public void updateDatasetOwners(@Nonnull String datasetUrn, @Nonnull List<DatasetOwner> owners,
@Nonnull String user) throws Exception {
OwnerArray ownerArray = new OwnerArray();
for (DatasetOwner owner : owners) {
ownerArray.add(toTmsOwner(owner));
}
_ownerships.createOwnership(toDatasetUrn(datasetUrn), new Ownership().setOwners(ownerArray),
new CorpuserUrn(user));
}
}

21
datahub-dao/src/main/java/com/linkedin/datahub/dao/table/DatasetsDao.java
View File

@ -1,21 +0,0 @@
package com.linkedin.datahub.dao.table;
import com.linkedin.dataset.client.Ownerships;
import javax.annotation.Nonnull;
import java.util.Arrays;
import java.util.List;
public class DatasetsDao {
private final Ownerships _ownership;
public DatasetsDao(@Nonnull Ownerships ownerships) {
this._ownership = ownerships;
}
public List<String> getDatasetOwnerTypes() {
return Arrays.asList("DataOwner", "Producer", "Delegate", "Stakeholder", "Consumer", "Developer");
}
}

47
datahub-dao/src/main/java/com/linkedin/datahub/dao/table/GmsDao.java
View File

@ -1,47 +0,0 @@
package com.linkedin.datahub.dao.table;
import com.linkedin.dataplatform.client.DataPlatforms;
import com.linkedin.dataset.client.Datasets;
import com.linkedin.dataset.client.Deprecations;
import com.linkedin.dataset.client.InstitutionalMemory;
import com.linkedin.dataset.client.Lineages;
import com.linkedin.dataset.client.Ownerships;
import com.linkedin.dataset.client.Schemas;
import com.linkedin.identity.client.CorpUsers;
import com.linkedin.metadata.restli.DefaultRestliClientFactory;
import com.linkedin.restli.client.Client;
import javax.annotation.Nonnull;
import lombok.Getter;
@Getter
public class GmsDao {
private final CorpUsers _corpUsers;
private final Datasets _datasets;
private final Ownerships _ownerships;
private final InstitutionalMemory _institutionalMemory;
private final Deprecations _deprecations;
private final Schemas _schemas;
private final Lineages _lineages;
private final DataPlatforms _dataPlatforms;
public GmsDao(@Nonnull Client restClient) {
_corpUsers = new CorpUsers(restClient);
_datasets = new Datasets(restClient);
_ownerships = new Ownerships(restClient);
_institutionalMemory = new InstitutionalMemory(restClient);
_deprecations = new Deprecations(restClient);
_schemas = new Schemas(restClient);
_lineages = new Lineages(restClient);
_dataPlatforms = new DataPlatforms(restClient);
}
public GmsDao(@Nonnull String restliHostName, @Nonnull int restliHostPort) {
this(DefaultRestliClientFactory.getRestLiClient(restliHostName, restliHostPort));
}
public GmsDao(@Nonnull String restliHostName, @Nonnull int restliHostPort, boolean useSSL, String sslProtocol) {
this(DefaultRestliClientFactory.getRestLiClient(restliHostName, restliHostPort, useSSL, sslProtocol));
}
}

57
datahub-dao/src/main/java/com/linkedin/datahub/dao/table/LineageDao.java
View File

@ -1,57 +0,0 @@
package com.linkedin.datahub.dao.table;
import com.linkedin.common.urn.DatasetUrn;
import com.linkedin.datahub.models.view.LineageView;
import com.linkedin.dataset.Dataset;
import com.linkedin.dataset.DownstreamArray;
import com.linkedin.dataset.UpstreamArray;
import com.linkedin.dataset.client.Datasets;
import com.linkedin.dataset.client.Lineages;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import static com.linkedin.datahub.util.DatasetUtil.*;
public class LineageDao {
private final Lineages _lineages;
private final Datasets _datasets;
public LineageDao(@Nonnull Lineages lineages, @Nonnull Datasets datasets) {
_lineages = lineages;
_datasets = datasets;
}
/**
* Gets the upstream datasets of a certain dataset with lineage metadata attached
* @param datasetUrn String
* @return List of LineageView
*/
public List<LineageView> getUpstreamLineage(@Nonnull String datasetUrn) throws Exception {
final UpstreamArray upstreamArray = _lineages.getUpstreamLineage(toDatasetUrn(datasetUrn)).getUpstreams();
final Map<DatasetUrn, Dataset> datasets = _datasets.batchGet(upstreamArray.stream().map(u -> u.getDataset())
.collect(Collectors.toSet()));
return upstreamArray.stream()
.map(us -> toLineageView(datasets.get(us.getDataset()), us.getType().name(), us.getAuditStamp()))
.collect(Collectors.toList());
}
/**
* Gets the downstream datasets of a certain dataset with lineage metadata attached
* @param datasetUrn String
* @return List of LineageView
*/
public List<LineageView> getDownstreamLineage(@Nonnull String datasetUrn) throws Exception {
final DownstreamArray downstreamArray = _lineages.getDownstreamLineage(toDatasetUrn(datasetUrn)).getDownstreams();
final Map<DatasetUrn, Dataset> datasets = _datasets.batchGet(downstreamArray.stream().map(u -> u.getDataset())
.collect(Collectors.toSet()));
return downstreamArray.stream()
.map(ds -> toLineageView(datasets.get(ds.getDataset()), ds.getType().name(), ds.getAuditStamp()))
.collect(Collectors.toList());
}
}

67
datahub-dao/src/main/java/com/linkedin/datahub/dao/view/BrowseDAO.java
View File

@ -1,67 +0,0 @@
package com.linkedin.datahub.dao.view;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.linkedin.chart.client.Charts;
import com.linkedin.dashboard.client.Dashboards;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.util.DatasetUtil;
import com.linkedin.dataset.client.Datasets;
import com.linkedin.metadata.query.BrowseResult;
import com.linkedin.metadata.query.BrowseResultEntityArray;
import com.linkedin.r2.RemoteInvocationException;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Map;
import javax.annotation.Nonnull;
import static com.linkedin.datahub.util.RestliUtil.*;
import static java.util.stream.Collectors.toList;
public class BrowseDAO<CLIENT> {
private final CLIENT _client;
public BrowseDAO(@Nonnull CLIENT client) {
this._client = client;
}
@Nonnull
public JsonNode browse(@Nonnull String path, @Nonnull Map<String, String> requestFilters,
int start, int count) throws RemoteInvocationException, IOException {
final BrowseResult resp;
if (_client instanceof Datasets) {
resp = ((Datasets) _client).browse(path, requestFilters, start, count);
} else if (_client instanceof Dashboards) {
resp = ((Dashboards) _client).browse(path, requestFilters, start, count);
} else if (_client instanceof Charts) {
resp = ((Charts) _client).browse(path, requestFilters, start, count);
} else {
throw new IllegalArgumentException("Unexpected client type: " + _client.getClass().getName());
}
return getJsonFromBrowseResult(resp);
}
@Nonnull
public JsonNode getBrowsePaths(@Nonnull String urn) throws RemoteInvocationException, URISyntaxException {
final StringArray response;
if (_client instanceof Datasets) {
response = ((Datasets) _client).getBrowsePaths(DatasetUtil.toDatasetUrn(urn));
} else {
throw new IllegalArgumentException("Unexpected client type: " + _client.getClass().getName());
}
return stringCollectionToArrayNode(response.stream().collect(toList()));
}
@Nonnull
public JsonNode getJsonFromBrowseResult(@Nonnull BrowseResult browseResult) throws IOException {
final ObjectNode node = OM.createObjectNode();
final BrowseResultEntityArray browseResultEntityArray = browseResult.getEntities();
node.set("elements", collectionToArrayNode(browseResultEntityArray.subList(0, browseResultEntityArray.size())));
node.put("start", browseResult.getFrom());
node.put("count", browseResult.getPageSize());
node.put("total", browseResult.getNumEntities());
node.set("metadata", toJsonNode(browseResult.getMetadata()));
return node;
}
}

63
datahub-dao/src/main/java/com/linkedin/datahub/dao/view/CorpUserViewDao.java
View File

@ -1,63 +0,0 @@
package com.linkedin.datahub.dao.view;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.datahub.util.CorpUserUtil;
import com.linkedin.identity.CorpUser;
import com.linkedin.identity.CorpUserEditableInfo;
import com.linkedin.identity.client.CorpUsers;
import lombok.extern.slf4j.Slf4j;
import javax.annotation.Nonnull;
import java.util.List;
@Slf4j
public class CorpUserViewDao {
private static final String MANAGER_FIELD_NAME = "managerName";
private final CorpUsers _corpUsers;
public CorpUserViewDao(@Nonnull CorpUsers corpUsers) {
this._corpUsers = corpUsers;
}
/**
* Adds manager name to {@link CorpUser} if managerUrn is present in info
* @param corpUserUrn corpUser urn string
* @return JSON node corresponding to the {@link CorpUser}
* @throws Exception
*/
@Nonnull
public CorpUser get(@Nonnull String corpUserUrn) throws Exception {
CorpUser corpUser = _corpUsers.get(CorpUserUtil.toCorpUserUrn(corpUserUrn));
if (corpUser.getInfo() == null) {
return corpUser;
}
CorpuserUrn managerUrn = corpUser.getInfo().getManagerUrn();
if (managerUrn == null) {
return corpUser;
}
CorpUser manager = _corpUsers.get(managerUrn);
String managerName = manager.getInfo() != null ? manager.getInfo().getFullName() : null;
if (managerName != null) {
corpUser.getInfo().data().put(MANAGER_FIELD_NAME, managerName);
}
return corpUser;
}
@Nonnull
public CorpUser getByUserName(@Nonnull String userName) throws Exception {
return get(new CorpuserUrn(userName).toString());
}
@Nonnull
public List<CorpUser> getAllCorpUsers() throws Exception {
return _corpUsers.getAll();
}
public void updateCorpUserEditableConfig(@Nonnull String corpUserUrn,
@Nonnull CorpUserEditableInfo corpUserEditableInfo) throws Exception {
_corpUsers.createEditableInfo(CorpUserUtil.toCorpUserUrn(corpUserUrn), corpUserEditableInfo);
}
}

144
datahub-dao/src/main/java/com/linkedin/datahub/dao/view/DatasetViewDao.java
View File

@ -1,144 +0,0 @@
package com.linkedin.datahub.dao.view;
import com.fasterxml.jackson.databind.JsonNode;
import com.linkedin.common.InstitutionalMemory;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.datahub.models.view.DatasetColumn;
import com.linkedin.datahub.models.view.DatasetSchema;
import com.linkedin.datahub.models.view.DatasetView;
import com.linkedin.dataset.DatasetDeprecation;
import com.linkedin.schema.SchemaField;
import com.linkedin.schema.SchemaFieldArray;
import com.linkedin.schema.SchemaMetadata;
import com.linkedin.dataset.client.Datasets;
import com.linkedin.dataset.client.Deprecations;
import com.linkedin.dataset.client.Schemas;
import com.linkedin.metadata.snapshot.DatasetSnapshot;
import lombok.extern.slf4j.Slf4j;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
import static com.linkedin.datahub.util.DatasetUtil.toDatasetUrn;
import static com.linkedin.datahub.util.DatasetUtil.toDatasetView;
import static com.linkedin.datahub.util.RestliUtil.toJsonNode;
@Slf4j
public class DatasetViewDao {
private final Datasets _datasets;
private final Deprecations _deprecations;
private final com.linkedin.dataset.client.InstitutionalMemory _institutionalMemory;
private final Schemas _schemas;
public DatasetViewDao(@Nonnull Datasets datasets,
@Nonnull Deprecations deprecations,
@Nonnull com.linkedin.dataset.client.InstitutionalMemory institutionalMemory,
@Nonnull Schemas schemas) {
this._datasets = datasets;
this._deprecations = deprecations;
this._institutionalMemory = institutionalMemory;
this._schemas = schemas;
}
@Nonnull
public DatasetView getDatasetView(@Nonnull String datasetUrn) throws Exception {
return toDatasetView(_datasets.get(toDatasetUrn(datasetUrn)));
}
@Nonnull
public JsonNode getSnapshot(@Nonnull String datasetUrn) throws Exception {
DatasetSnapshot snapshot = _datasets.getLatestFullSnapshot(toDatasetUrn(datasetUrn));
return toJsonNode(snapshot);
}
public void updateInstitutionalMemory(@Nonnull String datasetUrn, @Nonnull InstitutionalMemory institutionalMemory)
throws Exception {
_institutionalMemory.updateInstitutionalMemory(toDatasetUrn(datasetUrn), institutionalMemory);
}
@Nonnull
public JsonNode getInstitutionalMemory(@Nonnull String datasetUrn) throws Exception {
InstitutionalMemory institutionalMemory = _institutionalMemory.getInstitutionalMemory(toDatasetUrn(datasetUrn));
return toJsonNode(institutionalMemory);
}
public void setDatasetDeprecation(@Nonnull String datasetUrn, boolean isDeprecated, @Nonnull String deprecationNote,
@Nullable Long decommissionTime, @Nonnull String user) throws Exception {
DatasetDeprecation datasetDeprecation = new DatasetDeprecation()
.setDeprecated(isDeprecated)
.setNote(deprecationNote)
.setActor(new CorpuserUrn(user));
if (decommissionTime != null) {
datasetDeprecation.setDecommissionTime(decommissionTime);
}
_deprecations.updateDatasetDeprecation(toDatasetUrn(datasetUrn), datasetDeprecation);
}
/**
* Get dataset schema by dataset urn
* @param datasetUrn String
* @return dataset schema
*/
@Nullable
public DatasetSchema getDatasetSchema(@Nonnull String datasetUrn) throws Exception {
SchemaMetadata schema = _schemas.getLatestSchemaByDataset(toDatasetUrn(datasetUrn));
if (schema == null) {
return null;
}
SchemaMetadata.PlatformSchema platformSchema = schema.getPlatformSchema();
DatasetSchema dsSchema = new DatasetSchema();
dsSchema.setLastModified(schema.getLastModified().getTime());
if (platformSchema.isSchemaless()) {
dsSchema.setSchemaless(true);
dsSchema.setColumns(new ArrayList<>());
return dsSchema;
}
if (platformSchema.isPrestoDDL()) {
dsSchema.setRawSchema(platformSchema.getPrestoDDL().getRawSchema());
} else if (platformSchema.isOracleDDL()) {
dsSchema.setRawSchema(platformSchema.getOracleDDL().getTableSchema());
} else if (platformSchema.isMySqlDDL()) {
dsSchema.setRawSchema(platformSchema.getMySqlDDL().getTableSchema());
} else if (platformSchema.isKafkaSchema()) {
dsSchema.setRawSchema(platformSchema.getKafkaSchema().getDocumentSchema());
} else if (platformSchema.isOrcSchema()) {
dsSchema.setRawSchema(platformSchema.getOrcSchema().getSchema());
} else if (platformSchema.isBinaryJsonSchema()) {
dsSchema.setRawSchema(platformSchema.getBinaryJsonSchema().getSchema());
} else if (platformSchema.isEspressoSchema()) {
dsSchema.setKeySchema(platformSchema.getEspressoSchema().getTableSchema());
dsSchema.setRawSchema(platformSchema.getEspressoSchema().getDocumentSchema());
} else if (platformSchema.isKeyValueSchema()) {
dsSchema.setKeySchema(platformSchema.getKeyValueSchema().getKeySchema());
dsSchema.setRawSchema(platformSchema.getKeyValueSchema().getValueSchema());
}
dsSchema.setSchemaless(false);
if (schema.hasFields() && schema.getFields().size() > 0) {
dsSchema.setColumns(toWhDatasetColumns(schema.getFields()));
}
return dsSchema;
}
/**
* Convert TMS schema field array to WH list of DatasetColumn
*/
private List<DatasetColumn> toWhDatasetColumns(@Nonnull SchemaFieldArray fields) {
List<DatasetColumn> columns = new ArrayList<>();
for (SchemaField field : fields) {
DatasetColumn col = new DatasetColumn();
col.setFieldName(field.getFieldPath());
col.setFullFieldPath(field.getFieldPath());
col.setDataType(field.hasNativeDataType() ? field.getNativeDataType() : "");
col.setComment(field.hasDescription() ? field.getDescription() : "");
columns.add(col);
}
return columns;
}
}

54
datahub-dao/src/main/java/com/linkedin/datahub/dao/view/DocumentSearchDao.java
View File

@ -1,54 +0,0 @@
package com.linkedin.datahub.dao.view;
import com.fasterxml.jackson.databind.JsonNode;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.dataset.client.Datasets;
import com.linkedin.identity.client.CorpUsers;
import com.linkedin.metadata.query.AutoCompleteResult;
import com.linkedin.r2.RemoteInvocationException;
import com.linkedin.restli.common.CollectionResponse;
import java.io.IOException;
import java.util.Map;
import javax.annotation.Nonnull;
import static com.linkedin.datahub.util.RestliUtil.*;
public class DocumentSearchDao<CLIENT> {
private final CLIENT _client;
public DocumentSearchDao(@Nonnull CLIENT client) {
this._client = client;
}
@Nonnull
public JsonNode search(@Nonnull String input, @Nonnull Map<String, String> requestFilters, int start, int count)
throws RemoteInvocationException, IOException {
CollectionResponse<? extends RecordTemplate> resp = null;
if (_client instanceof Datasets) {
resp = ((Datasets) _client).search(input, requestFilters, start, count);
} else if (_client instanceof CorpUsers) {
resp = ((CorpUsers) _client).search(input, requestFilters, start, count);
} else {
throw new IllegalStateException("Unexpected client type: " + _client.getClass().getName());
}
return collectionResponseWithMetadataToJsonNode(resp);
}
@Nonnull
public JsonNode autoComplete(@Nonnull String input, @Nonnull String field,
@Nonnull Map<String, String> requestFilters, int limit) throws RemoteInvocationException, IOException {
AutoCompleteResult autoCompleteResult = null;
if (_client instanceof Datasets) {
autoCompleteResult = ((Datasets) _client).autoComplete(input, field, requestFilters, limit);
} else if (_client instanceof CorpUsers) {
autoCompleteResult = ((CorpUsers) _client).autocomplete(input, field, requestFilters, limit);
} else {
throw new IllegalStateException("Unexpected client type: " + _client.getClass().getName());
}
return toJsonNode(autoCompleteResult);
}
}

78
datahub-dao/src/main/java/com/linkedin/datahub/dao/view/OwnerViewDao.java
View File

@ -1,78 +0,0 @@
package com.linkedin.datahub.dao.view;
import com.linkedin.common.Owner;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.common.Ownership;
import com.linkedin.common.urn.DatasetUrn;
import com.linkedin.datahub.models.view.DatasetOwner;
import com.linkedin.datahub.models.view.DatasetOwnership;
import com.linkedin.datahub.util.CorpUserUtil;
import com.linkedin.datahub.util.OwnerUtil;
import com.linkedin.dataset.client.Ownerships;
import com.linkedin.identity.CorpUser;
import com.linkedin.identity.client.CorpUsers;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import lombok.extern.slf4j.Slf4j;
import static com.linkedin.datahub.util.DatasetUtil.*;
@Slf4j
public class OwnerViewDao {
private final Ownerships _ownerships;
private final CorpUsers _corpUsers;
public OwnerViewDao(@Nonnull Ownerships ownerships, @Nonnull CorpUsers corpUsers) {
_ownerships = ownerships;
_corpUsers = corpUsers;
}
@Nonnull
public DatasetOwnership getDatasetOwners(@Nonnull String datasetUrn) throws Exception {
final DatasetUrn urn = toDatasetUrn(datasetUrn);
Ownership ownership = _ownerships.getLatestOwnership(urn);
Map<CorpuserUrn, CorpUser> owners = _corpUsers.batchGet(getOwnerUrns(ownership));
DatasetOwnership datasetOwnership = new DatasetOwnership();
datasetOwnership.setDatasetUrn(datasetUrn);
datasetOwnership.setFromUpstream(false);
datasetOwnership.setOwners(fillDatasetOwner(ownership, owners));
datasetOwnership.setActor(ownership.getLastModified().getActor().getId());
datasetOwnership.setLastModified(ownership.getLastModified().getTime());
return datasetOwnership;
}
@Nonnull
private List<DatasetOwner> fillDatasetOwner(@Nonnull Ownership ownership,
@Nonnull Map<CorpuserUrn, CorpUser> owners) {
final Long modified = ownership.getLastModified().getTime();
return ownership.getOwners()
.stream()
.map(o -> OwnerUtil.toWhOwner(o, owners.get(getOwnerUrn(o))))
.peek(o -> o.setModifiedTime(modified))
.collect(Collectors.toList());
}
@Nullable
private CorpuserUrn getOwnerUrn(@Nonnull Owner owner) {
try {
return CorpUserUtil.toCorpUserUrn(owner.getOwner().toString());
} catch (URISyntaxException e) {
log.error("CorpuserUrn syntax error", e);
return null;
}
}
@Nonnull
private Set<CorpuserUrn> getOwnerUrns(@Nonnull Ownership ownership) {
return ownership.getOwners().stream()
.map(this::getOwnerUrn)
.collect(Collectors.toSet());
}
}

23
datahub-dao/src/main/java/com/linkedin/datahub/models/PagedCollection.java
View File

@ -1,23 +0,0 @@
package com.linkedin.datahub.models;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class PagedCollection<T> {
private int total;
private int start;
private int count;
private List<T> elements;
}

25
datahub-dao/src/main/java/com/linkedin/datahub/models/table/CompanyUser.java
View File

@ -1,25 +0,0 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package com.linkedin.datahub.models.table;
import lombok.Data;
@Data
public class CompanyUser {
private String userName;
private String pictureLink;
private String displayName;
}

40
datahub-dao/src/main/java/com/linkedin/datahub/models/table/Dataset.java
View File

@ -1,40 +0,0 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package com.linkedin.datahub.models.table;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.Date;
import java.util.List;
public class Dataset {
public long id;
public String name;
public String source;
public String urn;
public Date created;
public Date modified;
public String formatedModified;
public String schema;
public String hdfsBrowserLink;
public boolean isFavorite;
public boolean isOwned;
public long watchId;
public boolean isWatched;
public boolean hasSchemaHistory;
public JsonNode properties;
public List<User> owners;
public List<String> umpFrequency;
}

23
datahub-dao/src/main/java/com/linkedin/datahub/models/table/Group.java
View File

@ -1,23 +0,0 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package com.linkedin.datahub.models.table;
import lombok.Data;
@Data
public class Group {
private String name;
}

28
datahub-dao/src/main/java/com/linkedin/datahub/models/table/User.java
View File

@ -1,28 +0,0 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package com.linkedin.datahub.models.table;
import lombok.Data;
@Data
public class User {
private int id;
private String userName;
private int departmentNum;
private String email;
private String name; // display name
private String pictureLink;
private UserSetting userSetting;
}

25
datahub-dao/src/main/java/com/linkedin/datahub/models/table/UserEntity.java
View File

@ -1,25 +0,0 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package com.linkedin.datahub.models.table;
import lombok.Data;
@Data
public class UserEntity {
public String label;
public String category;
public String displayName;
public String pictureLink;
}

24
datahub-dao/src/main/java/com/linkedin/datahub/models/table/UserSetting.java
View File

@ -1,24 +0,0 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package com.linkedin.datahub.models.table;
import lombok.Data;
@Data
public class UserSetting {
private String detailDefaultView;
private String defaultWatch;
}

24
datahub-dao/src/main/java/com/linkedin/datahub/models/view/DataOriginView.java
View File

@ -1,24 +0,0 @@
package com.linkedin.datahub.models.view;
import lombok.Value;
/**
* The DataOriginView class is used to find our dataset data origin. Previously, the values EI, CORP, and PROD
* were hardcoded into the frontend UI, but users actually wanted even more information about the cluster the
* data lives on (e.g. Holdem, War).
*/
@Value
public class DataOriginView {
/**
* This will be the item that actually shows up on the UI in the list of possible data origins. It would be the
* more specific designation (e.g. Holdem)
*/
private String displayTitle;
/**
* Since our urn for datasets isn't to the same degree of specificity as displayTitle, this origin maps to the
* concepts of EI, CORP, PROD so that the UI can understand how to create the link for the dataset fabric
*/
private String origin;
}

46
datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetColumn.java
View File

@ -1,46 +0,0 @@
package com.linkedin.datahub.models.view;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class DatasetColumn {
private Long id;
private int sortID;
private int parentSortID;
private String fieldName;
private String parentPath;
private String fullFieldPath;
private String dataType;
private String comment;
private Long commentCount;
private String partitionedStr;
private boolean partitioned;
private String nullableStr;
private boolean nullable;
private String indexedStr;
private boolean indexed;
private String distributedStr;
private boolean distributed;
private String treeGridClass;
}

44
datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetOwner.java
View File

@ -1,44 +0,0 @@
package com.linkedin.datahub.models.view;
import java.io.Serializable;
import lombok.Data;
@Data
public class DatasetOwner {
private String userName;
private String source;
private String namespace;
private String name;
private String email;
private Boolean isGroup;
private Boolean isActive;
private String idType;
private String type;
private String subType;
private Integer sortId;
private String sourceUrl;
private String confirmedBy;
private Long modifiedTime;
private String pictureLink;
static class DatasetOwnerKey implements Serializable {
private String userName;
private String source;
}
}

23
datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetOwnership.java
View File

@ -1,23 +0,0 @@
package com.linkedin.datahub.models.view;
import java.util.List;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@NoArgsConstructor
public class DatasetOwnership {
private List<DatasetOwner> owners;
private Boolean fromUpstream;
private String datasetUrn;
private Long lastModified;
private String actor;
}

23
datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetSchema.java
View File

@ -1,23 +0,0 @@
package com.linkedin.datahub.models.view;
import java.util.List;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Setter
@Getter
@NoArgsConstructor
public class DatasetSchema {
private Boolean schemaless;
private String rawSchema;
private String keySchema;
private List<DatasetColumn> columns;
private Long lastModified;
}

41
datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetView.java
View File

@ -1,41 +0,0 @@
package com.linkedin.datahub.models.view;
import java.util.List;
import java.util.Map;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class DatasetView {
private String platform;
private String nativeName;
private String fabric;
private String uri;
private String description;
private String nativeType;
private String properties;
private List<String> tags;
private Boolean removed;
private Boolean deprecated;
private String deprecationNote;
private Long decommissionTime;
private Long createdTime;
private Long modifiedTime;
private Map<String, String> customProperties;
}

16
datahub-dao/src/main/java/com/linkedin/datahub/models/view/LineageView.java
View File

@ -1,16 +0,0 @@
package com.linkedin.datahub.models.view;
import lombok.Data;
@Data
public class LineageView {
private DatasetView dataset;
private String type;
private String actor;
private String modified;
}

78
datahub-dao/src/main/java/com/linkedin/datahub/util/CorpUserUtil.java
View File

@ -1,78 +0,0 @@
package com.linkedin.datahub.util;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.datahub.models.table.CompanyUser;
import com.linkedin.datahub.models.table.User;
import com.linkedin.datahub.models.table.UserEntity;
import com.linkedin.identity.CorpUser;
import java.net.URISyntaxException;
import javax.annotation.Nonnull;
public class CorpUserUtil {
private CorpUserUtil() {
}
/**
* Deserialize {@link CorpuserUrn} string
*
* @param urnStr urn string
* @return {@link CorpuserUrn}
*/
@Nonnull
public static CorpuserUrn toCorpUserUrn(@Nonnull String urnStr) throws URISyntaxException {
return CorpuserUrn.createFromString(urnStr);
}
/**
* Convert {@link CorpUser} to {@link User}
* @param corpUser {@link CorpUser}
* @return {@link User}
*/
@Nonnull
public static User toCorpUserView(CorpUser corpUser) {
User user = new User();
user.setUserName(corpUser.getUsername());
if (corpUser.getInfo() != null) {
user.setEmail(corpUser.getInfo().getEmail());
user.setName(corpUser.getInfo().getFullName());
}
if (corpUser.getEditableInfo() != null) {
user.setPictureLink(corpUser.getEditableInfo().getPictureLink().toString());
}
return user;
}
/**
* Convert {@link CorpUser} to {@link CompanyUser}
* @param corpUser {@link CorpUser}
* @return {@link CompanyUser}
*/
@Nonnull
public static CompanyUser toCompanyUserView(CorpUser corpUser) {
CompanyUser user = new CompanyUser();
user.setUserName(corpUser.getUsername());
user.setDisplayName(corpUser.getInfo().getDisplayName());
user.setPictureLink(corpUser.getEditableInfo().getPictureLink().toString());
return user;
}
/**
* Convert {@link CorpUser} to {@link UserEntity}
* @param corpUser {@link CorpUser}
* @return {@link UserEntity}
*/
@Nonnull
public static UserEntity toUserEntityView(CorpUser corpUser) {
UserEntity user = new UserEntity();
user.setCategory("person");
user.setLabel(corpUser.getUsername());
if (corpUser.hasInfo()) {
user.setDisplayName(corpUser.getInfo().getDisplayName());
}
if (corpUser.hasEditableInfo() && corpUser.getEditableInfo().hasPictureLink()) {
user.setPictureLink(corpUser.getEditableInfo().getPictureLink().toString());
}
return user;
}
}

101
datahub-dao/src/main/java/com/linkedin/datahub/util/DatasetUtil.java
View File

@ -1,101 +0,0 @@
package com.linkedin.datahub.util;
import com.linkedin.common.AuditStamp;
import com.linkedin.common.urn.DatasetUrn;
import com.linkedin.datahub.models.view.DatasetView;
import com.linkedin.datahub.models.view.LineageView;
import com.linkedin.dataset.Dataset;
import javax.annotation.Nonnull;
import java.net.URISyntaxException;
import static com.linkedin.datahub.util.UrnUtil.splitWhUrn;
public class DatasetUtil {
private DatasetUtil() {
}
/**
* Convert WhereHows dataset URN to DatasetUrn, set dataset origin as PROD.
* @param urn String WH dataset URN
* @return DatasetUrn
*/
public static DatasetUrn toDatasetUrnFromWhUrn(@Nonnull String urn) {
String[] urnParts = splitWhUrn(urn);
return com.linkedin.common.urn.UrnUtils.toDatasetUrn(urnParts[0], urnParts[1], "PROD");
}
/**
* Check input string to determine if WH URN or TMS URN, then convert to DatasetUrn
*/
public static DatasetUrn toDatasetUrn(@Nonnull String datasetUrn) throws URISyntaxException {
if (datasetUrn.contains(":///")) { // wherehows URN
return toDatasetUrnFromWhUrn(datasetUrn);
} else { // TMS URN
return DatasetUrn.createFromString(datasetUrn);
}
}
/**
* Convert TMS Dataset to WH DatasetView
* @param dataset Dataset
* @return DatasetView
*/
public static DatasetView toDatasetView(Dataset dataset) {
DatasetView view = new DatasetView();
view.setPlatform(dataset.getPlatform().getPlatformNameEntity());
view.setNativeName(dataset.getName());
view.setFabric(dataset.getOrigin().name());
view.setDescription(dataset.getDescription());
view.setTags(dataset.getTags());
// construct DatasetUrn and overwrite URI field for frontend use
view.setUri(new DatasetUrn(dataset.getPlatform(), dataset.getName(), dataset.getOrigin()).toString());
if (dataset.hasPlatformNativeType()) {
view.setNativeType(dataset.getPlatformNativeType().name());
}
if (dataset.getStatus() != null) {
view.setRemoved(dataset.getStatus().isRemoved());
}
if (dataset.hasDeprecation()) {
view.setDeprecated(dataset.getDeprecation().isDeprecated());
view.setDeprecationNote(dataset.getDeprecation().getNote());
if (dataset.getDeprecation().hasDecommissionTime()) {
view.setDecommissionTime(dataset.getDeprecation().getDecommissionTime());
}
}
if (dataset.hasCreated()) {
view.setCreatedTime(dataset.getCreated().getTime());
}
if (dataset.hasLastModified()) {
view.setModifiedTime(dataset.getLastModified().getTime());
}
if (dataset.hasProperties()) {
view.setCustomProperties(dataset.getProperties());
}
return view;
}
/**
* Converts TMS lineage response to WH LineageView which requires datasetView conversion
* for the dataset in the lineage response
* @param dataset dataset
* @param lineageType type of lineage
* @param auditStamp audit stamp
* @return LineageView
*/
public static LineageView toLineageView(Dataset dataset, String lineageType, AuditStamp auditStamp) {
LineageView view = new LineageView();
DatasetView datasetView = toDatasetView(dataset);
datasetView.setModifiedTime(auditStamp.getTime());
view.setDataset(datasetView);
view.setType(lineageType);
view.setActor(auditStamp.getActor().toString());
return view;
}
}

78
datahub-dao/src/main/java/com/linkedin/datahub/util/OwnerUtil.java
View File

@ -1,78 +0,0 @@
package com.linkedin.datahub.util;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.BiMap;
import com.google.common.collect.ImmutableBiMap;
import com.linkedin.common.Owner;
import com.linkedin.common.OwnershipType;
import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.models.view.DatasetOwner;
import com.linkedin.identity.CorpUser;
import java.net.URISyntaxException;
import javax.annotation.Nonnull;
public class OwnerUtil {
private OwnerUtil() {
}
/**
* Convert from TMS Owner to WhereHows DatasetOwner
* @param owner Owner
* @return DatasetOwner
*/
@VisibleForTesting
public static DatasetOwner toWhOwner(@Nonnull Owner owner, @Nonnull CorpUser corpUser) {
DatasetOwner dsOwner = new DatasetOwner();
dsOwner.setConfirmedBy("UI");
dsOwner.setIdType("USER");
dsOwner.setIsGroup(false);
dsOwner.setNamespace("urn:li:corpuser");
dsOwner.setSource("UI");
dsOwner.setUserName(corpUser.getUsername());
dsOwner.setType(OWNER_CATEGORY_MAP_INV.get(owner.getType()));
if (corpUser.hasInfo()) {
dsOwner.setEmail(corpUser.getInfo().getEmail());
dsOwner.setIsActive(corpUser.getInfo().isActive());
if (corpUser.getInfo().hasFullName()) {
dsOwner.setName(corpUser.getInfo().getFullName());
}
}
if (corpUser.hasEditableInfo() && corpUser.getEditableInfo().hasPictureLink()) {
dsOwner.setPictureLink(corpUser.getEditableInfo().getPictureLink().toString());
}
return dsOwner;
}
/**
* Convert from a WhereHows DatasetOwner to TMS Owner
* @param dsOwner dsOwner
* @return Owner
*/
@Nonnull
public static Owner toTmsOwner(@Nonnull DatasetOwner dsOwner) throws URISyntaxException {
return new Owner().setOwner(new Urn(dsOwner.getNamespace() + ":" + dsOwner.getUserName()))
.setType(OWNER_CATEGORY_MAP.get(dsOwner.getType()));
}
/**
* Mapping between WhereHows owner type values and TMS OwnerCategory
*/
private static final BiMap<String, OwnershipType> OWNER_CATEGORY_MAP =
new ImmutableBiMap.Builder<String, OwnershipType>()
// format
.put("DataOwner", OwnershipType.DATAOWNER)
.put("Producer", OwnershipType.PRODUCER)
.put("Delegate", OwnershipType.DELEGATE)
.put("Stakeholder", OwnershipType.STAKEHOLDER)
.put("Consumer", OwnershipType.CONSUMER)
.put("Developer", OwnershipType.DEVELOPER)
.build();
private static final BiMap<OwnershipType, String> OWNER_CATEGORY_MAP_INV = OWNER_CATEGORY_MAP.inverse();
}

149
datahub-dao/src/main/java/com/linkedin/datahub/util/RestliUtil.java
View File

@ -1,149 +0,0 @@
package com.linkedin.datahub.util;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.linkedin.common.UrnArray;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.datahub.models.PagedCollection;
import com.linkedin.metadata.dao.utils.RecordUtils;
import com.linkedin.metadata.query.SearchResultMetadata;
import com.linkedin.restli.common.CollectionResponse;
import java.io.IOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import javax.annotation.Nonnull;
public class RestliUtil {
public static final ObjectMapper OM = new ObjectMapper();
private RestliUtil() {
}
// convert restli collection response to PagedCollection
@Nonnull
public static <T extends RecordTemplate> PagedCollection<T> toPagedCollection(
@Nonnull CollectionResponse<T> response) {
PagedCollection<T> result = new PagedCollection<>();
result.setElements(response.getElements());
result.setStart(response.getPaging().getStart());
result.setCount(response.getPaging().getCount());
result.setTotal(response.getPaging().getTotal());
return result;
}
// convert restli RecordTemplate to JsonNode
@Nonnull
public static <T extends RecordTemplate> JsonNode toJsonNode(@Nonnull T record) throws IOException {
String jsonString = RecordUtils.toJsonString(record);
return OM.readTree(jsonString);
}
// convert restli collection response to JsonNode
@Nonnull
public static <T extends RecordTemplate> JsonNode collectionResponseToJsonNode(
@Nonnull CollectionResponse<T> response) throws IOException {
final ObjectNode node = OM.createObjectNode();
node.set("elements", collectionToArrayNode(response.getElements()));
node.put("start", response.getPaging().getStart());
node.put("count", response.getPaging().getCount());
node.put("total", response.getPaging().getTotal());
return node;
}
// Converts a collection to Json ArrayNode
@Nonnull
public static <T extends RecordTemplate> ArrayNode collectionToArrayNode(@Nonnull Collection<T> elements)
throws IOException {
final ArrayNode arrayNode = OM.createArrayNode();
for (T element : elements) {
arrayNode.add(toJsonNode(element));
}
return arrayNode;
}
// Converts a collection to Json ArrayNode. For each element in the collection, it adds the corresponding urn
@Nonnull
public static <T extends RecordTemplate> ArrayNode collectionToArrayNode(@Nonnull Collection<T> elements, @Nonnull UrnArray urns)
throws IOException {
if (elements.size() != urns.size()) {
throw new RuntimeException("Collection size and urn size should match");
}
final ArrayNode arrayNode = OM.createArrayNode();
final Iterator<Urn> urnIterator = urns.iterator();
for (T element : elements) {
final ObjectNode node = (ObjectNode) toJsonNode(element);
arrayNode.add(node.put("urn", urnIterator.next().toString()));
}
return arrayNode;
}
// Convert restli collection response with metadata to JsonNode
@Nonnull
public static <T extends RecordTemplate> JsonNode collectionResponseWithMetadataToJsonNode(
@Nonnull CollectionResponse<T> response) throws IOException {
final SearchResultMetadata searchResultMetadata = new SearchResultMetadata(response.getMetadataRaw());
final ObjectNode objectNode = (ObjectNode) collectionResponseToJsonNodeWithoutElements(response);
objectNode.set("elements", collectionToArrayNode(response.getElements(), searchResultMetadata.getUrns()));
if (response.getMetadataRaw() == null) {
return objectNode;
}
// Use searchResultMetadatas field as a section in the final result json node
final String name = SearchResultMetadata.fields().searchResultMetadatas().toString();
final String fieldName = name.startsWith("/") ? name.substring(1) : name;
objectNode.set(fieldName, toJsonNode(searchResultMetadata).get(fieldName));
return objectNode;
}
// convert restli collection response to JsonNode without populating elements
@Nonnull
public static <T extends RecordTemplate> JsonNode collectionResponseToJsonNodeWithoutElements(
@Nonnull CollectionResponse<T> response) throws IOException {
final ObjectNode node = OM.createObjectNode();
node.put("start", response.getPaging().getStart());
node.put("count", response.getPaging().getCount());
node.put("total", response.getPaging().getTotal());
return node;
}
// Convert restli collection response with metadata to JsonNode
@Nonnull
public static <T extends RecordTemplate, Y extends RecordTemplate> JsonNode collectionResponseToJsonNode(
@Nonnull CollectionResponse<T> response, @Nonnull Y metadata) throws IOException {
JsonNode node = collectionResponseToJsonNode(response);
ObjectNode objectNode = (ObjectNode) node;
objectNode.set("metadata", toJsonNode(metadata));
return objectNode;
}
// Converts restli map response to JsonNode
@Nonnull
public static <T extends RecordTemplate> JsonNode mapResponseToJsonNode(@Nonnull Map<?, T> map) throws IOException {
ObjectNode node = OM.createObjectNode();
for (Map.Entry<?, T> entry : map.entrySet()) {
node.set(entry.getKey().toString(), toJsonNode(entry.getValue()));
}
return node;
}
// Converts a string collection to Json ArrayNode
@Nonnull
public static ArrayNode stringCollectionToArrayNode(@Nonnull Collection<String> elements) {
final ArrayNode arrayNode = OM.createArrayNode();
elements.forEach(s -> arrayNode.add(s));
return arrayNode;
}
}

30
datahub-dao/src/main/java/com/linkedin/datahub/util/Search.java
View File

@ -1,30 +0,0 @@
package com.linkedin.datahub.util;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.nio.file.Files;
import java.nio.file.Paths;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Search {
private static final ObjectMapper _OM = new ObjectMapper();
private Search() {
}
public static String readJsonQueryFile(String jsonFile) {
try {
String contents = new String(Files.readAllBytes(Paths.get(jsonFile)));
JsonNode json = _OM.readTree(contents);
return json.toString();
} catch (Exception e) {
log.error("ReadJsonQueryFile failed. Error: " + e.getMessage());
e.printStackTrace();
return null;
}
}
}

45
datahub-dao/src/main/java/com/linkedin/datahub/util/UrnUtil.java
View File

@ -1,45 +0,0 @@
package com.linkedin.datahub.util;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class UrnUtil {
private UrnUtil() {
}
/**
* Get the entity (last part) of an URN, e.g. urn:li:user:abc -> abc
* If input null, return null
* @param urn colon separated URN String
* @return entity String
*/
public static String getUrnEntity(@Nullable String urn) {
return urn == null ? null : urn.substring(urn.lastIndexOf(':') + 1);
}
/**
* Split WhereHows dataset URN into two parts: platform + dataset name
* Also replace '/' with '.'getObjectArrayListBy in dataset name other than hdfs datasets
* E.g. oracle:///abc/def > [oracle, abc.def]
* @param urn String WhereHows dataset URN
* @return String[] platform + dataset name
*/
public static String[] splitWhUrn(@Nonnull String urn) {
if (urn == null) {
throw new IllegalArgumentException("URN is null");
}
final String[] parts = urn.split(":///"); // [platform, dataset name]
if ("hdfs".equalsIgnoreCase(parts[0])) {
parts[1] = "/" + parts[1];
} else {
parts[1] = parts[1].replace("/", ".");
}
return parts;
}
}

12
datahub-dao/src/main/resources/META-INF/persistence.xml
View File

@ -1,12 +0,0 @@
<!--
# DO NOT USE # this is a dummy config file required by hibernate,
All properties can be set programmatically when initialize EntityManagerFactory
-->
<persistence xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://java.sun.com/xml/ns/persistence"
xsi:schemaLocation="http://java.sun.com/xml/ns/persistence http://java.sun.com/xml/ns/persistence/persistence_2_0.xsd"
version="2.0">
<persistence-unit name="default">
</persistence-unit>
</persistence>

83
datahub-dao/src/test/java/com/linkedin/datahub/dao/view/CorpUserViewDaoTest.java
View File

@ -1,83 +0,0 @@
package com.linkedin.datahub.dao.view;
import com.fasterxml.jackson.databind.JsonNode;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.util.RestliUtil;
import com.linkedin.identity.CorpUser;
import com.linkedin.identity.CorpUserEditableInfo;
import com.linkedin.identity.CorpUserInfo;
import com.linkedin.identity.client.CorpUsers;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.Arrays;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNull;
public class CorpUserViewDaoTest {
private CorpUsers _corpUsers;
private CorpUserViewDao _corpUserViewDao;
@BeforeClass
public void init() {
_corpUsers = mock(CorpUsers.class);
_corpUserViewDao = new CorpUserViewDao(_corpUsers);
}
@Test
public void testGetManagerName() throws Exception {
String testUser = "testuser";
String testManager = "testManager";
CorpuserUrn corpUserUrn = new CorpuserUrn(testUser);
CorpuserUrn corpUserManagerUrn = new CorpuserUrn(testManager);
CorpUserEditableInfo corpUserEditableInfo = new CorpUserEditableInfo()
.setAboutMe("I am a test user")
.setSkills(new StringArray(Arrays.asList("skill1", "skill2")))
.setTeams(new StringArray(Arrays.asList("team1", "team2")));
CorpUser corpManager = new CorpUser().setUsername(testManager)
.setInfo(new CorpUserInfo().setActive(true).setFullName("Test Manager").setEmail("testManager@linkedin.com"));
when(_corpUsers.get(corpUserManagerUrn)).thenReturn(corpManager);
// test 1: corp user has the manager urn
CorpUserInfo corpUserInfo1 = new CorpUserInfo()
.setActive(true)
.setDisplayName("Test User")
.setEmail("testuser@linkedin.com")
.setManagerUrn(corpUserManagerUrn);
CorpUser corpUser1 = new CorpUser().setUsername(testUser)
.setEditableInfo(corpUserEditableInfo)
.setInfo(corpUserInfo1);
when(_corpUsers.get(corpUserUrn)).thenReturn(corpUser1);
JsonNode node1 = RestliUtil.toJsonNode(_corpUserViewDao.get(corpUserUrn.toString()));
assertEquals(node1.get("username").textValue(), testUser);
assertEquals(node1.get("info").get("managerName").textValue(), "Test Manager");
// test 2: corp user does not have manager urn
CorpUserInfo corpUserInfo2 = new CorpUserInfo()
.setActive(true)
.setDisplayName("Test User")
.setEmail("testuser@linkedin.com");
CorpUser corpUser2 = new CorpUser().setUsername(testUser)
.setEditableInfo(corpUserEditableInfo)
.setInfo(corpUserInfo2);
when(_corpUsers.get(corpUserUrn)).thenReturn(corpUser2);
JsonNode node2 = RestliUtil.toJsonNode(_corpUserViewDao.get(corpUserUrn.toString()));
assertEquals(node2.get("username").textValue(), testUser);
assertNull(node2.get("info").get("managerName"));
// test 3: corp user does not have info set
CorpUser corpUser3 = new CorpUser().setUsername(testUser)
.setEditableInfo(corpUserEditableInfo);
when(_corpUsers.get(corpUserUrn)).thenReturn(corpUser3);
JsonNode node3 = RestliUtil.toJsonNode(_corpUserViewDao.get(corpUserUrn.toString()));
assertEquals(node3.get("username").textValue(), testUser);
assertNull(node3.get("info"));
}
}

54
datahub-dao/src/test/java/com/linkedin/datahub/util/RestliUtilTest.java
View File

@ -1,54 +0,0 @@
package com.linkedin.datahub.util;
import com.fasterxml.jackson.databind.JsonNode;
import com.linkedin.data.DataList;
import com.linkedin.data.DataMap;
import com.linkedin.metadata.query.BrowseResultEntity;
import com.linkedin.metadata.query.BrowseResultGroup;
import com.linkedin.metadata.query.BrowseResultMetadata;
import com.linkedin.restli.common.CollectionMetadata;
import com.linkedin.restli.common.CollectionResponse;
import java.util.Collections;
import org.testng.annotations.Test;
import static com.linkedin.datahub.util.RestliUtil.*;
import static org.testng.Assert.*;
public class RestliUtilTest {
@Test
public void testCollectionResponseToJsonNodeForBrowse() throws Exception {
DataMap documentsDataMap = new DataMap();
documentsDataMap.put("elements", new DataList(Collections.singletonList(new BrowseResultEntity().data())));
documentsDataMap.put("paging", new CollectionMetadata().setStart(0).setCount(1).setTotal(10).data());
CollectionResponse<BrowseResultEntity> resp = new CollectionResponse<>(documentsDataMap, BrowseResultEntity.class);
DataMap dataMap = new DataMap();
dataMap.put("path", "/foo");
dataMap.put("groups",
new DataList(Collections.singletonList(new BrowseResultGroup().setName("/foo/bar").setCount(31L).data())));
dataMap.put("totalNumEntities", 100L);
DataMap metadataMap = new DataMap();
metadataMap.put("metadata", dataMap);
resp.setMetadataRaw(metadataMap);
JsonNode node = collectionResponseToJsonNode(resp, new BrowseResultMetadata(dataMap));
assertNotNull(node);
assertEquals(node.get("start").asInt(), 0);
assertEquals(node.get("count").asInt(), 1);
assertEquals(node.get("total").asInt(), 10);
assertTrue(node.get("elements").isArray());
assertEquals(node.get("elements").size(), 1);
assertEquals(node.get("elements").get(0), toJsonNode(new BrowseResultEntity()));
assertEquals(node.get("metadata").get("path").asText(), "/foo");
assertTrue(node.get("metadata").get("groups").isArray());
assertEquals(node.get("metadata").get("groups").size(), 1);
assertEquals(node.get("metadata").get("groups").get(0).get("name").asText(), "/foo/bar");
assertEquals(node.get("metadata").get("groups").get(0).get("count").asLong(), 31L);
assertEquals(node.get("metadata").get("totalNumEntities").asLong(), 100);
}
}

102
datahub-frontend/app/controllers/Application.java
View File

@ -61,37 +61,6 @@ public class Application extends Controller {
return ok("GOOD");
}
@Nonnull
public Result printDeps() {
final String appHome = System.getenv("WHZ_APP_HOME");
String libPath = appHome + "/lib";
String commitFile = appHome + "/commit";
String commit = "";
if (appHome == null) {
return ok("WHZ_APP_HOME environmental variable not defined");
}
try (BufferedReader reader = new BufferedReader(new FileReader(commitFile))) {
commit = reader.readLine();
} catch (IOException ioe) {
_logger.error("Error while reading commit file. Error message: " + ioe.getMessage());
}
//get all the files from /libs directory
StringBuilder sb = new StringBuilder();
try (Stream<Path> paths = Files.list(Paths.get(libPath))) {
paths.filter(Files::isRegularFile).
forEach(f -> sb.append(f.getFileName()).append("\n"));
} catch (IOException ioe) {
_logger.error("Error while traversing the directory. Error message: " + ioe.getMessage());
}
return ok("commit: " + commit + "\n" + "libraries: " + sb.toString());
}
/**
* index Action proxies to serveAsset
*
@ -171,7 +140,6 @@ public class Application extends Controller {
* @return Json object with internal wiki links
*/
@Nonnull
private ObjectNode wikiLinks() {
final ObjectNode wikiLinks = Json.newObject();
wikiLinks.put("appHelp", _config.getString("links.wiki.appHelp"));
@ -207,74 +175,4 @@ public class Application extends Controller {
trackingConfig.put("isEnabled", true);
return trackingConfig;
}
@Nonnull
public Result login() {
//You can generate the Csrf token such as String csrfToken = SecurityPlugin.getInstance().getCsrfToken();
String csrfToken = "";
return serveAsset("");
}
@BodyParser.Of(BodyParser.Json.class)
@Nonnull
public Result authenticate() throws NamingException {
JsonNode json = request().body().asJson();
// Extract username and password as String from JsonNode,
// null if they are not strings
String username = json.findPath("username").textValue();
String password = json.findPath("password").textValue();
if (StringUtils.isBlank(username)) {
return badRequest("Missing or invalid [username]");
}
if (password == null) {
password = "";
}
session().clear();
// Create a uuid string for this session if one doesn't already exist
// to be appended to the Result object
String uuid = session("uuid");
if (uuid == null) {
uuid = java.util.UUID.randomUUID().toString();
session("uuid", uuid);
}
try {
AuthenticationManager.authenticateUser(username, password);
} catch (AuthenticationException e) {
_logger.warn("Authentication error!", e);
return badRequest("Invalid Credential");
}
// Adds the username to the session cookie
session("user", username);
String secretKey = _config.getString("play.http.secret.key");
try {
//store hashed username to PLAY_SESSION cookie
String hashedUserName = AuthUtil.generateHash(username, secretKey.getBytes());
session("auth_token", hashedUserName);
} catch (NoSuchAlgorithmException | InvalidKeyException e) {
_logger.error("Failed to hash username", e);
}
// Construct an ObjectNode with the username and uuid token to be sent with the response
ObjectNode data = Json.newObject();
data.put("username", username);
data.put("uuid", uuid);
// Create a new response ObjectNode to return when authenticate request is successful
ObjectNode response = Json.newObject();
response.put("status", "ok");
response.set("data", data);
return ok(response);
}
@Nonnull
public Result logout() {
session().clear();
return ok();
}
}

32
datahub-frontend/app/controllers/api/v1/Dataset.java
View File

@ -1,32 +0,0 @@
package controllers.api.v1;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.linkedin.datahub.dao.DaoFactory;
import com.linkedin.datahub.dao.table.DatasetsDao;
import controllers.Secured;
import play.libs.Json;
import play.mvc.Controller;
import play.mvc.Result;
import play.mvc.Security;
import javax.annotation.Nonnull;
public class Dataset extends Controller {
private final DatasetsDao _datasetsDao;
public Dataset() {
_datasetsDao = DaoFactory.getDatasetsDao();
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getDatasetOwnerTypes() {
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("ownerTypes", Json.toJson(_datasetsDao.getDatasetOwnerTypes()));
return ok(result);
}
}

118
datahub-frontend/app/controllers/api/v1/User.java
View File

@ -1,118 +0,0 @@
package controllers.api.v1;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.linkedin.datahub.dao.DaoFactory;
import com.linkedin.datahub.dao.view.CorpUserViewDao;
import com.linkedin.datahub.models.table.CompanyUser;
import com.linkedin.datahub.models.table.Group;
import com.linkedin.datahub.models.table.UserEntity;
import com.linkedin.datahub.util.CorpUserUtil;
import controllers.Secured;
import jersey.repackaged.com.google.common.cache.Cache;
import jersey.repackaged.com.google.common.cache.CacheBuilder;
import play.libs.Json;
import play.mvc.Controller;
import play.mvc.Result;
import play.mvc.Security;
import utils.ControllerUtil;
import javax.annotation.Nonnull;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
public class User extends Controller {
private static final String CACHE_INTERNAL_USERS = "internal.users.cache";
private static final String CACHE_INTERNAL_GROUPS = "internal.groups.cache";
private static final String CACHE_INTERNAL_ENTITIES = "internal.entities.cache";
private final CorpUserViewDao _corpUserViewDao;
private static final Cache<String, List> CACHE = CacheBuilder.newBuilder()
.expireAfterWrite(24, TimeUnit.HOURS)
.maximumSize(3)
.build();
public User() {
_corpUserViewDao = DaoFactory.getCorpUserViewDao();
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getLoggedInUser() {
ObjectNode result = Json.newObject();
com.linkedin.identity.CorpUser corpUser;
String username = request().attrs().get(Security.USERNAME);
try {
corpUser = _corpUserViewDao.getByUserName(username);
} catch (Exception e) {
throw new RuntimeException(e);
}
if (corpUser == null || !corpUser.getUsername().equals(username)
|| !corpUser.hasInfo()) {
result.put("status", "failed");
result.put("message", "can't find user info");
return ok(result);
}
result.set("user", Json.toJson(CorpUserUtil.toCorpUserView(corpUser)));
result.put("status", "ok");
return ok(result);
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getAllCompanyUsers() {
List<CompanyUser> users = (List<CompanyUser>) CACHE.getIfPresent(CACHE_INTERNAL_USERS);
if (users == null || users.size() == 0) {
try {
users = _corpUserViewDao.getAllCorpUsers().stream()
.map(CorpUserUtil::toCompanyUserView)
.collect(Collectors.toList());
} catch (Exception e) {
return internalServerError(ControllerUtil.errorResponse(e));
}
CACHE.put(CACHE_INTERNAL_USERS, users);
}
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("employees", Json.toJson(users));
return ok(result);
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getAllGroups() {
List<Group> groups = (List<Group>) CACHE.getIfPresent(CACHE_INTERNAL_GROUPS);
if (groups == null || groups.size() == 0) {
groups = Collections.emptyList();
CACHE.put(CACHE_INTERNAL_GROUPS, groups);
}
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("groups", Json.toJson(groups));
return ok(result);
}
@Nonnull
public Result getAllUserEntities() {
List<UserEntity> entities = (List<UserEntity>) CACHE.getIfPresent(CACHE_INTERNAL_ENTITIES);
if (entities == null || entities.size() == 0) {
try {
entities = _corpUserViewDao.getAllCorpUsers().stream()
.map(CorpUserUtil::toUserEntityView)
.collect(Collectors.toList());
} catch (Exception e) {
return internalServerError(ControllerUtil.errorResponse(e));
}
CACHE.put(CACHE_INTERNAL_ENTITIES, entities);
}
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("userEntities", Json.toJson(entities));
return ok(result);
}
}

104
datahub-frontend/app/controllers/api/v2/Browse.java
View File

@ -1,104 +0,0 @@
package controllers.api.v2;
import com.google.common.collect.ImmutableMap;
import com.linkedin.datahub.dao.DaoFactory;
import com.linkedin.datahub.dao.view.BrowseDAO;
import controllers.Secured;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import play.Logger;
import play.mvc.Controller;
import play.mvc.Result;
import play.mvc.Security;
import utils.ControllerUtil;
import javax.annotation.Nonnull;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import static org.apache.commons.lang3.StringUtils.isBlank;
public class Browse extends Controller {
private static final String REQUEST_TYPE = "type";
private static final String REQUEST_PATH = "path";
private static final String REQUEST_START = "start";
private static final String REQUEST_COUNT = "count";
private static final String REQUEST_URN = "urn";
private static final String DATASET_ENTITY_TYPE = "dataset";
private static final String DEFAULT_PATH = "";
private static final int DEFAULT_START_VALUE = 0;
private static final int DEFAULT_PAGE_SIZE = 10;
private final Map<String, BrowseDAO> _browseDAOMap;
private static final ImmutableMap<String, Set<String>> FILTER_FIELDS =
ImmutableMap.of();
private static Map<String, String> buildRequestMap(String type) {
Map<String, String> requestMap = new HashMap<>();
Set<String> facets = FILTER_FIELDS.getOrDefault(type, Collections.emptySet());
facets.stream().forEach(facet -> requestMap.put(facet, request().getQueryString(facet)));
return requestMap;
}
public Browse() {
_browseDAOMap = ImmutableMap.<String, BrowseDAO>builder()
.put(DATASET_ENTITY_TYPE, DaoFactory.getDatasetBrowseDAO())
.build();
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result browse() {
try {
String type = request().getQueryString(REQUEST_TYPE);
if (isBlank(type)) {
return badRequest("Bad Request. type parameter can not be null");
}
String input = StringUtils.defaultIfBlank(request().getQueryString(REQUEST_PATH), DEFAULT_PATH);
int start = NumberUtils.toInt(request().getQueryString(REQUEST_START), DEFAULT_START_VALUE);
int count = NumberUtils.toInt(request().getQueryString(REQUEST_COUNT), DEFAULT_PAGE_SIZE);
Map<String, String> requestMap = buildRequestMap(type);
BrowseDAO browseDAO = _browseDAOMap.get(type);
if (browseDAO != null) {
return ok(browseDAO.browse(input, requestMap, start, count));
} else {
return badRequest("Bad Request. Type is not supported");
}
} catch (Exception e) {
Logger.error("Browse failed: ", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getBrowsePaths() {
try {
String type = request().getQueryString(REQUEST_TYPE);
String urn = request().getQueryString(REQUEST_URN);
if (isBlank(type) || isBlank(urn)) {
return badRequest("Bad Request. Type or urn parameter can not be null");
}
BrowseDAO browseDAO = _browseDAOMap.get(type);
if (browseDAO != null) {
return ok(browseDAO.getBrowsePaths(urn));
} else {
return badRequest("Bad Request. Type is not supported");
}
} catch (Exception e) {
Logger.error("Browse paths request failed: ", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
}
}

87
datahub-frontend/app/controllers/api/v2/CorpUser.java
View File

@ -1,87 +0,0 @@
package controllers.api.v2;
import com.fasterxml.jackson.databind.JsonNode;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.datahub.dao.DaoFactory;
import com.linkedin.datahub.dao.view.CorpUserViewDao;
import com.linkedin.datahub.util.CorpUserUtil;
import com.linkedin.identity.CorpUserEditableInfo;
import com.linkedin.metadata.dao.utils.RecordUtils;
import controllers.Secured;
import org.apache.commons.lang3.StringUtils;
import play.Logger;
import play.libs.Json;
import play.mvc.Controller;
import play.mvc.Result;
import play.mvc.Results;
import play.mvc.Security;
import utils.ControllerUtil;
import javax.annotation.Nonnull;
import java.net.URISyntaxException;
import static com.linkedin.datahub.util.RestliUtil.*;
public class CorpUser extends Controller {
private static final JsonNode EMPTY_RESPONSE = Json.newObject();
private final CorpUserViewDao _corpUserViewDao;
public CorpUser() {
_corpUserViewDao = DaoFactory.getCorpUserViewDao();
}
/**
* Get CorpUser given corpUser urn
* @param corpUserUrn String
* @return CorpUser
*/
@Security.Authenticated(Secured.class)
@Nonnull
public Result getCorpUser(@Nonnull String corpUserUrn) {
try {
return Results.ok(toJsonNode(_corpUserViewDao.get(corpUserUrn)));
} catch (Exception e) {
if (e.toString().contains("Response status 404")) {
return notFound(EMPTY_RESPONSE);
}
Logger.error("Failed to get corp user", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
}
/**
* Creates or Updates CorpUserEditableInfo aspect
*
* @param corpUserUrn CorpUser urn
*/
@Security.Authenticated(Secured.class)
@Nonnull
public Result updateCorpUserEditableInfo(@Nonnull String corpUserUrn) {
final String username = session("user");
if (StringUtils.isBlank(username)) {
return unauthorized(EMPTY_RESPONSE);
}
final CorpuserUrn corpUser;
try {
corpUser = CorpUserUtil.toCorpUserUrn(corpUserUrn);
} catch (URISyntaxException e) {
return unauthorized("Invalid urn");
}
if (!corpUser.getUsernameEntity().equals(username)) {
return unauthorized(EMPTY_RESPONSE);
}
final JsonNode requestBody = request().body().asJson();
try {
CorpUserEditableInfo corpUserEditableInfo =
RecordUtils.toRecordTemplate(CorpUserEditableInfo.class, requestBody.toString());
_corpUserViewDao.updateCorpUserEditableConfig(corpUserUrn, corpUserEditableInfo);
return Results.ok((JsonNode) Json.newObject().set("updated", Json.toJson(true)));
} catch (Exception e) {
Logger.error("Failed to upsert corp user editable info", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
}
}

286
datahub-frontend/app/controllers/api/v2/Dataset.java
View File

@ -1,286 +0,0 @@
package controllers.api.v2;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.linkedin.common.AuditStamp;
import com.linkedin.common.InstitutionalMemory;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.datahub.dao.DaoFactory;
import com.linkedin.datahub.dao.table.DataPlatformsDao;
import com.linkedin.datahub.dao.table.DatasetOwnerDao;
import com.linkedin.datahub.dao.table.LineageDao;
import com.linkedin.datahub.dao.view.DatasetViewDao;
import com.linkedin.datahub.dao.view.OwnerViewDao;
import com.linkedin.datahub.models.view.*;
import com.linkedin.metadata.dao.utils.RecordUtils;
import controllers.Secured;
import org.apache.commons.lang3.StringUtils;
import play.Logger;
import play.libs.Json;
import play.mvc.Controller;
import play.mvc.Result;
import play.mvc.Security;
import utils.ControllerUtil;
import javax.annotation.Nonnull;
import java.util.List;
public class Dataset extends Controller {
private final DatasetViewDao _datasetViewDao;
private final OwnerViewDao _ownerViewDao;
private final DatasetOwnerDao _datasetOwnerDao;
private final LineageDao _lineageDao;
private final DataPlatformsDao _dataPlatformsDao;
private static final JsonNode EMPTY_RESPONSE = Json.newObject();
public Dataset() {
_datasetViewDao = DaoFactory.getDatasetViewDao();
_ownerViewDao = DaoFactory.getOwnerViewDao();
_datasetOwnerDao = DaoFactory.getDatasetOwnerDao();
_lineageDao = DaoFactory.getLineageDao();
_dataPlatformsDao = DaoFactory.getDataPlatformsDao();
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getDataset(@Nonnull String datasetUrn) {
final DatasetView view;
try {
view = _datasetViewDao.getDatasetView(datasetUrn);
} catch (Exception e) {
if (ControllerUtil.checkErrorCode(e, 404)) {
return notFound(EMPTY_RESPONSE);
}
Logger.error("Failed to get dataset view", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
return ok(Json.toJson(view));
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result updateDatasetDeprecation(@Nonnull String datasetUrn) {
final String username = session("user");
if (StringUtils.isBlank(username)) {
return unauthorized(EMPTY_RESPONSE);
}
try {
JsonNode record = request().body().asJson();
boolean deprecated = record.get("deprecated").asBoolean();
String deprecationNote = record.hasNonNull("deprecationNote") ? record.get("deprecationNote").asText() : "";
long decommissionTime = record.hasNonNull("decommissionTime") ? record.get("decommissionTime").asLong() : 0;
if (deprecated && decommissionTime <= 0) {
throw new IllegalArgumentException("Invalid decommission time");
}
_datasetViewDao.setDatasetDeprecation(datasetUrn, deprecated, deprecationNote, decommissionTime, username);
} catch (Exception e) {
Logger.error("Update dataset deprecation fail", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
return ok(EMPTY_RESPONSE);
}
/**
* Creates or Updates {@link InstitutionalMemory} aspect given dataset urn
*
* <p>{@link com.linkedin.common.InstitutionalMemoryMetadata} record that does not contain audit stamp is filled
* here with session username as the actor and system time as the time</p>
*
* @param datasetUrn Dataset Urn
*/
@Security.Authenticated(Secured.class)
@Nonnull
public Result updateInstitutionalMemory(@Nonnull String datasetUrn) {
final String username = session("user");
if (StringUtils.isBlank(username)) {
return unauthorized(EMPTY_RESPONSE);
}
final JsonNode requestBody = request().body().asJson();
try {
InstitutionalMemory institutionalMemory =
RecordUtils.toRecordTemplate(InstitutionalMemory.class, requestBody.toString());
institutionalMemory.getElements().forEach(element -> {
if (!element.hasCreateStamp()) {
element.setCreateStamp(
new AuditStamp().setActor(new CorpuserUrn(username)).setTime(System.currentTimeMillis()));
}
});
_datasetViewDao.updateInstitutionalMemory(datasetUrn, institutionalMemory);
} catch (Exception e) {
Logger.error("Failed to update Institutional Memory aspect", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
return ok(EMPTY_RESPONSE);
}
/**
* Gets {@link InstitutionalMemory} aspect given dataset urn
*
* @param datasetUrn Dataset Urn
*/
@Security.Authenticated(Secured.class)
@Nonnull
public Result getInstitutionalMemory(String datasetUrn) {
try {
JsonNode responsenode = _datasetViewDao.getInstitutionalMemory(datasetUrn);
return ok(responsenode);
} catch (Exception e) {
if (ControllerUtil.checkErrorCode(e, 404)) {
return notFound(EMPTY_RESPONSE);
}
Logger.error("Failed to get Institutional Memory aspect", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getDatasetOwners(@Nonnull String datasetUrn) {
final DatasetOwnership ownership;
try {
ownership = _ownerViewDao.getDatasetOwners(datasetUrn);
} catch (Exception e) {
if (ControllerUtil.checkErrorCode(e, 404)) {
return notFound(EMPTY_RESPONSE);
}
Logger.error("Fetch owners fail", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
return ok(Json.toJson(ownership));
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result updateDatasetOwners(@Nonnull String datasetUrn) {
final String username = session("user");
if (StringUtils.isBlank(username)) {
return unauthorized(EMPTY_RESPONSE);
}
final JsonNode content = request().body().asJson();
// content should contain arraynode 'owners': []
if (content == null || !content.has("owners") || !content.get("owners").isArray()) {
return badRequest(ControllerUtil.errorResponse("Update dataset owners fail: missing owners field"));
}
try {
final List<DatasetOwner> owners = Json.mapper().readerFor(new TypeReference<List<DatasetOwner>>() {
}).readValue(content.get("owners"));
long confirmedOwnerUserCount = owners.stream()
.filter(s -> "DataOwner".equalsIgnoreCase(s.getType()) && "user".equalsIgnoreCase(s.getIdType())
&& "UI".equalsIgnoreCase(s.getSource()))
.count();
// enforce at least two UI (confirmed) USER DataOwner for a dataset before making any changes
if (confirmedOwnerUserCount < 2) {
return badRequest(ControllerUtil.errorResponse("Less than 2 UI USER owners"));
}
_datasetOwnerDao.updateDatasetOwners(datasetUrn, owners, username);
} catch (Exception e) {
Logger.error("Update Dataset owners fail", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
return ok(EMPTY_RESPONSE);
}
/**
* Gets latest DatasetSnapshot given DatasetUrn
*
* @param datasetUrn String
* @return DatasetSnapshot
*/
@Security.Authenticated(Secured.class)
@Nonnull
public Result getDatasetSnapshot(@Nonnull String datasetUrn) {
try {
return ok(_datasetViewDao.getSnapshot(datasetUrn));
} catch (Exception e) {
if (e.toString().contains("Response status 404")) {
return notFound(EMPTY_RESPONSE);
}
Logger.error("Failed to get dataset snapshot", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getDatasetSchema(@Nonnull String datasetUrn) {
final DatasetSchema schema;
try {
schema = _datasetViewDao.getDatasetSchema(datasetUrn);
} catch (Exception e) {
if (ControllerUtil.checkErrorCode(e, 404)) {
return notFound(EMPTY_RESPONSE);
}
Logger.error("Fetch schema fail", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
if (schema == null) {
return notFound(EMPTY_RESPONSE);
}
return ok((JsonNode) Json.newObject().set("schema", Json.toJson(schema)));
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getDatasetSuggestedOwners(@Nonnull String datasetUrn) {
return ok(EMPTY_RESPONSE);
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getDatasetUpstreams(@Nonnull String datasetUrn) {
final List<LineageView> upstreams;
try {
upstreams = _lineageDao.getUpstreamLineage(datasetUrn);
} catch (Exception e) {
if (ControllerUtil.checkErrorCode(e, 404)) {
int[] emptyUpstreams = new int[0];
return ok(Json.toJson(emptyUpstreams));
}
Logger.error("Fetch Dataset upstreams error", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
return ok(Json.toJson(upstreams));
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getDatasetDownstreams(@Nonnull String datasetUrn) {
final List<LineageView> downstreams;
try {
downstreams = _lineageDao.getDownstreamLineage(datasetUrn);
} catch (Exception e) {
Logger.error("Fetch Dataset downstreams error", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
return ok(Json.toJson(downstreams));
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result getDataPlatforms() {
try {
return ok(ControllerUtil.jsonNode("platforms", _dataPlatformsDao.getAllPlatforms()));
} catch (final Exception e) {
Logger.error("Fail to get data platforms", e);
return notFound(ControllerUtil.errorResponse(e));
}
}
}

135
datahub-frontend/app/controllers/api/v2/Search.java
View File

@ -1,135 +0,0 @@
package controllers.api.v2;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.linkedin.datahub.dao.DaoFactory;
import com.linkedin.datahub.dao.view.DocumentSearchDao;
import controllers.Secured;
import org.apache.commons.lang3.math.NumberUtils;
import play.Logger;
import play.mvc.Controller;
import play.mvc.Result;
import play.mvc.Security;
import utils.ControllerUtil;
import utils.SearchUtil;
import javax.annotation.Nonnull;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import static org.apache.commons.lang3.StringUtils.isBlank;
public class Search extends Controller {
private static final String REQUEST_TYPE = "type";
private static final String REQUEST_INPUT = "input";
private static final String REQUEST_FIELD = "field";
private static final String REQUEST_START = "start";
private static final String REQUEST_COUNT = "count";
private static final String REQUEST_LIMIT = "limit";
private static final String CORP_USER_TYPE = "corpuser";
private static final String DATASET_TYPE = "dataset";
private static final Set<String> CORP_USER_FACET_FIELDS = Collections.emptySet();
private static final Set<String> DATASET_FACET_FIELDS =
ImmutableSet.of("origin", "platform");
private static final ImmutableMap<String, Set<String>> FACET_FIELDS = ImmutableMap.of(
CORP_USER_TYPE, CORP_USER_FACET_FIELDS,
DATASET_TYPE, DATASET_FACET_FIELDS
);
private static final int _DEFAULT_START_VALUE = 0;
private static final int _DEFAULT_PAGE_SIZE = 10;
private static final int _DEFAULT_LIMIT_VALUE = 20;
private static Map<String, String> buildRequestMap(String type) {
Map<String, String> requestMap = new HashMap<>();
Set<String> facets = FACET_FIELDS.getOrDefault(type, Collections.emptySet());
facets.stream().forEach(facet -> requestMap.put(facet, request().getQueryString(facet)));
return requestMap;
}
private final Map<String, DocumentSearchDao> _searchDaoMap;
public Search() {
_searchDaoMap = ImmutableMap.<String, DocumentSearchDao>builder()
.put(CORP_USER_TYPE, DaoFactory.getCorpUserDocumentSearchDao())
.put(DATASET_TYPE, DaoFactory.getDatasetDocumentSearchDao())
.build();
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result search() {
try {
String type = request().getQueryString(REQUEST_TYPE);
if (isBlank(type)) {
return badRequest("Bad Request. type parameter can not be null or empty");
}
// escape forward slash since it is a reserved character in Elasticsearch
// TODO: Once apis for exact or advanced(with support for field specific/regex) search are exposed,
// update to call appropriate api based on indication from user.
final String input = SearchUtil
.escapeForwardSlash(Strings.nullToEmpty(request().getQueryString(REQUEST_INPUT)));
if (isBlank(input)) {
return badRequest("Bad Request. input parameter can not be null or empty");
}
int start = NumberUtils.toInt(request().getQueryString(REQUEST_START), _DEFAULT_START_VALUE);
int count = NumberUtils.toInt(request().getQueryString(REQUEST_COUNT), _DEFAULT_PAGE_SIZE);
Map<String, String> requestMap = buildRequestMap(type);
DocumentSearchDao documentSearchDao = _searchDaoMap.get(type);
if (documentSearchDao != null) {
return ok(documentSearchDao.search(input, requestMap, start, count));
} else {
return badRequest("Bad Request. Type is not supported");
}
} catch (Exception e) {
Logger.error("Failed in documents search.", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
}
@Security.Authenticated(Secured.class)
@Nonnull
public Result autoComplete() {
try {
String type = request().getQueryString(REQUEST_TYPE);
if (isBlank(type)) {
return badRequest("Bad Request. type parameter can not be null or empty");
}
String field = request().getQueryString(REQUEST_FIELD);
if (isBlank(field)) {
return badRequest("Bad Request. field parameter can not be null or empty");
}
// escape forward slash since it is a reserved character in Elasticsearch
// TODO: Once apis for exact or advanced(with support for field specific/regex) search are exposed,
// update to call appropriate api based on indication from user.
final String input = SearchUtil
.escapeForwardSlash(Strings.nullToEmpty(request().getQueryString(REQUEST_INPUT)));
int limit = NumberUtils.toInt(request().getQueryString(REQUEST_LIMIT), _DEFAULT_LIMIT_VALUE);
Map<String, String> requestMap = buildRequestMap(type);
DocumentSearchDao documentSearchDao = _searchDaoMap.get(type);
if (documentSearchDao != null) {
return ok(documentSearchDao.autoComplete(input, field, requestMap, limit));
} else {
return badRequest("Bad Request. Type is not supported");
}
} catch (Exception e) {
Logger.error("Failed in auto complete request.", e);
return internalServerError(ControllerUtil.errorResponse(e));
}
}
}

58
datahub-frontend/app/utils/ControllerUtil.java
View File

@ -1,58 +0,0 @@
package utils;
import com.fasterxml.jackson.databind.JsonNode;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.datahub.util.RestliUtil;
import com.linkedin.restli.client.RestLiResponseException;
import java.io.IOException;
import javax.annotation.Nonnull;
import play.Logger;
import play.libs.Json;
/**
* Helper class for controller APIs
*/
public class ControllerUtil {
private ControllerUtil() {
//utility class
}
@Nonnull
public static <E extends Throwable> JsonNode errorResponse(@Nonnull E e) {
return errorResponse(e.toString());
}
@Nonnull
public static JsonNode errorResponse(@Nonnull String msg) {
return Json.newObject().put("msg", msg);
}
public static boolean checkErrorCode(Exception e, int statusCode) {
return (e instanceof RestLiResponseException) && (((RestLiResponseException) e).getStatus() == statusCode);
}
/**
* Creates a generic jsonNode out of a key and value which is a very common use case
* @param key
* @param value
* @return
*/
@Nonnull
public static JsonNode jsonNode(@Nonnull final String key, @Nonnull final Object value) {
JsonNode node = null;
if (value instanceof RecordTemplate) {
try {
node = RestliUtil.toJsonNode((RecordTemplate) value);
} catch (final IOException e) {
Logger.error("Could not create a json", e);
}
} else if (value instanceof JsonNode) {
node = (JsonNode) value;
} else {
node = Json.toJson(value);
}
return Json.newObject().set(key, node);
}
}

29
datahub-frontend/conf/routes
View File

@ -7,36 +7,7 @@
GET / controllers.Application.index(path="index.html")
GET /admin controllers.Application.healthcheck()
POST /authenticate controllers.Application.authenticate()
GET /config controllers.Application.appConfig()
GET /deps controllers.Application.printDeps()
GET /login controllers.Application.login()
GET /logout controllers.Application.logout()
GET /api/v1/owner/types controllers.api.v1.Dataset.getDatasetOwnerTypes()
GET /api/v1/party/employees controllers.api.v1.User.getAllCompanyUsers()
GET /api/v1/party/entities controllers.api.v1.User.getAllUserEntities()
GET /api/v1/party/groups controllers.api.v1.User.getAllGroups()
GET /api/v1/user/me controllers.api.v1.User.getLoggedInUser()
GET /api/v2/autocomplete controllers.api.v2.Search.autoComplete()
GET /api/v2/browse controllers.api.v2.Browse.browse()
GET /api/v2/browsePaths controllers.api.v2.Browse.getBrowsePaths()
GET /api/v2/corpusers/:urn controllers.api.v2.CorpUser.getCorpUser(urn: String)
POST /api/v2/corpusers/:urn/editableInfo controllers.api.v2.CorpUser.updateCorpUserEditableInfo(urn: String)
GET /api/v2/datasets/:urn controllers.api.v2.Dataset.getDataset(urn: String)
PUT /api/v2/datasets/:urn/deprecate controllers.api.v2.Dataset.updateDatasetDeprecation(urn: String)
GET /api/v2/datasets/:urn/downstreams controllers.api.v2.Dataset.getDatasetDownstreams(urn: String)
GET /api/v2/datasets/:urn/institutionalmemory controllers.api.v2.Dataset.getInstitutionalMemory(urn: String)
POST /api/v2/datasets/:urn/institutionalmemory controllers.api.v2.Dataset.updateInstitutionalMemory(urn: String)
GET /api/v2/datasets/:urn/owners controllers.api.v2.Dataset.getDatasetOwners(urn: String)
POST /api/v2/datasets/:urn/owners controllers.api.v2.Dataset.updateDatasetOwners(urn: String)
GET /api/v2/datasets/:urn/owners/suggestion controllers.api.v2.Dataset.getDatasetSuggestedOwners(urn: String)
GET /api/v2/datasets/:urn/schema controllers.api.v2.Dataset.getDatasetSchema(urn: String)
GET /api/v2/datasets/:urn/snapshot controllers.api.v2.Dataset.getDatasetSnapshot(urn: String)
GET /api/v2/datasets/:urn/upstreams controllers.api.v2.Dataset.getDatasetUpstreams(urn: String)
GET /api/v2/list/platforms controllers.api.v2.Dataset.getDataPlatforms
GET /api/v2/search controllers.api.v2.Search.search()
# Routes used exclusively by the React application.

1
datahub-frontend/play.gradle
View File

@ -27,7 +27,6 @@ dependencies {
play('net.minidev:json-smart:2.4.1')
play('io.netty:netty-all:4.1.44.Final')
}
play project(":datahub-dao")
play project(":datahub-graphql-core")
play externalDependency.jettyJaas

19
docker/docker-compose.ember.yml
View File

@ -1,19 +0,0 @@
# Introduces datahub-frontend-ember service to serve the Ember UI on localhost:9001/
---
version: '3.8'
services:
datahub-frontend-ember:
build:
context: ../
dockerfile: docker/datahub-frontend/Dockerfile
args:
SERVER_PORT: 9001
ENABLE_EMBER: "true"
image: linkedin/datahub-frontend:${DATAHUB_VERSION:-head}
env_file: datahub-frontend/env/docker.env
hostname: datahub-frontend
container_name: datahub-frontend
ports:
- "9001:9001"
depends_on:
- datahub-gms

1
gms/client/build.gradle
View File

@ -4,7 +4,6 @@ dependencies {
compile project(':gms:api')
compile project(':gms:impl')
compile project(path: ':gms:api', configuration: 'restClient')
compile project(':metadata-dao-impl:restli-dao')
compile project(':metadata-events:mxe-schemas')
dataModel project(':li-utils')

1
metadata-jobs/mae-consumer/build.gradle
View File

@ -19,7 +19,6 @@ dependencies {
compile project(':metadata-utils')
compile project(":entity-registry")
compile project(':metadata-builders')
compile project(':metadata-dao-impl:restli-dao')
compile project(':metadata-events:mxe-schemas')
compile project(':metadata-events:mxe-avro-1.7')
compile project(':metadata-events:mxe-registration')

1
metadata-jobs/mce-consumer/build.gradle
View File

@ -18,7 +18,6 @@ dependencies {
compile project(':metadata-events:mxe-avro-1.7')
compile project(':metadata-events:mxe-registration')
compile project(':metadata-events:mxe-utils-avro-1.7')
compile project(':metadata-dao-impl:restli-dao')
compile project(':metadata-io')
compile project(':gms:client')
compile spec.product.pegasus.restliClient

1
settings.gradle
View File

@ -1,4 +1,3 @@
include 'datahub-dao'
include 'datahub-frontend'
include 'datahub-graphql-core'
include 'datahub-gms-graphql-service'

265
smoke-test/test_e2e.py
View File

@ -2,6 +2,7 @@ import time
import pytest
import requests
import urllib
from datahub.cli.docker import check_local_docker_containers
from datahub.ingestion.run.pipeline import Pipeline
@ -98,41 +99,12 @@ def test_run_ingestion(wait_for_healthchecks):
pass
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_gms_list_data_platforms():
response = requests.get(
f"{GMS_ENDPOINT}/dataPlatforms",
headers={
**restli_default_headers,
"X-RestLi-Method": "get_all",
},
)
response.raise_for_status()
data = response.json()
assert len(data["elements"]) > 10
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_gms_get_all_users():
response = requests.get(
f"{GMS_ENDPOINT}/corpUsers",
headers={
**restli_default_headers,
"X-RestLi-Method": "get_all",
},
)
response.raise_for_status()
data = response.json()
assert len(data["elements"]) >= 3
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_gms_get_user():
username = "jdoe"
urn = f"urn:li:corpuser:{username}"
response = requests.get(
f"{GMS_ENDPOINT}/corpUsers/($params:(),name:{username})",
f"{GMS_ENDPOINT}/entities/{urllib.parse.quote(urn)}",
headers={
**restli_default_headers,
},
@ -140,9 +112,9 @@ def test_gms_get_user():
response.raise_for_status()
data = response.json()
assert data["username"] == username
assert data["info"]["displayName"]
assert data["info"]["email"]
assert data["value"]
assert data["value"]["com.linkedin.metadata.snapshot.CorpUserSnapshot"]
assert data["value"]["com.linkedin.metadata.snapshot.CorpUserSnapshot"]["urn"] == urn
@pytest.mark.parametrize(
@ -172,19 +144,18 @@ def test_gms_get_dataset(platform, dataset_name, env):
urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
response = requests.get(
f"{GMS_ENDPOINT}/datasets/($params:(),name:{dataset_name},origin:{env},platform:{requests.utils.quote(platform)})",
f"{GMS_ENDPOINT}/entities/{urllib.parse.quote(urn)}",
headers={
**restli_default_headers,
"X-RestLi-Method": "get",
},
)
response.raise_for_status()
data = response.json()
res_data = response.json()
assert data["urn"] == urn
assert data["name"] == dataset_name
assert data["platform"] == platform
assert len(data["schemaMetadata"]["fields"]) >= 2
assert res_data["value"]
assert res_data["value"]["com.linkedin.metadata.snapshot.DatasetSnapshot"]
assert res_data["value"]["com.linkedin.metadata.snapshot.DatasetSnapshot"]["urn"] == urn
@pytest.mark.parametrize(
@ -196,19 +167,26 @@ def test_gms_get_dataset(platform, dataset_name, env):
)
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_gms_search_dataset(query, min_expected_results):
response = requests.get(
f"{GMS_ENDPOINT}/datasets?q=search&input={query}",
headers={
**restli_default_headers,
"X-RestLi-Method": "finder",
},
json = {
"input": f"{query}",
"entity": "dataset",
"start": 0,
"count": 10
}
print(json)
response = requests.post(
f"{GMS_ENDPOINT}/entities?action=search",
headers=restli_default_headers,
json=json
)
response.raise_for_status()
data = response.json()
res_data = response.json()
assert len(data["elements"]) >= min_expected_results
assert data["paging"]["total"] >= min_expected_results
assert data["elements"][0]["urn"]
assert res_data["value"]
assert res_data["value"]["numEntities"] >= min_expected_results
assert len(res_data["value"]["entities"]) >= min_expected_results
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
@ -253,7 +231,7 @@ def frontend_session(wait_for_healthchecks):
}
data = '{"username":"datahub", "password":"datahub"}'
response = session.post(
f"{FRONTEND_ENDPOINT}/authenticate", headers=headers, data=data
f"{FRONTEND_ENDPOINT}/logIn", headers=headers, data=data
)
response.raise_for_status()
@ -267,15 +245,43 @@ def test_frontend_auth(frontend_session):
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_frontend_browse_datasets(frontend_session):
response = frontend_session.get(
f"{FRONTEND_ENDPOINT}/api/v2/browse?type=dataset&path=/prod"
)
response.raise_for_status()
data = response.json()
assert data["metadata"]["totalNumEntities"] >= 4
assert len(data["metadata"]["groups"]) >= 4
assert len(data["metadata"]["groups"]) <= 8
json = {
"query": """query browse($input: BrowseInput!) {\n
browse(input: $input) {\n
start\n
count\n
total\n
groups {
name
}
entities {\n
... on Dataset {\n
urn\n
name\n
}\n
}\n
}\n
}""",
"variables": {
"input": {
"type": "DATASET",
"path": ["prod"]
}
}
}
response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status()
res_data = response.json()
assert res_data
assert res_data["data"]
assert res_data["data"]["browse"]
assert len(res_data["data"]["browse"]["entities"]) == 0
assert len(res_data["data"]["browse"]["groups"]) > 0
@pytest.mark.parametrize(
@ -286,36 +292,81 @@ def test_frontend_browse_datasets(frontend_session):
],
)
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_frontend_browse_datasets(frontend_session, query, min_expected_results):
response = frontend_session.get(
f"{FRONTEND_ENDPOINT}/api/v2/search?type=dataset&input={query}"
def test_frontend_search_datasets(frontend_session, query, min_expected_results):
json = {
"query": """query search($input: SearchInput!) {\n
search(input: $input) {\n
start\n
count\n
total\n
searchResults {\n
entity {\n
... on Dataset {\n
urn\n
name\n
}\n
}\n
}\n
}\n
}""",
"variables": {
"input": {
"type": "DATASET",
"query": f"{query}",
"start": 0,
"count": 10
}
}
}
response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status()
data = response.json()
res_data = response.json()
assert len(data["elements"]) >= min_expected_results
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_frontend_list_users(frontend_session):
response = frontend_session.get(f"{FRONTEND_ENDPOINT}/api/v1/party/entities")
response.raise_for_status()
data = response.json()
assert data["status"] == "ok"
assert len(data["userEntities"]) >= 3
assert res_data
assert res_data["data"]
assert res_data["data"]["search"]
assert res_data["data"]["search"]["total"] >= min_expected_results
assert len(res_data["data"]["search"]["searchResults"]) >= min_expected_results
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_frontend_user_info(frontend_session):
response = frontend_session.get(f"{FRONTEND_ENDPOINT}/api/v1/user/me")
response.raise_for_status()
data = response.json()
assert data["status"] == "ok"
assert data["user"]["userName"] == "datahub"
assert data["user"]["name"]
assert data["user"]["email"]
urn = f"urn:li:corpuser:datahub"
json = {
"query": """query corpUser($urn: String!) {\n
corpUser(urn: $urn) {\n
urn\n
username\n
editableInfo {\n
pictureLink\n
}\n
info {\n
firstName\n
fullName\n
title\n
email\n
}\n
}\n
}""",
"variables": {
"urn": urn
}
}
response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status()
res_data = response.json()
assert res_data
assert res_data["data"]
assert res_data["data"]["corpUser"]
assert res_data["data"]["corpUser"]["urn"] == urn
@pytest.mark.parametrize(
@ -336,31 +387,41 @@ def test_frontend_user_info(frontend_session):
],
)
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_frontend_user_info(frontend_session, platform, dataset_name, env):
def test_frontend_datasets(frontend_session, platform, dataset_name, env):
urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
json = {
"query": """query getDataset($urn: String!) {\n
dataset(urn: $urn) {\n
urn\n
name\n
description\n
platform {\n
urn\n
}\n
schemaMetadata {\n
name\n
version\n
createdAt\n
}\n
}\n
}""",
"variables": {
"urn": urn
}
}
# Basic dataset info.
response = frontend_session.get(f"{FRONTEND_ENDPOINT}/api/v2/datasets/{urn}")
response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status()
data = response.json()
res_data = response.json()
assert data["nativeName"] == dataset_name
assert data["fabric"] == env
assert data["uri"] == urn
# Schema info.
response = frontend_session.get(f"{FRONTEND_ENDPOINT}/api/v2/datasets/{urn}/schema")
response.raise_for_status()
data = response.json()
assert len(data["schema"]["columns"]) >= 2
# Ownership info.
response = frontend_session.get(f"{FRONTEND_ENDPOINT}/api/v2/datasets/{urn}/owners")
response.raise_for_status()
data = response.json()
assert len(data["owners"]) >= 1
assert res_data
assert res_data["data"]
assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["urn"] == urn
assert res_data["data"]["dataset"]["name"] == dataset_name
assert res_data["data"]["dataset"]["platform"]["urn"] == platform
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_ingest_with_system_metadata():