yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-28 10:28:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(deletes): Fixing system metadata index deletes (#3693)

Browse Source
This commit is contained in:
John Joyce 2021-12-08 14:03:24 -08:00 committed by GitHub
parent 1a5121a5ae
commit 3bac7f7c43
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 45 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ESSystemMetadataDAO.java
View File

@ -90,6 +90,30 @@ public class ESSystemMetadataDAO {
return null;
}
public BulkByScrollResponse deleteByUrnAspect(
@Nonnull final String urn,
@Nonnull final String aspect
) {
BoolQueryBuilder finalQuery = QueryBuilders.boolQuery();
finalQuery.must(QueryBuilders.termQuery("urn", urn));
finalQuery.must(QueryBuilders.termQuery("aspect", aspect));
DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest();
deleteByQueryRequest.setQuery(finalQuery);
deleteByQueryRequest.indices(indexConvention.getIndexName(INDEX_NAME));
try {
final BulkByScrollResponse deleteResponse = client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT);
return deleteResponse;
} catch (IOException e) {
log.error("ERROR: Failed to delete by query. See stacktrace for a more detailed error:");
e.printStackTrace();
}
return null;
}
public SearchResponse findByParams(Map<String, String> searchParams) {
SearchRequest searchRequest = new SearchRequest();

15
metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java
View File

@ -8,11 +8,11 @@ import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder;
import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
import com.linkedin.mxe.SystemMetadata;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@ -24,7 +24,6 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
@ -73,21 +72,19 @@ public class ElasticSearchSystemMetadataService implements SystemMetadataService
String rawDocId = urn + DOC_DELIMETER + aspect;
try {
byte[] bytesOfRawDocID = rawDocId.getBytes("UTF-8");
byte[] bytesOfRawDocID = rawDocId.getBytes(StandardCharsets.UTF_8);
MessageDigest md = MessageDigest.getInstance("MD5");
byte[] thedigest = md.digest(bytesOfRawDocID);
return new String(thedigest, StandardCharsets.US_ASCII);
} catch (NoSuchAlgorithmException | UnsupportedEncodingException e) {
return Base64.getEncoder().encodeToString(thedigest);
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
return rawDocId;
}
}
@Override
public Boolean delete(String urn, String aspect) {
String docId = toDocId(urn, aspect);
DeleteResponse response = _esDAO.deleteByDocId(docId);
return response.status().getStatus() >= 200 && response.status().getStatus() < 300;
public void deleteAspect(String urn, String aspect) {
_esDAO.deleteByUrnAspect(urn, aspect);
}
@Override

8
metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java
View File

@ -8,7 +8,13 @@ import javax.annotation.Nullable;
public interface SystemMetadataService {
Boolean delete(String urn, String aspect);
/**
* Deletes a specific aspect from the system metadata service.
*
* @param urn the urn of the entity
* @param aspect the aspect to delete
*/
void deleteAspect(String urn, String aspect);
void deleteUrn(String finalOldUrn);

7
metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java
View File

@ -261,9 +261,14 @@ public class MetadataChangeLogProcessor {
private void deleteSystemMetadata(Urn urn, AspectSpec aspectSpec, Boolean isKeyAspect) {
if (isKeyAspect) {
// Delete all aspects
log.debug(String.format("Deleting all system metadata for urn: %s", urn));
_systemMetadataService.deleteUrn(urn.toString());
} else {
// Delete all aspects from system metadata service
log.debug(String.format("Deleting system metadata for urn: %s, aspect: %s", urn, aspectSpec.getName()));
_systemMetadataService.deleteAspect(urn.toString(), aspectSpec.getName());
}
_systemMetadataService.delete(urn.toString(), aspectSpec.getName());
}
private void deleteGraphData(Urn urn, AspectSpec aspectSpec, RecordTemplate aspect, Boolean isKeyAspect) {

4
smoke-test/test_e2e.py
View File

@ -594,8 +594,8 @@ def test_frontend_list_policies(frontend_session):
assert res_data["data"]
assert res_data["data"]["listPolicies"]
assert res_data["data"]["listPolicies"]["start"] is 0
assert res_data["data"]["listPolicies"]["count"] is 8
assert len(res_data["data"]["listPolicies"]["policies"]) is 8 # Length of default policies.
assert res_data["data"]["listPolicies"]["count"] > 0
assert len(res_data["data"]["listPolicies"]["policies"]) > 0
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion", "test_frontend_list_policies"])
def test_frontend_update_policy(frontend_session):