yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 18:07:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(analytics): add support for AWS ES (#2539)

Browse Source
This commit is contained in:
Dexter Lee 2021-05-12 10:54:18 -07:00 committed by GitHub
parent 7fce505ffb
commit b4cf74bcb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 110 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
docker/elasticsearch-setup/create-indices.sh
View File

@ -3,6 +3,7 @@
set -e
: ${DATAHUB_ANALYTICS_ENABLED:=true}
: ${USE_AWS_ELASTICSEARCH:=false}
if [[ $ELASTICSEARCH_USE_SSL == true ]]; then
ELASTICSEARCH_PROTOCOL=https
@ -128,14 +129,32 @@ function create_datahub_usage_event_datastream() {
if [ $(curl -o /dev/null -s -w "%{http_code}" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_ilm/policy/datahub_usage_event_policy") -eq 404 ]
then
echo -e "\ncreating datahub_usage_event_policy"
curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_ilm/policy/datahub_usage_event_policy" -H 'Content-Type: application/json' --data @/index/$1
curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_ilm/policy/datahub_usage_event_policy" -H 'Content-Type: application/json' --data @/index/usage-event/policy.json
else
echo -e "\ndatahub_usage_event_policy exists"
fi
if [ $(curl -o /dev/null -s -w "%{http_code}" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_index_template/datahub_usage_event_index_template") -eq 404 ]
then
echo -e "\ncreating datahub_usage_event_index_template"
curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_index_template/datahub_usage_event_index_template" -H 'Content-Type: application/json' --data @/index/$2
curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_index_template/datahub_usage_event_index_template" -H 'Content-Type: application/json' --data @/index/usage-event/index_template.json
else
echo -e "\ndatahub_usage_event_index_template exists"
fi
}
function create_datahub_usage_event_aws_elasticsearch() {
if [ $(curl -o /dev/null -s -w "%{http_code}" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_opendistro/_ism/policies/datahub_usage_event_policy") -eq 404 ]
then
echo -e "\ncreating datahub_usage_event_policy"
curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_opendistro/_ism/policies/datahub_usage_event_policy" -H 'Content-Type: application/json' --data @/index/usage-event/aws_es_ism_policy.json
else
echo -e "\ndatahub_usage_event_policy exists"
fi
if [ $(curl -o /dev/null -s -w "%{http_code}" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_template/datahub_usage_event_index_template") -eq 404 ]
then
echo -e "\ncreating datahub_usage_event_index_template"
curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_template/datahub_usage_event_index_template" -H 'Content-Type: application/json' --data @/index/usage-event/aws_es_index_template.json
curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/datahub_usage_event-000001" -H 'Content-Type: application/json' --data "{\"aliases\":{\"datahub_usage_event\":{\"is_write_index\":true}}}"
else
echo -e "\ndatahub_usage_event_index_template exists"
fi
@ -153,6 +172,10 @@ create_index $(get_index_name tagdocument) tags/settings.json tags/mappings.json
create_index $(get_index_name glossaryterminfodocument) glossary/term/settings.json glossary/term/mappings.json || exit 1
create_index $(get_index_name glossarynodeinfodocument) glossary/node/settings.json glossary/node/mappings.json || exit 1
if [[ $DATAHUB_ANALYTICS_ENABLED == true ]]; then
create_datahub_usage_event_datastream usage-event/policy.json usage-event/index_template.json || exit 1
if [[ $USE_AWS_ELASTICSEARCH == false ]]; then
create_datahub_usage_event_datastream || exit 1
else
create_datahub_usage_event_aws_elasticsearch || exit 1
fi
fi

25
gms/impl/src/main/resources/index/usage-event/aws_es_index_template.json Normal file
View File

@ -0,0 +1,25 @@
{
"index_patterns": ["datahub_usage_event-*"],
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"type": {
"type": "keyword"
},
"timestamp": {
"type": "date"
},
"userAgent": {
"type": "keyword"
},
"browserId": {
"type": "keyword"
}
}
},
"settings": {
"index.opendistro.index_state_management.rollover_alias": "datahub_usage_event"
}
}

58
gms/impl/src/main/resources/index/usage-event/aws_es_ism_policy.json Normal file
View File

@ -0,0 +1,58 @@
{
"policy": {
"policy_id": "datahub_usage_event_policy",
"default_state": "Rollover",
"schema_version": 1,
"states": [
{
"name": "Rollover",
"actions": [
{
"rollover": {
"min_index_age": "1d"
}
}
],
"transitions": [
{
"state_name": "ReadOnly",
"conditions": {
"min_index_age": "7d"
}
}
]
},
{
"name": "ReadOnly",
"actions": [
{
"read_only": {}
}
],
"transitions": [
{
"state_name": "Delete",
"conditions": {
"min_index_age": "60d"
}
}
]
},
{
"name": "Delete",
"actions": [
{
"delete": {}
}
],
"transitions": []
}
],
"ism_template": {
"index_patterns": [
"datahub_usage_event-*"
],
"priority": 100
}
}
}

2
gms/impl/src/main/resources/index/usage-event/policy.json
View File

@ -9,7 +9,7 @@
}
},
"delete": {
"min_age": "90d",
"min_age": "60d",
"actions": {
"delete": {}
}