1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
#!/bin/sh
# This script is to re-index existing data from the old single-index "pages" into the new "pages-monthly-*" indices.
# It will ASK for user confirmation before doing ANY writing tasks.
# It works in 3 simple steps:
# 1. Query the single-index for all existing data, aggregate by month.
# 2. Parse this result into yyyy-MM format, store this
# 3. For each month, query the single-index to get only data for that month, and call the re-index api to store
# only that subset of data into the monthly index of the form page-monthly-yyyy-MM.
# !!DISCLAIMER!!: before you run this, you SHOULD configure the indices for effecient indexing: diable refresh, and disable replicas.
# This script does NOT take care of the above. You need to do manually. Remember to re-enable these settings, once this script is done.
CLUSTERNAME="$1"
SRC_INDEX="pages"
DEST_INDEX_PREFIX="pages-monthly-"
# Step 1,2:
OLD_DATES=`curl -s '-HContent-Type: application/json' -XGET -d '
{
"size": 0,
"query": {
"match_all": {}
}
, "aggs": {
"by_day": {
"date_histogram": {
"field": "date",
"interval": "1M",
"min_doc_count": 10
}
}
}
}
' "http://$CLUSTERNAME:9200/$SRC_INDEX/_search?pretty" | egrep 'key_as_string' | cut -d':' -f2 | cut -d'"' -f2 | cut -d'-' -f1,2`
echo "Months found:"
echo "${OLD_DATES}"
echo
echo -n "if this make sence, Y to continue N to exit [Y/N]:"
read INPUT
if [ "$INPUT" == "Y" ] || [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] || [ "$INPUT" == "YES" ]
then
# Step 3:
for index in "$OLD_DATES"; do
echo "Working on month: $index"
DEST_INDEX="$DEST_INDEX_PREFIX$index"
curl -s '-HContent-Type: application/json' -XPOST -d '
{
"conflicts": "proceed",
"source": {
"index": "'$SRC_INDEX'",
"query": {
"range": {
"date": {
"gte": "'$index'",
"lt": "'$index'||+1M",
"format": "yyyy-MM"
}
}
}
},
"dest": {
"index": "'$DEST_INDEX'"
}
}
' "http://$CLUSTERNAME:9200/_reindex"
echo "Done indexing $DEST_INDEX"
done
else
echo "SCRIPT CLOSED BY USER, BYE ..."
exit
fi
|
