initial commitHEADmaster

1 files changed, 74 insertions, 0 deletions

diff --git a/elastic_reindex_by_date.sh b/elastic_reindex_by_date.sh
new file mode 100755
index 0000000..36188bc
--- /dev/null
+++ b/elastic_reindex_by_date.sh
@@ -0,0 +1,74 @@
+#!/bin/sh
+# This script is to re-index existing data from the old single-index "pages" into the new "pages-monthly-*" indices.
+# It will ASK for user confirmation before doing ANY writing tasks.
+# It works in 3 simple steps:
+# 1. Query the single-index for all existing data, aggregate by month.
+# 2. Parse this result into yyyy-MM format, store this
+# 3. For each month, query the single-index to get only data for that month, and call the re-index api to store
+#       only that subset of data into the monthly index of the form page-monthly-yyyy-MM.
+
+# !!DISCLAIMER!!: before you run this, you SHOULD configure the indices for effecient indexing: diable refresh, and disable replicas.
+#       This script does NOT take care of the above. You need to do manually. Remember to re-enable these settings, once this script is done.
+
+
+CLUSTERNAME="$1"
+SRC_INDEX="pages"
+DEST_INDEX_PREFIX="pages-monthly-"
+
+# Step 1,2:
+OLD_DATES=`curl -s '-HContent-Type: application/json' -XGET -d '
+{
+  "size": 0,
+  "query": {
+      "match_all": {}
+  }
+  , "aggs": {
+    "by_day": {
+      "date_histogram": {
+        "field": "date",
+        "interval": "1M",
+        "min_doc_count": 10
+      }
+    }
+  }
+}
+' "http://$CLUSTERNAME:9200/$SRC_INDEX/_search?pretty" | egrep 'key_as_string' | cut -d':' -f2 | cut -d'"' -f2 | cut -d'-' -f1,2`
+echo "Months found:"
+echo "${OLD_DATES}"
+
+echo
+echo -n "if this make sence, Y to continue N to exit [Y/N]:"
+read INPUT
+if [ "$INPUT" == "Y" ] || [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] || [ "$INPUT" == "YES" ]
+then
+    # Step 3:
+    for index in "$OLD_DATES"; do
+        echo "Working on month: $index"
+        DEST_INDEX="$DEST_INDEX_PREFIX$index"
+
+        curl -s '-HContent-Type: application/json' -XPOST -d '
+        {
+          "conflicts": "proceed",
+          "source": {
+            "index": "'$SRC_INDEX'",
+            "query": {
+              "range": {
+                "date": {
+                  "gte": "'$index'",
+                  "lt": "'$index'||+1M",
+                  "format": "yyyy-MM"
+                }
+              }
+            }
+          },
+          "dest": {
+            "index": "'$DEST_INDEX'"
+          }
+        }
+        ' "http://$CLUSTERNAME:9200/_reindex"
+        echo "Done indexing $DEST_INDEX"
+    done
+else
+    echo "SCRIPT CLOSED BY USER, BYE ..."
+    exit
+fi