#!/bin/sh
# This script is to re-index existing data from the old single-index "pages" into the new "pages-monthly-*" indices.
# It will ASK for user confirmation before doing ANY writing tasks.
# It works in 3 simple steps:
# 1. Query the single-index for all existing data, aggregate by month.
# 2. Parse this result into yyyy-MM format, store this
# 3. For each month, query the single-index to get only data for that month, and call the re-index api to store
#       only that subset of data into the monthly index of the form page-monthly-yyyy-MM.

# !!DISCLAIMER!!: before you run this, you SHOULD configure the indices for effecient indexing: diable refresh, and disable replicas.
#       This script does NOT take care of the above. You need to do manually. Remember to re-enable these settings, once this script is done.


CLUSTERNAME="$1"
SRC_INDEX="pages"
DEST_INDEX_PREFIX="pages-monthly-"

# Step 1,2:
OLD_DATES=`curl -s '-HContent-Type: application/json' -XGET -d '
{
  "size": 0,
  "query": {
      "match_all": {}
  }
  , "aggs": {
    "by_day": {
      "date_histogram": {
        "field": "date",
        "interval": "1M",
        "min_doc_count": 10
      }
    }
  }
}
' "http://$CLUSTERNAME:9200/$SRC_INDEX/_search?pretty" | egrep 'key_as_string' | cut -d':' -f2 | cut -d'"' -f2 | cut -d'-' -f1,2`
echo "Months found:"
echo "${OLD_DATES}"

echo
echo -n "if this make sence, Y to continue N to exit [Y/N]:"
read INPUT
if [ "$INPUT" == "Y" ] || [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] || [ "$INPUT" == "YES" ]
then
    # Step 3:
    for index in "$OLD_DATES"; do
        echo "Working on month: $index"
        DEST_INDEX="$DEST_INDEX_PREFIX$index"

        curl -s '-HContent-Type: application/json' -XPOST -d '
        {
          "conflicts": "proceed",
          "source": {
            "index": "'$SRC_INDEX'",
            "query": {
              "range": {
                "date": {
                  "gte": "'$index'",
                  "lt": "'$index'||+1M",
                  "format": "yyyy-MM"
                }
              }
            }
          },
          "dest": {
            "index": "'$DEST_INDEX'"
          }
        }
        ' "http://$CLUSTERNAME:9200/_reindex"
        echo "Done indexing $DEST_INDEX"
    done
else
    echo "SCRIPT CLOSED BY USER, BYE ..."
    exit
fi