diff options
| -rwxr-xr-x | elastic_reindex_by_date.sh | 74 | ||||
| -rwxr-xr-x | vault-merge.sh | 112 |
2 files changed, 186 insertions, 0 deletions
diff --git a/elastic_reindex_by_date.sh b/elastic_reindex_by_date.sh new file mode 100755 index 0000000..36188bc --- /dev/null +++ b/elastic_reindex_by_date.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# This script is to re-index existing data from the old single-index "pages" into the new "pages-monthly-*" indices. +# It will ASK for user confirmation before doing ANY writing tasks. +# It works in 3 simple steps: +# 1. Query the single-index for all existing data, aggregate by month. +# 2. Parse this result into yyyy-MM format, store this +# 3. For each month, query the single-index to get only data for that month, and call the re-index api to store +# only that subset of data into the monthly index of the form page-monthly-yyyy-MM. + +# !!DISCLAIMER!!: before you run this, you SHOULD configure the indices for effecient indexing: diable refresh, and disable replicas. +# This script does NOT take care of the above. You need to do manually. Remember to re-enable these settings, once this script is done. + + +CLUSTERNAME="$1" +SRC_INDEX="pages" +DEST_INDEX_PREFIX="pages-monthly-" + +# Step 1,2: +OLD_DATES=`curl -s '-HContent-Type: application/json' -XGET -d ' +{ + "size": 0, + "query": { + "match_all": {} + } + , "aggs": { + "by_day": { + "date_histogram": { + "field": "date", + "interval": "1M", + "min_doc_count": 10 + } + } + } +} +' "http://$CLUSTERNAME:9200/$SRC_INDEX/_search?pretty" | egrep 'key_as_string' | cut -d':' -f2 | cut -d'"' -f2 | cut -d'-' -f1,2` +echo "Months found:" +echo "${OLD_DATES}" + +echo +echo -n "if this make sence, Y to continue N to exit [Y/N]:" +read INPUT +if [ "$INPUT" == "Y" ] || [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] || [ "$INPUT" == "YES" ] +then + # Step 3: + for index in "$OLD_DATES"; do + echo "Working on month: $index" + DEST_INDEX="$DEST_INDEX_PREFIX$index" + + curl -s '-HContent-Type: application/json' -XPOST -d ' + { + "conflicts": "proceed", + "source": { + "index": "'$SRC_INDEX'", + "query": { + "range": { + "date": { + "gte": "'$index'", + "lt": "'$index'||+1M", + "format": "yyyy-MM" + } + } + } + }, + "dest": { + "index": "'$DEST_INDEX'" + } + } + ' "http://$CLUSTERNAME:9200/_reindex" + echo "Done indexing $DEST_INDEX" + done +else + echo "SCRIPT CLOSED BY USER, BYE ..." + exit +fi diff --git a/vault-merge.sh b/vault-merge.sh new file mode 100755 index 0000000..507a04d --- /dev/null +++ b/vault-merge.sh @@ -0,0 +1,112 @@ +#!/bin/sh +# vault-merge + +# This script handles conflicts generated by attempts to merge encrypted +# Ansible Vault files. +# Run `git merge` as usual; when git warns of a merge conflict, +# run this command to attempt a merge on the unencrypted versions of +# the file. If there are conflicts, you will be given a chance to correct them +# in $EDITOR (vim). + +# Usage: +# $./vault-merge.sh -p <path/to/the/vault/KEY> <path/to/conflicted/vault> + +# First, we ensure we ensure that a suitable e$EDITOR is set +export EDITOR=vim + +# Next, we ensure we are inside the working directory of a git repo. + +GIT_ROOT=`git rev-parse --show-toplevel` +if [ $? != 0 ]; then + exit $? +fi + +# Next, we set a default location for a vault password file, and allow the user +# to override it if desired. + +VAULT_PASSWORD_FILE="$GIT_ROOT/vaults/dev-vault-id" + +while getopts "p:" opt; do + case $opt in + p) + VAULT_PASSWORD_FILE=$OPTARG + ;; + \?) + # Invalid option (e.g., -p without an argument) + exit 1 + ;; + esac +done +shift $(($OPTIND - 1)) + +VAULT_OPT="--vault-password-file=$VAULT_PASSWORD_FILE" +VAULT_FILE=$1 + +# If no vault has been provided, abort! + +if [ -z $VAULT_FILE ]; then + echo "Usage: $0 [-p PASSWORD_FILE] VAULT_FILE" + exit 1 +fi + +# If the password file doesn't exist, we prompt for the password and save it. + +if [ ! -e $VAULT_PASSWORD_FILE ]; then + read -s -p "Vault Password: " VAULT_PASSWORD + echo + echo "Remembering password in $VAULT_PASSWORD_FILE" + echo $VAULT_PASSWORD > $VAULT_PASSWORD_FILE +else + echo "Using password saved in $VAULT_PASSWORD_FILE" +fi + +# Fetch the base (common ancestor) version of the encrypted vault file, save +# it to a temporary location, and decrypt it. (Hat Tip to the git-merge manual +# page for tipping me off to the `git show :1:path` notation.) + +BASE=`mktemp ${VAULT_FILE}.base.XXXX` +git show :1:${VAULT_FILE} > $BASE 2> /dev/null +if [ $? != 0 ]; then + echo "Path '${VAULT_FILE}' does not have any conflicts." + rm $BASE + exit 1 +fi +ansible-vault decrypt $VAULT_OPT $BASE || exit $? + +# Do the same with the current (branch we are merging INTO) version of the vault +# file. + +CURRENT=`mktemp ${VAULT_FILE}.current.XXXX` +git show :2:${VAULT_FILE} > $CURRENT 2> /dev/null +ansible-vault decrypt $VAULT_OPT $CURRENT || exit $? + +# And finally, with the other (branch we a merging FROM) version of the vault. + +OTHER=`mktemp ${VAULT_FILE}.other.XXXX` +git show :3:${VAULT_FILE} > $OTHER 2> /dev/null +ansible-vault decrypt $VAULT_OPT $OTHER || exit $? + +# Now that we have all three versions decrypted, ask git to attempt the merge +# again. If it fails again due to a conflict, open $EDITOR and let the user +# perform a manual merge. + +git merge-file $CURRENT $BASE $OTHER +if [ $? == 0 ]; then + echo "Merge OK" +else + echo "Merge conflict; opening editor to resolve." + $EDITOR $CURRENT +fi + +# Now that we're done, encrypt the file and move it into the repo, and clean up +# the temporary files (they contain secrets!). + +ansible-vault encrypt $VAULT_OPT $CURRENT +cp $CURRENT $VAULT_FILE +rm $BASE $CURRENT $OTHER + +echo "$VAULT_FILE has been updated." +echo " (use \"git add $VAULT_FILE\" to mark as resolved)" +echo " (or re-run this command to retry the merge)" +exit 0 + |