#!/usr/bin/env bash # Author: Marcel Herrguth, Claude Code set -euo pipefail usage() { cat < -U -p [-k] -u Elasticsearch URL (default: https://127.0.0.1:9200) -U Username (default: elastic) -p Password (required, or set ES_PASSWORD env var) -k Allow insecure TLS (skip cert verification) EOF exit 1 } url="https://127.0.0.1:9200" user="elastic" password="${ES_PASSWORD:-}" insecure=() while getopts "u:U:p:kh" opt; do case "$opt" in u) url="$OPTARG" ;; U) user="$OPTARG" ;; p) password="$OPTARG" ;; k) insecure=(--insecure) ;; h|*) usage ;; esac done if [[ -z "$password" ]]; then echo "Password required (-p or ES_PASSWORD)" usage fi CURL=(curl "${insecure[@]}" -s -u "${user}:${password}") fetch_settings_mappings() { # $1 = index to read from settings=$("${CURL[@]}" "${url}/$1/_settings" | jq ".[\"$1\"].settings.index | del(.uuid, .creation_date, .version, .provided_name, .resize, .blocks)") mappings=$("${CURL[@]}" "${url}/$1/_mapping" | jq ".[\"$1\"].mappings") } create_index() { # $1 = index to create, uses $settings/$mappings from fetch_settings_mappings body=$(jq -n --argjson settings "$settings" --argjson mappings "$mappings" '{settings: {index: $settings}, mappings: $mappings}') "${CURL[@]}" -X PUT "${url}/$1" -H 'Content-Type: application/json' -d "$body" >/dev/null } reindex() { # $1 = source, $2 = dest; prints the summary line, sets $reindex_ok result=$("${CURL[@]}" -X POST "${url}/_reindex" -H 'Content-Type: application/json' -d "{\"source\":{\"index\":\"$1\"},\"dest\":{\"index\":\"$2\"}}") echo "$result" | jq -c '{took, total, created, failures: (.failures | length)}' failure_count=$(echo "$result" | jq '(.failures // []) | length') has_error=$(echo "$result" | jq 'has("error")') if [[ "$failure_count" != "0" || "$has_error" == "true" ]]; then reindex_ok=false else reindex_ok=true fi } indices=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.index_settings // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .key') failed=() if [[ -z "$indices" ]]; then echo "No plain indices require reindexing." else indices_csv=$(echo "$indices" | paste -sd, -) sizes=$("${CURL[@]}" "${url}/_cat/indices/${indices_csv}?h=index,store.size&bytes=b") echo "Indices to rebuild in place:" total_bytes=0 max_bytes=0 while read -r name bytes; do [[ -n "$name" ]] || continue human=$(numfmt --to=iec --suffix=B "$bytes" 2>/dev/null || echo "${bytes}B") printf ' - %-60s %s\n' "$name" "$human" total_bytes=$(( total_bytes + bytes )) (( bytes > max_bytes )) && max_bytes=$bytes done <<< "$sizes" echo echo "Total on-disk size across all listed indices: $(numfmt --to=iec --suffix=B "$total_bytes")" echo "Each index is rebuilt one at a time, so the extra headroom you actually need is roughly the size" echo "of the single largest index below (briefly held twice during its own rebuild), not the sum of all:" echo " largest single index: $(numfmt --to=iec --suffix=B "$max_bytes")" echo read -rp "Proceed with index rebuild? Each index is rebuilt under its original name (no aliases left behind). [y/N] " ok if [[ "$ok" == "y" ]]; then for idx in $indices; do tmp="${idx}-tmp-reindex" echo "=== ${idx}: rebuilding via ${tmp} ===" fetch_settings_mappings "$idx" "${CURL[@]}" -X PUT "${url}/${idx}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": true}' >/dev/null create_index "$tmp" reindex "$idx" "$tmp" if [[ "$reindex_ok" != "true" ]]; then echo "!!! first-hop reindex failed for ${idx} — leaving original untouched, removing ${tmp}" "${CURL[@]}" -X DELETE "${url}/${tmp}" >/dev/null "${CURL[@]}" -X PUT "${url}/${idx}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": false}' >/dev/null failed+=("$idx (first hop)") continue fi "${CURL[@]}" -X DELETE "${url}/${idx}" >/dev/null create_index "$idx" reindex "$tmp" "$idx" if [[ "$reindex_ok" != "true" ]]; then echo "!!! second-hop reindex failed for ${idx} — data is safe in ${tmp}, NOT deleting it. ${idx} may be partially populated." failed+=("$idx (second hop — recover from ${tmp})") continue fi "${CURL[@]}" -X DELETE "${url}/${tmp}" >/dev/null echo "=== done: ${idx} rebuilt (real index, no alias) ===" done else echo "Skipping index rebuild." fi fi migrate_data_stream() { local ds="$1" echo "=== data stream: ${ds} ===" start_resp=$("${CURL[@]}" -X POST "${url}/_migration/reindex" -H 'Content-Type: application/json' -d "{\"source\":{\"index\":\"${ds}\"},\"mode\":\"upgrade\"}") if echo "$start_resp" | jq -e 'has("error")' >/dev/null; then echo "!!! failed to start migration for data stream ${ds}:" echo "$start_resp" | jq -c . ds_failed+=("$ds") return fi while true; do status=$("${CURL[@]}" "${url}/_migration/reindex/${ds}/_status") complete=$(echo "$status" | jq -r '.complete // false') successes=$(echo "$status" | jq -r '.successes // 0') total=$(echo "$status" | jq -r '.total_indices_requiring_upgrade // 0') pending=$(echo "$status" | jq -r '.pending // 0') echo " progress: ${successes}/${total} backing indices upgraded, ${pending} pending" [[ "$complete" == "true" ]] && break sleep 5 done err_count=$(echo "$status" | jq '(.errors // []) | length') if [[ "$err_count" != "0" ]]; then echo "!!! data stream ${ds} finished with errors:" echo "$status" | jq -c '.errors' ds_failed+=("$ds") else echo "=== done: ${ds} (${successes}/${total} backing indices upgraded, history preserved) ===" fi } data_streams=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.data_streams // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .key') ds_failed=() if [[ -n "$data_streams" ]]; then backing_indices=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.data_streams // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .value[]._meta.indices_requiring_upgrade[]') backing_csv=$(echo "$backing_indices" | paste -sd, -) ds_sizes=$("${CURL[@]}" "${url}/_cat/indices/${backing_csv}?h=index,store.size&bytes=b") echo echo "Data streams to migrate (native _migration/reindex, no data discarded):" ds_total_bytes=0 ds_max_bytes=0 while read -r name bytes; do [[ -n "$name" ]] || continue human=$(numfmt --to=iec --suffix=B "$bytes" 2>/dev/null || echo "${bytes}B") printf ' - %-60s %s\n' "$name" "$human" ds_total_bytes=$(( ds_total_bytes + bytes )) (( bytes > ds_max_bytes )) && ds_max_bytes=$bytes done <<< "$ds_sizes" echo " ($(echo "$data_streams" | wc -l | tr -d ' ') data stream(s), backing indices totaling $(numfmt --to=iec --suffix=B "$ds_total_bytes"), largest single backing index $(numfmt --to=iec --suffix=B "$ds_max_bytes"))" echo read -rp "Migrate these data streams in place? [y/N] " ds_ok if [[ "$ds_ok" == "y" ]]; then for ds in $data_streams; do migrate_data_stream "$ds" done else echo "Skipping data stream migration." fi fi if (( ${#failed[@]} > 0 || ${#ds_failed[@]} > 0 )); then echo echo "Finished with failures:" printf ' - %s\n' "${failed[@]}" "${ds_failed[@]}" exit 1 fi