| 1 | #!/usr/bin/env bash
|
| 2 | # Author: Marcel Herrguth, Claude Code
|
| 3 | set -euo pipefail
|
| 4 |
|
| 5 | usage() {
|
| 6 | cat <<EOF
|
| 7 | Usage: $0 -u <url> -U <user> -p <password> [-k]
|
| 8 |
|
| 9 | -u Elasticsearch URL (default: https://127.0.0.1:9200)
|
| 10 | -U Username (default: elastic)
|
| 11 | -p Password (required, or set ES_PASSWORD env var)
|
| 12 | -k Allow insecure TLS (skip cert verification)
|
| 13 | EOF
|
| 14 | exit 1
|
| 15 | }
|
| 16 |
|
| 17 | url="https://127.0.0.1:9200"
|
| 18 | user="elastic"
|
| 19 | password="${ES_PASSWORD:-}"
|
| 20 | insecure=()
|
| 21 |
|
| 22 | while getopts "u:U:p:kh" opt; do
|
| 23 | case "$opt" in
|
| 24 | u) url="$OPTARG" ;;
|
| 25 | U) user="$OPTARG" ;;
|
| 26 | p) password="$OPTARG" ;;
|
| 27 | k) insecure=(--insecure) ;;
|
| 28 | h|*) usage ;;
|
| 29 | esac
|
| 30 | done
|
| 31 |
|
| 32 | if [[ -z "$password" ]]; then
|
| 33 | echo "Password required (-p or ES_PASSWORD)"
|
| 34 | usage
|
| 35 | fi
|
| 36 |
|
| 37 | CURL=(curl "${insecure[@]}" -s -u "${user}:${password}")
|
| 38 |
|
| 39 | fetch_settings_mappings() {
|
| 40 | # $1 = index to read from
|
| 41 | settings=$("${CURL[@]}" "${url}/$1/_settings" | jq ".[\"$1\"].settings.index | del(.uuid, .creation_date, .version, .provided_name, .resize, .blocks)")
|
| 42 | mappings=$("${CURL[@]}" "${url}/$1/_mapping" | jq ".[\"$1\"].mappings")
|
| 43 | }
|
| 44 |
|
| 45 | create_index() {
|
| 46 | # $1 = index to create, uses $settings/$mappings from fetch_settings_mappings
|
| 47 | body=$(jq -n --argjson settings "$settings" --argjson mappings "$mappings" '{settings: {index: $settings}, mappings: $mappings}')
|
| 48 | "${CURL[@]}" -X PUT "${url}/$1" -H 'Content-Type: application/json' -d "$body" >/dev/null
|
| 49 | }
|
| 50 |
|
| 51 | reindex() {
|
| 52 | # $1 = source, $2 = dest; prints the summary line, sets $reindex_ok
|
| 53 | result=$("${CURL[@]}" -X POST "${url}/_reindex" -H 'Content-Type: application/json' -d "{\"source\":{\"index\":\"$1\"},\"dest\":{\"index\":\"$2\"}}")
|
| 54 | echo "$result" | jq -c '{took, total, created, failures: (.failures | length)}'
|
| 55 |
|
| 56 | failure_count=$(echo "$result" | jq '(.failures // []) | length')
|
| 57 | has_error=$(echo "$result" | jq 'has("error")')
|
| 58 | if [[ "$failure_count" != "0" || "$has_error" == "true" ]]; then
|
| 59 | reindex_ok=false
|
| 60 | else
|
| 61 | reindex_ok=true
|
| 62 | fi
|
| 63 | }
|
| 64 |
|
| 65 | indices=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.index_settings // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .key')
|
| 66 |
|
| 67 | failed=()
|
| 68 |
|
| 69 | if [[ -z "$indices" ]]; then
|
| 70 | echo "No plain indices require reindexing."
|
| 71 | else
|
| 72 | indices_csv=$(echo "$indices" | paste -sd, -)
|
| 73 | sizes=$("${CURL[@]}" "${url}/_cat/indices/${indices_csv}?h=index,store.size&bytes=b")
|
| 74 |
|
| 75 | echo "Indices to rebuild in place:"
|
| 76 | total_bytes=0
|
| 77 | max_bytes=0
|
| 78 | while read -r name bytes; do
|
| 79 | [[ -n "$name" ]] || continue
|
| 80 | human=$(numfmt --to=iec --suffix=B "$bytes" 2>/dev/null || echo "${bytes}B")
|
| 81 | printf ' - %-60s %s\n' "$name" "$human"
|
| 82 | total_bytes=$(( total_bytes + bytes ))
|
| 83 | (( bytes > max_bytes )) && max_bytes=$bytes
|
| 84 | done <<< "$sizes"
|
| 85 |
|
| 86 | echo
|
| 87 | echo "Total on-disk size across all listed indices: $(numfmt --to=iec --suffix=B "$total_bytes")"
|
| 88 | echo "Each index is rebuilt one at a time, so the extra headroom you actually need is roughly the size"
|
| 89 | echo "of the single largest index below (briefly held twice during its own rebuild), not the sum of all:"
|
| 90 | echo " largest single index: $(numfmt --to=iec --suffix=B "$max_bytes")"
|
| 91 | echo
|
| 92 |
|
| 93 | read -rp "Proceed with index rebuild? Each index is rebuilt under its original name (no aliases left behind). [y/N] " ok
|
| 94 | if [[ "$ok" == "y" ]]; then
|
| 95 | for idx in $indices; do
|
| 96 | tmp="${idx}-tmp-reindex"
|
| 97 | echo "=== ${idx}: rebuilding via ${tmp} ==="
|
| 98 |
|
| 99 | fetch_settings_mappings "$idx"
|
| 100 | "${CURL[@]}" -X PUT "${url}/${idx}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": true}' >/dev/null
|
| 101 |
|
| 102 | create_index "$tmp"
|
| 103 | reindex "$idx" "$tmp"
|
| 104 | if [[ "$reindex_ok" != "true" ]]; then
|
| 105 | echo "!!! first-hop reindex failed for ${idx} — leaving original untouched, removing ${tmp}"
|
| 106 | "${CURL[@]}" -X DELETE "${url}/${tmp}" >/dev/null
|
| 107 | "${CURL[@]}" -X PUT "${url}/${idx}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": false}' >/dev/null
|
| 108 | failed+=("$idx (first hop)")
|
| 109 | continue
|
| 110 | fi
|
| 111 |
|
| 112 | "${CURL[@]}" -X DELETE "${url}/${idx}" >/dev/null
|
| 113 | create_index "$idx"
|
| 114 | reindex "$tmp" "$idx"
|
| 115 | if [[ "$reindex_ok" != "true" ]]; then
|
| 116 | echo "!!! second-hop reindex failed for ${idx} — data is safe in ${tmp}, NOT deleting it. ${idx} may be partially populated."
|
| 117 | failed+=("$idx (second hop — recover from ${tmp})")
|
| 118 | continue
|
| 119 | fi
|
| 120 |
|
| 121 | "${CURL[@]}" -X DELETE "${url}/${tmp}" >/dev/null
|
| 122 | echo "=== done: ${idx} rebuilt (real index, no alias) ==="
|
| 123 | done
|
| 124 | else
|
| 125 | echo "Skipping index rebuild."
|
| 126 | fi
|
| 127 | fi
|
| 128 |
|
| 129 | migrate_data_stream() {
|
| 130 | local ds="$1"
|
| 131 | echo "=== data stream: ${ds} ==="
|
| 132 |
|
| 133 | start_resp=$("${CURL[@]}" -X POST "${url}/_migration/reindex" -H 'Content-Type: application/json' -d "{\"source\":{\"index\":\"${ds}\"},\"mode\":\"upgrade\"}")
|
| 134 | if echo "$start_resp" | jq -e 'has("error")' >/dev/null; then
|
| 135 | echo "!!! failed to start migration for data stream ${ds}:"
|
| 136 | echo "$start_resp" | jq -c .
|
| 137 | ds_failed+=("$ds")
|
| 138 | return
|
| 139 | fi
|
| 140 |
|
| 141 | while true; do
|
| 142 | status=$("${CURL[@]}" "${url}/_migration/reindex/${ds}/_status")
|
| 143 | complete=$(echo "$status" | jq -r '.complete // false')
|
| 144 | successes=$(echo "$status" | jq -r '.successes // 0')
|
| 145 | total=$(echo "$status" | jq -r '.total_indices_requiring_upgrade // 0')
|
| 146 | pending=$(echo "$status" | jq -r '.pending // 0')
|
| 147 | echo " progress: ${successes}/${total} backing indices upgraded, ${pending} pending"
|
| 148 | [[ "$complete" == "true" ]] && break
|
| 149 | sleep 5
|
| 150 | done
|
| 151 |
|
| 152 | err_count=$(echo "$status" | jq '(.errors // []) | length')
|
| 153 | if [[ "$err_count" != "0" ]]; then
|
| 154 | echo "!!! data stream ${ds} finished with errors:"
|
| 155 | echo "$status" | jq -c '.errors'
|
| 156 | ds_failed+=("$ds")
|
| 157 | else
|
| 158 | echo "=== done: ${ds} (${successes}/${total} backing indices upgraded, history preserved) ==="
|
| 159 | fi
|
| 160 | }
|
| 161 |
|
| 162 | data_streams=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.data_streams // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .key')
|
| 163 |
|
| 164 | ds_failed=()
|
| 165 |
|
| 166 | if [[ -n "$data_streams" ]]; then
|
| 167 | backing_indices=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.data_streams // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .value[]._meta.indices_requiring_upgrade[]')
|
| 168 | backing_csv=$(echo "$backing_indices" | paste -sd, -)
|
| 169 | ds_sizes=$("${CURL[@]}" "${url}/_cat/indices/${backing_csv}?h=index,store.size&bytes=b")
|
| 170 |
|
| 171 | echo
|
| 172 | echo "Data streams to migrate (native _migration/reindex, no data discarded):"
|
| 173 | ds_total_bytes=0
|
| 174 | ds_max_bytes=0
|
| 175 | while read -r name bytes; do
|
| 176 | [[ -n "$name" ]] || continue
|
| 177 | human=$(numfmt --to=iec --suffix=B "$bytes" 2>/dev/null || echo "${bytes}B")
|
| 178 | printf ' - %-60s %s\n' "$name" "$human"
|
| 179 | ds_total_bytes=$(( ds_total_bytes + bytes ))
|
| 180 | (( bytes > ds_max_bytes )) && ds_max_bytes=$bytes
|
| 181 | done <<< "$ds_sizes"
|
| 182 | echo " ($(echo "$data_streams" | wc -l | tr -d ' ') data stream(s), backing indices totaling $(numfmt --to=iec --suffix=B "$ds_total_bytes"), largest single backing index $(numfmt --to=iec --suffix=B "$ds_max_bytes"))"
|
| 183 | echo
|
| 184 |
|
| 185 | read -rp "Migrate these data streams in place? [y/N] " ds_ok
|
| 186 | if [[ "$ds_ok" == "y" ]]; then
|
| 187 | for ds in $data_streams; do
|
| 188 | migrate_data_stream "$ds"
|
| 189 | done
|
| 190 | else
|
| 191 | echo "Skipping data stream migration."
|
| 192 | fi
|
| 193 | fi
|
| 194 |
|
| 195 | sf_failed=()
|
| 196 |
|
| 197 | before_sf=$("${CURL[@]}" "${url}/_migration/system_features")
|
| 198 | sf_status=$(echo "$before_sf" | jq -r '.migration_status')
|
| 199 |
|
| 200 | if [[ "$sf_status" == "NO_MIGRATION_NEEDED" ]]; then
|
| 201 | echo
|
| 202 | echo "No system feature migration needed."
|
| 203 | else
|
| 204 | echo
|
| 205 | echo "System features requiring migration:"
|
| 206 | echo "$before_sf" | jq -r '.features[] | select(.migration_status=="MIGRATION_NEEDED") | " - \(.feature_name): \(.indices | map(.index) | join(", "))"'
|
| 207 | echo
|
| 208 |
|
| 209 | read -rp "Trigger system feature migration (POST /_migration/system_features)? [y/N] " sf_ok
|
| 210 | if [[ "$sf_ok" == "y" ]]; then
|
| 211 | before_map=$(echo "$before_sf" | jq -c '[.features[] | {feature: .feature_name, indices: [.indices[].index]}]')
|
| 212 |
|
| 213 | post_resp=$("${CURL[@]}" -X POST "${url}/_migration/system_features")
|
| 214 | echo "$post_resp" | jq -c .
|
| 215 |
|
| 216 | if echo "$post_resp" | jq -e '.accepted == false' >/dev/null; then
|
| 217 | echo "!!! system feature migration was not accepted"
|
| 218 | sf_failed+=("system_features (not accepted)")
|
| 219 | else
|
| 220 | after_sf=""
|
| 221 | while true; do
|
| 222 | after_sf=$("${CURL[@]}" "${url}/_migration/system_features")
|
| 223 | top=$(echo "$after_sf" | jq -r '.migration_status')
|
| 224 | echo " status: ${top}"
|
| 225 | if [[ "$top" == "ERROR" ]]; then
|
| 226 | echo "!!! system feature migration reported ERROR:"
|
| 227 | echo "$after_sf" | jq -c '.features[] | select(.migration_status=="ERROR")'
|
| 228 | sf_failed+=("system_features (error)")
|
| 229 | break
|
| 230 | fi
|
| 231 | [[ "$top" == "NO_MIGRATION_NEEDED" ]] && break
|
| 232 | sleep 5
|
| 233 | done
|
| 234 |
|
| 235 | if [[ "$top" == "NO_MIGRATION_NEEDED" ]]; then
|
| 236 | after_map=$(echo "$after_sf" | jq -c '[.features[] | {feature: .feature_name, indices: [.indices[].index]}]')
|
| 237 |
|
| 238 | orphans=$(jq -n --argjson before "$before_map" --argjson after "$after_map" '
|
| 239 | ($after | map({(.feature): .indices}) | add) as $afterByFeature |
|
| 240 | [ $before[] | . as $b |
|
| 241 | ($afterByFeature[$b.feature] // []) as $a |
|
| 242 | select(($a | length) > 0) |
|
| 243 | ($b.indices - $a)[]
|
| 244 | ]')
|
| 245 |
|
| 246 | echo "Old pre-migration indices confirmed superseded (locking read-only, not deleting):"
|
| 247 | for old in $(echo "$orphans" | jq -r '.[]'); do
|
| 248 | exists=$("${CURL[@]}" -o /dev/null -w '%{http_code}' "${url}/${old}")
|
| 249 | if [[ "$exists" == "200" ]]; then
|
| 250 | "${CURL[@]}" -X PUT "${url}/${old}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": true}' >/dev/null
|
| 251 | echo " - ${old}: locked"
|
| 252 | fi
|
| 253 | done
|
| 254 | fi
|
| 255 | fi
|
| 256 | else
|
| 257 | echo "Skipping system feature migration."
|
| 258 | fi
|
| 259 | fi
|
| 260 |
|
| 261 | if (( ${#failed[@]} > 0 || ${#ds_failed[@]} > 0 || ${#sf_failed[@]} > 0 )); then
|
| 262 | echo
|
| 263 | echo "Finished with failures:"
|
| 264 | printf ' - %s\n' "${failed[@]}" "${ds_failed[@]}" "${sf_failed[@]}"
|
| 265 | exit 1
|
| 266 | fi
|
| 267 |
|
