| 1 | #!/usr/bin/env bash
|
| 2 | set -euo pipefail
|
| 3 |
|
| 4 | usage() {
|
| 5 | cat <<EOF
|
| 6 | Usage: $0 -u <url> -U <user> -p <password> [-k]
|
| 7 |
|
| 8 | -u Elasticsearch URL (default: https://127.0.0.1:9200)
|
| 9 | -U Username (default: elastic)
|
| 10 | -p Password (required, or set ES_PASSWORD env var)
|
| 11 | -k Allow insecure TLS (skip cert verification)
|
| 12 | EOF
|
| 13 | exit 1
|
| 14 | }
|
| 15 |
|
| 16 | url="https://127.0.0.1:9200"
|
| 17 | user="elastic"
|
| 18 | password="${ES_PASSWORD:-}"
|
| 19 | insecure=()
|
| 20 |
|
| 21 | while getopts "u:U:p:kh" opt; do
|
| 22 | case "$opt" in
|
| 23 | u) url="$OPTARG" ;;
|
| 24 | U) user="$OPTARG" ;;
|
| 25 | p) password="$OPTARG" ;;
|
| 26 | k) insecure=(--insecure) ;;
|
| 27 | h|*) usage ;;
|
| 28 | esac
|
| 29 | done
|
| 30 |
|
| 31 | if [[ -z "$password" ]]; then
|
| 32 | echo "Password required (-p or ES_PASSWORD)"
|
| 33 | usage
|
| 34 | fi
|
| 35 |
|
| 36 | CURL=(curl "${insecure[@]}" -s -u "${user}:${password}")
|
| 37 |
|
| 38 | fetch_settings_mappings() {
|
| 39 | # $1 = index to read from
|
| 40 | settings=$("${CURL[@]}" "${url}/$1/_settings" | jq ".[\"$1\"].settings.index | del(.uuid, .creation_date, .version, .provided_name, .resize, .blocks)")
|
| 41 | mappings=$("${CURL[@]}" "${url}/$1/_mapping" | jq ".[\"$1\"].mappings")
|
| 42 | }
|
| 43 |
|
| 44 | create_index() {
|
| 45 | # $1 = index to create, uses $settings/$mappings from fetch_settings_mappings
|
| 46 | body=$(jq -n --argjson settings "$settings" --argjson mappings "$mappings" '{settings: {index: $settings}, mappings: $mappings}')
|
| 47 | "${CURL[@]}" -X PUT "${url}/$1" -H 'Content-Type: application/json' -d "$body" >/dev/null
|
| 48 | }
|
| 49 |
|
| 50 | reindex() {
|
| 51 | # $1 = source, $2 = dest; prints the summary line, sets $reindex_ok
|
| 52 | result=$("${CURL[@]}" -X POST "${url}/_reindex" -H 'Content-Type: application/json' -d "{\"source\":{\"index\":\"$1\"},\"dest\":{\"index\":\"$2\"}}")
|
| 53 | echo "$result" | jq -c '{took, total, created, failures: (.failures | length)}'
|
| 54 |
|
| 55 | failure_count=$(echo "$result" | jq '(.failures // []) | length')
|
| 56 | has_error=$(echo "$result" | jq 'has("error")')
|
| 57 | if [[ "$failure_count" != "0" || "$has_error" == "true" ]]; then
|
| 58 | reindex_ok=false
|
| 59 | else
|
| 60 | reindex_ok=true
|
| 61 | fi
|
| 62 | }
|
| 63 |
|
| 64 | indices=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.index_settings // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .key')
|
| 65 |
|
| 66 | failed=()
|
| 67 |
|
| 68 | if [[ -z "$indices" ]]; then
|
| 69 | echo "No plain indices require reindexing."
|
| 70 | else
|
| 71 | indices_csv=$(echo "$indices" | paste -sd, -)
|
| 72 | sizes=$("${CURL[@]}" "${url}/_cat/indices/${indices_csv}?h=index,store.size&bytes=b")
|
| 73 |
|
| 74 | echo "Indices to rebuild in place:"
|
| 75 | total_bytes=0
|
| 76 | max_bytes=0
|
| 77 | while read -r name bytes; do
|
| 78 | [[ -n "$name" ]] || continue
|
| 79 | human=$(numfmt --to=iec --suffix=B "$bytes" 2>/dev/null || echo "${bytes}B")
|
| 80 | printf ' - %-60s %s\n' "$name" "$human"
|
| 81 | total_bytes=$(( total_bytes + bytes ))
|
| 82 | (( bytes > max_bytes )) && max_bytes=$bytes
|
| 83 | done <<< "$sizes"
|
| 84 |
|
| 85 | echo
|
| 86 | echo "Total on-disk size across all listed indices: $(numfmt --to=iec --suffix=B "$total_bytes")"
|
| 87 | echo "Each index is rebuilt one at a time, so the extra headroom you actually need is roughly the size"
|
| 88 | echo "of the single largest index below (briefly held twice during its own rebuild), not the sum of all:"
|
| 89 | echo " largest single index: $(numfmt --to=iec --suffix=B "$max_bytes")"
|
| 90 | echo
|
| 91 |
|
| 92 | read -rp "Proceed with index rebuild? Each index is rebuilt under its original name (no aliases left behind). [y/N] " ok
|
| 93 | if [[ "$ok" == "y" ]]; then
|
| 94 | for idx in $indices; do
|
| 95 | tmp="${idx}-tmp-reindex"
|
| 96 | echo "=== ${idx}: rebuilding via ${tmp} ==="
|
| 97 |
|
| 98 | fetch_settings_mappings "$idx"
|
| 99 | "${CURL[@]}" -X PUT "${url}/${idx}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": true}' >/dev/null
|
| 100 |
|
| 101 | create_index "$tmp"
|
| 102 | reindex "$idx" "$tmp"
|
| 103 | if [[ "$reindex_ok" != "true" ]]; then
|
| 104 | echo "!!! first-hop reindex failed for ${idx} — leaving original untouched, removing ${tmp}"
|
| 105 | "${CURL[@]}" -X DELETE "${url}/${tmp}" >/dev/null
|
| 106 | "${CURL[@]}" -X PUT "${url}/${idx}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": false}' >/dev/null
|
| 107 | failed+=("$idx (first hop)")
|
| 108 | continue
|
| 109 | fi
|
| 110 |
|
| 111 | "${CURL[@]}" -X DELETE "${url}/${idx}" >/dev/null
|
| 112 | create_index "$idx"
|
| 113 | reindex "$tmp" "$idx"
|
| 114 | if [[ "$reindex_ok" != "true" ]]; then
|
| 115 | echo "!!! second-hop reindex failed for ${idx} — data is safe in ${tmp}, NOT deleting it. ${idx} may be partially populated."
|
| 116 | failed+=("$idx (second hop — recover from ${tmp})")
|
| 117 | continue
|
| 118 | fi
|
| 119 |
|
| 120 | "${CURL[@]}" -X DELETE "${url}/${tmp}" >/dev/null
|
| 121 | echo "=== done: ${idx} rebuilt (real index, no alias) ==="
|
| 122 | done
|
| 123 | else
|
| 124 | echo "Skipping index rebuild."
|
| 125 | fi
|
| 126 | fi
|
| 127 |
|
| 128 | migrate_data_stream() {
|
| 129 | local ds="$1"
|
| 130 | echo "=== data stream: ${ds} ==="
|
| 131 |
|
| 132 | start_resp=$("${CURL[@]}" -X POST "${url}/_migration/reindex" -H 'Content-Type: application/json' -d "{\"source\":{\"index\":\"${ds}\"},\"mode\":\"upgrade\"}")
|
| 133 | if echo "$start_resp" | jq -e 'has("error")' >/dev/null; then
|
| 134 | echo "!!! failed to start migration for data stream ${ds}:"
|
| 135 | echo "$start_resp" | jq -c .
|
| 136 | ds_failed+=("$ds")
|
| 137 | return
|
| 138 | fi
|
| 139 |
|
| 140 | while true; do
|
| 141 | status=$("${CURL[@]}" "${url}/_migration/reindex/${ds}/_status")
|
| 142 | complete=$(echo "$status" | jq -r '.complete // false')
|
| 143 | successes=$(echo "$status" | jq -r '.successes // 0')
|
| 144 | total=$(echo "$status" | jq -r '.total_indices_requiring_upgrade // 0')
|
| 145 | pending=$(echo "$status" | jq -r '.pending // 0')
|
| 146 | echo " progress: ${successes}/${total} backing indices upgraded, ${pending} pending"
|
| 147 | [[ "$complete" == "true" ]] && break
|
| 148 | sleep 5
|
| 149 | done
|
| 150 |
|
| 151 | err_count=$(echo "$status" | jq '(.errors // []) | length')
|
| 152 | if [[ "$err_count" != "0" ]]; then
|
| 153 | echo "!!! data stream ${ds} finished with errors:"
|
| 154 | echo "$status" | jq -c '.errors'
|
| 155 | ds_failed+=("$ds")
|
| 156 | else
|
| 157 | echo "=== done: ${ds} (${successes}/${total} backing indices upgraded, history preserved) ==="
|
| 158 | fi
|
| 159 | }
|
| 160 |
|
| 161 | data_streams=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.data_streams // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .key')
|
| 162 |
|
| 163 | ds_failed=()
|
| 164 |
|
| 165 | if [[ -n "$data_streams" ]]; then
|
| 166 | backing_indices=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.data_streams // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .value[]._meta.indices_requiring_upgrade[]')
|
| 167 | backing_csv=$(echo "$backing_indices" | paste -sd, -)
|
| 168 | ds_sizes=$("${CURL[@]}" "${url}/_cat/indices/${backing_csv}?h=index,store.size&bytes=b")
|
| 169 |
|
| 170 | echo
|
| 171 | echo "Data streams to migrate (native _migration/reindex, no data discarded):"
|
| 172 | ds_total_bytes=0
|
| 173 | ds_max_bytes=0
|
| 174 | while read -r name bytes; do
|
| 175 | [[ -n "$name" ]] || continue
|
| 176 | human=$(numfmt --to=iec --suffix=B "$bytes" 2>/dev/null || echo "${bytes}B")
|
| 177 | printf ' - %-60s %s\n' "$name" "$human"
|
| 178 | ds_total_bytes=$(( ds_total_bytes + bytes ))
|
| 179 | (( bytes > ds_max_bytes )) && ds_max_bytes=$bytes
|
| 180 | done <<< "$ds_sizes"
|
| 181 | echo " ($(echo "$data_streams" | wc -l | tr -d ' ') data stream(s), backing indices totaling $(numfmt --to=iec --suffix=B "$ds_total_bytes"), largest single backing index $(numfmt --to=iec --suffix=B "$ds_max_bytes"))"
|
| 182 | echo
|
| 183 |
|
| 184 | read -rp "Migrate these data streams in place? [y/N] " ds_ok
|
| 185 | if [[ "$ds_ok" == "y" ]]; then
|
| 186 | for ds in $data_streams; do
|
| 187 | migrate_data_stream "$ds"
|
| 188 | done
|
| 189 | else
|
| 190 | echo "Skipping data stream migration."
|
| 191 | fi
|
| 192 | fi
|
| 193 |
|
| 194 | sf_failed=()
|
| 195 |
|
| 196 | before_sf=$("${CURL[@]}" "${url}/_migration/system_features")
|
| 197 | sf_status=$(echo "$before_sf" | jq -r '.migration_status')
|
| 198 |
|
| 199 | if [[ "$sf_status" == "NO_MIGRATION_NEEDED" ]]; then
|
| 200 | echo
|
| 201 | echo "No system feature migration needed."
|
| 202 | else
|
| 203 | echo
|
| 204 | echo "System features requiring migration:"
|
| 205 | echo "$before_sf" | jq -r '.features[] | select(.migration_status=="MIGRATION_NEEDED") | " - \(.feature_name): \(.indices | map(.index) | join(", "))"'
|
| 206 | echo
|
| 207 |
|
| 208 | read -rp "Trigger system feature migration (POST /_migration/system_features)? [y/N] " sf_ok
|
| 209 | if [[ "$sf_ok" == "y" ]]; then
|
| 210 | before_map=$(echo "$before_sf" | jq -c '[.features[] | {feature: .feature_name, indices: [.indices[].index]}]')
|
| 211 |
|
| 212 | post_resp=$("${CURL[@]}" -X POST "${url}/_migration/system_features")
|
| 213 | echo "$post_resp" | jq -c .
|
| 214 |
|
| 215 | if echo "$post_resp" | jq -e '.accepted == false' >/dev/null; then
|
| 216 | echo "!!! system feature migration was not accepted"
|
| 217 | sf_failed+=("system_features (not accepted)")
|
| 218 | else
|
| 219 | after_sf=""
|
| 220 | while true; do
|
| 221 | after_sf=$("${CURL[@]}" "${url}/_migration/system_features")
|
| 222 | top=$(echo "$after_sf" | jq -r '.migration_status')
|
| 223 | echo " status: ${top}"
|
| 224 | if [[ "$top" == "ERROR" ]]; then
|
| 225 | echo "!!! system feature migration reported ERROR:"
|
| 226 | echo "$after_sf" | jq -c '.features[] | select(.migration_status=="ERROR")'
|
| 227 | sf_failed+=("system_features (error)")
|
| 228 | break
|
| 229 | fi
|
| 230 | [[ "$top" == "NO_MIGRATION_NEEDED" ]] && break
|
| 231 | sleep 5
|
| 232 | done
|
| 233 |
|
| 234 | if [[ "$top" == "NO_MIGRATION_NEEDED" ]]; then
|
| 235 | after_map=$(echo "$after_sf" | jq -c '[.features[] | {feature: .feature_name, indices: [.indices[].index]}]')
|
| 236 |
|
| 237 | orphans=$(jq -n --argjson before "$before_map" --argjson after "$after_map" '
|
| 238 | ($after | map({(.feature): .indices}) | add) as $afterByFeature |
|
| 239 | [ $before[] | . as $b |
|
| 240 | ($afterByFeature[$b.feature] // []) as $a |
|
| 241 | select(($a | length) > 0) |
|
| 242 | ($b.indices - $a)[]
|
| 243 | ]')
|
| 244 |
|
| 245 | echo "Old pre-migration indices confirmed superseded (locking read-only, not deleting):"
|
| 246 | for old in $(echo "$orphans" | jq -r '.[]'); do
|
| 247 | exists=$("${CURL[@]}" -o /dev/null -w '%{http_code}' "${url}/${old}")
|
| 248 | if [[ "$exists" == "200" ]]; then
|
| 249 | "${CURL[@]}" -X PUT "${url}/${old}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": true}' >/dev/null
|
| 250 | echo " - ${old}: locked"
|
| 251 | fi
|
| 252 | done
|
| 253 | fi
|
| 254 | fi
|
| 255 | else
|
| 256 | echo "Skipping system feature migration."
|
| 257 | fi
|
| 258 | fi
|
| 259 |
|
| 260 | if (( ${#failed[@]} > 0 || ${#ds_failed[@]} > 0 || ${#sf_failed[@]} > 0 )); then
|
| 261 | echo
|
| 262 | echo "Finished with failures:"
|
| 263 | printf ' - %s\n' "${failed[@]}" "${ds_failed[@]}" "${sf_failed[@]}"
|
| 264 | exit 1
|
| 265 | fi
|
| 266 |
|
