Migrating Elasticsearch from 7 to 9

1-requiired-preperation-commands.sh · 1.5 KiB · Bash Raw

# Ensure dependencies
apt update && apt install unzip jq -y

# The below part is relevant, if you have Elasticsearch running with HTTP
# Create certificate directory
mkdir -p /etc/elasticsearch/certs/

# Generate a CA certificate for Elasticsearch
/usr/share/elasticsearch/bin/elasticsearch-certutil ca

# Generate a certificate package for Elasticsearchs internal transport (not facing to users)
/usr/share/elasticsearch/bin/elasticsearch-certutil cert --ca /etc/elasticsearch/certs/elastic-stack-ca.p12

# Generate a HTTP certificate for Elasticsearch (user facing)
# Either as CSR to sign with your own Certificate Authority -or- self-signed from the above CA file
/usr/share/elasticsearch/bin/elasticsearch-certutil http

# Safely store the certificate file passwords to Elasticsearch, so it can open the files
# Transport
/usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password
/usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password
# HTTP
/usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.keystore.secure_password
/usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.truststore.secure_password

# Generate passwords for Elasticsearch
# (automatic)
/usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto

# (you define them manually)
/usr/share/elasticsearch/bin/elasticsearch-setup-passwords interactive

1	# Ensure dependencies
2	apt update && apt install unzip jq -y
3
4	# The below part is relevant, if you have Elasticsearch running with HTTP
5	# Create certificate directory
6	mkdir -p /etc/elasticsearch/certs/
7
8	# Generate a CA certificate for Elasticsearch
9	/usr/share/elasticsearch/bin/elasticsearch-certutil ca
10
11	# Generate a certificate package for Elasticsearchs internal transport (not facing to users)
12	/usr/share/elasticsearch/bin/elasticsearch-certutil cert --ca /etc/elasticsearch/certs/elastic-stack-ca.p12
13
14	# Generate a HTTP certificate for Elasticsearch (user facing)
15	# Either as CSR to sign with your own Certificate Authority -or- self-signed from the above CA file
16	/usr/share/elasticsearch/bin/elasticsearch-certutil http
17
18	# Safely store the certificate file passwords to Elasticsearch, so it can open the files
19	# Transport
20	/usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password
21	/usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password
22	# HTTP
23	/usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.keystore.secure_password
24	/usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.truststore.secure_password
25
26	# Generate passwords for Elasticsearch
27	# (automatic)
28	/usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto
29
30	# (you define them manually)
31	/usr/share/elasticsearch/bin/elasticsearch-setup-passwords interactive

2-needed-commands-prior-upgrading.sh · 1.3 KiB · Bash Raw

# Check for upcoming deprecations you have to adjust before ugprading
curl --insecure -uelastic:${es_password} https://127.0.0.1:9200/_migration/deprecations| jq

# Check if indexes need migration
curl --insecure -uelastic:${es_password} https://127.0.0.1:9200/_migration/system_features| jq

# Migrate endpoints, if they say "MIGRATION NEEDED" or "ERROR"
curl --insecure -uelastic:${es_password} -XPOST https://127.0.0.1:9200/_migration/system_features | jq

# Remove ingest-attachment plugin (Elasticsearch has that on board starting with version 8)
/usr/share/elasticsearch/bin/elasticsearch-plugin remove ingest-attachment

# Switch repository source from Elasticsearch 7 to 8
rm /etc/apt/sources.list.d/elastic-7.x.list
curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | \
  gpg --dearmor | tee /usr/share/keyrings/elasticsearch-keyring.gpg \
  && chmod 644 /usr/share/keyrings/elasticsearch-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-8.x.list

# Switch repository source from Elasticsearch 8 to 9
rm /etc/apt/sources.list.d/elastic-8.x.list
echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/9.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-9.x.list

1	# Check for upcoming deprecations you have to adjust before ugprading
2	curl --insecure -uelastic:${es_password} https://127.0.0.1:9200/_migration/deprecations\| jq
3
4	# Check if indexes need migration
5	curl --insecure -uelastic:${es_password} https://127.0.0.1:9200/_migration/system_features\| jq
6
7	# Migrate endpoints, if they say "MIGRATION NEEDED" or "ERROR"
8	curl --insecure -uelastic:${es_password} -XPOST https://127.0.0.1:9200/_migration/system_features \| jq
9
10	# Remove ingest-attachment plugin (Elasticsearch has that on board starting with version 8)
11	/usr/share/elasticsearch/bin/elasticsearch-plugin remove ingest-attachment
12
13	# Switch repository source from Elasticsearch 7 to 8
14	rm /etc/apt/sources.list.d/elastic-7.x.list
15	curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch \| \
16	gpg --dearmor \| tee /usr/share/keyrings/elasticsearch-keyring.gpg \
17	&& chmod 644 /usr/share/keyrings/elasticsearch-keyring.gpg
18	echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" \| tee /etc/apt/sources.list.d/elastic-8.x.list
19
20	# Switch repository source from Elasticsearch 8 to 9
21	rm /etc/apt/sources.list.d/elastic-8.x.list
22	echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/9.x/apt stable main" \| tee /etc/apt/sources.list.d/elastic-9.x.list

3-re-attach-zammad.sh · 660 B · Bash Raw

1	# Switch Elasticsearch to HTTPs
2	zammad run rails r "Setting.set('es_url', 'https://localhost:9200')"
3
4	# Set Username and Password for Elasticsearch access
5	zammad run rails r "Setting.set('es_user', '<username>')"
6	zammad run rails r "Setting.set('es_password', '<password>')"
7
8	# Disable SSL verification for Elasticsearch
9	zammad run rails r "Setting.set('es_ssl_verify', false)"
10
11	# -OR- provide Zammad with the certificate for proper SSL verification
12	openssl pkcs12 -in /etc/elasticsearch/certs/elastic-stack-ca.p12 -nokeys -out - \
13	\| zammad run rails r 'SSLCertificate.create!(certificate: STDIN.read)'
14	zammad run rails r "Setting.set('es_ssl_verify', true)"

es-reindex-outdated.sh · 7.3 KiB · Bash Raw

#!/usr/bin/env bash
# Author: Marcel Herrguth, Claude Code
set -euo pipefail

usage() {
  cat <<EOF
Usage: $0 -u <url> -U <user> -p <password> [-k]

-u  Elasticsearch URL (default: https://127.0.0.1:9200)
  -U  Username          (default: elastic)
  -p  Password          (required, or set ES_PASSWORD env var)
  -k  Allow insecure TLS (skip cert verification)
EOF
  exit 1
}

url="https://127.0.0.1:9200"
user="elastic"
password="${ES_PASSWORD:-}"
insecure=()

while getopts "u:U:p:kh" opt; do
  case "$opt" in
    u) url="$OPTARG" ;;
    U) user="$OPTARG" ;;
    p) password="$OPTARG" ;;
    k) insecure=(--insecure) ;;
    h|*) usage ;;
  esac
done

if [[ -z "$password" ]]; then
  echo "Password required (-p or ES_PASSWORD)"
  usage
fi

CURL=(curl "${insecure[@]}" -s -u "${user}:${password}")

fetch_settings_mappings() {
  # $1 = index to read from
  settings=$("${CURL[@]}" "${url}/$1/_settings" | jq ".[\"$1\"].settings.index | del(.uuid, .creation_date, .version, .provided_name, .resize, .blocks)")
  mappings=$("${CURL[@]}" "${url}/$1/_mapping" | jq ".[\"$1\"].mappings")
}

create_index() {
  # $1 = index to create, uses $settings/$mappings from fetch_settings_mappings
  body=$(jq -n --argjson settings "$settings" --argjson mappings "$mappings" '{settings: {index: $settings}, mappings: $mappings}')
  "${CURL[@]}" -X PUT "${url}/$1" -H 'Content-Type: application/json' -d "$body" >/dev/null
}

reindex() {
  # $1 = source, $2 = dest; prints the summary line, sets $reindex_ok
  result=$("${CURL[@]}" -X POST "${url}/_reindex" -H 'Content-Type: application/json' -d "{\"source\":{\"index\":\"$1\"},\"dest\":{\"index\":\"$2\"}}")
  echo "$result" | jq -c '{took, total, created, failures: (.failures | length)}'

failure_count=$(echo "$result" | jq '(.failures // []) | length')
  has_error=$(echo "$result" | jq 'has("error")')
  if [[ "$failure_count" != "0" || "$has_error" == "true" ]]; then
    reindex_ok=false
  else
    reindex_ok=true
  fi
}

indices=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.index_settings // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .key')

failed=()

if [[ -z "$indices" ]]; then
  echo "No plain indices require reindexing."
else
  indices_csv=$(echo "$indices" | paste -sd, -)
  sizes=$("${CURL[@]}" "${url}/_cat/indices/${indices_csv}?h=index,store.size&bytes=b")

echo "Indices to rebuild in place:"
  total_bytes=0
  max_bytes=0
  while read -r name bytes; do
    [[ -n "$name" ]] || continue
    human=$(numfmt --to=iec --suffix=B "$bytes" 2>/dev/null || echo "${bytes}B")
    printf '  - %-60s %s\n' "$name" "$human"
    total_bytes=$(( total_bytes + bytes ))
    (( bytes > max_bytes )) && max_bytes=$bytes
  done <<< "$sizes"

echo
  echo "Total on-disk size across all listed indices: $(numfmt --to=iec --suffix=B "$total_bytes")"
  echo "Each index is rebuilt one at a time, so the extra headroom you actually need is roughly the size"
  echo "of the single largest index below (briefly held twice during its own rebuild), not the sum of all:"
  echo "  largest single index: $(numfmt --to=iec --suffix=B "$max_bytes")"
  echo

read -rp "Proceed with index rebuild? Each index is rebuilt under its original name (no aliases left behind). [y/N] " ok
  if [[ "$ok" == "y" ]]; then
    for idx in $indices; do
      tmp="${idx}-tmp-reindex"
      echo "=== ${idx}: rebuilding via ${tmp} ==="

fetch_settings_mappings "$idx"
      "${CURL[@]}" -X PUT "${url}/${idx}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": true}' >/dev/null

create_index "$tmp"
      reindex "$idx" "$tmp"
      if [[ "$reindex_ok" != "true" ]]; then
        echo "!!! first-hop reindex failed for ${idx} — leaving original untouched, removing ${tmp}"
        "${CURL[@]}" -X DELETE "${url}/${tmp}" >/dev/null
        "${CURL[@]}" -X PUT "${url}/${idx}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": false}' >/dev/null
        failed+=("$idx (first hop)")
        continue
      fi

"${CURL[@]}" -X DELETE "${url}/${idx}" >/dev/null
      create_index "$idx"
      reindex "$tmp" "$idx"
      if [[ "$reindex_ok" != "true" ]]; then
        echo "!!! second-hop reindex failed for ${idx} — data is safe in ${tmp}, NOT deleting it. ${idx} may be partially populated."
        failed+=("$idx (second hop — recover from ${tmp})")
        continue
      fi

"${CURL[@]}" -X DELETE "${url}/${tmp}" >/dev/null
      echo "=== done: ${idx} rebuilt (real index, no alias) ==="
    done
  else
    echo "Skipping index rebuild."
  fi
fi

migrate_data_stream() {
  local ds="$1"
  echo "=== data stream: ${ds} ==="

start_resp=$("${CURL[@]}" -X POST "${url}/_migration/reindex" -H 'Content-Type: application/json' -d "{\"source\":{\"index\":\"${ds}\"},\"mode\":\"upgrade\"}")
  if echo "$start_resp" | jq -e 'has("error")' >/dev/null; then
    echo "!!! failed to start migration for data stream ${ds}:"
    echo "$start_resp" | jq -c .
    ds_failed+=("$ds")
    return
  fi

while true; do
    status=$("${CURL[@]}" "${url}/_migration/reindex/${ds}/_status")
    complete=$(echo "$status" | jq -r '.complete // false')
    successes=$(echo "$status" | jq -r '.successes // 0')
    total=$(echo "$status" | jq -r '.total_indices_requiring_upgrade // 0')
    pending=$(echo "$status" | jq -r '.pending // 0')
    echo "  progress: ${successes}/${total} backing indices upgraded, ${pending} pending"
    [[ "$complete" == "true" ]] && break
    sleep 5
  done

err_count=$(echo "$status" | jq '(.errors // []) | length')
  if [[ "$err_count" != "0" ]]; then
    echo "!!! data stream ${ds} finished with errors:"
    echo "$status" | jq -c '.errors'
    ds_failed+=("$ds")
  else
    echo "=== done: ${ds} (${successes}/${total} backing indices upgraded, history preserved) ==="
  fi
}

data_streams=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.data_streams // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .key')

ds_failed=()

if [[ -n "$data_streams" ]]; then
  backing_indices=$("${CURL[@]}" "${url}/_migration/deprecations" | jq -r '(.data_streams // {}) | to_entries[] | select(.value[]?._meta.reindex_required == true) | .value[]._meta.indices_requiring_upgrade[]')
  backing_csv=$(echo "$backing_indices" | paste -sd, -)
  ds_sizes=$("${CURL[@]}" "${url}/_cat/indices/${backing_csv}?h=index,store.size&bytes=b")

echo
  echo "Data streams to migrate (native _migration/reindex, no data discarded):"
  ds_total_bytes=0
  ds_max_bytes=0
  while read -r name bytes; do
    [[ -n "$name" ]] || continue
    human=$(numfmt --to=iec --suffix=B "$bytes" 2>/dev/null || echo "${bytes}B")
    printf '  - %-60s %s\n' "$name" "$human"
    ds_total_bytes=$(( ds_total_bytes + bytes ))
    (( bytes > ds_max_bytes )) && ds_max_bytes=$bytes
  done <<< "$ds_sizes"
  echo "  ($(echo "$data_streams" | wc -l | tr -d ' ') data stream(s), backing indices totaling $(numfmt --to=iec --suffix=B "$ds_total_bytes"), largest single backing index $(numfmt --to=iec --suffix=B "$ds_max_bytes"))"
  echo

read -rp "Migrate these data streams in place? [y/N] " ds_ok
  if [[ "$ds_ok" == "y" ]]; then
    for ds in $data_streams; do
      migrate_data_stream "$ds"
    done
  else
    echo "Skipping data stream migration."
  fi
fi

if (( ${#failed[@]} > 0 || ${#ds_failed[@]} > 0 )); then
  echo
  echo "Finished with failures:"
  printf '  - %s\n' "${failed[@]}" "${ds_failed[@]}"
  exit 1
fi

1	#!/usr/bin/env bash
2	# Author: Marcel Herrguth, Claude Code
3	set -euo pipefail
4
5	usage() {
6	cat <<EOF
7	Usage: $0 -u <url> -U <user> -p <password> [-k]
8
9	-u Elasticsearch URL (default: https://127.0.0.1:9200)
10	-U Username (default: elastic)
11	-p Password (required, or set ES_PASSWORD env var)
12	-k Allow insecure TLS (skip cert verification)
13	EOF
14	exit 1
15	}
16
17	url="https://127.0.0.1:9200"
18	user="elastic"
19	password="${ES_PASSWORD:-}"
20	insecure=()
21
22	while getopts "u:U:p:kh" opt; do
23	case "$opt" in
24	u) url="$OPTARG" ;;
25	U) user="$OPTARG" ;;
26	p) password="$OPTARG" ;;
27	k) insecure=(--insecure) ;;
28	h\|*) usage ;;
29	esac
30	done
31
32	if [[ -z "$password" ]]; then
33	echo "Password required (-p or ES_PASSWORD)"
34	usage
35	fi
36
37	CURL=(curl "${insecure[@]}" -s -u "${user}:${password}")
38
39	fetch_settings_mappings() {
40	# $1 = index to read from
41	settings=$("${CURL[@]}" "${url}/$1/_settings" \| jq ".[\"$1\"].settings.index \| del(.uuid, .creation_date, .version, .provided_name, .resize, .blocks)")
42	mappings=$("${CURL[@]}" "${url}/$1/_mapping" \| jq ".[\"$1\"].mappings")
43	}
44
45	create_index() {
46	# $1 = index to create, uses $settings/$mappings from fetch_settings_mappings
47	body=$(jq -n --argjson settings "$settings" --argjson mappings "$mappings" '{settings: {index: $settings}, mappings: $mappings}')
48	"${CURL[@]}" -X PUT "${url}/$1" -H 'Content-Type: application/json' -d "$body" >/dev/null
49	}
50
51	reindex() {
52	# $1 = source, $2 = dest; prints the summary line, sets $reindex_ok
53	result=$("${CURL[@]}" -X POST "${url}/_reindex" -H 'Content-Type: application/json' -d "{\"source\":{\"index\":\"$1\"},\"dest\":{\"index\":\"$2\"}}")
54	echo "$result" \| jq -c '{took, total, created, failures: (.failures \| length)}'
55
56	failure_count=$(echo "$result" \| jq '(.failures // []) \| length')
57	has_error=$(echo "$result" \| jq 'has("error")')
58	if [[ "$failure_count" != "0" \|\| "$has_error" == "true" ]]; then
59	reindex_ok=false
60	else
61	reindex_ok=true
62	fi
63	}
64
65	indices=$("${CURL[@]}" "${url}/_migration/deprecations" \| jq -r '(.index_settings // {}) \| to_entries[] \| select(.value[]?._meta.reindex_required == true) \| .key')
66
67	failed=()
68
69	if [[ -z "$indices" ]]; then
70	echo "No plain indices require reindexing."
71	else
72	indices_csv=$(echo "$indices" \| paste -sd, -)
73	sizes=$("${CURL[@]}" "${url}/_cat/indices/${indices_csv}?h=index,store.size&bytes=b")
74
75	echo "Indices to rebuild in place:"
76	total_bytes=0
77	max_bytes=0
78	while read -r name bytes; do
79	[[ -n "$name" ]] \|\| continue
80	human=$(numfmt --to=iec --suffix=B "$bytes" 2>/dev/null \|\| echo "${bytes}B")
81	printf ' - %-60s %s\n' "$name" "$human"
82	total_bytes=$(( total_bytes + bytes ))
83	(( bytes > max_bytes )) && max_bytes=$bytes
84	done <<< "$sizes"
85
86	echo
87	echo "Total on-disk size across all listed indices: $(numfmt --to=iec --suffix=B "$total_bytes")"
88	echo "Each index is rebuilt one at a time, so the extra headroom you actually need is roughly the size"
89	echo "of the single largest index below (briefly held twice during its own rebuild), not the sum of all:"
90	echo " largest single index: $(numfmt --to=iec --suffix=B "$max_bytes")"
91	echo
92
93	read -rp "Proceed with index rebuild? Each index is rebuilt under its original name (no aliases left behind). [y/N] " ok
94	if [[ "$ok" == "y" ]]; then
95	for idx in $indices; do
96	tmp="${idx}-tmp-reindex"
97	echo "=== ${idx}: rebuilding via ${tmp} ==="
98
99	fetch_settings_mappings "$idx"
100	"${CURL[@]}" -X PUT "${url}/${idx}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": true}' >/dev/null
101
102	create_index "$tmp"
103	reindex "$idx" "$tmp"
104	if [[ "$reindex_ok" != "true" ]]; then
105	echo "!!! first-hop reindex failed for ${idx} — leaving original untouched, removing ${tmp}"
106	"${CURL[@]}" -X DELETE "${url}/${tmp}" >/dev/null
107	"${CURL[@]}" -X PUT "${url}/${idx}/_settings" -H 'Content-Type: application/json' -d '{"index.blocks.write": false}' >/dev/null
108	failed+=("$idx (first hop)")
109	continue
110	fi
111
112	"${CURL[@]}" -X DELETE "${url}/${idx}" >/dev/null
113	create_index "$idx"
114	reindex "$tmp" "$idx"
115	if [[ "$reindex_ok" != "true" ]]; then
116	echo "!!! second-hop reindex failed for ${idx} — data is safe in ${tmp}, NOT deleting it. ${idx} may be partially populated."
117	failed+=("$idx (second hop — recover from ${tmp})")
118	continue
119	fi
120
121	"${CURL[@]}" -X DELETE "${url}/${tmp}" >/dev/null
122	echo "=== done: ${idx} rebuilt (real index, no alias) ==="
123	done
124	else
125	echo "Skipping index rebuild."
126	fi
127	fi
128
129	migrate_data_stream() {
130	local ds="$1"
131	echo "=== data stream: ${ds} ==="
132
133	start_resp=$("${CURL[@]}" -X POST "${url}/_migration/reindex" -H 'Content-Type: application/json' -d "{\"source\":{\"index\":\"${ds}\"},\"mode\":\"upgrade\"}")
134	if echo "$start_resp" \| jq -e 'has("error")' >/dev/null; then
135	echo "!!! failed to start migration for data stream ${ds}:"
136	echo "$start_resp" \| jq -c .
137	ds_failed+=("$ds")
138	return
139	fi
140
141	while true; do
142	status=$("${CURL[@]}" "${url}/_migration/reindex/${ds}/_status")
143	complete=$(echo "$status" \| jq -r '.complete // false')
144	successes=$(echo "$status" \| jq -r '.successes // 0')
145	total=$(echo "$status" \| jq -r '.total_indices_requiring_upgrade // 0')
146	pending=$(echo "$status" \| jq -r '.pending // 0')
147	echo " progress: ${successes}/${total} backing indices upgraded, ${pending} pending"
148	[[ "$complete" == "true" ]] && break
149	sleep 5
150	done
151
152	err_count=$(echo "$status" \| jq '(.errors // []) \| length')
153	if [[ "$err_count" != "0" ]]; then
154	echo "!!! data stream ${ds} finished with errors:"
155	echo "$status" \| jq -c '.errors'
156	ds_failed+=("$ds")
157	else
158	echo "=== done: ${ds} (${successes}/${total} backing indices upgraded, history preserved) ==="
159	fi
160	}
161
162	data_streams=$("${CURL[@]}" "${url}/_migration/deprecations" \| jq -r '(.data_streams // {}) \| to_entries[] \| select(.value[]?._meta.reindex_required == true) \| .key')
163
164	ds_failed=()
165
166	if [[ -n "$data_streams" ]]; then
167	backing_indices=$("${CURL[@]}" "${url}/_migration/deprecations" \| jq -r '(.data_streams // {}) \| to_entries[] \| select(.value[]?._meta.reindex_required == true) \| .value[]._meta.indices_requiring_upgrade[]')
168	backing_csv=$(echo "$backing_indices" \| paste -sd, -)
169	ds_sizes=$("${CURL[@]}" "${url}/_cat/indices/${backing_csv}?h=index,store.size&bytes=b")
170
171	echo
172	echo "Data streams to migrate (native _migration/reindex, no data discarded):"
173	ds_total_bytes=0
174	ds_max_bytes=0
175	while read -r name bytes; do
176	[[ -n "$name" ]] \|\| continue
177	human=$(numfmt --to=iec --suffix=B "$bytes" 2>/dev/null \|\| echo "${bytes}B")
178	printf ' - %-60s %s\n' "$name" "$human"
179	ds_total_bytes=$(( ds_total_bytes + bytes ))
180	(( bytes > ds_max_bytes )) && ds_max_bytes=$bytes
181	done <<< "$ds_sizes"
182	echo " ($(echo "$data_streams" \| wc -l \| tr -d ' ') data stream(s), backing indices totaling $(numfmt --to=iec --suffix=B "$ds_total_bytes"), largest single backing index $(numfmt --to=iec --suffix=B "$ds_max_bytes"))"
183	echo
184
185	read -rp "Migrate these data streams in place? [y/N] " ds_ok
186	if [[ "$ds_ok" == "y" ]]; then
187	for ds in $data_streams; do
188	migrate_data_stream "$ds"
189	done
190	else
191	echo "Skipping data stream migration."
192	fi
193	fi
194
195	if (( ${#failed[@]} > 0 \|\| ${#ds_failed[@]} > 0 )); then
196	echo
197	echo "Finished with failures:"
198	printf ' - %s\n' "${failed[@]}" "${ds_failed[@]}"
199	exit 1
200	fi
201

es-user-mgm.sh · 3.7 KiB · Bash Raw

#!/usr/bin/env bash
# Author: Marcel Herrguth, Claude Code
set -euo pipefail

usage() {
  cat <<EOF
Usage: $0 <action> [options]

Actions:
  create-ro   Create a read-only user (and a matching role)
  create-rw   Create a read-write user (and a matching role)
  remove      Remove a single named user (refuses default/reserved users)

Options:
  -u <url>       Elasticsearch URL      (default: https://127.0.0.1:9200)
  -U <user>      Admin username to authenticate with (default: elastic)
  -p <password>  Admin password         (required, or set ES_PASSWORD env var)
  -k             Allow insecure TLS (skip cert verification)
  -n <name>      Username                        (required for all actions)
  -w <password>  New user's password             (required for create-ro / create-rw, or set ES_NEW_PASSWORD)
  -i <pattern>   Restrict access to indices matching "<pattern>*" (default: "*", all indices)

Examples:
  $0 create-ro -U elastic -p "\$ES_PASSWORD" -n zammad_reader -w "\$NEW_PW" -i zammad_production -k
  $0 create-rw -U elastic -p "\$ES_PASSWORD" -n zammad_writer -w "\$NEW_PW" -i zammad_production -k
  $0 remove -U elastic -p "\$ES_PASSWORD" -n zammad_reader -k
EOF
  exit 1
}

[[ $# -ge 1 ]] || usage
action="$1"; shift

url="https://127.0.0.1:9200"
admin_user="elastic"
admin_password="${ES_PASSWORD:-}"
insecure=()
new_user=""
new_password="${ES_NEW_PASSWORD:-}"
index_pattern="*"

while getopts "u:U:p:kn:w:i:h" opt; do
  case "$opt" in
    u) url="$OPTARG" ;;
    U) admin_user="$OPTARG" ;;
    p) admin_password="$OPTARG" ;;
    k) insecure=(--insecure) ;;
    n) new_user="$OPTARG" ;;
    w) new_password="$OPTARG" ;;
    i) index_pattern="${OPTARG}*" ;;
    h|*) usage ;;
  esac
done

[[ -n "$admin_password" ]] || { echo "Admin password required (-p or ES_PASSWORD)"; usage; }

CURL=(curl "${insecure[@]}" -s -u "${admin_user}:${admin_password}")

create_user() {
  local role_name="$1" privileges_json="$2"

[[ -n "$new_user" ]] || { echo "New username required (-n)"; usage; }
  [[ -n "$new_password" ]] || { echo "New user's password required (-w or ES_NEW_PASSWORD)"; usage; }

role_body=$(jq -n --argjson privileges "$privileges_json" --arg pattern "$index_pattern" \
    '{indices: [{names: [$pattern], privileges: $privileges}]}')
  role_resp=$("${CURL[@]}" -X PUT "${url}/_security/role/${role_name}" -H 'Content-Type: application/json' -d "$role_body")
  echo "role ${role_name}: $(echo "$role_resp" | jq -c .)"

user_body=$(jq -n --arg password "$new_password" --arg role "$role_name" \
    '{password: $password, roles: [$role]}')
  user_resp=$("${CURL[@]}" -X PUT "${url}/_security/user/${new_user}" -H 'Content-Type: application/json' -d "$user_body")
  echo "user ${new_user}: $(echo "$user_resp" | jq -c .)"
}

case "$action" in
  create-ro)
    create_user "${new_user}_ro" '["read","view_index_metadata"]'
    echo "Created read-only user '${new_user}', scoped to indices matching '${index_pattern}'."
    ;;

create-rw)
    create_user "${new_user}_rw" '["read","write","create_index","view_index_metadata"]'
    echo "Created read-write user '${new_user}', scoped to indices matching '${index_pattern}'."
    ;;

remove)
    [[ -n "$new_user" ]] || { echo "Username required (-n)"; usage; }

lookup=$("${CURL[@]}" "${url}/_security/user/${new_user}")
    if echo "$lookup" | jq -e 'has("error")' >/dev/null; then
      echo "User '${new_user}' not found."
      exit 1
    fi

if echo "$lookup" | jq -e --arg u "$new_user" '.[$u].metadata._reserved == true' >/dev/null; then
      echo "'${new_user}' is a default/reserved user — refusing to delete."
      exit 1
    fi

read -rp "Delete user '${new_user}'? [y/N] " ok
    [[ "$ok" == "y" ]] || exit 1
    "${CURL[@]}" -X DELETE "${url}/_security/user/${new_user}" | jq -c .
    ;;

*)
    usage
    ;;
esac

1	#!/usr/bin/env bash
2	# Author: Marcel Herrguth, Claude Code
3	set -euo pipefail
4
5	usage() {
6	cat <<EOF
7	Usage: $0 <action> [options]
8
9	Actions:
10	create-ro Create a read-only user (and a matching role)
11	create-rw Create a read-write user (and a matching role)
12	remove Remove a single named user (refuses default/reserved users)
13
14	Options:
15	-u <url> Elasticsearch URL (default: https://127.0.0.1:9200)
16	-U <user> Admin username to authenticate with (default: elastic)
17	-p <password> Admin password (required, or set ES_PASSWORD env var)
18	-k Allow insecure TLS (skip cert verification)
19	-n <name> Username (required for all actions)
20	-w <password> New user's password (required for create-ro / create-rw, or set ES_NEW_PASSWORD)
21	-i <pattern> Restrict access to indices matching "<pattern>" (default: "", all indices)
22
23	Examples:
24	$0 create-ro -U elastic -p "\$ES_PASSWORD" -n zammad_reader -w "\$NEW_PW" -i zammad_production -k
25	$0 create-rw -U elastic -p "\$ES_PASSWORD" -n zammad_writer -w "\$NEW_PW" -i zammad_production -k
26	$0 remove -U elastic -p "\$ES_PASSWORD" -n zammad_reader -k
27	EOF
28	exit 1
29	}
30
31	[[ $# -ge 1 ]] \|\| usage
32	action="$1"; shift
33
34	url="https://127.0.0.1:9200"
35	admin_user="elastic"
36	admin_password="${ES_PASSWORD:-}"
37	insecure=()
38	new_user=""
39	new_password="${ES_NEW_PASSWORD:-}"
40	index_pattern="*"
41
42	while getopts "u:U:p:kn:w:i:h" opt; do
43	case "$opt" in
44	u) url="$OPTARG" ;;
45	U) admin_user="$OPTARG" ;;
46	p) admin_password="$OPTARG" ;;
47	k) insecure=(--insecure) ;;
48	n) new_user="$OPTARG" ;;
49	w) new_password="$OPTARG" ;;
50	i) index_pattern="${OPTARG}*" ;;
51	h\|*) usage ;;
52	esac
53	done
54
55	[[ -n "$admin_password" ]] \|\| { echo "Admin password required (-p or ES_PASSWORD)"; usage; }
56
57	CURL=(curl "${insecure[@]}" -s -u "${admin_user}:${admin_password}")
58
59	create_user() {
60	local role_name="$1" privileges_json="$2"
61
62	[[ -n "$new_user" ]] \|\| { echo "New username required (-n)"; usage; }
63	[[ -n "$new_password" ]] \|\| { echo "New user's password required (-w or ES_NEW_PASSWORD)"; usage; }
64
65	role_body=$(jq -n --argjson privileges "$privileges_json" --arg pattern "$index_pattern" \
66	'{indices: [{names: [$pattern], privileges: $privileges}]}')
67	role_resp=$("${CURL[@]}" -X PUT "${url}/_security/role/${role_name}" -H 'Content-Type: application/json' -d "$role_body")
68	echo "role ${role_name}: $(echo "$role_resp" \| jq -c .)"
69
70	user_body=$(jq -n --arg password "$new_password" --arg role "$role_name" \
71	'{password: $password, roles: [$role]}')
72	user_resp=$("${CURL[@]}" -X PUT "${url}/_security/user/${new_user}" -H 'Content-Type: application/json' -d "$user_body")
73	echo "user ${new_user}: $(echo "$user_resp" \| jq -c .)"
74	}
75
76	case "$action" in
77	create-ro)
78	create_user "${new_user}_ro" '["read","view_index_metadata"]'
79	echo "Created read-only user '${new_user}', scoped to indices matching '${index_pattern}'."
80	;;
81
82	create-rw)
83	create_user "${new_user}_rw" '["read","write","create_index","view_index_metadata"]'
84	echo "Created read-write user '${new_user}', scoped to indices matching '${index_pattern}'."
85	;;
86
87	remove)
88	[[ -n "$new_user" ]] \|\| { echo "Username required (-n)"; usage; }
89
90	lookup=$("${CURL[@]}" "${url}/_security/user/${new_user}")
91	if echo "$lookup" \| jq -e 'has("error")' >/dev/null; then
92	echo "User '${new_user}' not found."
93	exit 1
94	fi
95
96	if echo "$lookup" \| jq -e --arg u "$new_user" '.[$u].metadata._reserved == true' >/dev/null; then
97	echo "'${new_user}' is a default/reserved user — refusing to delete."
98	exit 1
99	fi
100
101	read -rp "Delete user '${new_user}'? [y/N] " ok
102	[[ "$ok" == "y" ]] \|\| exit 1
103	"${CURL[@]}" -X DELETE "${url}/_security/user/${new_user}" \| jq -c .
104	;;
105
106	*)
107	usage
108	;;
109	esac
110

zzz-elasticsearch.yml · 465 B · YAML Raw

1	# What we ultimatively add to the Elasticsearch configuration during the upgrades
2	# xpack
3	xpack.security.enabled: true
4	xpack.security.transport.ssl.enabled: true
5	xpack.security.transport.ssl.verification_mode: certificate
6	xpack.security.transport.ssl.keystore.path: certs/elastic-certificates.p12
7	xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12
8	xpack.security.http.ssl.enabled: true
9	xpack.security.http.ssl.keystore.path: certs/http.p12