#!/bin/bash
+# reboot a ganeti cluster, making sure instances are moved around before and after
+
+# Copyright 2018, 2019 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
set -e
set -o pipefail
set -u
usage >&2
exit 1
}
+do_cleanup() {
+ local cnt
+ cnt=$((${#cleanup[*]}-1))
+ for i in $(seq ${cnt} -1 0); do
+ ${cleanup[$i]} || true
+ done
+}
+declare -a cleanup
+cleanup+=(":")
+trap do_cleanup EXIT
+
nodelist="node-list"
newmaster=""
${print_list} "$nodelist" | (
read tgt dummy
while read src dummy; do
- if has_instances "$tgt"; then
- echo "$tgt not empty."
- exit 1
- fi
reboot_host "$tgt"
if has_instances "$src"; then
)
}
+crossmigratemany() {
+ me=$(hostname -f)
+ if ! grep -q --line-regexp --fixed-strings "$me" "$nodelist"; then
+ echo >&2 "my hostname ($me) not found in nodelist"
+ exit 1
+ fi
+
+ # move ourselves last
+ newlist="$(tempfile)"
+ cleanup+=("rm -f '$newlist'")
+ grep -v --line-regexp --fixed-strings "$me" "$nodelist" > "$newlist"
+ echo "$me" >> "$newlist"
+
+ while read node ; do
+ if ! hbal -L -C -v -v --no-disk-moves --offline="$node" -X; then
+ echo >&2 "hbal failed at node $node. Bailing out."
+ exit 1
+ fi
+ if ! gnt-node migrate -f "$node"; then
+ echo >&2 "gnt-node migrate failed for node $node. Bailing out."
+ exit 1
+ fi
+ if [ "$node" = "$me" ] ; then
+ break
+ fi
+ reboot_host "$node"
+ # bring back disks
+ echo "Bringing back disks using the watcher"
+ ganeti-watcher
+ # wait for a cron-launched ganeti-watcher to finish
+ while pgrep ganeti-watcher > /dev/null ; do
+ echo -n "."
+ sleep 5
+ done
+ echo
+ done < "$newlist"
+
+ at 'now + 5 min' << 'EOF'
+screen -S hbal -d -m sh -c '
+ echo "Activating disks using the watcher.."
+ ganeti-watcher
+ while pgrep ganeti-watcher > /dev/null ; do
+ sleep 5
+ done
+ hbal -L -C -v -v --no-disk-moves -X
+ echo "done."
+ sleep 1h
+'
+EOF
+ reboot_host "$me"
+}
+
crossmigrate() {
me=$(hostname -f)
- if ! grep -q -F "$me" "$nodelist"; then
+ if ! grep -q --line-regexp --fixed-strings "$me" "$nodelist"; then
echo >&2 "my hostname ($me) not found in nodelist"
exit 1
fi
- them="$(grep -v -F "$me" "$nodelist")"
+ them="$(grep -v --line-regexp --fixed-strings "$me" "$nodelist")"
echo "Migrating from $them to $me."
if ! gnt-node migrate -f -n "$me" "$them"; then
if [ -e /proc/drbd ]; then
echo "Waiting for drbd to be consistent."
sleep 5
- while egrep -C2 --color -i 'iconsistent|finish' /proc/drbd; do
+ while egrep -C2 --color -i 'iconsistent|finish' /proc/drbd || ! /usr/lib/nagios/plugins/dsa-check-drbd -d All ; do
echo "Still waiting.."
sleep 5
done
###################
if ! [ -e "$nodelist" ]; then
tmp="$(tempfile)"
- trap "rm -f '$tmp'" EXIT
+ cleanup+=("rm -f '$tmp'")
gnt-node list --no-headers -o name > "$tmp"
nodelist="$tmp"
fi
2)
crossmigrate
;;
+ 3)
+ echo "WARNING: this is untested. ^C now if you want to stop"
+ read dummy
+ crossmigratemany
+ ;;
*)
moveupdown
;;