First attempt at a reboot script for the ubc arm cluster

[mirror/dsa-puppet.git] / modules / ganeti2 / files / ganeti-reboot-cluster

diff --git a/modules/ganeti2/files/ganeti-reboot-cluster b/modules/ganeti2/files/ganeti-reboot-cluster
index bc95de3..86b6dfd 100755 (executable)
--- a/modules/ganeti2/files/ganeti-reboot-cluster
+++ b/modules/ganeti2/files/ganeti-reboot-cluster
@@ -1,5 +1,29 @@
 #!/bin/bash
 
+# reboot a ganeti cluster, making sure instances are moved around before and after
+
+# Copyright 2018, 2019 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 set -e
 set -o pipefail
 set -u
@@ -12,6 +36,17 @@ error_usage() {
   usage >&2
   exit 1
 }
+do_cleanup() {
+    local cnt
+    cnt=$((${#cleanup[*]}-1))
+    for i in $(seq ${cnt} -1 0); do
+        ${cleanup[$i]} || true
+    done
+}
+declare -a cleanup
+cleanup+=(":")
+trap do_cleanup EXIT
+
 
 nodelist="node-list"
 newmaster=""
@@ -150,10 +185,6 @@ moveupdown() {
   ${print_list} "$nodelist" | (
     read tgt dummy
     while read src dummy; do
-      if has_instances "$tgt"; then
-        echo "$tgt not empty."
-        exit 1
-      fi
       reboot_host "$tgt"
 
       if has_instances "$src"; then
@@ -187,13 +218,65 @@ moveupdown() {
   )
 }
 
+crossmigratemany() {
+  me=$(hostname -f)
+  if ! grep -q --line-regexp --fixed-strings  "$me" "$nodelist"; then
+    echo >&2 "my hostname ($me) not found in nodelist"
+    exit 1
+  fi
+
+  # move ourselves last
+  newlist="$(tempfile)"
+  cleanup+=("rm -f '$newlist'")
+  grep -v --line-regexp --fixed-strings  "$me" "$nodelist" > "$newlist"
+  echo "$me" >> "$newlist"
+
+  while read node ; do
+    if ! hbal -L -C -v -v --no-disk-moves --offline="$node" -X; then
+      echo >&2 "hbal failed at node $node.  Bailing out."
+      exit 1
+    fi
+    if ! gnt-node migrate -f "$node"; then
+      echo >&2 "gnt-node migrate failed for node $node.  Bailing out."
+      exit 1
+    fi
+    if [ "$node" = "$me" ] ; then
+      break
+    fi
+    reboot_host "$node"
+    # bring back disks
+    echo "Bringing back disks using the watcher"
+    ganeti-watcher
+    # wait for a cron-launched ganeti-watcher to finish
+    while pgrep ganeti-watcher > /dev/null ; do
+      echo -n "."
+      sleep 5
+    done
+    echo
+  done < "$newlist"
+
+  at 'now + 5 min' << 'EOF'
+screen -S hbal -d -m sh -c '
+  echo "Activating disks using the watcher.."
+  ganeti-watcher
+  while pgrep ganeti-watcher > /dev/null ; do
+    sleep 5
+  done
+  hbal -L -C -v -v --no-disk-moves -X
+  echo "done."
+  sleep 1h
+'
+EOF
+  reboot_host "$me"
+}
+
 crossmigrate() {
   me=$(hostname -f)
-  if ! grep -q -F "$me" "$nodelist"; then
+  if ! grep -q --line-regexp --fixed-strings  "$me" "$nodelist"; then
     echo >&2 "my hostname ($me) not found in nodelist"
     exit 1
   fi
-  them="$(grep -v -F "$me" "$nodelist")"
+  them="$(grep -v --line-regexp --fixed-strings  "$me" "$nodelist")"
 
   echo "Migrating from $them to $me."
   if ! gnt-node migrate -f -n "$me" "$them"; then
@@ -283,7 +366,7 @@ fi
 ###################
 if ! [ -e "$nodelist" ]; then
   tmp="$(tempfile)"
-  trap "rm -f '$tmp'" EXIT
+  cleanup+=("rm -f '$tmp'")
   gnt-node list --no-headers -o name > "$tmp"
   nodelist="$tmp"
 fi
@@ -307,6 +390,11 @@ case "$lines" in
   2)
     crossmigrate
     ;;
+  3)
+    echo "WARNING: this is untested.  ^C now if you want to stop"
+    read dummy
+    crossmigratemany
+    ;;
   *)
     moveupdown
     ;;