#!/usr/bin/perl -w
#
# Copyright (c) 2000-2016, 2018 University of Utah and the Flux Group.
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#
use Getopt::Std;
use English;
use Errno;
use POSIX qw(strftime);

#
# Startup vnodes, if there are any!
#
# NB: This script should not be run in foreground mode on a remote node
# when booting; if boss is down the boot will hang. On local nodes, its
# okay to hang.
#
sub usage()
{
    print STDERR "Usage: bootvnodes [-w sec] [-d] [-f] [-k|-h|-r|-b|-c]\n";
    exit(1);
}
my $optlist = "kdfhrcbw:";

#
# Turn off line buffering on output
#
$| = 1;

# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }

#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself. 
# 
use libsetup;
use libtmcc;
use libtestbed qw(TBBackGround);

# Locals
my $logname = "$LOGDIR/bootvnodes.debug";
my $vndir   = "$VARDIR/jails";
my $debug   = 0;
my $daemon  = 1;
my $reconfig= 0;
my $waittime = 0;
my $action;

# Prototypes
sub prebootvnodes($$);
sub postbootvnodes($$);
sub bootvnode($$$;$);

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
# From rc.bootsetup:
#
# "-f"        first time boot, use vnode list from TMCC
# "-f -h"     shutdown, vnodes that are running
# "-f -c -h"  shutdown during reconfig, vnodes running but not in TMCC list
# "-f -c -b"  reboot during reconfig
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"d"})) {
    $debug  = 1;
}
if (defined($options{"f"})) {
    $daemon = 0;
}
if (defined($options{"r"})) {
    $action = "reboot";
}
if (defined($options{"k"})) {
    $action = "kill";
}
if (defined($options{"h"})) {
    $action = "halt";
}
if (defined($options{"b"})) {
    $action = "boot";
}
if (defined($options{"c"})) {
    $reconfig = 1;
}
if (defined($options{"w"})) {
    if ($options{"w"} !~ /^(\d+)$/) {
	print STDERR "Invalid wait time for -w\n";
	usage();
    }
    $waittime = $1;
}
if (@ARGV) {
    usage();
}

#
# Must be root.
# 
if ($UID != 0) {
    die("*** $0:\n".
	"    Must be root to run this script!\n");
}

# We need to know this below.
my $sysname = `uname -s`;
chomp($sysname);
my $islinux = ($sysname eq "Linux");

# ...
my $isfreenas = -e "/etc/rc.freenas" ? 1 : 0;

# Nodes come and go ...
configtmcc("nocache", 1);

#
# Put this into the background and log its output. We *must* do this cause
# we do not want to halt the boot if the testbed is down!
# 
if ($daemon && TBBackGround($logname)) {
    #
    # Parent exits normally
    #
    exit(0);
}

#
# Shortcut for xen shutdown
#
if (GENVNODE() &&
    GENVNODETYPE() eq "xen" && defined($action) && $action eq "halt") {
    my $toolstack = `grep TOOLSTACK /etc/default/xen`;
    my $XM = "/usr/sbin/xm";
    if ($toolstack =~ /xl$/) {
	$XM = "/usr/sbin/xl";
    }
    system("$XM shutdown -a -w");
    exit($? >> 8);
}

#
# Sort by vnode number if it follows are convention. We need this because
# we don't use a fixed field width for the vnode number.
# XXX I am sure there are more perl-ish ways to do this.
#
sub byvnode {
    if ($a =~ /^(.*)-(\d+)$/ && ($apre = $1) && ($anum = $2) &&
	$b =~ /^(.*)-(\d+)$/ && ($bpre = $1) && ($bnum = $2) &&
	$apre eq $bpre) {
	return $anum <=> $bnum;
    }
    return $a cmp $b;
}

#
# This applies to whatever vnodes are running. Do it and exit.
# 
if (defined($action) && !$reconfig) {
    opendir(DIR, "$vndir") or
	die("Cannot opendir $vndir: $!\n");
    my @files = readdir(DIR);
    closedir(DIR);

    #
    # XXX make a feeble effort to weed out random files.
    #
    # We used to do this by making sure the name was of the form
    # pcvm<foo>-<num>, but not all clusters use the "pc" naming scheme.
    # So now, the best we can do is ensure that the name is like
    # <foo>vm<bar>-<num> and is a directory.
    #
    my @vnfiles = ();
    foreach my $file (@files) {
	if ($file =~ /^([-\w]+vm[-\w]*-\d+)$/ && -d "$vndir/$file") {
	    push(@vnfiles, $1);
	}
    }
    #
    # This is probably true for Xen too, but in some cases, the
    # vnodesetup early-release hackwaitandexit timeout of 30 seconds
    # causes a race condition.  Normally, the first node sets up
    # significant network state, and sometimes flips MAC addresses
    # around from interface to interface -- OR puts a physical interface
    # into a bridge, then changes the bridge's MAC address.  There is a
    # short window of time where both the bridge and the new member
    # interface share a MAC address -- and if the tmcc ifconfig resolves
    # the wrong device's MAC address and uses that to flesh out the
    # ifconfig info, the vnodesetup will be in a world of hurt.  The
    # chance of this happening is miniscule, but I've seen it.
    #
    # So, at least for docker for now, we protect the first vnode against
    # the 30-second timeout in vnodesetup hackwaitandexit.
    #
    my $vht;
    if (GENVNODETYPE() eq 'docker') {
	$vht = 0;
    }
    foreach my $file (sort byvnode @vnfiles) {
	bootvnode($file, $action, (-e "$vndir/$file/fakejail" ? 0 : 1),$vht);
	$vht = undef;
    }
    exit(0);
}

my %newvnodelist = ();
my %curvnodelist = ();
my $fakejails = 0;

#
# Get the current set of vnodes that are supposed to be running on
# this node.
#
my @tmccresults;

if (tmcc(TMCCCMD_VNODELIST, undef, \@tmccresults) < 0) {
    die("*** WARNING: Could not get vnode list from server!\n");
}
foreach my $str (@tmccresults) {
    if ($str =~ /^VNODEID=([-\w]+) JAILED=(\d)$/) {
	$newvnodelist{$1} = $2;
	$fakejails++
	    if ($2 eq "0");
    }
    else {
	warn("*** WARNING: Skipping bad VNODEID: '$str'\n");
    }
}

#
# If doing a reconfig, get the current set of vnodes that are running.
# We want to compare that list against the list we just got from tmcd.
#
if ($reconfig) {
    opendir(DIR, "$vndir") or
	die("Cannot opendir $vndir: $!\n");
    my @files = readdir(DIR);
    closedir(DIR);

    # XXX make a feeble effort to weed out random files.
    my @vnfiles = ();
    foreach my $file (@files) {
	if ($file =~ /^([-\w]+vm[-\w]*-\d+)$/ && -d "$vndir/$file") {
	    push(@vnfiles, $1);
	}
    }
    foreach my $file (sort byvnode @vnfiles) {
	if (-e "$vndir/$file/fakejail") {
	    $fakejails++;
	    $curvnodelist{$file} = 0;
	} else {
	    $curvnodelist{$file} = 1;
	}
    }

    #
    # Reboot nodes that are running and in the current list from tmcd.
    # Kill nodes that are running but not in the current list from tmcd.
    #
    foreach my $vnode (sort byvnode keys(%curvnodelist)) {
	if (exists($newvnodelist{$vnode})) {
	    bootvnode($vnode,
		      (defined($action) ? $action : "reboot"),
		      $newvnodelist{$vnode});
	    delete($newvnodelist{$vnode});
	}
	else {
	    bootvnode($vnode, "kill", $curvnodelist{$vnode});
	}
    }
}
# XXX Exit if specific action requested, and it was not "boot"
exit(0)
    if (defined($action) && $action ne "boot");

# Exit if nothing left to do (no new vnodes).
exit(0)
    if (! scalar(keys(%newvnodelist)));

prebootvnodes(\%curvnodelist, \%newvnodelist)
    if (!$fakejails);

my $vht;
if (GENVNODETYPE() eq 'docker') {
    $vht = 0;
}
foreach my $vnode (sort byvnode keys(%newvnodelist)) {
    # Blocks until mostly setup. 
    bootvnode($vnode, "boot", $newvnodelist{$vnode},$vht);
    $vht = undef;
}

postbootvnodes(\%curvnodelist, \%newvnodelist)
    if (!$fakejails);

exit(0);

#
# Prepare the vnode host prior to individual vnode setup
# Takes references to the current and new vnode hashes.
#
sub prebootvnodes($$)
{
    my ($cvn, $nvn) = @_;

    #
    # XXX OS dependent stuff.  Should move elsewhere.
    #
    if ($islinux || $isfreenas) {
	# This is handled in the libvnode libraries.
	return;
    }

    #
    # For local vnodes, create the extra FS for filesystems and other state.
    # This will fail if it already exists, so just keep going.
    #
    if (!REMOTE()) {
	my $mntdir = $vndir;
	system("mkextrafs.pl $mntdir");
    }

    #
    # Symlink /local into it.
    #
    if (! -e "/local") {
	system("ln -s " . LOCALROOTFS() . " /local");
    }

    #
    # Make sure enough vn devices exist
    #
    for (my $i = 0; $i < scalar(keys(%$nvn)) + scalar(keys(%$cvn)); $i++) {
	my $dev = "vn${i}";
	if (! -e "/dev/${dev}c") {
	    system("(cd /dev; ./MAKEDEV $dev)");
	}
    }

    #
    # XXX tweak IP interrupt queue size to accomodate up to 8 x 100Mb
    # trivial links XXX turn on local hack to retry on NFS EACCES
    # errors (thanks mountd!)
    #
    system("sysctl net.inet.ip.intr_queue_maxlen=128 >/dev/null 2>&1");
    system("sysctl vfs.nfs.eacces_retry_enable=1 >/dev/null 2>&1");

    #
    # XXX grossed out yet?  Try this one: the mount command will HUP mountd
    # after every successful mount.  Thus if mounts in one jail overlap NFS
    # activity in another, the latter may be randomly killed off.
    #
    # Elegant solution: rename the mountd.pid file so mount cannot find it!
    # XXX hmm...mountd_sux hack above should take care of this, and we need
    # to hup mountd when we modify the exports file.
    #
    rename("/var/run/mountd.pid", "/var/run/mountd.thisreallysux")
	if (-e "/var/run/mountd.pid");
}

#
# Final setup of vnode host after individual vnode setup
# Takes references to the current and new vnode hashes.
#
sub postbootvnodes($$)
{
    my ($cvn, $nvn) = @_;

    #
    # XXX OS dependent stuff.  Should move elsewhere
    #
    if ($islinux || $isfreenas) {
	return;
    }

    my $PIDFILE = "/var/run/progagent.pid";

    foreach my $vnode (keys(%$nvn)) {
	system("rtprio 15 -`cat $vndir/$vnode/root/$PIDFILE`")
	    if (-e "$vndir/$vnode/root/$PIDFILE");
    }
}

#
# Helper function to boot/kill/halt/reboot a specific vnode.
#
sub bootvnode($$$;$)
{
    my ($vnode, $action, $jailed, $vnodesetup_hackwaitandexit_timeout) = @_;
    my $opt;
    my $act;
    my $extrawait = $waittime;

    if ($action eq "halt") {
	$opt = "-h";
	$act = "Halting";
    }
    elsif ($action eq "reboot") {
	$opt = "-r";
	$act = "Rebooting";
	## XXX should no longer be needed
	#$extrawait = 20 if (GENVNODETYPE() eq "xen");
    }
    elsif ($action eq "kill") {
	$opt = "-k";
	$act = "Killing";
    }
    else {
	$opt = "-b";
	$act = "Booting";
	## XXX should no longer be needed
	#$extrawait = 20 if (GENVNODETYPE() eq "xen");
    }
    $opt .= ($jailed ? " -jVt" : " -i");
    if (defined($vnodesetup_hackwaitandexit_timeout)) {
	$opt .= " -F $vnodesetup_hackwaitandexit_timeout";
    }
    
    print "$act vnode $vnode with options '$opt' at " .
	libsetup::TBTimeStamp() . "\n";
    system("vnodesetup $opt $vnode");
    return($?)
	if ($?);
    sleep($extrawait)
	if ($extrawait > 0);
    return 0;
}
