#!/bin/sh
# Copyright 2007, 2008, 2009 Michael Creel <michael.creel@uab.es>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
# set this to the network you'd like to use for the cluster

# make_pelican: this script allows you to make a custom version
# of the PelicanHPC live CD image for creation of high performance
# computing clusters. You need to install the live_helper package to use
# it. See http://pareto.uab.es/mcreel/PelicanHPC for more information.

# Version 2.1
# * major simplification of script, which I expect will be
#   more reliable. Uses information from the nice diskless cluster howto
#   at http://blog.signalnine.net/?p=12  Thanks!
# * back to a Lenny base. Too much adventure with squeeze.
# * Octave, Open MPI and openmpi_ext are compiled from source by entering
#   the chroot. This allows use of recent versions while sticking with a
#   Lenny base.
# * add gqview
# * add lbfgs minimizer for Octave: note, use requires citation - see README
# * remove ganglia, slurm, ifenslave. Let's keep it simple, please.
############ packages to add - place names of packages you want here ####################
cat <<PACKAGELIST > pelicanhpc.list
# basic stuff needed for cluster setup
dnsmasq syslinux nfs-kernel-server nfs-common
tftpd-hpa xinetd ssh

# configuration and tools
wget bzip2 dialog less net-tools rsync fping screen
make htop fail2ban locales console-common
gqview vim hal libhal1 libhal-storage1
subversion

# X stuff
xorg xfce4 ksysguard ksysguardd okular
gnuplot

# stuff to  build octave from source
libx11-dev build-essential gfortran texinfo
libatlas-base-dev libarpack2-dev libglpk-dev
libfftw3-dev libsuitesparse-dev libreadline5-dev
libpcre3-dev
PACKAGELIST
################## END OF PACKAGELIST ################

PELICAN_NETWORK="10.11.12"
MAXNODES="100"
ARCHITECTURE="amd64"
KERNEL="amd64"
#ARCHITECTURE="i386"
#KERNEL="686"
IMAGETYPE="iso"
#IMAGETYPE="usb-hdd"
DISTRIBUTION="lenny"
MIRROR="de"

# leave the rest of this alone unless you really know what you're up to
THISDIR="`pwd`"
mkdir "$ARCHITECTURE"
cd "$THISDIR/$ARCHITECTURE"

# frontend configuration
LIVECDDIR="frontend"
mkdir "$THISDIR/$ARCHITECTURE/$LIVECDDIR"
cd "$THISDIR/$ARCHITECTURE/$LIVECDDIR"
lh config \
-p "pelicanhpc.list" \
--apt apt --apt-recommends disabled \
-a "$ARCHITECTURE" \
-b "$IMAGETYPE" \
-d "$DISTRIBUTION" \
-k "$KERNEL" \
--mirror-binary http://ftp.$MIRROR.debian.org/debian/ \
--mirror-chroot http://ftp.$MIRROR.debian.org/debian/ \
--mirror-bootstrap http://ftp.$MIRROR.debian.org/debian/ \
--mirror-binary-security http://security.eu.debian.org/ \
--mirror-chroot-security http://security.eu.debian.org \
--hostname pelican \
--iso-volume PelicanHPC \
--syslinux-timeout 20 \
--syslinux-menu enabled \
--bootappend-live "ramdisk_size=200000 noautologin noxautologin"
lh clean
# SYSLINUX SPLASH SCREEN
if [ -e "$THISDIR/splash.rle" ]; then
  install -d config/binary_syslinux/	
  cp "$THISDIR"/splash.rle config/binary_syslinux/splash.rle
  lh config --syslinux-splash "config/binary_syslinux/splash.rle"
fi
########## make directories on chroot #########
install -d config/chroot_local-includes/usr/bin
install -d config/chroot_local-packagelists
install -d config/chroot_local-packages
install -d config/chroot_local-includes/etc/skel
########## copy stuff to be added to chroot ######
mv "$THISDIR"/pelicanhpc.list config/chroot_local-packageslists/pelicanhpc.list
rsync -az "$THISDIR"/pelicanhome/ config/chroot_local-includes/etc/skel
rsync -az "$THISDIR"/packages/ config/chroot_local-packages

# chroot hook: makes initram fs for netboot, installs some software from source
cat <<CHROOTHOOK > config/chroot_local-hooks/script.sh
#!/bin/bash
update-initramfs -u -k all
install -d /var/lib/tftpboot
cp -a /etc/initramfs-tools/ /etc/initramfs-pxe
sed -i 's/BOOT=local/BOOT=nfs/g' /etc/initramfs-pxe/initramfs.conf 
cd /root
# get, build, and install qrupdate
wget http://downloads.sourceforge.net/project/qrupdate/qrupdate/1.1/qrupdate-1.1.1.tar.gz?use_mirror=ovh
tar xfz qrupdate-1.1.1.tar.gz
cd qrupdate-1.1.1/
make solib
make install
cd ../
rm -R -f qrupdate*
ldconfig
# get, build, and install Octave
wget ftp://ftp.octave.org/pub/octave/octave-3.2.4.tar.bz2
tar xfj octave-3.2.4.tar.bz2
cd octave-3.2.4/
./configure
make -j8
make -j8 # this is not an error - sometimes compilation hangs and a restart is needed
make install-strip
cd ../
rm -R -f octave*
# get, build, and install Open MPI
wget http://www.open-mpi.org/software/ompi/v1.4/downloads/openmpi-1.4.1.tar.bz2
tar xfj openmpi-1.4.1.tar.bz2
cd openmpi-1.4.1/
./configure
make -j8 all
make install
sync
cd ../
rm -R -f openmpi*
ldconfig
# get, build, and install openmpi_ext for Octave
mv /usr/bin/mpiCC /usr/bin/mpiCC.do_not_use # we want to use our Open MPI, not the one Debian installs due to libarpack2 dependency
svn co https://octave.svn.sourceforge.net/svnroot/octave/trunk/octave-forge/main/openmpi_ext openmpi_ext
tar cfz openmpi_ext.tar.gz openmpi_ext/
octave --eval "pkg install openmpi_ext.tar.gz"
rm -R -f ./openmpi_ext*
if [ -d "/etc/skel/Econometrics" ]; then
  LOCATION="/etc/skel/Econometrics"
  ############ my oct files############
  cd \$LOCATION/MyOctaveFiles/OctFiles
  echo "making my .cc files"
  make clean
  make all
  FILES="*.oct"
  for f in "\$FILES"; do strip \$f; done
  ######### PEA ###########
  cd \$LOCATION/Examples/Parallel/pea/example
  mkoctfile *.cc
  FILES="*.oct"
  for f in "\$FILES"; do strip \$f; done
  ########### lbfgsb ###############
  cd \$LOCATION/lbfgs
  octave --eval install_lbfgs
fi
CHROOTHOOK
chmod +x config/chroot_local-hooks/script.sh




######## WRITE THE PELICAN SETUP SCRIPTS #############
cat <<01SETUP > config/chroot_local-includes/usr/bin/pelican_boot_setup
#!/bin/bash
PATH="/bin:/sbin:/usr/bin:/usr/sbin"
export PATH
TMP="/tmp/pelican_setup.tmp\$\$"
DIALOG="dialog"
TITLE="Pelican Setup"
exec >/dev/console </dev/console 2>&1
NORMAL=""
RED=""
GREEN=""
YELLOW=""
BLUE=""
MAGENTA=""
CYAN=""
WHITE=""
bailout(){
  # set user password
  \$DIALOG --title "\$TITLE" --inputbox "Welcome to PelicanHPC. Please type in a password, and then press <enter> to continue" 15 50 "PleaseChangeMe!" 2>/home/user/pw
  chmod 644 /home/user/pw
  read PASSWORD </home/user/pw
  echo "user:\$PASSWORD" | chpasswd
  chown -R user.user /home/user
  echo ""
  echo -n ""
  echo "\${GREEN}Welcome to \${CYAN}PelicanHPC\${GREEN}!"
  echo ""
  echo "To log in, enter \${YELLOW}user\${GREEN} as the username, and the password you just specified."
  echo "After you're logged in, you can:"
  echo ""
  echo "\${BLUE}* create a cluster: type \${YELLOW}pelican_setup\${GREEN}"
  echo "\${BLUE}* enter a desktop environment: type \${YELLOW}startx\${GREEN}"
  echo ""
  echo "For more information, visit http://pelicanhpc.org. Have fun!\${NORMAL}"
  echo ""
  exit \$1
}
bailout2(){
  read PASSWORD </home/user/pw
  echo "user:\$PASSWORD" | /usr/sbin/chpasswd
  echo "All done setting up this PelicanHPC compute node"
  echo ""
  echo -n ""
  echo "\${GREEN}This is a \${CYAN}PelicanHPC \${GREEN}compute node. It is part of a cluster of computers that is doing some \${YELLOW}REALLY important stuff\${GREEN}."
  echo ""
  echo "Please don't try to use it, and \${RED}DON'T TURN IT OFF!"
  echo ""
  echo "\${GREEN}THANKS!\${BLUE}"
  exit \$1
}
echo "Welcome to PelicanHPC!"
# things for both frontend and nodes
echo "" > /etc/motd
echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
# null pointer dereference issue that affects kernels < 2.6.32, see http://wiki.debian.org/mmap_min_addr Remove this in future?
echo "vm.mmap_min_addr = 4096" > /etc/sysctl.d/mmap_min_addr.conf
/etc/init.d/procps restart
# test for master or compute node: $PELICAN_NETWORK will only be there at boot if this is a compute node
computenode=\`dmesg | grep -c nfsroot=$PELICAN_NETWORK\`
if [ \$computenode = 0 ]; then
# master node
echo "making initrd.img for netboot, this will take a little while"
mkinitramfs -d /etc/initramfs-pxe -o /var/lib/tftpboot/initrd.img
rm -R -f /home
mkdir /home
mkfs.ext2 -q -m 0 /dev/ram1
rm -f "\$TMP"
\$DIALOG --title "\$TITLE" --inputbox "Do NOT type anything into this input box, it \
is only for the convenience of advanced users who have experience using PelicanHPC, and who have read the documentation to \
learn what it is used for. Press <enter> to continue" 15 50 "ram1" 2>"\$TMP"
read HOMELOCATION <"\$TMP" ; rm -f "\$TMP"
mount /dev/\$HOMELOCATION /home
install -d /home/user
MESSAGE="Copy user configuration and examples /home/user? You should probably choose YES unless you are using a specially made vesion of PelicanHPC that uses a permanent storage device, and you already have a setup you would like to keep."
\$DIALOG --defaultno --title "\$TITLE" --yesno "\$MESSAGE" 10 50 || bailout
rsync -avz /etc/skel/ /home/user/
mkdir /home/user/backup # for vim backup files
# Start up xfce4
install -d /home/user/.config/autostart
echo "[Desktop Entry]" >> /home/user/.config/autostart/xfce4-tips-autostart.desktop
echo "Hidden=true"  >> /home/user/.config/autostart/xfce4-tips-autostart.desktop
# echo "/usr/bin/startx" >> /home/user/.bash_profile # uncomment to make entry into GUI automatic
# support software compiled using older Open MPI 
ln -s /usr/lib/openmpi/lib/libopen-rte.so.0.0.0 /usr/lib/openmpi/lib/liborte.so.0
ln -s /usr/lib/openmpi/lib/libopen-pal.so.0.0.0 /usr/lib/openmpi/lib/libopal.so.0 
bailout
# for the compute nodes
else
rm -R -f /home
mkdir /home
mount $PELICAN_NETWORK.1:/home /home
myip=(\`/sbin/ifconfig | grep -i "Ethernet" -A 1|grep "inet addr"|cut -d  " " -f 12|cut -d ":" -f 2\`)
echo "\$myip is up" > /home/user/tmp/\$myip
# support software compiled using older Open MPI 
ln -s /usr/lib/openmpi/lib/libopen-rte.so.0.0.0 /usr/lib/openmpi/lib/liborte.so.0
ln -s /usr/lib/openmpi/lib/libopen-pal.so.0.0.0 /usr/lib/openmpi/lib/libopal.so.0 
bailout2
fi
01SETUP

cat <<02USERSETUP > config/chroot_local-includes/usr/bin/pelican_setup
  #!/bin/bash	
  pelican_setup_user
  pelican_setup_netdevice
  pelican_terminalserver
  pelican_restart_hpc
02USERSETUP

cat <<03USER > config/chroot_local-includes/usr/bin/pelican_setup_user
#!/bin/bash
bailout(){
exit \$1
}
cd /home/user
HOME="/home/user"
PKTMP="\$HOME/tmp"
echo "Creating temporary directory"
rm -R -f \$PKTMP
mkdir \$PKTMP
chown user.user \$PKTMP
chmod 777 \$PKTMP
# generate keys if not there 
if [ -d "/home/user/.ssh" ]; then
  echo "ssh already configured"
else 
  echo "Generating new RSA keys"
  ssh-keygen  -q -t rsa -N "" -f "\$HOME/.ssh/id_rsa"
  cp \$HOME/.ssh/id_rsa.pub \$HOME/.ssh/authorized_keys
  chmod 600 \$HOME/.ssh/authorized_keys
fi
# make list of hosts to fping
echo "$PELICAN_NETWORK.2" > \$HOME/fpinghosts
i=2
while [ \$i -lt $MAXNODES ]
do
  i=\`expr \$i + 1\`
  echo $PELICAN_NETWORK.\$i >> \$HOME/fpinghosts
done
bailout
03USER

cat <<04NETDEVICE > config/chroot_local-includes/usr/bin/pelican_setup_netdevice
#!/bin/bash
# modification of netcardconfig from Knoppix.
PATH="/bin:/sbin:/usr/bin:/usr/sbin"
export PATH
DIALOG="dialog"
[ "\`id -u\`" != "0" ] && exec sudo "\$0" "\$@"
TMP="/tmp/netconfig.tmp\$\$"
bailout(){
rm -f "\$TMP"
exit \$1
}
TITLE="Pelican Setup"
MESSAGE0="No supported network cards found."
MESSAGE1="Please select the network device that connects to the cluster."
NETDEVICES="\$(cat /proc/net/dev | awk -F: '/eth.:|br.:|tr.:|wlan.:/{print \$1}')"
if [ -z "\$NETDEVICES" ]; then
\$DIALOG --title "\$TITLE" --msgbox "\$MESSAGE0" 5 30
bailout
fi
count="\$(echo "\$NETDEVICES" | wc -w)"
if [ "\$count" -gt 1 ]; then
DEVICELIST=""
for DEVICE in \$NETDEVICES; do DEVICELIST="\$DEVICELIST \${DEVICE} network_device_\${DEVICE}"; done
rm -f "\$TMP"
echo "Hints about your net devices"
dmesg|grep eth
rm -f "\$TMP"
\$DIALOG --menu "\$MESSAGE1" 18 45 12 \$DEVICELIST 2>"\$TMP" || bailout
read DV <"\$TMP" ; rm -f "\$TMP"
else
# Remove additional spaces
DV="\$(echo \$NETDEVICES)"
fi
IP="$PELICAN_NETWORK.1"
NM="255.255.255.0 "
BC="$PELICAN_NETWORK.255"
ifdown \$DV
sleep 4
CMD="ifconfig \$DV \$IP netmask \$NM broadcast \$BC up"
\$CMD
sleep 4
bailout
04NETDEVICE

cat <<05TERMINALSERVER > config/chroot_local-includes/usr/bin/pelican_terminalserver
#!/bin/sh
PATH="/bin:/sbin:/usr/bin:/usr/sbin"
export PATH
DIALOG="dialog"
[ "\`id -u\`" != "0" ] && exec sudo "\$0" "\$@"
bailout(){
exit \$1
}

cat <<DNSMASQ > /etc/dnsmasq.conf
dhcp-range=$PELICAN_NETWORK.2,$PELICAN_NETWORK.$MAXNODES,255.255.255.0,12h
dhcp-boot=pxelinux.0,pelican,$PELICAN_NETWORK.1
DNSMASQ

cat <<TFTP > /etc/xinetd.d/tftp-hpa
service tftp
{
	disable         = no
	id              = chargen-dgram
        socket_type     = dgram
        protocol        = udp
        user            = root
        wait            = yes
	server          = /usr/sbin/in.tftpd
	server_args     = -s /var/lib/tftpboot/
}
TFTP

# set up PXE service
install -d /var/lib/tftpboot/pxelinux.cfg
cp /usr/lib/syslinux/pxelinux.0 /var/lib/tftpboot/
cp /boot/vmlinuz-\`uname -r\` /var/lib/tftpboot/
cat << PXECONFIG > /var/lib/tftpboot/pxelinux.cfg/default
LABEL linux
KERNEL vmlinuz-\`uname -r\`
APPEND initrd=initrd.img  nfsroot=$PELICAN_NETWORK.1:/live/image ip=dhcp rw noautologin noxautologin union=aufs netboot=nfs boot=live
PXECONFIG

# generate /etc/exports
cat <<EXPORTS > /etc/exports
/live/image *(ro,async,no_subtree_check,no_root_squash,fsid=12345)
/home $PELICAN_NETWORK.0/255.255.255.0(rw,root_squash,async,no_subtree_check)
EXPORTS

#--------------------------------------------------
# # this is needed for sid, as of Sept 2009
# cat <<PORTMAP > /etc/default/portmap
# OPTIONS=
# PORTMAP
#-------------------------------------------------- 

# start services
TITLE="Start Pelican HPC netboot services"
MESSAGE="We now get ready to set up the cluster by starting services that will allow the compute nodes to netboot. \
IMPORTANT: do not proceed if your cluster is on an existing network, or PelicanHPC's dhcp server may conflict \
with a running dhcp server. Continue?"
\$DIALOG --title "\$TITLE" --yesno "\$MESSAGE" 15 90 || bailout
sync
/etc/init.d/portmap restart
/etc/init.d/xinetd restart
/etc/init.d/dnsmasq restart
/etc/init.d/nfs-kernel-server restart
sleep 5
bailout
05TERMINALSERVER

cat <<06HPC > config/chroot_local-includes/usr/bin/pelican_restart_hpc
#!/bin/sh
PKTMP="/home/user/tmp"
PATH="/bin:/sbin:/usr/bin:/usr/sbin"
export PATH
DIALOG="dialog"
bailout(){
exit 0
}
# check which nodes are up
checknodes(){
rm \$PKTMP/bhosts
sudo fping -a -q -r0 -f /home/user/fpinghosts > \$PKTMP/bhosts
}
retry(){
checknodes
NNODES="\$(grep -c "" \$PKTMP/bhosts)"
MESSAGENODES="\nGo turn on your compute nodes now. \n\nAt the moment \$NNODES compute nodes (not counting this frontend node) are available. \n\nClick no to rescan the available nodes. Click yes when the desired number of nodes are available. You might want to wait a bit if some nodes are still finishing booting up."
\$DIALOG --title "\$TITLE" --defaultno --yesno "\$MESSAGENODES" 20 50 || retry
}
trap bailout 1 2 3 15
TITLE="Restart Pelican HPC"
MESSAGE="\nTime to bring the compute nodes into the cluster. If you are resizing a running cluster, be aware that continuing will interrupt any running MPI jobs. Continue?"
\$DIALOG --title "\$TITLE" --yesno "\$MESSAGE" 15 90 || bailout
retry
# master must be last in the list
echo "$PELICAN_NETWORK.1" | cat >> \$PKTMP/bhosts
# display success message
NNODES="\$(grep -c "" \$PKTMP/bhosts)"
# final report
SUCCESS="\nYour cluster of \$NNODES nodes is (probably) lambooted. If there was a problem, just re-run the script.\nThe nodes in the cluster are listed in the file ~/tmp/bhosts. If you add or remove compute nodes, re-run this script (pelican_restart_hpc) whenever you like."
\$DIALOG --title "\$TITLE" --msgbox "\$SUCCESS" 15 50
bailout
06HPC
######## END OF PELICAN SETUP SCRIPTS #############


######## POST BOOT SCRIPT ####################
install -d config/chroot_local-includes/usr/share/initramfs-tools/scripts/live-bottom/
cat <<99START > config/chroot_local-includes/usr/share/initramfs-tools/scripts/live-bottom/99script
#!/bin/sh
cat <<RC_LOCAL > /root/etc/rc.local
#! /bin/sh
pelican_boot_setup
RC_LOCAL
chmod a+x /root/etc/rc.local
99START
chmod +x config/chroot_local-includes/usr/share/initramfs-tools/scripts/live-bottom/99script

# make the scripts in /usr/bin executable
chmod +x config/chroot_local-includes/usr/bin/*

# build the ISO image
nice ionice -c2 lh build --debug

# safeguard against crashes
umount proc-live
umount sysfs-live
umount devpts-live
sync