#!/bin/bash
# Path to all the resources on the CD.
ISOMOUNT="/mnt/cdrom"
MANIFEST_FILE=$ISOMOUNT/build/manifest.json
ZENOSS_TYPE=$(python -c "import json; print json.load(open('$MANIFEST_FILE'))['type']")
LOGFILE="/tmp/upgrade-$ZENOSS_TYPE-$(date +'%Y-%m%d-%H%M%S').log"
> $LOGFILE
exec > >(tee -a $LOGFILE)
exec 2> >(tee -a $LOGFILE >&2)
## PREREQUISITES AND CONTEXT
function red () {
tput setaf 1
echo $@
tput sgr0
}
function green () {
tput setaf 2
echo $@
tput sgr0
}
function failure_prompt () {
red $@ >&2
red Upgrade log stored at $LOGFILE.
read -n1 -r -p "Upgrade failed. Press any key to return."
exit 1
}
green Upgrade log stored at $LOGFILE.
THINPOOL_DEVICE=$(source /etc/default/serviced; echo $SERVICED_DM_THINPOOLDEV)
if [ "$THINPOOL_DEVICE" = "" ]; then
red "Serviced appears to be using a loopback thinpool device."
red "It is highly recommended that you migrate to an LVM device for improved performance."
echo
read -n1 -r -p "Press Escape to abort now, or any other key to continue." II
echo
if [ "$II" = $'\e' ]; then
exit 1
fi
fi
# Setup variables from manifest file.
DOCKER_IMAGES=$(python -c "import json; print ' '.join(json.load(open('$MANIFEST_FILE'))['images'].values())")
RPM_IMPORT4=$(python -c "import json; print json.load(open('$MANIFEST_FILE'))['rpms']['import4']" 2>/dev/null)
RPM_SERVICED=$(python -c "import json; print json.load(open('$MANIFEST_FILE'))['rpms']['serviced']")
RPM_SVCDEF=$(python -c "import json; print json.load(open('$MANIFEST_FILE'))['rpms']['svcdef']")
ZENOSS_IMAGE=$(python -c "import json; print json.load(open('$MANIFEST_FILE'))['images']['zenoss'][len('install-zenoss-'):-len('.run')]")
ZENOSS_SHORT=$(python -c "import json; print json.load(open('$MANIFEST_FILE'))['images']['zenoss'][len('install-zenoss-'):].split(':')[0]")
ZENOSS_VERSION=$(python -c "import json; print json.load(open('$MANIFEST_FILE'))['zenoss-version']")
IS_MASTER=$(source /etc/default/serviced; echo $SERVICED_MASTER)
if [ "$IS_MASTER" = "" ]; then
IS_MASTER="1"
fi
# Update admin menu.
\cp -r -f $ISOMOUNT/common/appliance/standard/* /root/appliance
\cp -r -f $ISOMOUNT/common/appliance/$ZENOSS_TYPE/* /root/appliance
## UTILITY FUNCTIONS
function stop_all_services () {
serviced service stop $ZENOSS_TYPE 2>/dev/null
RETRIES=60
STATUS=""
until [ "$STATUS" = "Stopped" ] || [ "$RETRIES" -le 0 ]; do
STATUS=$(serviced service status --show-fields 'Status' 2>/dev/null | grep -v "Stopped" | grep -v "Status" | grep -v ^[[:space:]]*$)
if [ -z "$STATUS" ]; then
STATUS="Stopped"
fi
sleep 5
((RETRIES--))
done
if [ "$RETRIES" -le 0 ]; then
failure_prompt "Failed to stop all $ZENOSS_TYPE services."
fi
}
function restart_serviced () {
green "Restarting $ZENOSS_TYPE..."
systemctl unmask serviced --quiet
systemctl enable serviced --quiet
systemctl restart serviced --quiet
STATUS=""
RETRIES=60
until (serviced healthcheck &> /dev/null) || [ "$RETRIES" -le 0 ]; do
sleep 5
((RETRIES--))
done
if [ "$RETRIES" -le 0 ]; then
failure_prompt "Failed to restart serviced."
fi
green "Restarted $ZENOSS_TYPE."
}
function create_thinpool_check() {
if [ ! -f "$1" ]; then
cat <<EOF > $1
#!/bin/bash
function check_for_thinpools () {
if [ ! -e /dev/mapper/docker-docker--pool ]; then
exit 1
fi
if [ ! -e /dev/mapper/serviced-serviced--pool ]; then
exit 1
fi
}
while ( ! check_for_thinpools ); do
sleep 5
done
EOF
chmod +x $1
fi
}
## UPGRADE PROCEDURE
green "Stopping $ZENOSS_TYPE services..."
stop_all_services
green "$ZENOSS_TYPE services stopped."
green "Stopping serviced..."
systemctl stop serviced --quiet
STATUS=""
RETRIES=60
until [ "$STATUS" = "inactive" ] || [ "$STATUS" = "failed" ] || [ "$RETRIES" -le 0 ]; do
STATUS=$(systemctl show serviced 2>/dev/null | awk -F= '/ActiveState/ { print $2 }')
sleep 5
((RETRIES--))
done
if [ "$RETRIES" -le 0 ]; then
failure_prompt "Failed to stop serviced."
fi
systemctl mask serviced --quiet
green "Serviced stopped."
green "Updating system packages..."
cat <<EOF > /etc/yum.repos.d/centos-update-mirror.repo
[centos-update-mirror]
name=Centos update mirror
baseurl=file://$ISOMOUNT/centos-repo
enabled=0
gpgcheck=0
EOF
yum update -y --disablerepo=\* --enablerepo=centos-update-mirror
rm -f /etc/yum.repos.d/centos-update-mirror.repo
green "System packages updated."
cd $ISOMOUNT/build
# Replace old yum-mirror, if any, with new one.
RHEL_VERSION=$(awk '{print $4}' /etc/redhat-release)
YUM_MIRROR=$(python -c "import json; print json.load(open('$MANIFEST_FILE'))['yum-mirror-centos$RHEL_VERSION']")
OLD_MIRROR=$(yum list --disablerepo=\* | awk '/^yum-mirror-centos/ { print $1}')
if [ ! -z "$OLD_MIRROR" ]; then
yum remove -y $OLD_MIRROR &> /dev/null
fi
yum remove -y yum-mirror-ucspm-unstable
yum install -y $YUM_MIRROR 2>/dev/null
yum clean all
# Modify the serviced configs before it's upgraded and restarted.
if ! grep -xq 'SERVICED_DOCKER_REGISTRY=.*' /etc/default/serviced; then
echo 'SERVICED_DOCKER_REGISTRY=localhost:5000' >> /etc/default/serviced
fi
if [ "$THINPOOL_DEVICE" = "" ]; then
echo 'SERVICED_ALLOW_LOOP_BACK=true' >> /etc/default/serviced
fi
green "Importing $ZENOSS_TYPE Docker images..."
systemctl restart docker --quiet
for IMAGE in $DOCKER_IMAGES; do
./$IMAGE -y &> /dev/null
IMAGE_NV=${IMAGE%%.run}
IMAGE_NV=${IMAGE_NV##install-}
IFS=":" read -ra IMAGE_INFO <<< "$IMAGE_NV"
IMAGE_NAME=$(echo ${IMAGE_INFO[0]} | tr '-' '.')
IMAGE_VERSION=${IMAGE_INFO[1]}
IMAGE_INSTALLED=$(docker images | grep $IMAGE_NAME | grep $IMAGE_VERSION)
if [ -z "$IMAGE_INSTALLED" ]; then
failure_prompt "Error importing $IMAGE."
fi
done
green "Imported $ZENOSS_TYPE Docker images."
# Updating serviced will also upgrade Docker.
green "Upgrading serviced and dependencies..."
yum install -y --disablerepo=\* --enablerepo=zenoss-mirror ${RPM_SERVICED%%.rpm}
if [ $? -ne 0 ]; then
failure_prompt "Failed to upgrade serviced binary."
fi
green "Upgraded serviced and dependencies."
DOCKER_CONF="/etc/systemd/system/docker.service.d/docker.conf"
if [ -f "$DOCKER_CONF" ]; then
# Switch Docker DNS to support Docker 1.9 upgrade.
DOCKER_DNS=$(ip addr show docker0 | grep inet | grep -v inet6 | awk '{print $2}' | awk -F / '{print $1}')
DOCKER_BIP=$DOCKER_DNS/24
sed -i -e "s;--dns=.*;--dns=$DOCKER_DNS --bip=$DOCKER_BIP;" "$DOCKER_CONF"
# Set Docker startup timeout
TimeoutSec=$(source $DOCKER_CONF &>/dev/null; echo $TimeoutSec)
if [ -z "$TimeoutSec" ]; then
echo "TimeoutSec=300" >> "$DOCKER_CONF"
elif [ "$TimeoutSec" -lt 300 ]; then
sed -i -e "s/TimeoutSec=.*/TimeoutSec=300/" "$DOCKER_CONF"
fi
# Add devicemapper wait if not using thinpools
if [ "$THINPOOL_DEVICE" != "" ]; then
# Create thinpool check script
THINPOOL_SCRIPT=/root/appliance/docker-thinpool-check.sh
create_thinpool_check $THINPOOL_SCRIPT
THINPOOL_PRE=$(grep $THINPOOL_SCRIPT $DOCKER_CONF)
if [ -z "$THINPOOL_PRE" ]; then
echo "ExecStartPre=$THINPOOL_SCRIPT" >> "$DOCKER_CONF"
fi
systemctl daemon-reload
fi
fi
if [ "$IS_MASTER" = "1" ]; then
# Serviced must be running for 'serviced docker sync' to succeed.
restart_serviced
serviced docker sync
green "Installing new service definition..."
yum install -y --disablerepo=\* --enablerepo=zenoss-mirror $RPM_IMPORT4 $RPM_SVCDEF 2> /dev/null
green "New service definition installed."
stop_all_services
# If this is the master, we cannot proceed with the upgrade until the agents have been upgraded first.
if [ "$IS_MASTER" = "1" ]; then
OLD_AGENTS=""
MY_INFO=$(serviced host list --show-fields=Addr,Release | grep $(ifconfig | awk '/inet / { print "-e" $2 }'))
if [ -z "$MY_INFO" ]; then
red "Could not determine this host's address and release. Cannot determine if all agents are updated."
echo
read -n1 -r -p "Press Enter to continue."
else
MY_IP=$(awk '{print $1}' <<< "$MY_INFO")
MY_RELEASE=$(awk '{print $2}' <<< "$MY_INFO")
AGENT_RELEASES=$(serviced host list --show-fields=Addr,Release | grep -v Addr | grep -v -F "$MY_IP")
OLD_AGENTS=$(grep -v -F $MY_RELEASE <<< "$AGENT_RELEASES")
fi
if [ ! -z "$OLD_AGENTS" ]; then
red "This host appears to be the master host."
red "One or more agents do not appear to have been upgraded yet."
serviced host list --show-fields=Name,Addr,Release | grep -v -F $MY_IP
echo
failure_prompt "Please upgrade agent hosts first before upgrading this host."
fi
fi
# Pull the zenoss image, rsync to /root/5.Z.x
green "Updating" $ZENOSS_TYPE "..."
ZENOSS_5DOT=${ZENOSS_VERSION%?}x
docker run -it --rm -v /root:/mnt/root zenoss/${ZENOSS_IMAGE} rsync -a /root/${ZENOSS_5DOT} /mnt/root
sed -i -e "s/^SVC_USE zenoss\/${ZENOSS_SHORT}:5.* /SVC_USE zenoss\/${ZENOSS_IMAGE} /g" /root/${ZENOSS_5DOT}/upgrade-${ZENOSS_TYPE}.txt
/root/${ZENOSS_5DOT}/upgrade-${ZENOSS_TYPE}-${ZENOSS_5DOT}.sh
if [ $? -ne 0 ]; then
red "Error upgrading $ZENOSS_TYPE." >&2
failure_prompt "You may need to remove one or more preupgrade snapshots before retrying this upgrade."
fi
green "Updated $ZENOSS_TYPE."
stop_all_services
fi
systemctl stop serviced --quiet
green "$ZENOSS_TYPE upgraded successfully."
read -n1 -r -p "Press any key to reboot..."
systemctl unmask serviced
systemctl enable serviced
green Upgrade log stored at $LOGFILE.
reboot