cml-community/scripts/migration-tool/virl2-migrate-data.sh

#!/bin/bash
#
#  This file is part of VIRL2
#  Cisco (c) 2021
#

source /etc/default/virl2

SRC_DIRS="${BASE_DIR}/images ${CFG_DIR}"
KEY_FILE="migration_key"
SSH_PORT="1122"

set_virl_version() {
    product_file=/PRODUCT
    if [ $# = 1 ]; then
        product_file=$1
    fi

    vers=$(jq -r ".PRODUCT_VERSION" < "${product_file}")

    if [ $# = 1 ]; then
        echo ${vers}
    else
        VIRL_VERSION=${vers}
    fi
}

build_local_src_dirs() {
    if [ "${REF_PLAT}" != "${LIBVIRT_IMAGES}" ]; then
        MOUNT_POINT=${REF_PLAT}/cdrom
    else
        MOUNT_POINT=${LIBVIRT_IMAGES}
    fi

    if [ -d "${REF_PLAT_DIR}" ]; then
        MOUNT_POINT="${REF_PLAT_DIR}"
    fi

    WRKDIR="${REF_PLAT}"/diff

    # Find all custom node defs
    new_node_defs=$(find "${WRKDIR}"/node-definitions/ -maxdepth 1 -type f)
    old_IFS=${IFS}
    IFS='
'
    for nd in ${new_node_defs}; do
        fname=$(basename "${nd}")
        if [ ! -f "${MOUNT_POINT}"/node-definitions/"${fname}" ]; then
            SRC_DIRS="${SRC_DIRS} ${nd}"
        else
            if ! diff -q "${nd}" "${MOUNT_POINT}"/node-definitions/"${fname}" >/dev/null 2>&1; then
                SRC_DIRS="${SRC_DIRS} ${nd}"
            fi
        fi
    done

    # Find all custom image defs
    new_image_defs=$(find "${WRKDIR}"/virl-base-images/ -maxdepth 1 -type d)

    for id in ${new_image_defs}; do
        if [ "${id}" = "${WRKDIR}"/virl-base-images/ ]; then
            continue
        fi
        dname=$(basename "${id}")
        if [ ! -d "${MOUNT_POINT}"/virl-base-images/"${dname}" ]; then
            SRC_DIRS="${SRC_DIRS} ${id}"
        fi
    done

    IFS=${old_IFS}
}

export_libvirt_domains() {
    ddir=$(mktemp -d /tmp/libvirt_domains.XXXXX)
    domains=$(virsh list --all --name)
    if [ $? != 0 ]; then
        return $?
    fi

    old_IFS=${IFS}
    IFS='
'

    for domain in ${domains}; do
        virsh dumpxml "${domain}" > "${ddir}"/"${domain}".xml
    done

    IFS=${old_IFS}

    echo "${ddir}"
}

define_domains() {
    ddir=$1

    old_IFS=${IFS}
    IFS='
'

    for domain in "${ddir}"/*.xml; do
        virsh define "${domain}"
        if [ $? != 0 ]; then
            rc=$?
            IFS=${old_IFS}
            return ${rc}
        fi
    done

    IFS=${old_IFS}
}

delete_libvirt_domains() {
    for domain in $(virsh list --all --name); do
        virsh undefine "${domain}" >/dev/null
    done
}

check_disk_space() {
    source=$1
    target=$2

    if [ -z "${source}" ] && [ $# = 3 ]; then
        total_needed=$3
    else
        total_needed=$(du -B1 -sc ${source} | grep total | sed -E -s 's|\s+total||')
    fi

    total_available=$(df -B1 --output=avail "${target}" | grep -E '[0-9]')

    if [ "${total_needed}" -gt "${total_available}" ]; then
        echo "Insufficient disk space required in ${target}; ${total_needed} bytes required but only ${total_available} bytes available."
        return 1
    fi

    return 0
}

prepare_as_remote_host() {
    systemctl enable sshd.service && systemctl start sshd
    rc=$?
    if [ ${rc} = 0 ]; then
        echo "Host is now ready to be a migration source.  Run '${ME} --host' on the remote host and point to this host's IP."
    else
        echo "Failed to start OpenSSH.  See output above."
    fi

    return ${rc}
}

unprepare_as_remote_host() {
    systemctl stop sshd && systemctl disable sshd.service
    rc=$?
    if [ ${rc} = 0 ]; then
        echo "Host is no longer usable as a migration source."
    else
        echo "Failed to shutdown OpenSSH.  See output above."
    fi

    return ${rc}
}

wait_for_vms_to_stop() {
    # this is only needed if VMs are running on the
    # controller where the ISO is mounted.
    loops=5   # max 5x5 = 25s
    done=0
    while (( loops > 0 && !done )); do
    done=1
    for vm in $(virsh -c qemu:///system list --all --name); do
        if [[ "$vm" =~ ^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12} ]]; then
            vm_state=$(virsh dominfo "$vm")
            if [[ "$vm_state" =~ State:\ +running ]]; then
                echo "still running: $vm"
                done=0
            fi
        fi
    done
    (( loops -= 1 ))
    (( !done )) && sleep 5
    done
    if (( !done )); then
        echo "some VMs are still running, giving up..."
    fi
}

stop_cml_services() {
    wait_for_vms_to_stop
    systemctl stop virl2.target
    rc=$?
    if [ ${rc} != 0 ]; then
        return ${rc}
    fi

    echo "Waiting for controller to become inactive..."
    while [ "$(systemctl is-active virl2-controller)" = "active" ]; do
        echo "  Still active..."
        sleep 5
    done
    return ${rc}
}

restart_cml_services() {
    echo "Restarting CML services..."
    systemctl start virl2.target
}

backup_local_files() {
    # Backup each source directory, just in case.
    # Do this with mv to avoid running out of disk space.
    for dir in ${SRC_DIRS}; do
        if [ -e "${dir}" ]; then
            rm -rf "${dir}".bak
            mv -f "${dir}" "${dir}".bak
        fi
    done
}

restore_local_files() {
    for dir in ${SRC_DIRS}; do
        if [ -e "${dir}" ]; then
            rm -rf "${dir}"
            mv -f "${dir}".bak "${dir}"
        fi
    done
}

generate_ssh_key() {
    # Generate an SSH key we can use to avoid re-prompting for a password.
    tempd=$(mktemp -d /tmp/migration.XXXXX)
    ssh-keygen -b 4096 -q -N "" -f "${tempd}"/${KEY_FILE}
    echo "${tempd}"
}

cleanup_from_host() {
    key_dir=$1
    backup_ddir=$2

    rm -rf "${key_dir}"
    restore_local_files
    define_domains "${backup_ddir}"
    restart_cml_services
    rm -rf "${backup_ddir}"
}

sync_from_host() {
    host=$1

    if ! nc -z "${host}" ${SSH_PORT}; then
        echo "Remote host does not have OpenSSH enabled; run '${ME} --prep' on the remote host first."
        return 1
    fi

    read -r -p "Do you wish to import data from ${host}? [y/N] " confirm
    echo
    if echo "${confirm}" | grep -viq '^y'; then
        echo "Terminating import."
        return 0
    fi

    stop_cml_services || ( echo "Failed to stop CML services." ; exit $? )
    backup_local_files
    backup_ddir=$(export_libvirt_domains)
    delete_libvirt_domains

    key_dir=$(generate_ssh_key)
    key=$(cat "${key_dir}"/${KEY_FILE}.pub)

    printf "\nThe next prompt will be for sysadmin's password on %s.\n" "${host}"
    printf "The prompt following that will be for sysadmin's password on %s to enter sudo mode.\n\n" "${host}"
    # Stop the service on the remote host and make sure we don't need
    # a password for sudo.  We also install the SSH pubkey for subsequent logins.
    if ! ssh -o "StrictHostKeyChecking=no" -t -p ${SSH_PORT} sysadmin@"${host}" "sudo /usr/local/bin/${ME} --stop && echo '%sysadmin        ALL=(ALL)       NOPASSWD: ALL' | sudo tee /etc/sudoers.d/cml-migrate >/dev/null 2>&1 && mkdir -p ~/.ssh && \
        chmod 0700 ~/.ssh && (cp -fa ~/.ssh/authorized_keys ~/.ssh/authorized_keys.migration >/dev/null 2>&1 || true) && echo ${key} | tee -a ~/.ssh/authorized_keys >/dev/null 2>&1 && chmod 0600 ~/.ssh/authorized_keys"; then
        rc=$?
        cleanup_from_host "${key_dir}" "${backup_ddir}"
        echo "Error preparing ${host} for migration.  The original local data have been restored."
        return ${rc}
    fi

    # Check CML versions on both hosts.
    output=$( (ssh -o "StrictHostKeyChecking=no" -i "${key_dir}"/${KEY_FILE} -p ${SSH_PORT} sysadmin@"${host}" "sudo /usr/local/bin/${ME} --version") 2>/dev/null)
    if [ $? != 0 ]; then
        rc=$?
        cleanup_from_host "${key_dir}" "${backup_ddir}"
        echo "Failed to determine remote CML version.  Make sure /usr/local/bin/${ME} is installed on the remote machine."
        exit ${rc}
    fi

    if [ "${VIRL_VERSION}" != "${output}" ]; then
        cleanup_from_host "${key_dir}" "${backup_ddir}"
        echo "Versions do not match.  Source server version: ${VIRL_VERSION}, Dest server version: ${output}.  The original local data has been restored."
        return 1
    fi

    # Build the list of remote src dirs.
    output=$( (ssh -o "StrictHostKeyChecking=no" -i "${key_dir}"/${KEY_FILE} -p ${SSH_PORT} sysadmin@"${host}" "sudo /usr/local/bin/${ME} --src-dirs") 2>/dev/null)
    if [ $? != 0 ]; then
        rc=$?
        cleanup_from_host "${key_dir}" "${backup_ddir}"
        echo "Failed to obtain remote src dirs.  Make sure /usr/local/bin/${ME} is installed on the remote machine."
        exit ${rc}
    fi

    SRC_DIRS=${output}

    # Get the list of domains from virsh.
    libvirt_domains=$( (ssh -o "StrictHostKeyChecking=no" -i "${key_dir}"/${KEY_FILE} -p ${SSH_PORT} sysadmin@"${host}" "sudo /usr/local/bin/${ME} --get-domains") 2>/dev/null)
    if [ $? != 0 ]; then
        rc=$?
        cleanup_from_host "${key_dir}" "${backup_ddir}"
        echo "Failed to get libvirt domains from ${host}: ${libvirt_domains}"
        exit ${rc}
    fi

    SRC_DIRS="${SRC_DIRS} ${libvirt_domains}"

    echo "Migrating ${SRC_DIRS} to this CML server..."

    # Get required disk space from remote host
    output=$( (ssh -o "StrictHostKeyChecking=no" -i "${key_dir}"/${KEY_FILE} -p ${SSH_PORT} sysadmin@"${host}" "sudo du -B1 -sc ${SRC_DIRS} | grep total | sed -E -s 's|\s+total||'") 2>/dev/null)
    if check_disk_space "" ${BASE_DIR} "${output}"; then
        printf "Starting migration.  Please be patient, migration may take a while....\n\n\n"
        output=$( (ssh -o "StrictHostKeyChecking=no" -i "${key_dir}"/${KEY_FILE} -p ${SSH_PORT} sysadmin@"${host}" "sudo tar --acls --selinux -cpf - ${SRC_DIRS}" | tar -C / --acls --selinux -xpf -) 2>&1 )
        rc=$?
        if [ ${rc} != 0 ]; then
            restore_local_files
            define_domains "${backup_ddir}"
            echo "Migration completed with errors:"
            printf '%s\n\n' "${output}"
            echo "The original local data have been restored."
        else
            # For each of the libvirt domains, migrate the XML.
            echo "Migrating libvirt domains..."
            output=$(define_domains "${libvirt_domains}" 2>&1)
            rc=$?
            if [ ${rc} != 0 ]; then
                restore_local_files
                define_domains "${backup_ddir}"
                echo "Libvirt domain import completed with errors:"
                printf '%s\n\n' "${output}"
                echo "The original local data have been restored."
            else
                echo "Migration completed SUCCESSFULLY."
                echo "Please make sure you have either mounted the same refplat ISO on or copied its contents to this CML server."
            fi
        fi
    else
        rc=$?
    fi

    printf "\nFinishing up with the remote host..."
    if ! ssh -o "StrictHostKeyChecking=no" -i "${key_dir}"/${KEY_FILE} -p ${SSH_PORT} sysadmin@"${host}" "sudo /usr/local/bin/${ME} --start && sudo rm -rf ${libvirt_domains} && sudo rm -f /etc/sudoers.d/cml-migrate && (cp -fa ~/.ssh/authorized_keys.migration ~/.ssh/authorized_keys >/dev/null 2>&1 || true)"; then
        printf "FAILED.\n"
        echo "Error finishing up on remote host.  Check to make sure the CML services are running on ${host}."
        rc=$?
    else
        printf "DONE.\n"
    fi

    rm -rf "${key_dir}"
    rm -rf "${libvirt_domains}"
    rm -rf "${backup_ddir}"

    restart_cml_services

    return ${rc}
}

ME=$(basename "$0")
set_virl_version

if [ "$EUID" != 0 ]; then
    echo "This script must be run as root.  Use 'sudo' to run it."
    exit 1
fi

opts=$(getopt -o brpuf:h:vd --long host:,file:,prep,unprep,backup,restore,version,src-dirs,stop,start,get-domains -- "$@")
if [ $? != 0 ]; then
    echo "usage: $0 -h|--host HOST_TO_MIGRATE_FROM"
    echo "       OR"
    echo "       $0 -b|--backup|-r|--restore -f|--file PATH_TO_BACKUP_FILE"
    echo "       OR"
    echo "       $0 -p|--prep"
    echo "       OR"
    echo "       $0 -u|--unprep"
    exit 1
fi

REMOTE_HOST=
BACKUP_FILE=
PREP=0
UNPREP=0
BACKUP=0
RESTORE=0

eval set -- "$opts"
while true; do
    case "$1" in
    -h | --host)
        shift
        REMOTE_HOST=$1
        ;;
    -f | --file)
        shift
        BACKUP_FILE=$1
        ;;
    -p | --prep)
        PREP=1
        ;;
    -u | --unprep)
        UNPREP=1
        ;;
    -b | --backup)
        BACKUP=1
        ;;
    -r | --restore)
        RESTORE=1
        ;;
    -v | --version)
        echo ${VIRL_VERSION}
        exit 0
        ;;
    -d | --src-dirs)
        build_local_src_dirs
        echo ${SRC_DIRS}
        exit 0
        ;;
    --stop)
        stop_cml_services
        exit $?
        ;;
    --start)
        restart_cml_services
        exit $?
        ;;
    --get-domains)
        export_libvirt_domains
        exit $?
        ;;
    --)
        shift
        break
        ;;
    esac
    shift
done

if [ ${PREP} = 1 ] && [ ${UNPREP} = 1 ]; then
    echo "Only one of --prep or --unprep may be specified."
    exit 1
fi

if [ ${PREP} = 1 ]; then
    prepare_as_remote_host
    exit $?
elif [ ${UNPREP} = 1 ]; then
    unprepare_as_remote_host
    exit $?
fi

if [ -n "${REMOTE_HOST}" ] && [ -n "${BACKUP_FILE}" ]; then
    echo "Only one of --host or --file may be specified."
    exit 1
fi

if [ -z "${REMOTE_HOST}" ] && [ -z "${BACKUP_FILE}" ]; then
    echo "One of --host or --file must be specified."
    exit 1
fi

if [ -n "${REMOTE_HOST}" ]; then
    sync_from_host "${REMOTE_HOST}"
    exit $?
fi

if [ ${BACKUP} = 1 ] && [ ${RESTORE} = 1 ]; then
    echo "Only one of --backup or --restore can be specified."
    exit 1
fi

if [ ${BACKUP} = 0 ] && [ ${RESTORE} = 0 ]; then
    echo "One of --backup or --restore must be specified."
    exit 1
fi

if [ ${RESTORE} = 1 ]; then
    if [ ! -f "${BACKUP_FILE}" ]; then
        echo "Backup file ${BACKUP_FILE} does not exist or is not a file."
        exit 1
    fi

    # While not all data may go to the product root, the way CML's file system
    # is laid out means it's almost certainly going to be on the same FS.
    if ! check_disk_space "${BACKUP_FILE}" ${BASE_DIR}; then
        exit $?
    fi

    read -r -p "Do you wish to restore from ${BACKUP_FILE}?  This will overwrite current local data. [y/N] " confirm
    echo
    if echo "${confirm}" | grep -qiv '^y'; then
        echo "Terminating restore."
        exit 0
    fi

    # First, extract /PRODUCT from the backup and check that the version matches the current version.
    tempd=$(mktemp -d /tmp/migration.XXXXX)
    output=$(tar -C "${tempd}" --acls --selinux -xpf "${BACKUP_FILE}" PRODUCT libvirt_domains.dat 2>&1)
    rc=$?
    if [ ${rc} != 0 ]; then
        echo "Failed to extract /PRODUCT from backup:"
        printf '%s\n\n' "${output}"
        exit ${rc}
    fi

    virl_version=$(set_virl_version "${tempd}"/PRODUCT)
    if [ "${VIRL_VERSION}" != "${virl_version}" ]; then
        rm -rf "${tempd}"
        echo "Versions do not match.  Source server version: ${VIRL_VERSION}, Dest server version: ${virl_version}."
        exit 1
    fi

    ddir=$(cat "${tempd}"/libvirt_domains.dat)

    rm -rf "${tempd}"

    stop_cml_services || ( echo "Failed to stop CML services." ; exit $? )
    backup_local_files
    backup_ddir=$(export_libvirt_domains)
    delete_libvirt_domains

    echo "Restoring ${BACKUP_FILE} to local CML.  Please be patient, this may take a while..."
    output=$(tar -C / --acls --selinux --exclude=PRODUCT -xpf "${BACKUP_FILE}" 2>&1)
    rc=$?
    if [ ${rc} != 0 ]; then
        restore_local_files
        define_domains "${backup_ddir}"
        echo "Restore failed with error:"
        printf '%s\n\n' "${output}"
        echo "The original local data has been restored."
    else
        output=$(define_domains "${ddir}" 2>&1)
        rc=$?
        if [ ${rc} != 0 ]; then
            echo "Libvirt domain import completed with errors:"
            printf '%s\n\n' "${output}"
            echo "The original local data have been restored."
        else
            echo "Restore completed SUCCESSFULLY."
            echo "Please make sure you have either mounted the same refplat ISO on or copied its contents to this CML server."
        fi
    fi

    rm -rf "${backup_ddir}"

    restart_cml_services

    exit ${rc}
fi

build_local_src_dirs

BACKUP_FILE=$(realpath "${BACKUP_FILE}")

# We are doing a dump to a single tar file.
if ! check_disk_space "${SRC_DIRS}" "$(dirname "${BACKUP_FILE}")"; then
    exit $?
fi

read -r -p "Are you sure you want to backup?  Doing so will restart the CML services. [y/N] " confirm
echo
if echo "${confirm}" | grep -qiv '^y'; then
        echo "Terminating backup."
        exit 0
fi

stop_cml_services || ( echo "Failed to stop CML services." ; exit $? )

ddir=$(export_libvirt_domains)
tempd=$(mktemp -d /tmp/migration.XXXXX)
cd "${tempd}"
echo "${ddir}" > libvirt_domains.dat

SRC_DIRS="${SRC_DIRS} ${ddir}"

echo "Backing up ${SRC_DIRS}..."

echo "Backing up CML data to ${BACKUP_FILE}.  Please be patient, this may take a while..."
output=$(tar -C "${tempd}" --acls --selinux -cpf "${BACKUP_FILE}" /PRODUCT ${SRC_DIRS} libvirt_domains.dat 2>&1)
rc=$?
if [ ${rc} != 0 ]; then
    rm -f "${BACKUP_FILE}"
    echo "Backup completed with errors:"
    printf '%s\n\n' "${output}"
else
    echo "Backup completed SUCCESSFULLY."
fi

rm -rf "${tempd}"
rm -rf "${ddir}"

restart_cml_services

exit ${rc}