freedombone/src/freedombone-backup-remote

#!/bin/bash
#
# .---.                  .              .
# |                      |              |
# |--- .--. .-.  .-.  .-.|  .-. .--.--. |.-.  .-. .--.  .-.
# |    |   (.-' (.-' (   | (   )|  |  | |   )(   )|  | (.-'
# '    '     --'  --'  -' -  -' '  '   -' -'   -' '   -  --'
#
#                    Freedom in the Cloud
#
# Backup to remote servers - the web of backups

# License
# =======
#
# Copyright (C) 2015 Bob Mottram <bob@robotics.uk.to>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

PROJECT_NAME='freedombone'
COMPLETION_FILE=$HOME/${PROJECT_NAME}-completed.txt
CONFIG_FILE=$HOME/${PROJECT_NAME}.cfg
BACKUP_EXTRA_DIRECTORIES=/root/backup-extra-dirs.csv
ENABLE_VERIFICATION="no"

export TEXTDOMAIN=${PROJECT_NAME}-backup-remote
export TEXTDOMAINDIR="/usr/share/locale"

# Temporary location for data to be backed up to other servers
SERVER_DIRECTORY=/root/remotebackup

ADMIN_USERNAME=$(cat $COMPLETION_FILE | grep "Admin user" | awk -F ':' '{print $2}')
ADMIN_NAME=$(getent passwd $ADMIN_USERNAME | cut -d: -f5 | cut -d, -f1)
ADMIN_EMAIL_ADDRESS=${ADMIN_USERNAME}@${HOSTNAME}
if [ ! -f /etc/ssl/private/backup.key ]; then
    echo $"Creating backup key"
    ${PROJECT_NAME}-addcert -h backup --dhkey 2048
fi

if [ ! -f /home/${ADMIN_USERNAME}/backup.list ]; then
    exit 1
fi

# MariaDB password
DATABASE_PASSWORD=''
if [ -f /root/dbpass ]; then
    DATABASE_PASSWORD=$(cat /root/dbpass)
fi

# local directory where the backup will be made
if [ ! -d $SERVER_DIRECTORY ]; then
    mkdir $SERVER_DIRECTORY
fi

if [ ! -d $SERVER_DIRECTORY/backup ]; then
    mkdir -p $SERVER_DIRECTORY/backup
fi

# The name of a currently suspended site
# Sites are suspended so that verification should work
SUSPENDED_SITE=

function suspend_site {
    # suspends a given website
    if [[ $ENABLE_VERIFICATION != "yes" ]]; then
        return
    fi
    SUSPENDED_SITE="$1"
    nginx_dissite $SUSPENDED_SITE
    service nginx reload
}

function restart_site {
    # restarts a given website
    if [ ! $SUSPENDED_SITE ]; then
        return
    fi
    nginx_ensite $SUSPENDED_SITE
    service nginx reload
    SUSPENDED_SITE=
}

function backup_directory_to_friend {
    BACKUP_KEY_EXISTS=$(gpg --list-keys "$ADMIN_NAME (backup key)")
    if [ ! "$?" = "0" ]; then
        echo $"Backup key could not be found"
        restart_site
        exit 43382
    fi
    ADMIN_BACKUP_KEY_ID=$(gpg --list-keys "$ADMIN_NAME (backup key)" | grep 'pub ' | awk -F ' ' '{print $2}' | awk -F '/' '{print $2}')
    if [ ! -d $SERVER_DIRECTORY/backup/${2} ]; then
        mkdir -p $SERVER_DIRECTORY/backup/${2}
    fi
    obnam force-lock -r $SERVER_DIRECTORY/backup/${2} --encrypt-with ${ADMIN_BACKUP_KEY_ID} ${1}
    obnam backup -r $SERVER_DIRECTORY/backup/${2} --encrypt-with ${ADMIN_BACKUP_KEY_ID} ${1}
    if [[ $ENABLE_VERIFICATION == "yes" ]]; then
        obnam verify -r $SERVER_DIRECTORY/backup/${2} --encrypt-with ${ADMIN_BACKUP_KEY_ID} ${1}
        if [ ! "$?" = "0" ]; then
            if [[ ${1} == "/root/temp"* || ${1} == *"tempbackup" ]]; then
                shred -zu /root/temp${2}/*
                rm -rf /root/temp${2}
            fi
            # Send a warning email
            echo "Unable to verify ${2}" | mail -s "${PROJECT_NAME} backup to friends" ${ADMIN_EMAIL_ADDRESS}
            restart_site
            exit 953
        fi
    fi
    obnam forget --keep=30d -r $SERVER_DIRECTORY/backup/${2} --encrypt-with ${ADMIN_BACKUP_KEY_ID}
    if [ ! "$?" = "0" ]; then
        if [[ ${1} == "/root/temp"* || ${1} == *"tempbackup" ]]; then
            shred -zu /root/temp${2}/*
            rm -rf /root/temp${2}
        fi
        # Send a warning email
        echo "Unable to backup ${2}" | mail -s "${PROJECT_NAME} backup to friends" ${ADMIN_EMAIL_ADDRESS}
        restart_site
        exit 853
    fi
    if [[ ${1} == "/root/temp"* || ${1} == *"tempbackup" ]]; then
        shred -zu /root/temp${2}/*
        rm -rf /root/temp${2}
    fi
}

function backup_database_to_friend {
    if [ ${#DATABASE_PASSWORD} -lt 2 ]; then
        echo $"No MariaDB password was given"
        restart_site
        exit 5783
    fi
    if [ ! -d $SERVER_DIRECTORY/backup/${1} ]; then
        mkdir -p $SERVER_DIRECTORY/backup/${1}
    fi
    if [ ! -d $SERVER_DIRECTORY/backup/${1}data ]; then
        mkdir -p $SERVER_DIRECTORY/backup/${1}data
    fi
    if [ ! -d /root/temp${1}data ]; then
        mkdir -p /root/temp${1}data
    fi
    echo "Obtaining ${1} database backup"
    mysqldump --password=$DATABASE_PASSWORD ${1} > /root/temp${1}data/${1}.sql
    if [ ! -s /root/temp${1}data/${1}.sql ]; then
        echo $"${1} database could not be saved"
        shred -zu /root/temp${1}data/*
        rm -rf /root/temp${1}data
        # Send a warning email
        echo $"Unable to export ${1} database" | mail -s $"${PROJECT_NAME} backup to friends" $ADMIN_EMAIL_ADDRESS
        restart_site
        exit 5738
    fi
}

function backup_configuration {
    echo $"Backing up ${PROJECT_NAME} configuration files"
    if [ ! -d /root/tempbackupconfig ]; then
        mkdir -p /root/tempbackupconfig
    fi
    cp -f $CONFIG_FILE /root/tempbackupconfig
    cp -f $COMPLETION_FILE /root/tempbackupconfig
    if [ -f $BACKUP_EXTRA_DIRECTORIES ]; then
        cp -f $BACKUP_EXTRA_DIRECTORIES /root/tempbackupconfig
    fi
    backup_directory_to_friend /root/tempbackupconfig config
}

function backup_users {
    for d in /home/*/ ; do
        USERNAME=$(echo "$d" | awk -F '/' '{print $3}')
        if [[ $USERNAME != "git" ]]; then

            # personal settings
            if [ -d /home/$USERNAME/personal ]; then
                echo $"Backing up personal settings for $USERNAME"
                backup_directory_to_friend /home/$USERNAME/personal personal/$USERNAME
            fi

            # gpg keys
            if [ -d /home/$USERNAME/.gnupg ]; then
                echo $"Backing up gpg keys for $USERNAME"
                backup_directory_to_friend /home/$USERNAME/.gnupg gnupg/$USERNAME
            fi

            # ssh keys
            if [ -d /home/$USERNAME/.ssh ]; then
                echo $"Backing up ssh keys for $USERNAME"
                backup_directory_to_friend /home/$USERNAME/.ssh ssh/$USERNAME
            fi

            # config files
            if [ -d /home/$USERNAME/.config ]; then
                echo $"Backing up config files for $USERNAME"
                backup_directory_to_friend /home/$USERNAME/.config config/$USERNAME
            fi

            # mutt settings
            if [ -f /home/$USERNAME/.muttrc ]; then
                echo $"Backing up Mutt settings for $USERNAME"
                if [ ! -d /home/$USERNAME/tempbackup ]; then
                    mkdir -p /home/$USERNAME/tempbackup
                fi
                cp /home/$USERNAME/.muttrc /home/$USERNAME/tempbackup
                if [ -f /etc/Muttrc ]; then
                    cp /etc/Muttrc /home/$USERNAME/tempbackup
                fi
                backup_directory_to_friend /home/$USERNAME/tempbackup mutt/$USERNAME
            fi

            # procmail settings
            if [ -f /home/$USERNAME/.procmailrc ]; then
                echo $"Backing up procmail settings for $USERNAME"
                if [ ! -d /home/$USERNAME/tempbackup ]; then
                    mkdir -p /home/$USERNAME/tempbackup
                fi
                cp /home/$USERNAME/.procmailrc /home/$USERNAME/tempbackup
                backup_directory_to_friend /home/$USERNAME/tempbackup procmail/$USERNAME
            fi

            # spamassassin settings
            if [ -d /home/$USERNAME/.spamassassin ]; then
                echo $"Backing up spamassassin settings for $USERNAME"
                backup_directory_to_friend /home/$USERNAME/.spamassassin spamassassin/$USERNAME
            fi

            # email
            if [ -d /home/$USERNAME/Maildir ]; then
                echo $"Creating an email archive"
                if [ ! -d /root/backupemail/$USERNAME ]; then
                    mkdir -p /root/backupemail/$USERNAME
                fi
                tar -czvf /root/backupemail/$USERNAME/maildir.tar.gz /home/$USERNAME/Maildir
                echo $"Backing up emails for $USERNAME"
                backup_directory_to_friend /root/backupemail/$USERNAME mail/$USERNAME
            fi
        fi
    done
}

function backup_letsencrypt {
    if [ -d /etc/letsencrypt ]; then
        echo $"Backing up Lets Encrypt settings"
        backup_directory_to_friend /etc/letsencrypt letsencrypt
    fi
}

function backup_gnusocial {
    if grep -q "GNU Social domain" $COMPLETION_FILE; then
        MICROBLOG_DOMAIN_NAME=$(cat $COMPLETION_FILE | grep "GNU Social domain" | awk -F ':' '{print $2}')
        if [ -d /var/www/${MICROBLOG_DOMAIN_NAME}/htdocs ]; then
            suspend_site ${MICROBLOG_DOMAIN_NAME}
            backup_database_to_friend gnusocial
            backup_directory_to_friend /root/tempgnusocialdata gnusocialdata
            echo $"Backing up GNU social installation"
            backup_directory_to_friend /var/www/${MICROBLOG_DOMAIN_NAME}/htdocs gnusocial
            restart_site
        else
            echo $"GNU Social domain specified but not found in /var/www/${MICROBLOG_DOMAIN_NAME}"
        fi
    fi
}

function backup_hubzilla {
    if grep -q "Hubzilla domain" $COMPLETION_FILE; then
        HUBZILLA_DOMAIN_NAME=$(cat $COMPLETION_FILE | grep "Hubzilla domain" | awk -F ':' '{print $2}')
        if [ -d /var/www/${HUBZILLA_DOMAIN_NAME} ]; then
            suspend_site ${HUBZILLA_DOMAIN_NAME}
            backup_database_to_friend hubzilla
            backup_directory_to_friend /root/temphubzilladata hubzilladata
            echo "Backing up Hubzilla installation"
            backup_directory_to_friend /var/www/${HUBZILLA_DOMAIN_NAME}/htdocs hubzilla
            restart_site
        else
            echo $"Hubzilla domain specified but not found in /var/www/${HUBZILLA_DOMAIN_NAME}"
            exit 2578
        fi
    fi
}

function backup_owncloud {
    if [ -d /etc/owncloud ]; then
        OWNCLOUD_DOMAIN_NAME=$(cat $COMPLETION_FILE | grep "Owncloud domain" | awk -F ':' '{print $2}')
        suspend_site ${OWNCLOUD_DOMAIN_NAME}
        backup_database_to_friend owncloud
        backup_directory_to_friend /root/tempownclouddata ownclouddata
        echo $"Backing up Owncloud data"
        backup_directory_to_friend /var/lib/owncloud owncloud
        backup_directory_to_friend /etc/owncloud owncloud2
        restart_site
    fi
}

function backup_gogs {
    if [ -d /home/git/go/src/github.com/gogits ]; then
        GIT_DOMAIN_NAME=$(cat $COMPLETION_FILE | grep "Gogs domain" | awk -F ':' '{print $2}')
        suspend_site ${GIT_DOMAIN_NAME}
        backup_database_to_friend gogs
        backup_directory_to_friend /root/tempgogsdata gogsdata
        echo $"Obtaining Gogs settings backup"
        backup_directory_to_friend /home/git/go/src/github.com/gogits/gogs/custom gogs
        echo $"Obtaining Gogs repos backup"
        mv /home/git/gogs-repositories/*.git /home/git/gogs-repositories/bob
        backup_directory_to_friend /home/git/gogs-repositories gogsrepos
        echo $"Obtaining Gogs authorized_keys backup"
        backup_directory_to_friend /home/git/.ssh gogsssh
        restart_site
    fi
}

function backup_wiki {
    if [ -d /etc/dokuwiki ]; then
        echo $"Backing up wiki"
        backup_directory_to_friend /var/lib/dokuwiki wiki
        backup_directory_to_friend /etc/dokuwiki wiki2
    fi
}

function backup_blog {
    if grep -q "Blog domain" $COMPLETION_FILE; then
        FULLBLOG_DOMAIN_NAME=$(cat $COMPLETION_FILE | grep "Blog domain" | awk -F ':' '{print $2}')
        if [ -d /var/www/${FULLBLOG_DOMAIN_NAME} ]; then
            echo $"Backing up blog"
            backup_directory_to_friend /var/www/${FULLBLOG_DOMAIN_NAME}/htdocs blog
        else
            echo $"Blog domain specified but not found in /var/www/${FULLBLOG_DOMAIN_NAME}"
            exit 2578
        fi
    fi
}

function backup_certs {
    if [ -d /etc/ssl ]; then
        echo $"Backing up certificates"
        backup_directory_to_friend /etc/ssl ssl
    fi
}

function backup_mailing_list {
    if [ -d /var/spool/mlmmj ]; then
        echo $"Backing up the public mailing list"
        backup_directory_to_friend /var/spool/mlmmj mailinglist
    fi
}

function backup_xmpp {
    if [ -d /var/lib/prosody ]; then
        echo $"Backing up the XMPP settings"
        backup_directory_to_friend /var/lib/prosody xmpp
    fi
}

function backup_web_server {
    if [ -d /etc/nginx ]; then
        echo $"Backing up web settings"
        backup_directory_to_friend /etc/nginx/sites-available web
    fi
}

function backup_admin_readme {
    if [ -f /home/$ADMIN_USERNAME/README ]; then
        echo $"Backing up README"
        if [ ! -d /home/$ADMIN_USERNAME/tempbackup ]; then
            mkdir -p /home/$ADMIN_USERNAME/tempbackup
        fi
        cp -f /home/$ADMIN_USERNAME/README /home/$ADMIN_USERNAME/tempbackup
        backup_directory_to_friend /home/$ADMIN_USERNAME/tempbackup readme
    fi
}

function backup_ipfs {
    if [ -d /home/$ADMIN_USERNAME/.ipfs ]; then
        echo $"Backing up IPFS"
        backup_directory_to_friend /home/$ADMIN_USERNAME/.ipfs ipfs
    fi
}

function backup_dlna {
    if [ -d /var/cache/minidlna ]; then
        echo $"Backing up DLNA cache"
        backup_directory_to_friend /var/cache/minidlna dlna
    fi
}

function backup_voip {
    if [ -f /etc/mumble-server.ini ]; then
        echo $"Backing up VoIP settings"
        if [ ! -d /root/tempvoipbackup ]; then
            mkdir -p /root/tempvoipbackup
        fi
        cp -f /etc/mumble-server.ini /root/tempvoipbackup
        cp -f /var/lib/mumble-server/mumble-server.sqlite /root/tempvoipbackup
        cp -f /etc/sipwitch.conf /root/tempvoipbackup
        backup_directory_to_friend /root/tempvoipbackup voip
    fi
}

function backup_tox {
    if [ -d /var/lib/tox-bootstrapd ]; then
        echo "Backing up Tox node settings"
        if [ -d /var/lib/tox-bootstrapd/Maildir ]; then
            rm -rf /var/lib/tox-bootstrapd/Maildir
        fi
        cp /etc/tox-bootstrapd.conf /var/lib/tox-bootstrapd
        backup_directory_to_friend /var/lib/tox-bootstrapd tox
    fi
}

function backup_mariadb {
    if [ ${#DATABASE_PASSWORD} -gt 1 ]; then
        if [ ! -d /root/tempmariadb ]; then
            mkdir /root/tempmariadb
        fi
        mysqldump --password=$DATABASE_PASSWORD mysql user > /root/tempmariadb/mysql.sql
        if [ ! -s /root/tempmariadb/mysql.sql ]; then
            echo $"Unable to backup MariaDB settings"
            rm -rf /root/tempmariadb
            # Send a warning email
            echo $"Unable to export database settings" | mail -s "${PROJECT_NAME} backup to friends" $ADMIN_EMAIL_ADDRESS
            exit 653
        fi
        echo "$DATABASE_PASSWORD" > /root/tempmariadb/db
        chmod 400 /root/tempmariadb/db
        backup_directory_to_friend /root/tempmariadb mariadb
    fi
}

# Returns the filename of a key share
function get_key_share {
    no_of_shares=$1
    USERNAME="$2"
    REMOTE_DOMAIN="$3"

    # Get a share index based on the supplied domain name
    # This ensures that the same share is always given to the same domain
    sharenumstr=$(md5sum <<< "$REMOTE_DOMAIN")
    share_index=$(echo $((0x${sharenumstr%% *} % ${no_of_shares})) | tr -d -)

    # get the filename
    share_files=(/home/$USERNAME/.gnupg_fragments/keyshare.asc.*)
    share_filename=${share_files[share_index]}

    echo "$share_filename"
}

function disperse_key_shares {
    USERNAME=$1
    REMOTE_DOMAIN=$2
    REMOTE_SSH_PORT=$3
    REMOTE_PASSWORD=$4
    REMOTE_SERVER=$5

    if [ -d /home/$USERNAME/.gnupg_fragments ]; then
        if [ $REMOTE_DOMAIN ]; then
            cd /home/$USERNAME/.gnupg_fragments
            no_of_shares=$(ls -afq keyshare.asc.* | wc -l)
            if (( no_of_shares > 1 )); then
                share_filename=$(get_key_share $no_of_shares "$USERNAME" "$REMOTE_DOMAIN")

                # create a temp directory containing the share
                temp_key_share_dir=/home/$USERNAME/tempkey
                temp_key_share_fragments=$temp_key_share_dir/.gnupg_fragments_${USERNAME}
                mkdir -p $temp_key_share_fragments
                cp $share_filename $temp_key_share_fragments/

                # copy the fragments directory to the remote server
                /usr/bin/sshpass -p "$REMOTE_PASSWORD" \
                    scp -r -P $REMOTE_SSH_PORT $temp_key_share_fragments $REMOTE_SERVER
                if [ ! "$?" = "0" ]; then
                    # Send a warning email
                    echo "Key share to $REMOTE_SERVER failed" | \
                        mail -s "${PROJECT_NAME} social key management" $MY_EMAIL_ADDRESS
                else
                    # Send a confirmation email
                    echo "Key ${share_filename} shared to $REMOTE_SERVER" | \
                        mail -s "${PROJECT_NAME} social key management" $MY_EMAIL_ADDRESS
                fi

                # remove the temp file/directory
                shred -zu $temp_key_share_fragments/*
                rm -rf $temp_key_share_dir
            fi
        fi
    fi
}

function valid_backup_destination {
    destination_dir="$1"
    is_valid="yes"

    if [[ "$destination_dir" == "hubzilla" || \
          "$destination_dir" == "hubzilladata" || \
          "$destination_dir" == "gogs" || \
          "$destination_dir" == "gogsrepos" || \
          "$destination_dir" == "gogsssh" || \
          "$destination_dir" == "gnusocial" || \
          "$destination_dir" == "gnusocialdata" || \
          "$destination_dir" == "mariadb" || \
          "$destination_dir" == "config" || \
          "$destination_dir" == "letsencrypt" || \
          "$destination_dir" == "wiki" || \
          "$destination_dir" == "wiki2" || \
          "$destination_dir" == "xmpp" || \
          "$destination_dir" == "ipfs" || \
          "$destination_dir" == "dlna" || \
          "$destination_dir" == "tox" || \
          "$destination_dir" == "ssl" || \
          "$destination_dir" == "blog" || \
          "$destination_dir" == "owncloud" || \
          "$destination_dir" == "owncloud2" || \
          "$destination_dir" == "ownclouddata" || \
          "$destination_dir" == "mailinglist" ]]; then
        is_valid="no"
    fi

    echo $is_valid
}

function backup_extra_directories {
    if [ ! -f $BACKUP_EXTRA_DIRECTORIES ]; then
        return
    fi

    echo $"Backing up some additional directories"
    while read backup_line
    do
        backup_dir=$(echo "$backup_line" | awk -F ',' '{print $1}' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
        if [ -d "$backup_dir" ]; then
            destination_dir=$(echo "$backup_line" | awk -F ',' '{print $2}' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
            if [[ $(valid_backup_destination "$destination_dir") == "yes" ]]; then
                backup_directory_to_friend "$backup_dir" "$destination_dir"
            else
                echo $"WARNING: The backup directory $destination_dir is already used."
                echo $"Choose a different destination name for backing up $backup_dir"
            fi
        else
            echo $"WARNING: Directory $backup_dir does not exist"
        fi
    done <$BACKUP_EXTRA_DIRECTORIES
}

TEST_MODE="no"
if [[ "$1" == "test" ]]; then
    TEST_MODE="yes"
fi

backup_configuration
if [[ $TEST_MODE == "no" ]]; then
    backup_users
    backup_letsencrypt
    backup_gnusocial
    backup_hubzilla
    backup_owncloud
    backup_gogs
    backup_wiki
    backup_blog
    backup_certs
    backup_mailing_list
    backup_xmpp
    backup_web_server
    backup_admin_readme
    backup_ipfs
    backup_dlna
    backup_voip
    backup_tox
    backup_mariadb
    backup_extra_directories
fi

# For each remote server
while read remote_server
do
    # Get the server and its password
    # Format is:
    #   username@domain <port number> /home/username <ssh password>
    REMOTE_SERVER=$(echo "${remote_server}" | awk -F ' ' '{print $1}')
    if [ $REMOTE_SERVER ]; then
        REMOTE_DOMAIN=$(echo "${remote_server}" | awk -F ' ' '{print $1}' | awk -F '@' '{print $2}')
        REMOTE_SSH_PORT=$(echo "${remote_server}" | awk -F ' ' '{print $2}')
        REMOTE_DIRECTORY=$(echo "${remote_server}" | awk -F ' ' '{print $3}')
        REMOTE_PASSWORD=$(echo "${remote_server}" | awk -F ' ' '{print $4}')
        NOW=$(date +"%Y-%m-%d %H:%M:%S")
        REMOTE_SERVER=$REMOTE_SERVER:$REMOTE_DIRECTORY

        echo "$NOW Starting backup to $REMOTE_SERVER" >> /var/log/remotebackups.log

        # Social key management
        for d in /home/*/ ; do
            USERNAME=$(echo "$d" | awk -F '/' '{print $3}')
            if [[ $USERNAME != "git" ]]; then
                disperse_key_shares $USERNAME $REMOTE_DOMAIN $REMOTE_SSH_PORT "$REMOTE_PASSWORD" $REMOTE_SERVER
            fi
        done

        if [[ $TEST_MODE == "yes" ]]; then
            echo "rsync -ratlzv --rsh=\"/usr/bin/sshpass -p '$REMOTE_PASSWORD' ssh -p $REMOTE_SSH_PORT -o StrictHostKeyChecking=no\" $SERVER_DIRECTORY/backup $REMOTE_SERVER"
        fi
        rsync -ratlzv --rsh="/usr/bin/sshpass -p \"$REMOTE_PASSWORD\" ssh -p $REMOTE_SSH_PORT -o StrictHostKeyChecking=no" $SERVER_DIRECTORY/backup $REMOTE_SERVER
        if [ ! "$?" = "0" ]; then
            echo "$NOW Backup to $REMOTE_SERVER failed" >> /var/log/remotebackups.log
            # Send a warning email
            echo "Backup to $REMOTE_SERVER failed" | mail -s "${PROJECT_NAME} backup to friends" $ADMIN_EMAIL_ADDRESS
        else
            echo "$NOW Backed up to $REMOTE_SERVER" >> /var/log/remotebackups.log
        fi
    fi

done < /home/${ADMIN_USERNAME}/backup.list

exit 0