This script for backing up websites securely using Google Cloud Storage (Nearline)
Cost: $0.01 per GB per month
You will need to read these guides before starting with this one:
/install-gcsfuse-inside-a-google-cloud-compute-engine-running-ubuntu-15-10/
/how-to-mount-a-google-cloud-storage-bucket-as-a-drive-in-ubuntu-15-10-wily-using-fuse/
/how-to-mount-a-google-drive-as-a-drive-in-ubuntu-15-10-wily-using-fuse/
Install in home directory
~/backups
Run as a normal user:
~/backups/bin/backup > backuplog.txt 2> errorlog.txt
#!/bin/bash
#Change the variables below to reflect your information
echo "------------------------------------------------------------------"
echo " Hang Left Hosting Client Backups"
echo "------------------------------------------------------------------"
echo "------------------------------------------------------------------" >&2
echo " Hang Left Hosting Client Backups Error Log" >&2
echo "------------------------------------------------------------------" >&2
echo "Creating variables..."
#Create variables
#GREEN='\033[0;32m'
#NC='\033[0m' # No Color
printf "$0 <-- Name and path of this script.\n"
WD="/opt/backups" #Working directory
printf "$WD <-- Working Directory on local server.\n"
BIN_DIR="$WD/bin"
printf "$BIN_DIR/ <-- Script directory on local server.\n"
DOMAIN="example.com"
printf "$DOMAIN <-- Hostname or IP of server that has clients' live data (Data we are backing up)\n"
DIR_LIST="*"
printf "$DIR_LIST <-- Directories we are backing up under /home/{client}/ on $DOMAIN.\n"
echo "If a wildcard \"*\" is used, we will backup every file and folder in their home directory."
shopt -s dotglob #This is needed incase you use the wildcard, and you want it to back up hidden files like .htaccess
echo "------------------------------------------------------------------"
#Google Cloud storage bucket
echo "A Google Cloud storage bucket is used for longterm storage of incremental backups."
BUCKET="$DOMAIN-backups"
printf "$BUCKET <-- Google Cloud storage bucket name\n"
BUCKET_DIR="$WD/$BUCKET"
printf "$BUCKET_DIR <-- Google Cloud storage bucket mount point\n"
echo "------------------------------------------------------------------"
#Google Drive
echo "Google Drive is used for secure shared access to the SSH key directory"
GDRIVE="$WD/google-drive"
printf "$GDRIVE <-- Google Drive mount point\n"
KEY_DIR_SYNC="$GDRIVE/keys"
PASS_DIR_SYNC="$GDRIVE/passwords"
TEST_KEY_DIR_SYNC="/home/$username/keys"
printf "$KEY_DIR_SYNC <-- Path to directory of SSH keys on Google Drive\n"
KEY_DIR="$WD/keys"
printf "$KEY_DIR <-- Path to local mirror of SSH keys\n"
echo "------------------------------------------------------------------"
#Google Cloud compute presistant disks
#The name of the persistant disk mkfs.ext4 -L <label>
echo "A Google Cloud compute disk is used to store a local mirror of clients' data"
PDISK="clients"
printf "$PDISK <-- Google Cloud compute persistant disk name (mkfs.ext4 -L <label>)\n"
SYNC_DIR="/mnt/$PDISK"
printf "$SYNC_DIR <-- Google Cloud compute persistant disk mount point\n"
echo "------------------------------------------------------------------"
echo "Creating directories..."
echo "------------------------------------------------------------------"
#Create directories
#sudo mkdir -p $WD
#sudo chmod a+w $WD
mkdir -p $SYNC_DIR $BUCKET_DIR $GDRIVE
echo "------------------------------------------------------------------"
echo "Mounting Google drive..."
echo "------------------------------------------------------------------"
#Mount Google GDRIVE
google-drive-ocamlfuse $GDRIVE
#Copy the keys folder from Google GDRIVE
mkdir -p $KEY_DIR
echo "------------------------------------------------------------------"
echo "Syncing SSH keys from Google Drive..."
echo "------------------------------------------------------------------"
echo ""
rsync -azr --delete --progress "$KEY_DIR_SYNC" $WD
#rsync -azr --delete --progress "$TEST_KEY_DIR_SYNC" $WD
chmod 600 $KEY_DIR/*
echo "------------------------------------------------------------------"
echo "Mounting Google Cloud storage bucket..."
#Mount Google storage bucket
gcsfuse $BUCKET $WD/$BUCKET
echo "------------------------------------------------------------------"
echo "Mounting Google Cloud persistant disk..."
echo "------------------------------------------------------------------"
#Mount Google Persistant disk
#sudo mount -o discard,defaults /dev/disk/by-label/$PDISK $SYNC_DIR
#sudo chmod a+w $SYNC_DIR
for i in `ls -1 $KEY_DIR`; do
#Identify client in error log
echo "Client: $i" >&2
#Craft the SSH command
SSH="ssh -p 2222 -o StrictHostKeyChecking=no -o IdentityFile=$KEY_DIR/$i"
echo "------------------------------------------------------------------"
echo ""
echo "[$i] <-- Syncing client" #Display client
echo "------------------------------------------------------------------"
echo "Syncing public_html from live server to a mounted Google cloud persistant disk using rsync..."
echo "------------------------------------------------------------------"
#Copy password into mysql backup script
#$BIN_DIR/findpw.sh $i > "$PASS_DIR_SYNC/$i.txt"
PASS=`cat "$PASS_DIR_SYNC/$i.txt"`
cp $WD/mysqlbackup.sh $BIN_DIR/
sed -i 's/love/"'"$PASS"'"/g' $BIN_DIR/mysqlbackup.sh
chmod u+x $BIN_DIR/mysqlbackup.sh
#Sync bin directory
rsync -azhv --progress --delete -e "$SSH" $BIN_DIR/ $i@$DOMAIN:/home/$i/bin/
mkdir -p $BUCKET_DIR/$i
# #Mount the SSH Share
# BU_DIR="$WD/inmotion-clients"
# mkdir $BU_DIR/$i
# sshfs -p 2222 -o StrictHostKeyChecking=no -o IdentityFile=$KEY_DIR/$i $i@$DOMAIN:/home/$i $BU_DIR/$i
# #Unmount the share
# #sleep 5
# #fusermount -u $BU_DIR/$i
# #Copy old backups
# if [ ! -d $BUCKET_DIR/$i/archives ]; then
# echo "------------------------------------------------------------------"
# echo "Copying current backups to archives..."
# echo "------------------------------------------------------------------"
# mkdir $BUCKET_DIR/$i/archives -p
# $SSH $i@$DOMAIN "mkdir -p /home/$i/backups/archives && tar czf /home/$i/backups/archives/2016-04-01.tar.gz /home/$i/backups/2016-04-01"
# $SSH $i@$DOMAIN "mkdir -p /home/$i/backups/archives && tar czf /home/$i/backups/archives/2016-04-12.tar.gz /home/$i/backups/2016-04-12"
# rsync -azhv --stats --progress -e "$SSH" $i@$DOMAIN:/home/$i/backups/archives/*.tar.gz $BUCKET_DIR/$i/archives/
#
# #Delete old backups
# echo "Deleting old backups..."
# $SSH $i@$DOMAIN "rm -rf /home/$i/backups/2016* /home/$i/backups/archives"
$SSH $i@$DOMAIN "crontab -r"
# fi
#Back up MySQL
echo "Backing up MySQL database..."
echo "------------------------------------------------------------------"
mkdir $BUCKET_DIR/$i/mysql -p
$SSH $i@$DOMAIN "/home/$i/bin/mysqlbackup.sh"
rsync -azh --progress --stats -e "$SSH" $i@$DOMAIN:/home/$i/backups/*.sql.gz $BUCKET_DIR/$i/mysql/
#Sync the Google storage cloud with our presistant disk mirror
#gsutil -m rsync -d -r -e $SYNC_DIR/$i gs://$BUCKET/$i
#Sync our clients website with a persistanct disk mirror using rsync
#for n in $DIR_LIST; do
echo "Backing up files..."
echo "------------------------------------------------------------------"
rsync -azh --stats --progress -e "$SSH" --exclude "*.zip" --exclude ".sql" --exclude "sql.gz" --exclude "tar.gz" $i@$DOMAIN:/home/$i/$DIR_LIST $SYNC_DIR/$i #2>> error.log
if [ $? != 0 ]; then
echo $i >> error.log
echo "ERROR!"
fi
#done
echo ""
echo "[$i] <-- Backing up client" #Display client
echo "------------------------------------------------------------------"
echo "Backing up data to a Google Gloud storage bucket using duplicity.."
echo "------------------------------------------------------------------"
echo "An incremental backup every day. A full backup every month."
echo "We keep 12 full backups and their corresponding incrementals."
echo "One year of daily back up data!"
echo "------------------------------------------------------------------"
#Clean up in case of aborted backups
duplicity cleanup --no-encryption --force gs://$BUCKET/$i
#Backup data to bucket
duplicity --progress --no-encryption --full-if-older-than 1M $SYNC_DIR/$i gs://$BUCKET/$i
#Delete old backups
#duplicity remove-older-than 1Y --force $SYNC_DIR/$i gs://$BUCKET/$i
duplicity remove-all-but-n-full 12 --force gs://$BUCKET/$i
#sshfs -p 2222 -o allow_other,defer_permissions,IdentityFile=$DIR/$i [email protected]:/home/$i/backups $BU_DIR/$i
#scp -p 2222 -i $DIR/$1 filebackup.sh $i@$DOMAIN:bin/
#cat filebackup.sh | $SSH 'cat > bin/filebackup.sh'
#$SSH '~/bin/install_backup.sh; rm -rf ~/backups/MONTHLY; rm -rf ~/backups/2015-*; rm -rf ~/backups/WEEKLY-*; rsync -ah ~/public_html/ ~/backups/MONTHLY; ~/bin/filebackup.sh'
#rsync -am --stats $BU_DIR/$i/public_html $BUCKET_DIR/$1/
done
sleep 10
fusermount -u $GDRIVE
fusermount -u $BUCKET_DIR
#sudo umount $SYNC_DIR