The best Bash backup script ever! ~ Jeffrey Pine Hein

Posted on by Jeffrey Hein

This script for backing up websites securely using Google Cloud Storage (Nearline)
Cost: $0.01 per GB per month

You will need to read these guides before starting with this one:

/install-gcsfuse-inside-a-google-cloud-compute-engine-running-ubuntu-15-10/

/how-to-mount-a-google-cloud-storage-bucket-as-a-drive-in-ubuntu-15-10-wily-using-fuse/

/how-to-mount-a-google-drive-as-a-drive-in-ubuntu-15-10-wily-using-fuse/

/how-to-fix-the-device-or-resource-busy-error-when-trying-to-mount-your-google-drive-on-your-headless-google-cloud-compute-engine-server-running-ubuntu-15-10-wily/

/how-to-use-duplicity-to-backup-directly-to-a-google-cloud-storage-bucket-to-achieve-time-machine-like-incremental-backups-on-ubuntu-15-10-wily/

Install in home directory

~/backups

Run as a normal user:

~/backups/bin/backup > backuplog.txt 2> errorlog.txt

#!/bin/bash
#Change the variables below to reflect your information
echo "------------------------------------------------------------------"
echo "                 Hang Left Hosting Client Backups"
echo "------------------------------------------------------------------"
echo "------------------------------------------------------------------" >&2
echo "          Hang Left Hosting Client Backups Error Log" >&2
echo "------------------------------------------------------------------" >&2

echo "Creating variables..."

#Create variables
#GREEN='\033[0;32m'
#NC='\033[0m' # No Color
printf "$0 <-- Name and path of this script.\n"
WD="/opt/backups" #Working directory
printf "$WD <-- Working Directory on local server.\n"
BIN_DIR="$WD/bin"
printf "$BIN_DIR/ <-- Script directory on local server.\n"
DOMAIN="example.com"
printf "$DOMAIN <-- Hostname or IP of server that has clients' live data (Data we are backing up)\n"
DIR_LIST="*"
printf "$DIR_LIST <-- Directories we are backing up under /home/{client}/ on $DOMAIN.\n"
echo "If a wildcard \"*\" is used, we will backup every file and folder in their home directory."
shopt -s dotglob #This is needed incase you use the wildcard, and you want it to back up hidden files like .htaccess
echo "------------------------------------------------------------------"

#Google Cloud storage bucket
echo "A Google Cloud storage bucket is used for longterm storage of incremental backups."
BUCKET="$DOMAIN-backups"
printf "$BUCKET <-- Google Cloud storage bucket name\n"
BUCKET_DIR="$WD/$BUCKET"
printf "$BUCKET_DIR <-- Google Cloud storage bucket mount point\n"
echo "------------------------------------------------------------------"

#Google Drive
echo "Google Drive is used for secure shared access to the SSH key directory"
GDRIVE="$WD/google-drive"
printf "$GDRIVE <-- Google Drive mount point\n"
KEY_DIR_SYNC="$GDRIVE/keys"
PASS_DIR_SYNC="$GDRIVE/passwords"
TEST_KEY_DIR_SYNC="/home/$username/keys"
printf "$KEY_DIR_SYNC <-- Path to directory of SSH keys on Google Drive\n"
KEY_DIR="$WD/keys"
printf "$KEY_DIR <-- Path to local mirror of SSH keys\n"
echo "------------------------------------------------------------------"

#Google Cloud compute presistant disks
#The name of the persistant disk mkfs.ext4 -L <label>
echo "A Google Cloud compute disk is used to store a local mirror of clients' data"
PDISK="clients"
printf "$PDISK <-- Google Cloud compute persistant disk name (mkfs.ext4 -L <label>)\n"
SYNC_DIR="/mnt/$PDISK"
printf "$SYNC_DIR <-- Google Cloud compute persistant disk mount point\n"
echo "------------------------------------------------------------------"

echo "Creating directories..."
echo "------------------------------------------------------------------"
#Create  directories
#sudo mkdir -p $WD
#sudo chmod a+w $WD
mkdir -p $SYNC_DIR $BUCKET_DIR $GDRIVE

echo "------------------------------------------------------------------"
echo "Mounting Google drive..."
echo "------------------------------------------------------------------"
#Mount Google GDRIVE
google-drive-ocamlfuse $GDRIVE

#Copy the keys folder from Google GDRIVE
mkdir -p $KEY_DIR
echo "------------------------------------------------------------------"
echo "Syncing SSH keys from Google Drive..."
echo "------------------------------------------------------------------"
echo ""
rsync -azr --delete --progress "$KEY_DIR_SYNC" $WD
#rsync -azr --delete --progress "$TEST_KEY_DIR_SYNC" $WD
chmod 600 $KEY_DIR/*

echo "------------------------------------------------------------------"
echo "Mounting Google Cloud storage bucket..."
#Mount Google storage bucket
gcsfuse $BUCKET $WD/$BUCKET
echo "------------------------------------------------------------------"
echo "Mounting Google Cloud persistant disk..."
echo "------------------------------------------------------------------"

#Mount Google Persistant disk
#sudo mount -o discard,defaults /dev/disk/by-label/$PDISK $SYNC_DIR
#sudo chmod a+w $SYNC_DIR

for i in `ls -1 $KEY_DIR`; do
	#Identify client in error log
	echo "Client: $i" >&2
	#Craft the SSH command
	SSH="ssh -p 2222 -o StrictHostKeyChecking=no -o IdentityFile=$KEY_DIR/$i"

	echo "------------------------------------------------------------------"
	echo ""
	echo "[$i] <-- Syncing client" #Display client
	echo "------------------------------------------------------------------"
	echo "Syncing public_html from live server to a mounted Google cloud persistant disk using rsync..."
	echo "------------------------------------------------------------------"

	#Copy password into mysql backup script
	#$BIN_DIR/findpw.sh $i > "$PASS_DIR_SYNC/$i.txt"
	PASS=`cat "$PASS_DIR_SYNC/$i.txt"`
	cp $WD/mysqlbackup.sh $BIN_DIR/
	sed -i 's/love/"'"$PASS"'"/g' $BIN_DIR/mysqlbackup.sh
	chmod u+x $BIN_DIR/mysqlbackup.sh
	#Sync bin directory
	rsync -azhv --progress --delete -e "$SSH" $BIN_DIR/ $i@$DOMAIN:/home/$i/bin/

	mkdir -p $BUCKET_DIR/$i

#	#Mount the SSH Share
#	BU_DIR="$WD/inmotion-clients"
#	mkdir $BU_DIR/$i
#	sshfs -p 2222 -o StrictHostKeyChecking=no -o IdentityFile=$KEY_DIR/$i $i@$DOMAIN:/home/$i $BU_DIR/$i
#	#Unmount the share
#	#sleep 5
#	#fusermount -u $BU_DIR/$i


#	#Copy old backups
#	if [ ! -d $BUCKET_DIR/$i/archives ]; then
#	echo "------------------------------------------------------------------"
#	echo "Copying current backups to archives..."
#	echo "------------------------------------------------------------------"
#	mkdir $BUCKET_DIR/$i/archives -p
#	$SSH $i@$DOMAIN "mkdir -p /home/$i/backups/archives && tar czf /home/$i/backups/archives/2016-04-01.tar.gz /home/$i/backups/2016-04-01"
#	$SSH $i@$DOMAIN "mkdir -p /home/$i/backups/archives && tar czf /home/$i/backups/archives/2016-04-12.tar.gz /home/$i/backups/2016-04-12"
#	rsync -azhv --stats --progress -e "$SSH" $i@$DOMAIN:/home/$i/backups/archives/*.tar.gz $BUCKET_DIR/$i/archives/
#
#	#Delete old backups
#	echo "Deleting old backups..."
#	$SSH $i@$DOMAIN "rm -rf /home/$i/backups/2016* /home/$i/backups/archives"
	$SSH $i@$DOMAIN "crontab -r"
#	fi

	#Back up MySQL
	echo "Backing up MySQL database..."
	echo "------------------------------------------------------------------"
	mkdir $BUCKET_DIR/$i/mysql -p
	$SSH $i@$DOMAIN "/home/$i/bin/mysqlbackup.sh"
	rsync -azh --progress --stats -e "$SSH" $i@$DOMAIN:/home/$i/backups/*.sql.gz $BUCKET_DIR/$i/mysql/
	#Sync the Google storage cloud with our presistant disk mirror
	#gsutil -m rsync -d -r -e $SYNC_DIR/$i gs://$BUCKET/$i


	#Sync our clients website with a persistanct disk mirror using rsync
	#for n in $DIR_LIST; do
	echo "Backing up files..."
	echo "------------------------------------------------------------------"
	rsync -azh --stats --progress -e "$SSH" --exclude "*.zip" --exclude ".sql" --exclude "sql.gz" --exclude "tar.gz" $i@$DOMAIN:/home/$i/$DIR_LIST $SYNC_DIR/$i #2>> error.log
	if [ $? != 0 ]; then
		echo $i >> error.log
		echo "ERROR!"
	fi
	#done

	echo ""
	echo "[$i] <-- Backing up client" #Display client
	echo "------------------------------------------------------------------"
	echo "Backing up data to a Google Gloud storage bucket using duplicity.."
	echo "------------------------------------------------------------------"
	echo "An incremental backup every day. A full backup every month."
	echo "We keep 12 full backups and their corresponding incrementals."
	echo "One year of daily back up data!"
	echo "------------------------------------------------------------------"

	#Clean up in case of aborted backups
	duplicity cleanup --no-encryption --force gs://$BUCKET/$i

	#Backup data to bucket
	duplicity --progress --no-encryption --full-if-older-than 1M $SYNC_DIR/$i gs://$BUCKET/$i

	#Delete old backups
	#duplicity remove-older-than 1Y --force $SYNC_DIR/$i gs://$BUCKET/$i
	duplicity remove-all-but-n-full 12 --force gs://$BUCKET/$i

	#sshfs -p 2222 -o allow_other,defer_permissions,IdentityFile=$DIR/$i $i@jphein.com:/home/$i/backups $BU_DIR/$i
	#scp -p 2222 -i $DIR/$1 filebackup.sh $i@$DOMAIN:bin/
	#cat filebackup.sh | $SSH 'cat > bin/filebackup.sh'
	#$SSH '~/bin/install_backup.sh; rm -rf ~/backups/MONTHLY; rm -rf ~/backups/2015-*; rm -rf ~/backups/WEEKLY-*; rsync -ah ~/public_html/ ~/backups/MONTHLY; ~/bin/filebackup.sh' 
	#rsync -am --stats  $BU_DIR/$i/public_html $BUCKET_DIR/$1/

done
sleep 10
fusermount -u $GDRIVE
fusermount -u $BUCKET_DIR
#sudo umount $SYNC_DIR
Tags: , , ,

Leave a Reply

Your email address will not be published. Required fields are marked *

This site uses Akismet to reduce spam. Learn how your comment data is processed.

  • Hello human, I am a GPT3.5 Turbo powered AI chat bot. Ask me anything!

AI thinking ...