#!/bin/bash # s3putsecurefolder # # Synchronise a folder with Amazon s3 securely (s3putsecurefolder --help) # Note that you should install s3cmd before using this script, and configure it # using "s3cmd --configure". # In particular you need to configure GPG encryption via the 'Encryption # password' option during the configure process; this means that your data is # not readable on S3 by anyone who doesn't have the password. # I'm aware that s3cmd has a 'sync' option, but this synchronises in both # directions, and also performs a remote operations to sync. This tool only ever # uploads, giving you the certainty that you will never accidentally download # any data and overwrite local files, and requires no remote access to determine # whether any files need uploading (local timestamps are used). # # I personally use this script to upload my backup volumes to Amazon S3 as an # offsite storage. The files are re-used (pseudo tape volume cycling) so that # I only use a finite amount of space on S3. # Copyright (c) 2009 Steve Streeting # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. TIMESTAMPFILENAME=.s3lastupload function usage { echo "Usage: " echo " $0 [-n] " echo "Options:" echo " -n : Just list actions, don't perform them." echo " -c FILE : Use FILE as S3 config instead of ~/.s3cfg" } function help { usage echo echo "s3putsecurefolder is a tool for synchronising a folder with Amazon S3" echo "in a single direction (upload) and always using encryption." } function checkAndUploadFile { filename=$1 prefix=$2 bucket=$3 dryrun=$4 cfg=$5 lenprefix=${#prefix} targetpath=${filename:$lenprefix} if [[ "$cfg" != "" ]] then cfgopt="-c $cfg" fi if [[ `basename $filename` != $TIMESTAMPFILENAME ]] then # Hmm, s3cmd -n still seems to create files of 0 size in bucket! # do our own dry run if [[ $dryrun = 1 ]] then echo "Would execute: s3cmd -e $cfgopt put $filename s3://$bucket/$targetpath" else # upload (encrypted) s3cmd -e $cfgopt put $filename s3://$bucket/$targetpath fi fi } if [[ "$1" == "--help" ]] then help exit fi # Deal with optional named arguments DRYRUN=0 CFGFILE="" while getopts "nc:" Opt do case $Opt in n) DRYRUN=1;; c) CFGFILE=$OPTARG;; esac done shift $(($OPTIND - 1)) if [[ $# < 2 ]] then usage exit fi if [[ ! -e "$1" ]] then echo "First parameter needs to be source folder" exit fi if [[ "$CFGFILE" != "" && ! -e "$CFGFILE" ]] then echo "Configuration file $CFGFILE not found." exit fi BACKUP_FOLDER=$1 TARGET_BUCKET=$2 INFOFILE=$BACKUP_FOLDER/$TIMESTAMPFILENAME echo "Synchronising $BACKUP_FOLDER to s3://$TARGET_BUCKET" if [[ $DRYRUN = 1 ]] then echo "Simulation only, not uploading" fi # Timestamp at start TMPINFOFILE=`tempfile` touch $TMPINFOFILE if [[ -e $INFOFILE ]] then # All files since last time for filename in `find $BACKUP_FOLDER -type f -cnewer $INFOFILE` do checkAndUploadFile $filename $BACKUP_FOLDER $TARGET_BUCKET $DRYRUN $CFGFILE done else # Do all files for filename in `find $BACKUP_FOLDER -type f` do checkAndUploadFile $filename $BACKUP_FOLDER $TARGET_BUCKET $DRYRUN $CFGFILE done fi if [[ $DRYRUN == 0 ]] then # Update final info file, preserve timestamp cp -p $TMPINFOFILE $INFOFILE fi rm $TMPINFOFILE