bin/rsync-snapshot

   1 #!/bin/bash
   2 # ----------------------------------------------------------------------
   3 # created by francois scheurer on 20070323
   4 # derivate from mikes handy rotating-filesystem-snapshot utility
   5 # see http://www.mikerubel.org/computers/rsync_snapshots
   6 # ----------------------------------------------------------------------
   7 #rsync note:
   8 #    1) rsync -avz /src/foo  /dest      => ok, creates /dest/foo, like cp -a /src/foo /dest
   9 #    2) rsync -avz /src/foo/ /dest/foo  => ok, creates /dest/foo, like cp -a /src/foo/. /dest/foo (or like cp -a /src/foo /dest)
  10 #    3) rsync -avz /src/foo/ /dest/foo/ => ok, same as 2)
  11 #    4) rsync -avz /src/foo/ /dest      => dangerous!!! overwrite dest content, like cp -a /src/foo/. /dest
  12 #      solution: remove trailing / at /src/foo/ => 1)
  13 #      minor problem: rsync -avz /src/foo /dest/foo => creates /dest/foo/foo, like mkdir /dest/foo && cp -a /src/foo /dest/foo
  14 #    main options:
  15 #      -H --hard-links
  16 #      -a equals -rlptgoD (no -H,-A,-X)
  17 #        -r --recursive
  18 #        -l --links
  19 #        -p --perms
  20 #        -t --times
  21 #        -g --group
  22 #        -o --owner
  23 #        -D --devices --specials
  24 #      -x --one-file-system
  25 #      -S --sparse
  26 #      --numeric-ids
  27 #    useful options:
  28 #      -n --dry-run
  29 #      -z --compress
  30 #      -y --fuzzy
  31 #      --bwlimit=X limit disk IO to X kB/s
  32 #      -c --checksum
  33 #      -I --ignore-times
  34 #      --size-only
  35 #    other options:
  36 #      -v --verbose
  37 #      -P equals --progress --partial
  38 #      -h --human-readable
  39 #      --stats
  40 #      -e'ssh -o ServerAliveInterval=60'
  41 #      --delete
  42 #      --delete-delay
  43 #      --delete-excluded
  44 #      --ignore-existing
  45 #      -i --itemize-changes
  46 #      --stop-at
  47 #      --time-limit
  48 #      --rsh=\"ssh -p ${HOST_PORT} -i /root/.ssh/rsync_rsa -l root\"
  49 #      --rsync-path=\"/usr/bin/rsync\""
  50 #    quickcheck options:
  51 #      the default behavior is to skip files with same size & mtime on destination
  52 #      mtime = last data write access
  53 #      atime = last data read access (can be ignored with noatime mount option or with chattr +A)
  54 #      ctime = last inode change (write access, change of permission or ownership)
  55 #      note that a checksum is always done after a file synchronization/transfer
  56 #      --modify-window=X ignore mtime differences less or equal to X sec
  57 #      --size-only skip files with same size on destination (ignore mtime)
  58 #      -c --checksum skip files with same MD5 checksum on destination (ignore size & mtime, all files are read once, then the list of files to be resynchronized is read a second time, there is a lot of disk IO but network trafic is minimal if many files are identical; log includes only different files)
  59 #      -I --ignore-times never skip files (all files are resynchronized, all files are read once, there is more network trafic than with --checksum but less disk IO and hence is faster than --checksum if net is fast or if most files are different; log includes all files)
  60 #      --link-dest does the quickcheck on another reference-directory and makes hardlinks if quickcheck succeeds
  61 #        (however, if mtime is different and --perms is used, the reference file is copied in a new inode)
  62 #    see also this link for a rsync tutorial: http://www.thegeekstuff.com/2010/09/rsync-command-examples/
  63 #todo:
  64 #                 'du' slow on many snapshot.X..done
  65 #  autokill after n minutes.
  66 #                 if disk full, its better to replace the snapshot.001 than to cancel and have a very old backup (even if it may fail to create the snapshot and ends with 0 backups)..done
  67 #                 rsync-snapshot for oracle redo logs..old
  68 #                 'find'-list with md5 signatures -> .gz file stored aside rsync.log.gz inside the snapshot.X folder; this file will be move to parent dir /backup/snapshot/localhost/ before deletion of a snapshot; this file will also be used to extract an incremental backup with tape-arch.sh..done (md5sum calculation with rsync-list.sh for acm14=18m58 and only 5m27 with a reference file. speedup is ~250-300%)
  69 #  realtime freedisk display with echo $(($(stat -f -c "%f" /backup/snapshot/) * 4096 / 1024))
  70 #  use authorized_keys with restriction of bash (command=) and set sshd_config with PermitRootLogin=forced-commands-only, see http://troy.jdmz.net/rsync/index.html http://www.snailbook.com/faq/restricted-scp.auto.html
  71 #  note: rsync lists all files in snapshot.X disregarding inclusion patterns, this is slow.
  72
  73
  74
  75
  76 # ------------- the help page ------------------------------------------
  77 if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
  78   cat << "EOF"
  79 Version 2.01 2013-01-16
  80
  81 USAGE: rsync-snapshot.sh HOST [--recheck]
  82
  83 PURPOSE: create a snapshot backup of the whole filesystem into the folder
  84   '/backup/snapshot/HOST/snapshot.001'.
  85   If HOST is 'localhost' it is replaced with the local hostname.
  86   If HOST is a remote host then rsync over ssh is used to transfer the files
  87   with a delta-transfer algorithm to transfer only minimal parts of the files
  88   and improve speed; rsync uses for this the previous backup as reference.
  89   This reference is also used to create hard links instead of files when
  90   possible and thus save disk space. If original and reference file have
  91   identical content but different timestamps or permissions then no hard link
  92   is created.
  93   A rotation of all backups renames snapshot.X into snapshot.X+1 and removes
  94   backups with X>512. About 10 backups with non-linear distribution are kept
  95   in rotation; for example with X=1,2,3,4,8,16,32,64,128,256,512.
  96   The snapshots folders are protected read-only against all users including
  97   root using 'chattr'.
  98   The --recheck option forces a sync of all files even if they have same mtime
  99   & size; it is can verify a backup and fix corrupted files;
 100   --recheck recalculates also the MD5 integrity signatures without using the
 101   last signature-file as precalculation.
 102   Some features like filter rules, MD5, chattr, bwlimit and per server retention
 103   policy can be configured by modifying the scripts directly.
 104
 105 FILES:
 106     /backup/snapshot/rsync/rsync-snapshot.sh  the backup script
 107     /backup/snapshot/rsync/rsync-list.sh      the md5 signature script
 108     /backup/snapshot/rsync/rsync-include.txt  the filter rules
 109
 110 Examples:
 111   (nice -5 ./rsync-snapshot.sh >log &) ; tail -f log
 112   cd /backup/snapshot; for i in $(ls -A); do nice -10 /backup/snapshot/rsync/rsync-snapshot.sh $i; done
 113 EOF
 114   exit 1
 115 fi
 116
 117
 118
 119
 120 # ------------- tuning options, file locations and constants -----------
 121 SRC="$1" #name of backup source, may be a remote or local hostname
 122 OPT="$2" #options (--recheck)
 123 HOST_PORT=22 #port of source of backup
 124 SCRIPT_PATH="/backup/snapshot/rsync"
 125 SNAPSHOT_DST="/backup/snapshot" #destination folder
 126 NAME="snapshot" #backup name
 127 LOG="rsync.log"
 128 MIN_MIBSIZE=5000 # older snapshots (except snapshot.001) are removed if free disk <= MIN_MIBSIZE. the script may exit without performing a backup if free disk is still short.
 129 OVERWRITE_LAST=0 # if free disk space is too small, then this option let us remove snapshot.001 as well and retry once
 130 MAX_MIBSIZE=80000 # older snapshots (except snapshot.001) are removed if their size >= MAX_MIBSIZE. the script performs a backup even if their size is too big.
 131 #old: SPEED=5 # 1 is slow, 100 is fast, 100000 faster and 0 does not use slow-down. this allows to avoid rsync consuming too much system performance
 132 BWLIMIT=100000 # bandwidth limit in KiB/s. 0 does not use slow-down. this allows to avoid rsync consuming too much system performance
 133 BACKUPSERVER="rembk" # this server connects to all other to download filesystems and create remote snapshot backups
 134 MD5LIST=0 #to compute a list of md5 integrity signatures of all backuped files, need 'rsync-list.sh'
 135 CHATTR=1 # to use 'chattr' command and protect the backups again modification and deletion
 136 DU=1 # to use 'du' command and calculate the size of existing backups, disable it if you have many backups and it is getting too slow (for example on BACKUPSERVER)
 137 SOURCE="/" #source folder to backup
 138
 139 HOST_LOCAL="$(hostname -s)" #local hostname
 140 #HOST_SRC="${SRC:-${HOST_LOCAL}}" #explicit source hostname, default is local hostname
 141 if [ -z "${SRC}" ] || [ "${SRC}" == "localhost" ]; then
 142   HOST_SRC="${HOST_LOCAL}" #explicit source hostname, default is local hostname
 143 else
 144   HOST_SRC="${SRC}" #explicit source hostname
 145 fi
 146
 147 if [ "${HOST_LOCAL}" == "${BACKUPSERVER}" ]; then #if we are on BACKUPSERVER then do some fine tuning
 148   MD5LIST=1
 149   MIN_MIBSIZE=35000 #needed free space for chunk-file tape-arch.sh
 150   MAX_MIBSIZE=12000
 151   DU=0 # NB: 'du' is currently disabled on BACKUPSERVER for performance reasons
 152 elif [ "${HOST_LOCAL}" == "${HOST_SRC}" ]; then #else if we are on a generic server then do other some fine tuning
 153   if [ "${HOST_SRC}" == "ZRHSV-TST01" ]; then
 154     MIN_MIBSIZE=500; CHATTR=0; DU=0; MD5LIST=0
 155   fi
 156 fi
 157
 158
 159
 160
 161 # ------------- initialization -----------------------------------------
 162 shopt -s extglob                                            #enable extended pattern matching operators
 163
 164 OPTION="--stats \
 165   --recursive \
 166   --links \
 167   --perms \
 168   --times \
 169   --group \
 170   --owner \
 171   --devices \
 172   --hard-links \
 173   --numeric-ids \
 174   --delete \
 175   --delete-excluded \
 176   --bwlimit=${BWLIMIT}"
 177 #  --progress
 178 #  --size-only
 179 #  --stop-at
 180 #  --time-limit
 181 #  --sparse
 182
 183 if [ "${HOST_SRC}" != "${HOST_LOCAL}" ]; then #option for a remote server
 184   SOURCE="${HOST_SRC}:${SOURCE}"
 185   OPTION="${OPTION} \
 186   --compress \
 187   --rsh=\"ssh -p ${HOST_PORT} -i /root/.ssh/rsync_rsa -l root\" \
 188   --rsync-path=\"/usr/bin/rsync\""
 189 fi
 190 if [ "${OPT}" == "--recheck" ]; then
 191   OPTION="${OPTION} \
 192   --ignore-times"
 193 elif [ -n "${OPT}" ]; then
 194   echo "Try rsync-snapshot.sh --help ."
 195   exit 2
 196 fi
 197
 198
 199
 200
 201 # ------------- check conditions ---------------------------------------
 202 echo "$(date +%Y-%m-%d_%H:%M:%S) ${HOST_SRC}: === Snapshot backup is created into ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.001 ==="
 203 STARTDATE=$(date +%s)
 204
 205 # make sure we're running as root
 206 if (($(id -u) != 0)); then
 207   echo "Sorry, must be root. Exiting..."
 208   echo "$(date +%Y-%m-%d_%H:%M:%S) ${HOST_SRC}: === Snapshot failed. ==="
 209   exit 2
 210 fi
 211
 212 # make sure we have a correct snapshot folder
 213 if [ ! -d "${SNAPSHOT_DST}/${HOST_SRC}" ]; then
 214   echo "Sorry, folder ${SNAPSHOT_DST}/${HOST_SRC} is missing. Exiting..."
 215   echo "$(date +%Y-%m-%d_%H:%M:%S) ${HOST_SRC}: === Snapshot failed. ==="
 216   exit 2
 217 fi
 218
 219 # make sure we do not have started already rsync-snapshot.sh or rsync process (started by rsync-cp.sh or by a remote rsync-snapshot.sh) in the background.
 220 if [ "${HOST_LOCAL}" != "${BACKUPSERVER}" ]; then #because BACKUPSERVER need sometimes to perform an rsync-cp.sh it must disable the check of "already started".
 221   #RSYNCPID=$(pgrep -f "/bin/bash .*rsync-snapshot.sh")
 222   #if ([ -n "${RSYNCPID}" ] && [ "${RSYNCPID}" != "$$" ]) #|| pgrep -x "rsync"
 223   if pgrep -f "/bin/\w*sh \w*rsync-snapshot\.sh" | grep -qv "$$"; then
 224     echo "Sorry, rsync is already running in the background. Exiting..."
 225     echo "$(date +%Y-%m-%d_%H:%M:%S) ${HOST_SRC}: === Snapshot failed. ==="
 226     exit 2
 227   fi
 228 fi
 229
 230
 231
 232
 233 # ------------- remove some old backups --------------------------------
 234 # remove certain snapshots to achieve an exponential distribution in time of the backups (1,2,4,8,...)
 235 for b in 512 256 128 64 32 16 8 4; do
 236   let a=b/2+1
 237   let f=0 #this flag is set to 1 when we find the 1st snapshot in the range b..a
 238   for i in $(seq -f'%03g' "${b}" -1 "${a}"); do
 239     if [ -d "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i}" ]; then
 240       if [ "${f}" -eq 0 ]; then
 241         let f=1
 242       else
 243         echo "$(date +%Y-%m-%d_%H:%M:%S) Removing ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i} ..."
 244         [ "${CHATTR}" -eq 1 ] && chattr -R -i "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i}" &>/dev/null
 245         rm -rf "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i}"
 246       fi
 247     fi
 248   done
 249 done
 250
 251 # remove additional backups if free disk space is short
 252 remove_snapshot() {
 253   local MIN_MIBSIZE2=$1
 254   local MAX_MIBSIZE2=$2
 255   for i in $(seq -f'%03g' 512 -1 001); do
 256     if [ -d "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i}" ] || [ ${i} -eq 1 ]; then
 257       [ ! -h "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last" ] && [ -d "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i}" ] && ln -s "${NAME}.${i}" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last"
 258       let d=0 #disk space used by snapshots and free disk space are ok
 259       echo -n "$(date +%Y-%m-%d_%H:%M:%S) Checking free disk space... "
 260       FREEDISK=$(df -m ${SNAPSHOT_DST} | tail -1 | sed -e 's/  */ /g' | cut -d" " -f4 | sed -e 's/M*//g')
 261       echo -n "${FREEDISK} MiB free. "
 262       if [ ${FREEDISK} -ge ${MIN_MIBSIZE2} ]; then
 263         echo "Ok, bigger than ${MIN_MIBSIZE2} MiB."
 264         if [ "${DU}" -eq 0 ]; then #avoid slow 'du'
 265           break
 266         else
 267           echo -n "$(date +%Y-%m-%d_%H:%M:%S) Checking disk space used by ${SNAPSHOT_DST}/${HOST_SRC} ... "
 268           USEDDISK=$(du -ms "${SNAPSHOT_DST}/${HOST_SRC}/" | cut -f1)
 269           echo -n "${USEDDISK} MiB used. "
 270           if [ ${USEDDISK} -le ${MAX_MIBSIZE2} ]; then
 271             echo "Ok, smaller than ${MAX_MIBSIZE2} MiB."
 272             break
 273           else
 274             let d=2 #disk space used by snapshots is too big
 275           fi
 276         fi
 277       else
 278         let d=1 #free disk space is too small
 279       fi
 280       if [ ${d} -ne 0 ]; then #we need to remove snapshots
 281         if [ ${i} -ne 1 ]; then
 282           echo "Removing ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i} ..."
 283           [ "${CHATTR}" -eq 1 ] && chattr -R -i "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i}" &>/dev/null
 284           rm -rf "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i}"
 285           [ -h "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last" ] && rm -f "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last"
 286         else #all snapshots except snapshot.001 are removed
 287           if [ ${d} -eq 1 ]; then #snapshot.001 causes that free space is too small
 288             if [ "${OVERWRITE_LAST}" -eq 1 ]; then #last chance: remove snapshot.001 and retry once
 289               OVERWRITE_LAST=0
 290               echo "Warning, free disk space will be smaller than ${MIN_MIBSIZE} MiB."
 291               echo "$(date +%Y-%m-%d_%H:%M:%S) OVERWRITE_LAST enabled. Removing ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.001 ..."
 292               rm -rf "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.001"
 293               [ -h "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last" ] && rm -f "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last"
 294             else
 295               for j in ${LNKDST//--link-dest=/}; do
 296                 if [ -d "${j}" ] && [ "${CHATTR}" -eq 1 ] && [ $(lsattr -d "${j}" | cut -b5) != "i" ]; then
 297                   chattr -R +i "${j}" &>/dev/null #undo unprotection that was needed to use hardlinks
 298                 fi
 299               done
 300               [ ! -h "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last" ] && ln -s "${NAME}.${j}" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last"
 301               echo "Sorry, free disk space will be smaller than ${MIN_MIBSIZE} MiB. Exiting..."
 302               echo "$(date +%Y-%m-%d_%H:%M:%S) ${HOST_SRC}: === Snapshot failed. ==="
 303               exit 2
 304             fi
 305           elif [ ${d} -eq 2 ]; then #snapshot.001 causes that disk space used by snapshots is too big
 306             echo "Warning, disk space used by ${SNAPSHOT_DST}/${HOST_SRC} will be bigger than ${MAX_MIBSIZE} MiB. Continuing anyway..."
 307           fi
 308         fi
 309       fi
 310     fi
 311   done
 312 }
 313
 314 # perform an estimation of required disk space for the new backup
 315 while :; do #this loop is executed a 2nd time if OVERWRITE_LAST was ==1 and snapshot.001 got removed
 316   OOVERWRITE_LAST="${OVERWRITE_LAST}"
 317   echo -n "$(date +%Y-%m-%d_%H:%M:%S) Testing needed free disk space ..."
 318   mkdir -p "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.test-free-disk-space"
 319   chmod -R 775 "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.test-free-disk-space"
 320   cat /dev/null >"${SNAPSHOT_DST}/${HOST_SRC}/${LOG}"
 321   LNKDST=$(find "${SNAPSHOT_DST}/" -maxdepth 2 -type d -name "${NAME}.001" -printf " --link-dest=%p")
 322   for i in ${LNKDST//--link-dest=/}; do
 323     if [ -d "${i}" ] && [ "${CHATTR}" -eq 1 ] && [ $(lsattr -d "${i}" | cut -b5) == "i" ]; then
 324       chattr -R -i "${i}" &>/dev/null #unprotect last snapshots to use hardlinks
 325     fi
 326   done
 327   eval rsync \
 328     --dry-run \
 329     ${OPTION} \
 330     --include-from="${SCRIPT_PATH}/rsync-include.txt" \
 331     ${LNKDST} \
 332     "${SOURCE}" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.test-free-disk-space" >>"${SNAPSHOT_DST}/${HOST_SRC}/${LOG}"
 333   RES=$?
 334   if [ "${RES}" -ne 0 ] && [ "${RES}" -ne 23 ] && [ "${RES}" -ne 24 ]; then
 335     echo "Sorry, error in rsync execution (value ${RES}). Exiting..."
 336     echo "$(date +%Y-%m-%d_%H:%M:%S) ${HOST_SRC}: === Snapshot failed. ==="
 337     exit 2
 338   fi
 339   let i=$(tail -100 "${SNAPSHOT_DST}/${HOST_SRC}/${LOG}" | grep 'Total transferred file size:' | cut -d " " -f5)/1048576
 340   echo " ${i} MiB needed."
 341   rm -rf "${SNAPSHOT_DST}/${HOST_SRC}/${LOG}" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.test-free-disk-space"
 342   remove_snapshot $((${MIN_MIBSIZE} + ${i})) $((${MAX_MIBSIZE} - ${i}))
 343   if [ "${OOVERWRITE_LAST}" == "${OVERWRITE_LAST}" ]; then #no need to retry
 344     break
 345   fi
 346 done
 347
 348
 349
 350
 351 # ------------- create the snapshot backup -----------------------------
 352 # perform the filesystem backup using rsync and hard-links to the latest snapshot
 353 # Note:
 354 #   -rsync behaves like cp --remove-destination by default, so the destination
 355 #    is unlinked first.  If it were not so, this would copy over the other
 356 #    snapshot(s) too!
 357 #   -use --link-dest to hard-link when possible with previous snapshot,
 358 #    timestamps, permissions and ownerships are preserved
 359 echo "$(date +%Y-%m-%d_%H:%M:%S) Creating folder ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000 ..."
 360 mkdir -p "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000"
 361 chmod 775 "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000"
 362 cat /dev/null >"${SNAPSHOT_DST}/${HOST_SRC}/${LOG}"
 363 echo -n "$(date +%Y-%m-%d_%H:%M:%S) Creating backup of ${HOST_SRC} into ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000"
 364 if [ -n "${LNKDST}" ]; then
 365   echo " hardlinked with${LNKDST//--link-dest=/} ..."
 366 else
 367   echo " not hardlinked ..."
 368 fi
 369 eval rsync \
 370   -vv \
 371   ${OPTION} \
 372   --include-from="${SCRIPT_PATH}/rsync-include.txt" \
 373   ${LNKDST} \
 374   "${SOURCE}" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000" >>"${SNAPSHOT_DST}/${HOST_SRC}/${LOG}"
 375 RES=$?
 376 if [ "${RES}" -ne 0 ] && [ "${RES}" -ne 23 ] && [ "${RES}" -ne 24 ]; then
 377   echo "Sorry, error in rsync execution (value ${RES}). Exiting..."
 378   echo "$(date +%Y-%m-%d_%H:%M:%S) ${HOST_SRC}: === Snapshot failed. ==="
 379   exit 2
 380 fi
 381 for i in ${LNKDST//--link-dest=/}; do
 382   if [ -d "${i}" ] && [ "${CHATTR}" -eq 1 ] && [ $(lsattr -d "${i}" | cut -b5) != "i" ]; then
 383     chattr -R +i "${i}" &>/dev/null #undo unprotection that was needed to use hardlinks
 384   fi
 385 done
 386 mv "${SNAPSHOT_DST}/${HOST_SRC}/${LOG}" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000/${LOG}"
 387 gzip -f "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000/${LOG}"
 388
 389
 390
 391
 392 # ------------- create the MD5 integrity signature ---------------------
 393 # create a gziped 'find'-list of all snapshot files (including md5 signatures)
 394 if [ "${MD5LIST}" -eq 1 ]; then
 395   echo "$(date +%Y-%m-%d_%H:%M:%S) Computing filelist with md5 signatures of ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000 ..."
 396   OWD="$(pwd)"
 397   cd "${SNAPSHOT_DST}"
 398 #  NOW=$(date "+%s")
 399 #  MYTZ=$(date "+%z")
 400 #  let NOW${MYTZ:0:1}=3600*${MYTZ:1:2}+60*${MYTZ:3:2} # convert localtime to UTC
 401 #  DATESTR=$(date -d "1970-01-01 $((${NOW} - 1)) sec" "+%Y-%m-%d_%H:%M:%S") # 'now - 1s' to avoid missing files
 402   DATESTR=$(date -d "1970-01-01 UTC $(($(date +%s) - 1)) seconds" "+%Y-%m-%d_%H:%M:%S") # 'now - 1s' to avoid missing files
 403   REF_LIST="$(find ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.001/ -maxdepth 1 -type f -name 'snapshot.*.list.gz' 2>/dev/null)"
 404   if [ -n "${REF_LIST}" ] && [ "${OPT}" != "--recheck" ]; then
 405     REF_LIST2="/tmp/rsync-reflist.tmp"
 406     gzip -dc "${REF_LIST}" >"${REF_LIST2}"
 407     touch -r "${REF_LIST}" "${REF_LIST2}"
 408     ${SCRIPT_PATH}/rsync-list.sh "${HOST_SRC}/${NAME}.000" 0 "${REF_LIST2}" | sort -u | gzip -c >"${HOST_SRC}/${NAME}.${DATESTR}.list.gz"
 409     rm -f "${REF_LIST2}"
 410   else
 411     ${SCRIPT_PATH}/rsync-list.sh "${HOST_SRC}/${NAME}.000" 0 | sort -u | gzip -c >"${HOST_SRC}/${NAME}.${DATESTR}.list.gz"
 412   fi
 413   touch -d "${DATESTR/_/ }" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${DATESTR}.list.gz"
 414   cd "${OWD}"
 415   [ ! -d "${SNAPSHOT_DST}/${HOST_SRC}/md5-log" ] && mkdir -p "${SNAPSHOT_DST}/${HOST_SRC}/md5-log"
 416   cp -al "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${DATESTR}.list.gz" "${SNAPSHOT_DST}/${HOST_SRC}/md5-log/${NAME}.${DATESTR}.list.gz"
 417   mv "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${DATESTR}.list.gz" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000/${NAME}.${DATESTR}.list.gz"
 418   touch -d "${DATESTR/_/ }" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000"
 419 fi
 420
 421
 422
 423
 424 # ------------- finish and clean up ------------------------------------
 425 # protect the backup against modification with chattr +immutable
 426 if [ "${CHATTR}" -eq 1 ]; then
 427   echo "$(date +%Y-%m-%d_%H:%M:%S) Setting recursively immutable flag of ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000 ..."
 428   chattr -R +i "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.000" &>/dev/null
 429 fi
 430
 431 # rotate the backups
 432 if [ -d "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.512" ]; then #remove snapshot.512
 433   echo "Removing ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.512 ..."
 434   [ "${CHATTR}" -eq 1 ] && chattr -R -i "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.512" &>/dev/null
 435   rm -rf "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.512"
 436 fi
 437 [ -h "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last" ] && rm -f "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last"
 438 for i in $(seq -f'%03g' 511 -1 000); do
 439   if [ -d "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i}" ]; then
 440     let j=${i##+(0)}+1
 441     j=$(printf "%.3d" "${j}")
 442     echo "Renaming ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i} into ${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${j} ..."
 443     [ "${CHATTR}" -eq 1 ] && chattr -i "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i}" &>/dev/null
 444     mv "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${i}" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${j}"
 445     [ "${CHATTR}" -eq 1 ] && chattr +i "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.${j}" &>/dev/null
 446     [ ! -h "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last" ] && ln -s "${NAME}.${j}" "${SNAPSHOT_DST}/${HOST_SRC}/${NAME}.last"
 447   fi
 448 done
 449
 450 # remove additional backups if free disk space is short
 451 OVERWRITE_LAST=0 #next call of remove_snapshot() will not remove snapshot.001
 452 remove_snapshot ${MIN_MIBSIZE} ${MAX_MIBSIZE}
 453 echo "$(date +%Y-%m-%d_%H:%M:%S) ${HOST_SRC}: === Snapshot backup successfully done in $(($(date +%s) - ${STARTDATE})) sec. ==="
 454 exit 0
 455 #eof
 456