+++ /dev/null
-#!/bin/bash
-#
-# mount-media
-#
-
-# Parameters
-MEDIA="$1"
-VOLUME="$2"
-BASENAME="`basename $0`"
-MOUNTPOINT="/media/$MEDIA"
-
-# Check media config
-if [ -z "$MEDIA" ]; then
- echo "usage: $BASENAME <media> [volume]"
- echo "example: $BASENAME mymedia sdb1"
- exit 1
-fi
-
-# Check volume config
-if [ -z "$VOLUME" ]; then
- VOLUME="sdb1"
-fi
-
-DISK="`echo ${VOLUME} | sed -e s/[0-9]\$//g`"
-
-# Set sudo config
-if [ "`whoami`" != 'root' ]; then
- sudo="sudo"
-fi
-
-if [ "$BASENAME" == "mount-media" ]; then
- echo "Checking drive health status..."
- $sudo smartctl -H /dev/$DISK
- echo "Disabling STANDBY on drive..."
- $sudo sdparm --clear STANDBY -6 /dev/$DISK
- echo "Initializing crypto layer..."
- $sudo cryptsetup luksOpen /dev/$VOLUME $MEDIA && \
- echo "Checking filesystem..."
- $sudo fsck -v -y /dev/mapper/$MEDIA && \
- echo "Mounting volume at $MOUNTPOINT..."
- $sudo mkdir -p $MOUNTPOINT
- $sudo mount /dev/mapper/$MEDIA $MOUNTPOINT
-elif [ "$BASENAME" == "umount-media" ]; then
- $sudo umount $MOUNTPOINT && \
- $sudo cryptsetup luksClose $MEDIA
-fi
+++ /dev/null
-#!/bin/bash
-#
-# sync a removable volume with system backups.
-#
-
-# Parameters.
-BASENAME="`basename $0`"
-VOLUME="$1"
-MEDIA="/media/$VOLUME"
-BWLIMIT=${BWLIMIT:=32000}
-IMAGES="/var/data/crypt/"
-RSYNC="ionice -c 3 nice -n 19 rsync -avH --delete --bwlimit=$BWLIMIT"
-CP="ionice -c 3 nice -n 19 cp"
-
-# Sync backups for a node.
-function sync_backups_node {
- if [ ! -z "$NODE" ]; then
- # Get full node hostname.
- NODE_HOSTNAME="`cat /var/vservers/$NODE/etc/hostname`"
- if [ -z "$NODE_HOSTNAME" ]; then
- NODE_HOSTNAME="$NODE"
- fi
-
- # Sync local encrypted backup.
- echo "Syncing /var/vservers/$NODE/var/backups/duplicity/..."
- mkdir -p /$MEDIA/$NODE_HOSTNAME/duplicity
- $RSYNC /var/vservers/$NODE/var/backups/duplicity/ /$MEDIA/$NODE_HOSTNAME/duplicity/
-
- # Sync remote backups.
- for node in `ls /var/vservers/$NODE/var/backups/remote/`; do
- echo "Syncing /var/vservers/$NODE/var/backups/remote/$node/..."
- mkdir -p /$MEDIA/$NODE_HOSTNAME/remote/$node
- $RSYNC /var/vservers/$NODE/var/backups/remote/$node/ /$MEDIA/$NODE_HOSTNAME/remote/$node/
- done
- else
- # Sync local encrypted backup.
- echo "Syncing /var/backups/duplicity/..."
- mkdir -p /$MEDIA/$HOSTNAME/duplicity
- $RSYNC /var/backups/duplicity/ /$MEDIA/$HOSTNAME/duplicity/
-
- # Sync remote backups.
- for node in `ls /var/backups/remote/`; do
- echo "Syncing /var/backups/remote/$node/..."
- mkdir -p /$MEDIA/$HOSTNAME/remote/$node/
- $RSYNC /var/backups/remote/$node/ /$MEDIA/$HOSTNAME/remote/$node/
- done
-
- # Copy encrypted images.
- if [ -d "$IMAGES" ]; then
- for image in `find $IMAGES -name '*.img' -type f`; do
- echo "Copying image to /$MEDIA/$HOSTNAME/images/`dirname $image`"
- mkdir -p /$MEDIA/$HOSTNAME/images/`dirname $image`
- $CP $image /$MEDIA/$HOSTNAME/images/`dirname $image`
- done
- fi
- fi
-}
-
-# Parsing.
-if [ -z "$VOLUME" ]; then
- echo "usage: $BASENAME <media> [nodes]"
- exit 1
-else
- shift
-fi
-
-# Sync backups for each node.
-if [ ! -z "$*" ]; then
- for NODE in $*; do
- sync_backups_node
- done
-else
- sync_backups_node
-fi
+++ /dev/null
-#!/bin/bash
-#
-# sync-media assets using git-annex
-#
-
-REMOTE="$1"
-VOLUME="/media/$REMOTE"
-DOMAIN="`facter domain`"
-HOST="`facter hostname`"
-CACHE="/var/cache/$HOST/media"
-MEDIA="media.$DOMAIN"
-INCOMING="$CACHE/incoming"
-WHOAMI="`whoami`"
-OPTIONS="$*"
-
-# Fix identity
-function sync_media_identity {
- if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then
- repo="$(basename `pwd`)"
- git config user.name "${repo^} Archive"
- git config user.email "$repo@localhost"
- fi
-}
-
-# Add files into the annex
-function sync_media_add {
- git annex add .
-
- # Adding hidden files and symlinks, git+while version
- git status --porcelain -u | sed -e 's/?? //' | while read file; do
- if [ -h "$file" ]; then
- git add "$file"
- else
- git annex add "$file"
- fi
- done
-}
-
-# If there is a playlists folder, make sure mpd user can write to it
-function sync_media_playlist_perms {
- if [ -d "playlists" ]; then
- $sudo chmod 775 playlists
- $sudo chown -R mpd.audio playlists
- find playlists -type f -exec sudo chmod 664 {} \;
- find playlists -type d -exec sudo chmod 775 {} \;
- fi
-}
-
-# Fix incoming permissions
-function sync_media_incoming_perms {
- if [ -d "$INCOMING" ]; then
- echo "Fixing $INCOMING permissions..."
- $sudo find $INCOMING -type f -exec chmod 664 {} \;
- $sudo find $INCOMING -type d -exec chmod 775 {} \;
- $sudo chown -R $WHOAMI.incoming $INCOMING
- fi
-}
-
-# Run fsck
-function sync_media_fsck {
- if [ "$FSCK" == "true" ]; then
- git annex fsck --fast
- fi
-}
-
-# Run dropunused
-function sync_media_dropunused {
- if [ "$DROPUNUSED" == "true" ]; then
- git annex unused
- git annex dropunused 1-1000
- fi
-}
-
-# Get copies of annexed files
-function sync_media_get {
- local repo="$1"
- local numcopies
-
- if [ "`git -C $repo config sync-media.get`" != "false" ]; then
- if git -C $repo config sync-media.numcopies &> /dev/null; then
- numcopies="`git -C $repo config sync-media.numcopies`"
- else
- numcopies="3"
- fi
-
- git annex get . --numcopies=$numcopies
- fi
-}
-
-# Control whether the repository should have a copy of everything
-function sync_media_getall {
- local repo="$1"
-
- if [ "`git -C $repo config sync-media.getall`" == "true" ]; then
- git annex get .
- fi
-}
-
-# Ensure we have a reference to the remote repository
-function sync_media_ensure_remote {
- local remote="$1"
- local path="$2"
-
- if [ -z "$remote" ] || [ -z "$path" ]; then
- return
- fi
-
- # Check for local or remote repo
- if [ -z "$DRIVE" ]; then
- path="$REMOTE.$DOMAIN:$path"
- elif [ ! -d "$path/.git/annex" ]; then
- return
- fi
-
- if ! git remote | grep -q "^$remote$"; then
- echo git remote add $remote $path
- fi
-}
-
-# Set sudo config
-if [ "$WHOAMI" != 'root' ]; then
- sudo="sudo"
-else
- echo "Sorry, cannot run as root"
- exit 1
-fi
-
-# Set fsck config
-if echo $OPTIONS | grep -q -- "--fsck"; then
- FSCK="true"
-fi
-
-# Set unused config
-if echo $OPTIONS | grep -q -- "--dropunused"; then
- DROPUNUSED="true"
-fi
-
-# Set drive config
-if [ ! -z "$REMOTE" ]; then
- # Check storage media
- MOUNT="`mount | grep $VOLUME`"
- if [ ! -z "$MOUNT" ]; then
- DRIVE="$(basename `echo $MOUNT | awk '{ print $1 }'`)"
- fi
-fi
-
-# Commit changes
-if [ -d "$CACHE" ]; then
- # Fix cache permissions
- #echo "Fixing $CACHE permissions..."
- #$sudo find $CACHE -type f -exec chmod 644 {} \;
- #$sudo find $CACHE -type d -exec chmod 755 {} \;
-
- sync_media_incoming_perms
-
- # Add and update local repositories
- for folder in `ls $CACHE`; do
- if [ -d "$CACHE/$folder/.git/annex" ]; then
- if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
- continue
- fi
-
- #if [ "`git -C $CACHE/$folder config sync-media.ready`" != "true" ]; then
- # echo "Skipping $CACHE/$folder: not sync-media ready, please config your repo."
- # continue
- #fi
-
- (
- cd $CACHE/$folder
- echo "Syncing $CACHE/$folder..."
-
- sync_media_playlist_perms
- sync_media_ensure_remote $REMOTE $VOLUME/$MEDIA/$folder
- sync_media_identity
- sync_media_add
- git annex sync
- sync_media_getall $CACHE/$folder
- sync_media_fsck
- sync_media_dropunused
- )
- fi
- done
-
- if [ ! -z "$DRIVE" ]; then
- if [ ! -d "$VOLUME/$MEDIA" ]; then
- echo "Folder $VOLUME/$MEDIA does not exist..."
- else
- for folder in `ls $CACHE`; do
- if [ -d "$CACHE/$folder/.git/annex" ]; then
- if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
- continue
- fi
-
- #if [ "`git -C $CACHE/$folder config sync-media.ready`" != "true" ]; then
- # echo "Skipping $CACHE/$folder: not sync-media ready, please config your repo."
- # continue
- #fi
-
- if [ ! -d "$VOLUME/$MEDIA/$folder" ]; then
- (
- cd $VOLUME/$MEDIA
- echo "Initializing $VOLUME/$MEDIA/$folder..."
- git clone $CACHE/$folder && cd $folder && sync_media_identity && git annex init $DRIVE && \
- git remote rename origin $HOST && cd $CACHE/$folder && git remote add $DRIVE $VOLUME/$MEDIA/$folder
- )
- fi
- elif [ ! -d "$VOLUME/$MEDIA/$folder" ]; then
- if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then
- echo "Syncing $VOLUME/$MEDIA/$folder..."
- rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/
- fi
- fi
- done
- fi
- fi
-fi
-
-# Retrieve changes at media volumes
-if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then
- for folder in `ls $VOLUME/$MEDIA`; do
- if [ -d "$VOLUME/$MEDIA/$folder/.git/annex" ]; then
- if [ "`git -C $VOLUME/$MEDIA/$folder config sync-media.skip`" == "true" ]; then
- continue
- fi
-
- (
- cd $VOLUME/$MEDIA/$folder
- echo "Syncing $VOLUME/$MEDIA/$folder..."
-
- sync_media_ensure_remote $HOST $CACHE/$folder
- sync_media_playlist_perms
- sync_media_identity
- sync_media_add
- git annex sync
- sync_media_get $VOLUME/$MEDIA/$folder
- sync_media_getall $VOLUME/$MEDIA/$folder
- sync_media_fsck
- sync_media_dropunused
- #git annex drop --auto --numcopies=2
- )
- elif [ -d "$CACHE/$folder" ]; then
- if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then
- echo "Syncing $VOLUME/$MEDIA/$folder..."
- rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/
- fi
- fi
- done
-elif [ ! -z "$REMOTE" ]; then
- # Try to copy to a remote
- for folder in `ls $CACHE`; do
- if [ -d "$CACHE/$folder/.git/annex" ]; then
- if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
- continue
- fi
-
- sync_media_ensure_remote $REMOTE $CACHE/$folder
-
- (
- cd $CACHE/$folder
- git annex copy . --to $REMOTE
- git annex sync
- )
- else
- if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then
- echo "Syncing $VOLUME/$MEDIA/$folder..."
- rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $REMOTE.$DOMAIN:$CACHE/$folder/
- fi
- fi
- done
-fi
+++ /dev/null
-#!/bin/bash
-#
-# Copy git-annex repositories to remote server.
-#
-
-# Parameters
-BASENAME="`basename $0`"
-DESTINATION="$1"
-DOMAIN="`facter DOMAIN`"
-MEDIA="/var/cache/media"
-
-# Syntax check
-if [ -z "$DESTINATION" ]; then
- echo "usage: $BASENAME <DESTINATION>"
- exit 1
-fi
-
-# Process each repository
-for file in `ls $MEDIA`; do
- if [ -d "$MEDIA/$file/.git/annex" ]; then
- (
- echo Processing "$MEDIA/$file..."
-
- cd $MEDIA
-
- if ssh $DESTINATION if [ -d \"/var/cache/media/$file\" ] \; then echo exists\; fi | grep -q exists; then
- echo "Remote $file already exists, skipping..."
- continue;
- fi
-
- git clone $file $file.git && \
- rsync -avz $file.git/ $DESTINATION:/var/cache/media/$file/ && \
- rm -rf $file.git
- echo ""
- )
- fi
-done
+++ /dev/null
-#!/bin/bash
-#
-# Add git-annex remotes to repository in removable media or local cache.
-#
-
-# Parameters
-BASENAME="`basename $0`"
-VOLUME="$1"
-TYPES="$2"
-DOMAIN="`facter DOMAIN`"
-HOST="`facter hostname`"
-CACHES=""
-VOLUMES=""
-BOXES=""
-
-# Fix identity
-function sync_media_identity {
- if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then
- repo="$(basename `pwd`)"
- git config user.name "${repo^} Archive"
- git config user.email "$repo@localhost"
- fi
-}
-
-# Syntax check
-if [ -z "$VOLUME" ]; then
- echo "usage: $BASENAME <volume> [<caches|volumes|boxes> [remotes]]"
- exit 1
-fi
-
-# Determine media folder
-if [ "$VOLUME" == "$HOST" ] || [ "$VOLUME" == "localhost" ]; then
- MEDIA="/var/cache/media"
-else
- MEDIA="/media/$VOLUME/media.$DOMAIN"
-fi
-
-# Determine remote type
-shift 2
-if [ "$TYPES" == 'caches' ]; then
- # Remotes are caches of local boxes
- CACHES="$*"
-elif [ "$TYPES" == 'volumes' ]; then
- # Remotes are removable media
- VOLUMES="$*"
-elif [ "$TYPES" == 'boxes' ]; then
- # Remotes are remote boxes
- BOXES="$*"
-fi
-
-# Process each repository
-for file in `ls $MEDIA`; do
- if [ -d "$MEDIA/$file/.git" ]; then
- (
- echo Processing "$MEDIA/$file..."
- cd $MEDIA/$file
- sync_media_identity
-
- if git remote | grep -q "^origin$"; then
- echo "Removing origin..."
- git remote rm origin
- fi
-
- if [ ! -d "$MEDIA/$file/.git/annex" ]; then
- git annex init $VOLUME
- fi
-
- for remote in $CACHES; do
- echo "Adding /var/cache/$remote/media/$file remote..."
- git remote add $remote /var/cache/$remote/media/$file
- done
-
- for remote in $VOLUMES; do
- echo "Adding /media/$remote/media.$DOMAIN/$file..."
- git remote add $remote /media/$remote/media.$DOMAIN/$file
- done
-
- for remote in $BOXES; do
- if ! echo $remote | grep -q '\.'; then
- host="$remote.$DOMAIN"
- else
- host="$remote"
- fi
-
- echo "Adding ssh://$host/var/cache/media/$file..."
- git remote add $remote ssh://$host/var/cache/media/$file
- done
-
- echo ""
- )
- fi
-done
require => Package['backupninja'],
}
+ # TODO: remove in the future
# sync-backups script
file { "/usr/local/sbin/sync-backups":
owner => root,
group => root,
mode => 0755,
- ensure => present,
- source => "puppet:///modules/backup/sync-backups",
+ ensure => absent,
}
+ # TODO: remove in the future
# sync-media script
file { "/usr/local/sbin/sync-media":
owner => root,
group => root,
mode => 0755,
- ensure => present,
- source => "puppet:///modules/backup/sync-media",
+ ensure => absent,
}
+ # TODO: remove in the future
# sync-media-export script
file { "/usr/local/sbin/sync-media-export":
owner => root,
group => root,
mode => 0755,
- ensure => present,
- source => "puppet:///modules/backup/sync-media-export",
+ ensure => absent,
}
+ # TODO: remove in the future
# sync-media-init script
file { "/usr/local/sbin/sync-media-init":
owner => root,
ensure => absent,
}
+ # TODO: remove in the future
# sync-media-init script
file { "/usr/local/sbin/sync-media-initremotes":
owner => root,
group => root,
mode => 0755,
- ensure => present,
- source => "puppet:///modules/backup/sync-media-initremotes",
+ ensure => absent,
}
+ # TODO: remove in the future
# mount-media script
file { "/usr/local/sbin/mount-media":
owner => root,
group => root,
mode => 0755,
- ensure => present,
- source => "puppet:///modules/backup/mount-media",
+ ensure => absent,
}
+ # TODO: remove in the future
# umount-media script
file { "/usr/local/sbin/umount-media":
owner => root,
group => root,
- ensure => '/usr/local/sbin/mount-media',
- require => File['/usr/local/sbin/mount-media'],
+ ensure => absent,
}
}
projects / puppet-backup.git / commitdiff