Overview

Basic data

Hostname	`b.fluid.tuwien.ac.at`
Purpose	backup server
Operating system	Debian Stretch
Management interface	http://mb.fluid.tuwien.ac.at
Backup storage capacity	70TB
Client data storage capacity	3.5TB
Authentication options	local accounts, TU password
Manuals	Quick Reference Sheet, User Manual

Admins

Thomas Loimer
Iris Fula
Francesco Zonta
Rudolf Ladner (ZID)
Sebastian Boehm (ZID)

Backups

Clients synchronise data to /home on b.fluid using rsync.
A daily cronjob on b.fluid saves versioned backups of all data in /home to /mnt/backups.
The daily backup is performed using Borg.
Daily backups are kept for a week, weekly backups are kept for a month, monthly backups are kept for a year. Data older than one year will be discarded at the discretion of the admins.
Due to the amount of data on s15.fluid and s16.fluid, these hosts do not use rsync, but directly target /mnt/backups using borg instead.

Directory structure

/mnt/backup/
├── b
│   └── home
├── gs2
├── s15
│   └── home
├── s16
│   ├── data
│   └── home
├── user1
│   ├── pc1
│   └── pc2
└── user2
    └── pcname

Configuration

Basic shell setup, etckeeper

cat > /etc/etckeeper.conf <<EOF
VCS=git
HIGHLEVEL_PACKAGE_MANAGER=apt
LOWLEVEL_PACKAGE_MANAGER=dpkg
EOF
apt-get -y install etckeeper zsh emacs24-nox vim mg git tmux screen htop mc sudo
git config --global user.name root
export MAILNAME=`cat /etc/mailname`
git config --global user.email "root@${MAILNAME}"
cat > /etc/apt/sources.list.d/grml.list <<EOF
deb http://deb.grml.org/ grml-stable main #Grml
EOF
cat > /etc/apt/preferences.d/grml-pin << EOF
Package: *
Pin: release a=grml-stable
Pin-Priority: 200
EOF
apt-key --keyring /etc/apt/trusted.gpg.d/grml.gpg adv \
  --keyserver keyserver.ubuntu.com \
  --recv-keys ECDEA787
apt-get update
apt-get -y install grml-etc-core grml-debian-keyring
wget https://raw.githubusercontent.com/sometimesfood/chef-admin-essentials/master/files/default/tmux.conf -O /etc/tmux.conf
chsh -s /bin/zsh
chsh -s /bin/zsh oswat
touch ~oswat/.zshrc
cat > /etc/default/locale <<EOF
LANG=en_US.utf8
LC_CTYPE=en_US.utf8
LC_ALL=en_US.utf8
EOF

megacli installation, RAID tweaks

apt-key --keyring /etc/apt/trusted.gpg.d/hwraid.gpg adv \
  --keyserver keyserver.ubuntu.com \
  --recv-keys 23B3D3B4
cat > /etc/apt/sources.list.d/hwraid.list <<EOF
deb http://hwraid.le-vert.net/debian stretch main
EOF
apt-get update
apt-get -y install megacli
megacli -AdpSetProp -DsblSpinDownHSP 1 -a0

borg installation

apt-get -y install borgbackup python-llfuse

TISS authentication

export BINDPW=secret
apt-get -y install libpam-cap libpam-ldapd libnss-ldapd nslcd
cat > /etc/nslcd.conf <<EOF
# /etc/nslcd.conf
# nslcd configuration file. See nslcd.conf(5)
# for details.

# The user and group nslcd should run as.
uid nslcd
gid nslcd

# The location at which the LDAP server(s) should be reachable.
uri ldaps://dc.intern.tuwien.ac.at

# The search base that will be used for all queries.
base ou=tu,dc=intern,dc=tuwien,dc=ac,dc=at

# The LDAP protocol version to use.
#ldap_version 3

# The DN to bind with for normal lookups.
binddn cn=E322_LDAP,ou=interactive,ou=exchange,ou=IT-services,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at
bindpw ${BINDPW}

# The DN used for password modifications by root.
#rootpwmoddn cn=admin,dc=example,dc=com

# SSL options
#ssl off
tls_reqcert never
tls_cacertfile /etc/ssl/certs/ca-certificates.crt

# The search scope.
#scope sub

# Customize certain database lookups
base    group   ou=groups,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at
base    passwd  ou=people,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at
base    group   ou=people,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at

# Mappings
filter  passwd  (memberOf=cn=E322,ou=groups,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at)
map     passwd  uid             sAMAccountName
map     passwd  uidNumber       employeeID
map     passwd  gecos           cn
map     passwd  homeDirectory   "/home/\${sAMAccountName}"
map     passwd  loginShell      "/bin/bash"
map     passwd  gidNumber       "2153"

filter  shadow  (memberOf=cn=E322,ou=groups,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at)
map     shadow  uid             sAMAccountName

# Get the objectSid by decoding the base64-decoded string returned by an
# ldap-query, and swapping each pair of byes, and considering least- and
# most significant bit ordering, see
# http://www.selfadsi.org/deep-inside/microsoft-sid-attributes.htm
#   objectSid:: AAU--base64string---AA==
# $ echo AAU--base64string---AA== | base64 -d | od -x
# 000000 0501 0000 0000 0500 0015 0000 097f 1f75
# 000020 1c1d 1a1b 2c2d 2a2b
# The correct number is then, in hex,
# 1-5-21-1f75097f-1a1b1c1d-2a2b2c2d
filter  group   (cn=E322*)
map     group   userPassword    ""
map     group   gidNumber       objectSid:S-1-5-21-527763839-1561677997-902985232
EOF
chown oswat /etc/nslcd.conf
service nslcd restart
# note: files in pam.d are managed by pam-auth-update(8)

perl -pi \
  -e 's/^#\tstat-user\t\tsomebody$/\tstat-user\t\toswat/g;' \
  -e 's/^\tpositive-time-to-live\tpasswd\t\t600$/\tpositive-time-to-live\tpasswd\t\t86400/g;' \
  -e 's/^\tnegative-time-to-live\tpasswd\t\t20$/\tnegative-time-to-live\tpasswd\t\t3600/g;' \
  -e 's/^\tsuggested-size\t\tpasswd\t\t211$/\tsuggested-size\t\tpasswd\t\t401/g;' \
  -e 's/^\tpersistent\t\tpasswd\t\tyes$/\tpersistent\t\tpasswd\t\tno/g;' \
  -e 's/^\tpositive-time-to-live\tgroup\t\t3600$/\tpositive-time-to-live\tgroup\t\t86400/g;' \
  -e 's/^\tnegative-time-to-live\tgroup\t\t60$/\tnegative-time-to-live\tgroup\t\t3600/g;' \
  -e 's/^\tsuggested-size\t\tgroup\t\t211$/\tsuggested-size\t\tgroup\t\t401/g;' \
  -e 's/^\tpersistent\t\tgroup\t\tyes$/\tpersistent\t\tgroup\t\tno/g;' \
  -e 's/^\tenable-cache\t\tnetgroup\tyes$/\tenable-cache\t\tnetgroup\tno/g;' \
  /etc/nscd.conf

User directories

User directories must be created and deleted manually.

mkdir -p \
    /home/hsobiecz /home/hsockel /home/wschneid /home/pgittler /home/ikroenke \
    /home/hschima /home/akluwick /home/mrein /home/hmarek /home/hsteinru \
    /home/creichl /home/kcernoho /home/tloimer /home/sbraun /home/bpraunra \
    /home/jstrecha /home/hkuhlman /home/sscheich /home/umundum /home/smasoudi \
    /home/fromano /home/clechner /home/bscheich /home/ganestis /home/mmuellne \
    /home/jkuehnen /home/bbeladi /home/dkuzdas /home/dmursche /home/pgartleh \
    /home/hwu /home/imansky /home/hneth /home/cgoessni /home/aguszich \
    /home/fraderma /home/aaloy /home/fegner /home/asoldati /home/fzonta \
    /home/cschmidr /home/mdepaoli /home/cmader /home/rkarimi /home/phadisic \
    /home/hveliogl /home/tnanz /home/aroccon /home/sahmadi1 /home/fmehl \
    /home/pdesbosc
chgrp E322 \
    /home/hsobiecz /home/hsockel /home/wschneid /home/pgittler /home/ikroenke \
    /home/hschima /home/akluwick /home/mrein /home/hmarek /home/hsteinru \
    /home/creichl /home/kcernoho /home/tloimer /home/sbraun /home/bpraunra \
    /home/jstrecha /home/hkuhlman /home/sscheich /home/umundum /home/smasoudi \
    /home/fromano /home/clechner /home/bscheich /home/ganestis /home/mmuellne \
    /home/jkuehnen /home/bbeladi /home/dkuzdas /home/dmursche /home/pgartleh \
    /home/hwu /home/imansky /home/hneth /home/cgoessni /home/aguszich \
    /home/fraderma /home/aaloy /home/fegner /home/asoldati /home/fzonta \
    /home/cschmidr /home/mdepaoli /home/cmader /home/rkarimi /home/phadisic \
    /home/hveliogl /home/tnanz /home/aroccon /home/sahmadi1 /home/fmehl \
    /home/pdesbosc
for d in /home/^lost+found/; do chown $(basename $d) $d; done

Backup users

id -u s15-root &> /dev/null || useradd -mr -d /mnt/backup/s15 s15-root
id -u s16-root &> /dev/null || useradd -mr -d /mnt/backup/s16 s16-root
useradd -mr -d /mnt/backup/gs2 gs2-root
mkdir -p ~s15-root/.ssh/
mkdir -p ~s16-root/.ssh/
runuser -u gs2-root -- mkdir -m 700  ~gs2-root/.ssh
cat > ~s15-root/.ssh/authorized_keys << EOF
command="borg serve --restrict-to-path /mnt/backup",restrict ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJwa/VwQXhMtDU6YLDdP5qsb0dyp4grUnipuQ43ouihd root@s15
EOF
cat > ~s16-root/.ssh/authorized_keys << EOF
command="borg serve --restrict-to-path /mnt/backup",restrict ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIODF3X5KE0P66SYdOfmePdK/Wp2puusNQsX1HbMBjv5V root@s16
EOF
chown s15-root:s15-root ~s15-root/.ssh/authorized_keys
chown s16-root:s16-root ~s16-root/.ssh/authorized_keys
chmod 600 ~s15-root/.ssh/authorized_keys
chmod 600 ~s16-root/.ssh/authorized_keys
mkdir -p /mnt/backup/s15/home
mkdir -p /mnt/backup/s16/{home,data}
mkdir -p /mnt/backup/b/home
chown -R s15-root:s15-root /mnt/backup/s15
chown -R s16-root:s16-root /mnt/backup/s16

Backup of home directories

[[ -d /opt/borgscripts ]] || git clone oswat@b.fluid.tuwien.ac.at:backup-scripts.git /opt/borgscripts
cat > /etc/cron.d/borg-backup <<EOF
SHELL=/bin/bash
BASHOPTS=extglob

35 05 * * * root /opt/borgscripts/borg-backup /home/!(lost+found) /mnt/backup/b/home
EOF

chgrp oswat /etc/cron.daily/borg-backup

Backup pruning

cat > /etc/cron.d/borg-prune <<EOF
MAILTO=oswat
# prune borg repositories on the first weekend of the month
20 14 * * 5  gs2-root  test `date +\%d` -lt 8 && /opt/borgscripts/borg-prune /mnt/backup/gs2/*.borg && echo done
27 15 * * 5  s15-root  test `date +\%d` -lt 8 && /opt/borgscripts/borg-prune /mnt/backup/s15/*/*.borg && echo done
27 16 * * 5  s16-root  test `date +\%d` -lt 8 && /opt/borgscripts/borg-prune /mnt/backup/s16/*/*.borg && echo done
# User backup, user names obfuscated
# 27 19 * * *  cuser1  /opt/borgscripts/borg-prune /mnt/backup/cuser1/*.borg
# 27 21 * * *  fuser2   /opt/borgscripts/borg-prune /mnt/backup/fuser2/*.borg
27 23 * * *  root      /opt/borgscripts/borg-prune /mnt/backup/b/*/*.borg
EOF
chgrp oswat /etc/cron.d/borg-prune

misc

cat > /etc/systemd/timesyncd.conf <<EOF
[Time]
Servers=tutimea.tuwien.ac.at tutimeb.tuwien.ac.at tutimec.tuwien.ac.at
EOF
timedatectl set-ntp true

cat >/etc/cron.d/find-old-backups <<EOF
MAILTO=oswat
@daily  root      find /home/ -mindepth 3 -maxdepth 3 -name '.TIMESTAMP' -mtime +30
EOF

Appendix

Notes on tool selection and performance

Test results with Obnam indicated several issues with large repositories.
obnam was not able to complete a backup of a 600GB repository with ~3.2M files within 115h.
borg completed a backup of the same repository and some additional data (670GB, ~3.3M files) in under 7 hours.
Enabling compression increases run time to 200% of the original run while reducing the data volume to ~60% for the test repository.
A complete backup for s15 (currently 2.4TB of data, 12M files) takes about 18 hours without compression and 45 hours with compression. The resulting compressed backup uses 1.5TB of disk space in 270,000 files on b.fluid.
A complete run of the daily backup takes approximately 30min on s15.fluid.
Pruning the resulting backup takes approximately 2min.