Contents

  1. Overview
    1. Basic data
    2. Admins
  2. Backups
    1. Directory structure
  3. Configuration
    1. Basic shell setup, etckeeper
    2. megacli installation, RAID tweaks
    3. borg installation
    4. TISS authentication
    5. User directories
    6. Backup users
    7. Backup of home directories
    8. Backup pruning
    9. misc
  4. Appendix
    1. Notes on tool selection and performance

Overview

Basic data

Hostname

b.fluid.tuwien.ac.at

Purpose

backup server

Operating system

Debian Stretch

Management interface

http://mb.fluid.tuwien.ac.at

Backup storage capacity

70TB

Client data storage capacity

3.5TB

Authentication options

local accounts, TU password

Manuals

Quick Reference Sheet, User Manual

Admins

Backups

Directory structure

/mnt/backup/
├── b
│   └── home
├── gs2
├── s15
│   └── home
├── s16
│   ├── data
│   └── home
├── user1
│   ├── pc1
│   └── pc2
└── user2
    └── pcname

Configuration

Basic shell setup, etckeeper

cat > /etc/etckeeper.conf <<EOF
VCS=git
HIGHLEVEL_PACKAGE_MANAGER=apt
LOWLEVEL_PACKAGE_MANAGER=dpkg
EOF
apt-get -y install etckeeper zsh emacs24-nox vim mg git tmux screen htop mc sudo
git config --global user.name root
export MAILNAME=`cat /etc/mailname`
git config --global user.email "root@${MAILNAME}"
cat > /etc/apt/sources.list.d/grml.list <<EOF
deb http://deb.grml.org/ grml-stable main #Grml
EOF
cat > /etc/apt/preferences.d/grml-pin << EOF
Package: *
Pin: release a=grml-stable
Pin-Priority: 200
EOF
apt-key --keyring /etc/apt/trusted.gpg.d/grml.gpg adv \
  --keyserver keyserver.ubuntu.com \
  --recv-keys ECDEA787
apt-get update
apt-get -y install grml-etc-core grml-debian-keyring
wget https://raw.githubusercontent.com/sometimesfood/chef-admin-essentials/master/files/default/tmux.conf -O /etc/tmux.conf
chsh -s /bin/zsh
chsh -s /bin/zsh oswat
touch ~oswat/.zshrc
cat > /etc/default/locale <<EOF
LANG=en_US.utf8
LC_CTYPE=en_US.utf8
LC_ALL=en_US.utf8
EOF

megacli installation, RAID tweaks

apt-key --keyring /etc/apt/trusted.gpg.d/hwraid.gpg adv \
  --keyserver keyserver.ubuntu.com \
  --recv-keys 23B3D3B4
cat > /etc/apt/sources.list.d/hwraid.list <<EOF
deb http://hwraid.le-vert.net/debian stretch main
EOF
apt-get update
apt-get -y install megacli
megacli -AdpSetProp -DsblSpinDownHSP 1 -a0

borg installation

apt-get -y install borgbackup python-llfuse

TISS authentication

export BINDPW=secret
apt-get -y install libpam-cap libpam-ldapd libnss-ldapd nslcd
cat > /etc/nslcd.conf <<EOF
# /etc/nslcd.conf
# nslcd configuration file. See nslcd.conf(5)
# for details.

# The user and group nslcd should run as.
uid nslcd
gid nslcd

# The location at which the LDAP server(s) should be reachable.
uri ldaps://dc.intern.tuwien.ac.at

# The search base that will be used for all queries.
base ou=tu,dc=intern,dc=tuwien,dc=ac,dc=at

# The LDAP protocol version to use.
#ldap_version 3

# The DN to bind with for normal lookups.
binddn cn=E322_LDAP,ou=interactive,ou=exchange,ou=IT-services,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at
bindpw ${BINDPW}

# The DN used for password modifications by root.
#rootpwmoddn cn=admin,dc=example,dc=com

# SSL options
#ssl off
tls_reqcert never
tls_cacertfile /etc/ssl/certs/ca-certificates.crt

# The search scope.
#scope sub

# Customize certain database lookups
base    group   ou=groups,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at
base    passwd  ou=people,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at
base    group   ou=people,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at

# Mappings
filter  passwd  (memberOf=cn=E322,ou=groups,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at)
map     passwd  uid             sAMAccountName
map     passwd  uidNumber       employeeID
map     passwd  gecos           cn
map     passwd  homeDirectory   "/home/\${sAMAccountName}"
map     passwd  loginShell      "/bin/bash"
map     passwd  gidNumber       "2153"

filter  shadow  (memberOf=cn=E322,ou=groups,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at)
map     shadow  uid             sAMAccountName

# Get the objectSid by decoding the base64-decoded string returned by an
# ldap-query, and swapping each pair of byes, and considering least- and
# most significant bit ordering, see
# http://www.selfadsi.org/deep-inside/microsoft-sid-attributes.htm
#   objectSid:: AAU--base64string---AA==
# $ echo AAU--base64string---AA== | base64 -d | od -x
# 000000 0501 0000 0000 0500 0015 0000 097f 1f75
# 000020 1c1d 1a1b 2c2d 2a2b
# The correct number is then, in hex,
# 1-5-21-1f75097f-1a1b1c1d-2a2b2c2d
filter  group   (cn=E322*)
map     group   userPassword    ""
map     group   gidNumber       objectSid:S-1-5-21-527763839-1561677997-902985232
EOF
chown oswat /etc/nslcd.conf
service nslcd restart
# note: files in pam.d are managed by pam-auth-update(8)

perl -pi \
  -e 's/^#\tstat-user\t\tsomebody$/\tstat-user\t\toswat/g;' \
  -e 's/^\tpositive-time-to-live\tpasswd\t\t600$/\tpositive-time-to-live\tpasswd\t\t86400/g;' \
  -e 's/^\tnegative-time-to-live\tpasswd\t\t20$/\tnegative-time-to-live\tpasswd\t\t3600/g;' \
  -e 's/^\tsuggested-size\t\tpasswd\t\t211$/\tsuggested-size\t\tpasswd\t\t401/g;' \
  -e 's/^\tpersistent\t\tpasswd\t\tyes$/\tpersistent\t\tpasswd\t\tno/g;' \
  -e 's/^\tpositive-time-to-live\tgroup\t\t3600$/\tpositive-time-to-live\tgroup\t\t86400/g;' \
  -e 's/^\tnegative-time-to-live\tgroup\t\t60$/\tnegative-time-to-live\tgroup\t\t3600/g;' \
  -e 's/^\tsuggested-size\t\tgroup\t\t211$/\tsuggested-size\t\tgroup\t\t401/g;' \
  -e 's/^\tpersistent\t\tgroup\t\tyes$/\tpersistent\t\tgroup\t\tno/g;' \
  -e 's/^\tenable-cache\t\tnetgroup\tyes$/\tenable-cache\t\tnetgroup\tno/g;' \
  /etc/nscd.conf

User directories

User directories must be created and deleted manually. 

mkdir -p \
    /home/hsobiecz /home/hsockel /home/wschneid /home/pgittler /home/ikroenke \
    /home/hschima /home/akluwick /home/mrein /home/hmarek /home/hsteinru \
    /home/creichl /home/kcernoho /home/tloimer /home/sbraun /home/bpraunra \
    /home/jstrecha /home/hkuhlman /home/sscheich /home/umundum /home/smasoudi \
    /home/fromano /home/clechner /home/bscheich /home/ganestis /home/mmuellne \
    /home/jkuehnen /home/bbeladi /home/dkuzdas /home/dmursche /home/pgartleh \
    /home/hwu /home/imansky /home/hneth /home/cgoessni /home/aguszich \
    /home/fraderma /home/aaloy /home/fegner /home/asoldati /home/fzonta \
    /home/cschmidr /home/mdepaoli /home/cmader /home/rkarimi /home/phadisic \
    /home/hveliogl /home/tnanz /home/aroccon /home/sahmadi1 /home/fmehl \
    /home/pdesbosc
chgrp E322 \
    /home/hsobiecz /home/hsockel /home/wschneid /home/pgittler /home/ikroenke \
    /home/hschima /home/akluwick /home/mrein /home/hmarek /home/hsteinru \
    /home/creichl /home/kcernoho /home/tloimer /home/sbraun /home/bpraunra \
    /home/jstrecha /home/hkuhlman /home/sscheich /home/umundum /home/smasoudi \
    /home/fromano /home/clechner /home/bscheich /home/ganestis /home/mmuellne \
    /home/jkuehnen /home/bbeladi /home/dkuzdas /home/dmursche /home/pgartleh \
    /home/hwu /home/imansky /home/hneth /home/cgoessni /home/aguszich \
    /home/fraderma /home/aaloy /home/fegner /home/asoldati /home/fzonta \
    /home/cschmidr /home/mdepaoli /home/cmader /home/rkarimi /home/phadisic \
    /home/hveliogl /home/tnanz /home/aroccon /home/sahmadi1 /home/fmehl \
    /home/pdesbosc
for d in /home/^lost+found/; do chown $(basename $d) $d; done

Backup users

id -u s15-root &> /dev/null || useradd -mr -d /mnt/backup/s15 s15-root
id -u s16-root &> /dev/null || useradd -mr -d /mnt/backup/s16 s16-root
useradd -mr -d /mnt/backup/gs2 gs2-root
mkdir -p ~s15-root/.ssh/
mkdir -p ~s16-root/.ssh/
runuser -u gs2-root -- mkdir -m 700  ~gs2-root/.ssh
cat > ~s15-root/.ssh/authorized_keys << EOF
command="borg serve --restrict-to-path /mnt/backup",restrict ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJwa/VwQXhMtDU6YLDdP5qsb0dyp4grUnipuQ43ouihd root@s15
EOF
cat > ~s16-root/.ssh/authorized_keys << EOF
command="borg serve --restrict-to-path /mnt/backup",restrict ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIODF3X5KE0P66SYdOfmePdK/Wp2puusNQsX1HbMBjv5V root@s16
EOF
chown s15-root:s15-root ~s15-root/.ssh/authorized_keys
chown s16-root:s16-root ~s16-root/.ssh/authorized_keys
chmod 600 ~s15-root/.ssh/authorized_keys
chmod 600 ~s16-root/.ssh/authorized_keys
mkdir -p /mnt/backup/s15/home
mkdir -p /mnt/backup/s16/{home,data}
mkdir -p /mnt/backup/b/home
chown -R s15-root:s15-root /mnt/backup/s15
chown -R s16-root:s16-root /mnt/backup/s16

Backup of home directories

[[ -d /opt/borgscripts ]] || git clone oswat@b.fluid.tuwien.ac.at:backup-scripts.git /opt/borgscripts
cat > /etc/cron.d/borg-backup <<EOF
SHELL=/bin/bash
BASHOPTS=extglob

35 05 * * * root /opt/borgscripts/borg-backup /home/!(lost+found) /mnt/backup/b/home
EOF

chgrp oswat /etc/cron.daily/borg-backup

Backup pruning

cat > /etc/cron.d/borg-prune <<EOF
MAILTO=oswat
# prune borg repositories on the first weekend of the month
20 14 * * 5  gs2-root  test `date +\%d` -lt 8 && /opt/borgscripts/borg-prune /mnt/backup/gs2/*.borg && echo done
27 15 * * 5  s15-root  test `date +\%d` -lt 8 && /opt/borgscripts/borg-prune /mnt/backup/s15/*/*.borg && echo done
27 16 * * 5  s16-root  test `date +\%d` -lt 8 && /opt/borgscripts/borg-prune /mnt/backup/s16/*/*.borg && echo done
# User backup, user names obfuscated
# 27 19 * * *  cuser1  /opt/borgscripts/borg-prune /mnt/backup/cuser1/*.borg
# 27 21 * * *  fuser2   /opt/borgscripts/borg-prune /mnt/backup/fuser2/*.borg
27 23 * * *  root      /opt/borgscripts/borg-prune /mnt/backup/b/*/*.borg
EOF
chgrp oswat /etc/cron.d/borg-prune

misc

cat > /etc/systemd/timesyncd.conf <<EOF
[Time]
Servers=tutimea.tuwien.ac.at tutimeb.tuwien.ac.at tutimec.tuwien.ac.at
EOF
timedatectl set-ntp true


cat >/etc/cron.d/find-old-backups <<EOF
MAILTO=oswat
@daily  root      find /home/ -mindepth 3 -maxdepth 3 -name '.TIMESTAMP' -mtime +30
EOF

Appendix

Notes on tool selection and performance