#language en
#refresh 999999
## nearly two weeks until an update occurs

[[TableOfContents(2)]]

= Overview =
== Basic data ==
|| Hostname || `b.fluid.tuwien.ac.at` ||
|| Purpose || backup server ||
|| Operating system || Debian Stretch ||
|| Management interface || http://mb.fluid.tuwien.ac.at ||
|| Backup storage capacity || 70TB ||
|| Client data storage capacity || 3.5TB ||
|| Authentication options || local accounts, TU password ||
|| Manuals || [attachment:QRG-1785.pdf Quick Reference Sheet], [attachment:MNL-1785.pdf User Manual] ||

== Admins ==
 * Thomas Loimer
 * Iris Fula
 * Francesco Zonta
 * Rudolf Ladner (ZID)
 * Sebastian Boehm (ZID)

= Backups =
 * Clients synchronise data to `/home` on `b.fluid` using `rsync`.
 * A daily cronjob on `b.fluid` saves versioned backups of all data in `/home` to `/mnt/backups`.
 * The daily backup is performed using [http://borgbackup.readthedocs.io Borg].
 * Daily backups are kept for a week, weekly backups are kept for a month, monthly backups are kept for a year. Data older than one year will be discarded at the discretion of the admins.
 * Due to the amount of data on `s15.fluid` and `s16.fluid`, these hosts do not use `rsync`, but directly target `/mnt/backups` using `borg` instead.

== Directory structure ==
{{{
/mnt/backup/
├── b
│   └── home
├── gs2
├── s15
│   └── home
├── s16
│   ├── data
│   └── home
├── user1
│   ├── pc1
│   └── pc2
└── user2
    └── pcname
}}}

= Configuration =
== Basic shell setup, etckeeper ==
{{{
cat > /etc/etckeeper.conf <<EOF
VCS=git
HIGHLEVEL_PACKAGE_MANAGER=apt
LOWLEVEL_PACKAGE_MANAGER=dpkg
EOF
apt-get -y install etckeeper zsh emacs24-nox vim mg git tmux screen htop mc sudo
git config --global user.name root
export MAILNAME=`cat /etc/mailname`
git config --global user.email "root@${MAILNAME}"
cat > /etc/apt/sources.list.d/grml.list <<EOF
deb http://deb.grml.org/ grml-stable main #Grml
EOF
cat > /etc/apt/preferences.d/grml-pin << EOF
Package: *
Pin: release a=grml-stable
Pin-Priority: 200
EOF
apt-key --keyring /etc/apt/trusted.gpg.d/grml.gpg adv \
  --keyserver keyserver.ubuntu.com \
  --recv-keys ECDEA787
apt-get update
apt-get -y install grml-etc-core grml-debian-keyring
wget https://raw.githubusercontent.com/sometimesfood/chef-admin-essentials/master/files/default/tmux.conf -O /etc/tmux.conf
chsh -s /bin/zsh
chsh -s /bin/zsh oswat
touch ~oswat/.zshrc
cat > /etc/default/locale <<EOF
LANG=en_US.utf8
LC_CTYPE=en_US.utf8
LC_ALL=en_US.utf8
EOF
}}}

== megacli installation, RAID tweaks ==
{{{
apt-key --keyring /etc/apt/trusted.gpg.d/hwraid.gpg adv \
  --keyserver keyserver.ubuntu.com \
  --recv-keys 23B3D3B4
cat > /etc/apt/sources.list.d/hwraid.list <<EOF
deb http://hwraid.le-vert.net/debian stretch main
EOF
apt-get update
apt-get -y install megacli
megacli -AdpSetProp -DsblSpinDownHSP 1 -a0
}}}

== borg installation ==
{{{
apt-get -y install borgbackup python-llfuse
}}}

== TISS authentication ==
{{{
export BINDPW=secret
apt-get -y install libpam-cap libpam-ldapd libnss-ldapd nslcd
cat > /etc/nslcd.conf <<EOF
# /etc/nslcd.conf
# nslcd configuration file. See nslcd.conf(5)
# for details.

# The user and group nslcd should run as.
uid nslcd
gid nslcd

# The location at which the LDAP server(s) should be reachable.
uri ldaps://dc.intern.tuwien.ac.at

# The search base that will be used for all queries.
base ou=tu,dc=intern,dc=tuwien,dc=ac,dc=at

# The LDAP protocol version to use.
#ldap_version 3

# The DN to bind with for normal lookups.
binddn cn=E322_LDAP,ou=interactive,ou=exchange,ou=IT-services,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at
bindpw ${BINDPW}

# The DN used for password modifications by root.
#rootpwmoddn cn=admin,dc=example,dc=com

# SSL options
#ssl off
tls_reqcert never
tls_cacertfile /etc/ssl/certs/ca-certificates.crt

# The search scope.
#scope sub

# Customize certain database lookups
base    group   ou=groups,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at
base    passwd  ou=people,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at
base    group   ou=people,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at

# Mappings
filter  passwd  (memberOf=cn=E322,ou=groups,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at)
map     passwd  uid             sAMAccountName
map     passwd  uidNumber       employeeID
map     passwd  gecos           cn
map     passwd  homeDirectory   "/home/\${sAMAccountName}"
map     passwd  loginShell      "/bin/bash"
map     passwd  gidNumber       "2153"

filter  shadow  (memberOf=cn=E322,ou=groups,ou=TU,dc=intern,dc=tuwien,dc=ac,dc=at)
map     shadow  uid             sAMAccountName

# Get the objectSid by decoding the base64-decoded string returned by an
# ldap-query, and swapping each pair of byes, and considering least- and
# most significant bit ordering, see
# http://www.selfadsi.org/deep-inside/microsoft-sid-attributes.htm
#   objectSid:: AAU--base64string---AA==
# $ echo AAU--base64string---AA== | base64 -d | od -x
# 000000 0501 0000 0000 0500 0015 0000 097f 1f75
# 000020 1c1d 1a1b 2c2d 2a2b
# The correct number is then, in hex,
# 1-5-21-1f75097f-1a1b1c1d-2a2b2c2d
filter  group   (cn=E322*)
map     group   userPassword    ""
map     group   gidNumber       objectSid:S-1-5-21-527763839-1561677997-902985232
EOF
chown oswat /etc/nslcd.conf
service nslcd restart
# note: files in pam.d are managed by pam-auth-update(8)

perl -pi \
  -e 's/^#\tstat-user\t\tsomebody$/\tstat-user\t\toswat/g;' \
  -e 's/^\tpositive-time-to-live\tpasswd\t\t600$/\tpositive-time-to-live\tpasswd\t\t86400/g;' \
  -e 's/^\tnegative-time-to-live\tpasswd\t\t20$/\tnegative-time-to-live\tpasswd\t\t3600/g;' \
  -e 's/^\tsuggested-size\t\tpasswd\t\t211$/\tsuggested-size\t\tpasswd\t\t401/g;' \
  -e 's/^\tpersistent\t\tpasswd\t\tyes$/\tpersistent\t\tpasswd\t\tno/g;' \
  -e 's/^\tpositive-time-to-live\tgroup\t\t3600$/\tpositive-time-to-live\tgroup\t\t86400/g;' \
  -e 's/^\tnegative-time-to-live\tgroup\t\t60$/\tnegative-time-to-live\tgroup\t\t3600/g;' \
  -e 's/^\tsuggested-size\t\tgroup\t\t211$/\tsuggested-size\t\tgroup\t\t401/g;' \
  -e 's/^\tpersistent\t\tgroup\t\tyes$/\tpersistent\t\tgroup\t\tno/g;' \
  -e 's/^\tenable-cache\t\tnetgroup\tyes$/\tenable-cache\t\tnetgroup\tno/g;' \
  /etc/nscd.conf
}}}

== User directories ==
User directories must be created and deleted manually.
{{{
mkdir -p \
    /home/hsobiecz /home/hsockel /home/wschneid /home/pgittler /home/ikroenke \
    /home/hschima /home/akluwick /home/mrein /home/hmarek /home/hsteinru \
    /home/creichl /home/kcernoho /home/tloimer /home/sbraun /home/bpraunra \
    /home/jstrecha /home/hkuhlman /home/sscheich /home/umundum /home/smasoudi \
    /home/fromano /home/clechner /home/bscheich /home/ganestis /home/mmuellne \
    /home/jkuehnen /home/bbeladi /home/dkuzdas /home/dmursche /home/pgartleh \
    /home/hwu /home/imansky /home/hneth /home/cgoessni /home/aguszich \
    /home/fraderma /home/aaloy /home/fegner /home/asoldati /home/fzonta \
    /home/cschmidr /home/mdepaoli /home/cmader /home/rkarimi /home/phadisic \
    /home/hveliogl /home/tnanz /home/aroccon /home/sahmadi1 /home/fmehl \
    /home/pdesbosc
chgrp E322 \
    /home/hsobiecz /home/hsockel /home/wschneid /home/pgittler /home/ikroenke \
    /home/hschima /home/akluwick /home/mrein /home/hmarek /home/hsteinru \
    /home/creichl /home/kcernoho /home/tloimer /home/sbraun /home/bpraunra \
    /home/jstrecha /home/hkuhlman /home/sscheich /home/umundum /home/smasoudi \
    /home/fromano /home/clechner /home/bscheich /home/ganestis /home/mmuellne \
    /home/jkuehnen /home/bbeladi /home/dkuzdas /home/dmursche /home/pgartleh \
    /home/hwu /home/imansky /home/hneth /home/cgoessni /home/aguszich \
    /home/fraderma /home/aaloy /home/fegner /home/asoldati /home/fzonta \
    /home/cschmidr /home/mdepaoli /home/cmader /home/rkarimi /home/phadisic \
    /home/hveliogl /home/tnanz /home/aroccon /home/sahmadi1 /home/fmehl \
    /home/pdesbosc
for d in /home/^lost+found/; do chown $(basename $d) $d; done
}}}

== Backup users ==
{{{
id -u s15-root &> /dev/null || useradd -mr -d /mnt/backup/s15 s15-root
id -u s16-root &> /dev/null || useradd -mr -d /mnt/backup/s16 s16-root
useradd -mr -d /mnt/backup/gs2 gs2-root
mkdir -p ~s15-root/.ssh/
mkdir -p ~s16-root/.ssh/
runuser -u gs2-root -- mkdir -m 700  ~gs2-root/.ssh
cat > ~s15-root/.ssh/authorized_keys << EOF
command="borg serve --restrict-to-path /mnt/backup",restrict ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJwa/VwQXhMtDU6YLDdP5qsb0dyp4grUnipuQ43ouihd root@s15
EOF
cat > ~s16-root/.ssh/authorized_keys << EOF
command="borg serve --restrict-to-path /mnt/backup",restrict ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIODF3X5KE0P66SYdOfmePdK/Wp2puusNQsX1HbMBjv5V root@s16
EOF
chown s15-root:s15-root ~s15-root/.ssh/authorized_keys
chown s16-root:s16-root ~s16-root/.ssh/authorized_keys
chmod 600 ~s15-root/.ssh/authorized_keys
chmod 600 ~s16-root/.ssh/authorized_keys
mkdir -p /mnt/backup/s15/home
mkdir -p /mnt/backup/s16/{home,data}
mkdir -p /mnt/backup/b/home
chown -R s15-root:s15-root /mnt/backup/s15
chown -R s16-root:s16-root /mnt/backup/s16
}}}

== Backup of home directories ==
{{{
[[ -d /opt/borgscripts ]] || git clone oswat@b.fluid.tuwien.ac.at:backup-scripts.git /opt/borgscripts
cat > /etc/cron.d/borg-backup <<EOF
SHELL=/bin/bash
BASHOPTS=extglob

35 05 * * * root /opt/borgscripts/borg-backup /home/!(lost+found) /mnt/backup/b/home
EOF

chgrp oswat /etc/cron.daily/borg-backup
}}}

== Backup pruning ==
{{{
cat > /etc/cron.d/borg-prune <<EOF
MAILTO=oswat
# prune borg repositories on the first weekend of the month
20 14 * * 5  gs2-root  test `date +\%d` -lt 8 && /opt/borgscripts/borg-prune /mnt/backup/gs2/*.borg && echo done
27 15 * * 5  s15-root  test `date +\%d` -lt 8 && /opt/borgscripts/borg-prune /mnt/backup/s15/*/*.borg && echo done
27 16 * * 5  s16-root  test `date +\%d` -lt 8 && /opt/borgscripts/borg-prune /mnt/backup/s16/*/*.borg && echo done
# User backup, user names obfuscated
# 27 19 * * *  cuser1  /opt/borgscripts/borg-prune /mnt/backup/cuser1/*.borg
# 27 21 * * *  fuser2   /opt/borgscripts/borg-prune /mnt/backup/fuser2/*.borg
27 23 * * *  root      /opt/borgscripts/borg-prune /mnt/backup/b/*/*.borg
EOF
chgrp oswat /etc/cron.d/borg-prune
}}}

== misc ==
{{{
cat > /etc/systemd/timesyncd.conf <<EOF
[Time]
Servers=tutimea.tuwien.ac.at tutimeb.tuwien.ac.at tutimec.tuwien.ac.at
EOF
timedatectl set-ntp true
}}}

{{{
cat >/etc/cron.d/find-old-backups <<EOF
MAILTO=oswat
@daily  root      find /home/ -mindepth 3 -maxdepth 3 -name '.TIMESTAMP' -mtime +30
EOF
}}}

= Appendix =
== Notes on tool selection and performance ==
 * Test results with [http://obnam.org Obnam] indicated several issues with large repositories.
 * obnam was not able to complete a backup of a 600GB repository with ~3.2M files within 115h.
 * borg completed a backup of the same repository and some additional data (670GB, ~3.3M files) in under 7 hours.
 * Enabling compression increases run time to 200% of the original run while reducing the data volume to ~60% for the test repository.
 * A complete backup for s15 (currently 2.4TB of data, 12M files) takes about 18 hours without compression and 45 hours with compression. The resulting compressed backup uses 1.5TB of disk space in 270,000 files on `b.fluid`.
 * A complete run of the daily backup takes approximately 30min on `s15.fluid`.
 * Pruning the resulting backup takes approximately 2min.