Backup to True1

Common

ensure metal1 is booted up and true1 is accessible

Backup far1

NB only backs-up /home

from far1 run _/home/baz/bin/backupToTrue1.sh

There are a few other directions to backup. There is a script named /root/backups.sh to help.

cd /home
rsync -avP . true1:/mnt/Pool-1/devHomeBackup
cd /etc
rsync -avCP . true1:/mnt/Pool-1/far1/etc
cd /var
rsync -avCPx --exclude=cache . true1:/mnt/Pool-1/far1/var

This is now in a script:

sudo -s
. /root/backups.sh

Backup bazone

2 cronjobs are setup to backup bazone to true1.

on true1: 0 * * * * /usr/bin/ssh -o ExitOnForwardFailure=yes -R 32222:localhost:22222 bazone “while true ; do sleep 15 ; done” # this creates a tunnel so bazone can connect to true1 WARNING an upgrade of Truenas caused the cronjob to be deleted **
on bazone: 58 6 * * * /usr/bin/python3 /home/baz/bin/backup.py &> /home/baz/bin/lastBackup.log # run python script to rsync the 3 main directories: /var/lib/docker /home and /etc.

NB the script retries for 3 hours, then fails adn exists sending an ntfy notification.

Implementation Notes

There is no route from bazone to true1, so we need to create one using an ssh tunnel:

from true1: ssh -R 32222:localhost:22222 bazone
then on bazone use the python script /home/baz/bin/backup.py to rsync the 3 main directories: /var/lib/docker /home and /etc.

#!/usr/bin/env python3

from pexpect import pxssh, spawn, exceptions
import time, requests

ntfyURL = 'https://<must set this>'

def main():
    print(f"Starting backup.py at {time.asctime()}")
    if waitForPortOpen():    
        syncFiles('/','home','/mnt/Pool-1/bazone/home')
        syncFiles('/','etc','/mnt/Pool-1/bazone/etc')
        syncFiles('/var/lib','docker','/mnt/Pool-1/bazone/varLibDocker')
    else:
        print("failed to get open port to connect to, aborting.")
        requests.post(ntfyURL, data="bazone backup failed, could not connect to NAS", \
                      headers={"Priority": "urgent","Tags":"skull","Title":"bazone backup failed" })
    print(f"Finished backup.py at {time.asctime()}")

def syncFiles(fromDir, source, target) -> None:
    startTime = time.time()
    session = spawn("sudo -s")
    session.expect("root@bazone:/.*#")
    session.sendline(f"cd {fromDir}")
    session.expect(f":{fromDir}#")
    session.sendline(f"rsync -avP -e 'ssh -p 32222' {source}/ root@localhost:{target}")
    session.expect(r"total size is ([0-9,]+)  speedup is ([0-9,.]+)", timeout=900)
    endTime = time.time()
    matches = session.match.groups()
    print(f"rsync on {source} completed in {(endTime-startTime):0.1f} secs, Total Size = {matches[0].decode()}, Speedup = {matches[1].decode()}")

def waitForPortOpen() -> None:
    port = "32222"
    portClosedNotReported, numTimesPortClosed = True, 0
    startTime = time.time()
    while True:
        try:
            session = spawn(f"ss -anp sport = :{port}")
            session.expect(r"tcp\s+LISTEN.*\d(:\d+)\s")
        except exceptions.EOF:
            numTimesPortClosed += 1
            if portClosedNotReported or numTimesPortClosed % 100 == 0:
                portClosedNotReported = False
                if numTimesPortClosed < 2:
                    print(f"port {port} is not open, waiting for NAS to connect")
                    startTime = time.asctime()
                else:
                    print(f"port {port} still closed, retrying connect to NAS, since {startTime}.")

            time.sleep(10)
            continue
        matches = session.match.groups()
        if matches[0].decode() == f":{port}":
            return True
        if time.time() - startTime > 3 * 3600:
            return False

if __name__ == "__main__":
    main()