Restic scripting plus jq and minio client

1
# cat restic-backup.sh
2
#!/bin/bash
3
source /root/.restic-keys
4
resticprog=/usr/local/bin/restic-custom
5
#rcloneargs="serve restic --stdio --b2-hard-delete --cache-workers 64 --transfers 64 --retries 21"
6
region="s3_phx"
7
rundate=$(date +"%Y-%m-%d-%H%M")
8
logtop=/reports
9
logyear=$(date +"%Y")
10
logmonth=$(date +"%m")
11
logname=$logtop/$logyear/$logmonth/restic/$rundate-restic-backup
12
jsonspool=/tmp/restic-fss-jobs
13

14
## Backing up some OCI FSS (same as AWS EFS) NFS folders
15
FSS=(
16
"fs-oracle-apps|fs-oracle-apps|.snapshot"           ## backup all exclude .snapshot tree
17
"fs-app1|fs-app1|.snapshot"                         ## backup all exclude .snapshot tree
18
"fs-sw|fs-sw/oracle_sw,fs-sw/restic_pkg|.snapshot"  ## backup two folders exclude .snapshot tree
19
"fs-tifs|fs-tifs|.snapshot,.tif"                  ## backup all exclude .snapshot tree and *.tif files
20
)
21

22
## test commands especially before kicking off large backups
23
function verify_cmds
24
{
25
  f=$1
26
  restic_cmd=$2
27
  printf "\n$rundate and cmd: $restic_cmd\n"
28
}
29

30
function backup
31
{
32
 f=$1
33
 restic_cmd=$2
34

35
 jobstart=$(date +"%Y-%m-%d-%H%M")
36

37
 mkdir $jsonspool/$f
38
 jsonfile=$jsonspool/$f/$jobstart-restic-backup.json
39
 printf "$jobstart with cmd: $restic_cmd\n"
40

41
 mkdir /mnt/$f
42
 mount -o ro xx.xx.xx.xx:/$f /mnt/$f
43

44
 ## TODO: shell issue with passing exclude from variable. verify exclude .snapshot is working
45
 ## TODO: not passing *.tif exclude fail?  howto pass *?
46
 $restic_cmd > $jsonfile
47

48
 #cat $jsonfile >> $logname-$f.log
49
 umount /mnt/$f
50
 rmdir /mnt/$f
51

52
## Using rclone to copy to OCI object storage bucket.
53
## Note the extra level folder so rclone can simulate
54
## a server/20190711-restic.log style.
55
## Very useful with using minio client to analyze logs.
56
 rclone copy $jsonspool s3_ash:restic-backup-logs
57

58
 rm $jsonfile
59
 rmdir $jsonspool/$f
60

61
 jobfinish=$(date +"%Y-%m-%d-%H%M")
62
 printf "jobfinish $jobfinish\n"
63
}
64

65
for fss in "${FSS[@]}"; do
66
 arrFSS=(${fss//|/ })
67

68
 folders=""
69
 f=${arrFSS[0]}
70
 IFS=',' read -ra folderarr <<< ${arrFSS[1]}
71
 for folder in ${folderarr[@]};do folders+="/mnt/${folder} "; done
72

73
 excludearg=""
74
 IFS=',' read -ra excludearr <<< ${arrFSS[2]}
75
 for exclude in ${excludearr[@]};do excludearg+=" --exclude ${exclude}"; done
76

77
 backup_cmd="$resticprog -r rclone:$region:restic-$f backup ${folders} $excludearg --json"
78

79
## play with verify_cmds first before actual backups
80
 verify_cmds "$f" "$backup_cmd"
81
 #backup "$f" "$backup_cmd"
82
done

1
# cat restic-check-logs.sh
2
#!/bin/bash
3

4
fss=(
5
 fs-oracle-apps
6
)
7

8
#checkdate="2019-07-11"
9
checkdate=$(date +"%Y-%m-%d")
10

11
for f in ${fss[@]}; do
12
  echo
13
  echo
14
  printf "$f:  "
15
  name=$(mc find s3-ash/restic-backup-logs/$f -name "*$checkdate*" | head -1)
16
  if [ -n "$name" ]
17
  then
18
    echo $name
19
    # play with sql --query later
20
    #mc sql --query "select * from S3Object"  --json-input .message_type=summary s3-ash/restic-backup-logs/$f/2019-07-09-1827-restic-backup.json
21
    mc cat $name  | jq -r 'select(.message_type=="summary")'
22
  else
23
    echo "Fail - no file found"
24
  fi
25
done