Project

General

Profile

Download (886 Bytes)

Files ยป hifis-storage-sync.sh

Script for mirroring HIFIS dCache to local storage - Peter Evans, 2023-10-25 10:07 AM

 

# Update the mirror using `rclone`.

cd /mnt/kosciusko
find mirror/datasets/ -type f -name "*partial" -ls
logdir=$(mktemp -d ${TMPDIR:-/tmp}/rclone.XXXX)
for item in auxiliary models datasets ; do
echo "Syncing ${item}"
rclone sync hifis-storage:${item} mirror/${item} \
--log-file ${logdir}/rclone_${item}.log --stats 10s
done
# Excludes "accounting", "test.dir.1" etc. present on the upstream respository.

# Find a better way to mirror only [A-Z]*:
for item in CITATION LICENSE README ; do
echo "Syncing ${item}"
rclone sync hifis-storage:${item} mirror/${item} \
--log-file ${logdir}/rclone_${item}.log --stats 10s
done


ls -la ${logdir}

# Remove non-problem log reports:
find $logdir -empty -delete

# Find any problem left-overs:
find mirror -type f -name "*partial" -ls

# Find any recent files:
find . -mtime -7 -type f -ls

    (1-1/1)