From 3408e2435369d00e94530f74cdc2154cb6159ad9 Mon Sep 17 00:00:00 2001 From: ctr <639838+ctr@users.noreply.github.com> Date: Thu, 6 Jun 2019 12:13:07 +0100 Subject: [PATCH 1/5] Allow backing up a whole user's repositories. --- GitLabBackup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GitLabBackup.py b/GitLabBackup.py index fc05b3b..45c45a8 100755 --- a/GitLabBackup.py +++ b/GitLabBackup.py @@ -36,8 +36,8 @@ while True: # Request GitLab API to retrieve projects - gitlab_url = args.url + "/api/v3/projects?page=" + str(page) + "&private_token=" + args.token - r = requests.get(gitlab_url, verify=False) + gitlab_url = args.url + "/api/v4/projects?visibility=private&page=" + str(page) + "&private_token=" + args.token + r = requests.get(gitlab_url, verify=True) if r.status_code != 200: r.raise_for_status() @@ -47,7 +47,7 @@ url = project["ssh_url_to_repo"] if args.ssh_port: url = "ssh://" + url.replace(":", ":" + args.ssh_port + "/") - localPath = backup_directory + "/" + project["path"] + ".git" + localPath = backup_directory + "/" + project["path_with_namespace"] + ".git" if not os.path.exists(localPath): print("Create backup for " + localPath) os.system("git clone --mirror " + url + " " + localPath) From 455076b194cd543684bd96422ea670863bd98f8e Mon Sep 17 00:00:00 2001 From: ctr <639838+ctr@users.noreply.github.com> Date: Thu, 6 Jun 2019 12:16:25 +0100 Subject: [PATCH 2/5] Script to run from cron job. (This is a template - you must edit to insert API key etc.) --- .gitignore | 3 +++ run_backup_TEMPLATE.sh | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 .gitignore create mode 100755 run_backup_TEMPLATE.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8c662b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +run_backup.sh +bkup* + diff --git a/run_backup_TEMPLATE.sh b/run_backup_TEMPLATE.sh new file mode 100755 index 0000000..ac7cbfe --- /dev/null +++ b/run_backup_TEMPLATE.sh @@ -0,0 +1,9 @@ +## DELETE THIS LINE, AND INSERT YOUR PRIVATE KEY FILENAME AND API KEY BELOW +#!/bin/bash +cd /home/.../GitLabBackup +mkdir bkup-auto-$(date +%A) + +export GIT_SSH_COMMAND='ssh -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -i /home/.../backup-key' +exec /usr/bin/python3 GitLabBackup.py --backup_dir bkup-auto-$(date +%A) -- https://gitlab.com "INSERT_API_KEY_HERE" + + From 317bc24b0664e40a4a57d280c0217e9c45f931e4 Mon Sep 17 00:00:00 2001 From: ctr <639838+ctr@users.noreply.github.com> Date: Tue, 25 Feb 2020 12:25:46 +0000 Subject: [PATCH 3/5] Extra logging. --- run_backup_TEMPLATE.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/run_backup_TEMPLATE.sh b/run_backup_TEMPLATE.sh index ac7cbfe..a0ea7da 100755 --- a/run_backup_TEMPLATE.sh +++ b/run_backup_TEMPLATE.sh @@ -1,9 +1,13 @@ ## DELETE THIS LINE, AND INSERT YOUR PRIVATE KEY FILENAME AND API KEY BELOW #!/bin/bash -cd /home/.../GitLabBackup +cd /home/ctr28/wbic-git-backup/GitLabBackup + +date "+GitLabBackup running at %Y%m%d-%H%M%S" >> /home/ctr28/wbic-git-backup/GitLabBackup/GitLabBackup.log +echo " running as $(id)" >> /home/ctr28/wbic-git-backup/GitLabBackup/GitLabBackup.log + mkdir bkup-auto-$(date +%A) -export GIT_SSH_COMMAND='ssh -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -i /home/.../backup-key' +export GIT_SSH_COMMAND='ssh -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -i /home/ctr28/wbic-git-backup/keys/wbic-backup-key' exec /usr/bin/python3 GitLabBackup.py --backup_dir bkup-auto-$(date +%A) -- https://gitlab.com "INSERT_API_KEY_HERE" From 42b60b7a96c17972ac977603067a0d790f60b513 Mon Sep 17 00:00:00 2001 From: ctr <639838+ctr@users.noreply.github.com> Date: Thu, 9 May 2024 10:44:48 +0100 Subject: [PATCH 4/5] Rename run_backup_TEMPLATE.sh to run_backup.sh, fix ignores, add executable perm to python script. --- .gitignore | 6 ++++-- GitLabBackup.py | 0 run_backup_TEMPLATE.sh => run_backup.sh | 0 3 files changed, 4 insertions(+), 2 deletions(-) mode change 100755 => 100644 GitLabBackup.py rename run_backup_TEMPLATE.sh => run_backup.sh (100%) mode change 100755 => 100644 diff --git a/.gitignore b/.gitignore index 8c662b6..2d9c87c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ -run_backup.sh -bkup* +bkup/ +fromCygwin/ +api.key +log/ diff --git a/GitLabBackup.py b/GitLabBackup.py old mode 100755 new mode 100644 diff --git a/run_backup_TEMPLATE.sh b/run_backup.sh old mode 100755 new mode 100644 similarity index 100% rename from run_backup_TEMPLATE.sh rename to run_backup.sh From bb61111b52cc6bb52404aeb5f49dc0ac50f50b69 Mon Sep 17 00:00:00 2001 From: ctr <639838+ctr@users.noreply.github.com> Date: Thu, 9 May 2024 11:28:40 +0100 Subject: [PATCH 5/5] Switch to storing api key in the keys folder. --- .gitignore | 1 - run_backup.sh | 29 ++++++++++++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) mode change 100644 => 100755 run_backup.sh diff --git a/.gitignore b/.gitignore index 2d9c87c..f81d0b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ bkup/ fromCygwin/ -api.key log/ diff --git a/run_backup.sh b/run_backup.sh old mode 100644 new mode 100755 index a0ea7da..316cccd --- a/run_backup.sh +++ b/run_backup.sh @@ -1,13 +1,28 @@ -## DELETE THIS LINE, AND INSERT YOUR PRIVATE KEY FILENAME AND API KEY BELOW #!/bin/bash -cd /home/ctr28/wbic-git-backup/GitLabBackup +# This is run from cron by /etc/cron.d/CTR-git-backup +# Or run immediately with +# # sudo -i -u gitbackup /data/backups/wbic-git-backup/GitLabBackup/run_backup.sh -date "+GitLabBackup running at %Y%m%d-%H%M%S" >> /home/ctr28/wbic-git-backup/GitLabBackup/GitLabBackup.log -echo " running as $(id)" >> /home/ctr28/wbic-git-backup/GitLabBackup/GitLabBackup.log +# EDIT CONFIGURATION HERE: +ROOTDIR=/data/backups/wbic-git-backup/GitLabBackup +# API KEY must be stored in api.key file with limited permissions +APIKEY=$(< $ROOTDIR/../keys/api.key) +LOGDIR="$ROOTDIR/log" +LOGFILE="$LOGDIR/GitLabBackup.log" -mkdir bkup-auto-$(date +%A) -export GIT_SSH_COMMAND='ssh -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -i /home/ctr28/wbic-git-backup/keys/wbic-backup-key' -exec /usr/bin/python3 GitLabBackup.py --backup_dir bkup-auto-$(date +%A) -- https://gitlab.com "INSERT_API_KEY_HERE" +cd "$ROOTDIR" +[ ! -d "$LOGDIR" ] && mkdir "$LOGDIR" && echo "Log folder '$LOGDIR' created successfully." + +date "+GitLabBackup running at %Y%m%d-%H%M%S" >> "$LOGFILE" +echo " running as $(id)" >> "$LOGFILE" + +DESTDIR="$ROOTDIR/bkup/auto-$(date +%A)" +echo " DESTDIR=$DESTDIR" >> "$LOGFILE" + +[ ! -d "$DESTDIR" ] && mkdir "$ROOTDIR/bkup/auto-$(date +%A)" + +export GIT_SSH_COMMAND="ssh -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -i $ROOTDIR/../keys/wbic-backup-key" +exec /usr/bin/python3 GitLabBackup.py --backup_dir bkup/auto-$(date +%A) -- https://gitlab.com "$APIKEY" >> "$LOGFILE" 2>&1