From 5711aafa78c07f37bf5adf98013812fbd0f90ad1 Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Tue, 11 Aug 2026 13:37:34 +0200 Subject: [PATCH] infra: forgejo sync tooling and a nightly backup timer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems this session surfaced on the Forgejo box, both now addressed in tracked source rather than in shell history. forgejo-sync.sh pushes the local repositories to git.levineuwirth.org over HTTPS. It exists because the only Forgejo remote anywhere used ssh://…:2222, which is filtered on most public networks, so eight of nine repos had drifted three to four months behind while the CV cited them as canonical. Dry-run by default, never force-pushes, never touches `origin`, creates new repos private, and refuses to push when the remote holds commits the local machine does not — the check that caught `levshell`, where Forgejo's copy turned out to be the abandoned prototype rather than an older version of the current project. The divergence check authenticates via GIT_ASKPASS. An earlier version ran ls-remote unauthenticated, which fails silently on private repositories and returned empty — indistinguishable from "remote has nothing we lack", i.e. a false all-clear on exactly the repos where the check matters. A failed listing now blocks instead of passing. forgejo-backup.sh plus its timer close the other gap: there were no backups at all, of an instance holding nine repositories, four of them cited by URL in a CV PDF already in circulation. Nightly, no downtime, SQLite `.backup` for a consistent snapshot rather than cp, integrity-checked before it is trusted, archive verified readable before retention prunes anything, and 14 days kept. Aborts non-zero on any of those checks so a bad run surfaces as a failed unit instead of a corrupt archive displacing a good one. Both are still local to that box. Copying an archive off-site remains the open half of the backup story. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SUGesXiMmACsLBTGG1xuEU --- systemd/forgejo-backup.service | 26 ++++ systemd/forgejo-backup.timer | 20 +++ tools/forgejo-backup.sh | 80 ++++++++++++ tools/forgejo-sync.sh | 228 +++++++++++++++++++++++++++++++++ 4 files changed, 354 insertions(+) create mode 100644 systemd/forgejo-backup.service create mode 100644 systemd/forgejo-backup.timer create mode 100644 tools/forgejo-backup.sh create mode 100755 tools/forgejo-sync.sh diff --git a/systemd/forgejo-backup.service b/systemd/forgejo-backup.service new file mode 100644 index 0000000..6a1df18 --- /dev/null +++ b/systemd/forgejo-backup.service @@ -0,0 +1,26 @@ +# Nightly Forgejo backup on the VPS. Unlike archive-check.service, which is +# a --user unit on the laptop, this is a system unit on the server: +# +# scp tools/forgejo-backup.sh root@:/usr/local/bin/forgejo-backup.sh +# ssh root@ chmod 755 /usr/local/bin/forgejo-backup.sh +# scp systemd/forgejo-backup.{service,timer} root@:/etc/systemd/system/ +# ssh root@ systemctl daemon-reload +# ssh root@ systemctl enable --now forgejo-backup.timer +# +# The script aborts non-zero on a failed integrity check or an unreadable +# archive, so a bad run shows up as a failed unit rather than as a corrupt +# file that quietly displaces a good one. + +[Unit] +Description=Forgejo backup (consistent SQLite snapshot + data tarball) +Documentation=https://git.levineuwirth.org/neuwirth/levineuwirth.org +# The script checks for the container itself and fails loudly if it is +# absent, but ordering after docker avoids a guaranteed-failed run at boot. +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/forgejo-backup.sh +Nice=15 +IOSchedulingClass=idle diff --git a/systemd/forgejo-backup.timer b/systemd/forgejo-backup.timer new file mode 100644 index 0000000..209e0e3 --- /dev/null +++ b/systemd/forgejo-backup.timer @@ -0,0 +1,20 @@ +# Nightly trigger for forgejo-backup.service. See that unit's header for +# the install commands. + +[Unit] +Description=Nightly Forgejo backup + +[Timer] +# 03:30 UTC: after any realistic evening of work, before the morning. The +# backup does not stop the service, so the hour matters less than being +# consistently outside the window where a push might be mid-flight. +OnCalendar=*-*-* 03:30 +RandomizedDelaySec=20min +# The VPS is not always up (it rebooted this morning). Persistent=true runs +# a missed backup shortly after the next boot instead of silently skipping +# the night — the failure mode that turns "we have nightly backups" into +# "we had nightly backups until the reboot". +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/tools/forgejo-backup.sh b/tools/forgejo-backup.sh new file mode 100644 index 0000000..daacb5c --- /dev/null +++ b/tools/forgejo-backup.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# forgejo-backup.sh — nightly backup of the Forgejo instance on the VPS. +# +# Installed at /usr/local/bin/forgejo-backup.sh and driven by +# systemd/forgejo-backup.timer. Source of truth is this file in the repo. +# +# The database is SQLite, so the whole instance is a directory and a 2 MB +# file — but a plain `cp` of a live SQLite database can capture a torn +# write. `.backup` takes a consistent snapshot of a running database, and +# the snapshot is integrity-checked before anything is allowed to depend on +# it. If the check fails the run aborts non-zero, so systemd marks the unit +# failed rather than quietly writing a corrupt archive over a good one. +# +# Nothing is stopped: Forgejo stays up for the duration. +set -euo pipefail + +SRC=/root/forgejo-server +DEST=/root/forgejo-backups +KEEP=${KEEP:-14} +TS=$(date -u +%Y%m%dT%H%M%SZ) +SNAP_REL="gitea/hot-$TS.db" +SNAP="$SRC/forgejo-data/$SNAP_REL" + +mkdir -p "$DEST" + +# Remove the in-data snapshot on any exit path, success or failure, so a +# crashed run cannot leave stray database copies inside the live data +# directory where the next tar would pick them up. +cleanup() { rm -f "$SNAP"; } +trap cleanup EXIT + +echo "forgejo-backup: starting $TS" + +if ! docker ps --format '{{.Names}}' | grep -qx forgejo; then + echo "forgejo-backup: container 'forgejo' is not running — aborting" >&2 + exit 1 +fi + +docker exec forgejo sqlite3 /data/gitea/gitea.db ".backup '/data/$SNAP_REL'" + +INTEGRITY=$(docker exec forgejo sqlite3 "/data/$SNAP_REL" "PRAGMA integrity_check;") +if [ "$INTEGRITY" != "ok" ]; then + echo "forgejo-backup: snapshot failed integrity_check ($INTEGRITY) — aborting" >&2 + exit 1 +fi + +REPOS=$(docker exec forgejo sqlite3 "/data/$SNAP_REL" "SELECT COUNT(*) FROM repository;") +echo "forgejo-backup: snapshot ok, $REPOS repositories" + +ARCHIVE="$DEST/forgejo-$TS.tar.gz" +tar czf "$ARCHIVE" -C /root forgejo-server + +# Prove the archive reads back before it is allowed to count as a backup and +# push an older one out of the retention window. +if ! tar tzf "$ARCHIVE" >/dev/null 2>&1; then + echo "forgejo-backup: archive is unreadable — removing and aborting" >&2 + rm -f "$ARCHIVE" + exit 1 +fi + +sha256sum "$ARCHIVE" > "$ARCHIVE.sha256" +ln -sfn "$ARCHIVE" "$DEST/LATEST" +date -u +%Y-%m-%dT%H:%M:%SZ > "$DEST/last-success" + +SIZE=$(du -h "$ARCHIVE" | cut -f1) +echo "forgejo-backup: wrote $ARCHIVE ($SIZE)" + +# Retention. Prune only fully-formed archives (each has a .sha256 beside +# it), so a partial file from an interrupted run is never counted as one of +# the copies being kept. +mapfile -t OLD < <(ls -1t "$DEST"/forgejo-*.tar.gz 2>/dev/null | tail -n +$((KEEP + 1))) +for f in "${OLD[@]:-}"; do + [ -n "$f" ] || continue + echo "forgejo-backup: pruning $(basename "$f")" + rm -f "$f" "$f.sha256" +done + +COUNT=$(ls -1 "$DEST"/forgejo-*.tar.gz 2>/dev/null | wc -l) +echo "forgejo-backup: done — $COUNT archive(s) retained, keeping $KEEP" diff --git a/tools/forgejo-sync.sh b/tools/forgejo-sync.sh new file mode 100755 index 0000000..f74ec14 --- /dev/null +++ b/tools/forgejo-sync.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# +# forgejo-sync.sh — bring git.levineuwirth.org up to date with the local +# repositories, over HTTPS. +# +# Context: Forgejo is advertised as the canonical home (it is cited by URL +# in the CV PDF) but is three to four months behind on eight of nine repos, +# because the only configured Forgejo remote used ssh://…:2222, which is +# filtered on most public networks. HTTPS works everywhere. +# +# SAFETY PROPERTIES, all deliberate: +# +# * Dry run by default. Nothing is created, pushed, or modified unless +# --execute is passed. Without it the script only reads. +# * Never force-pushes. Plain `git push`. If Forgejo holds commits this +# machine does not have, the push is REFUSED and reported rather than +# resolved — losing someone else's commits is not this script's call. +# * Never touches `origin`. It adds a separate `forgejo` remote, so the +# existing GitHub workflow keeps working and every change here is +# undone by `git remote remove forgejo`. +# * New repositories are created PRIVATE. Some of these (oneiros-config, +# levshell) are not public on GitHub, and a wrong default here would +# publish them. Flip individual repos to public in the UI afterwards. +# * No mirrors are configured. A Forgejo push-mirror force-pushes to its +# target, so enabling one while Forgejo is behind would overwrite +# GitHub with stale history. Mirrors come after this has run clean. +# +# Usage: +# export FORGEJO_TOKEN=… # Settings → Applications, scope write:repository +# bash tools/forgejo-sync.sh # survey only +# bash tools/forgejo-sync.sh --execute # do it +# bash tools/forgejo-sync.sh --execute --create-missing +# +# The token is required even for the survey: it lists your private repos, +# and a tokenless listing would report them as missing and propose creating +# duplicates. + +set -uo pipefail + +FORGEJO_URL="https://git.levineuwirth.org" +FORGEJO_USER="neuwirth" +REPO_ROOT="${REPO_ROOT:-$HOME/Repos/personal}" + +# Repositories to leave alone entirely. They are still listed in the output, +# marked HELD, so that skipping one stays a visible decision rather than a +# repo that quietly disappears from the report. +# +# oneiros-config — personal configuration, not ready to leave this machine. +SKIP_REPOS=( + oneiros-config +) + +EXECUTE=false +CREATE_MISSING=false + +for arg in "$@"; do + case "$arg" in + --execute) EXECUTE=true ;; + --create-missing) CREATE_MISSING=true ;; + -h|--help) sed -n '2,40p' "$0"; exit 0 ;; + *) echo "unknown argument: $arg" >&2; exit 2 ;; + esac +done + +if [ -z "${FORGEJO_TOKEN:-}" ]; then + echo "FORGEJO_TOKEN is not set. Generate one at:" >&2 + echo " $FORGEJO_URL/user/settings/applications (scope: write:repository)" >&2 + echo "Then: export FORGEJO_TOKEN=…" >&2 + exit 2 +fi + +$EXECUTE || echo "### DRY RUN — nothing will be changed. Pass --execute to act. ###" +echo + +api() { + curl -sS -H "Authorization: token $FORGEJO_TOKEN" \ + -H "Content-Type: application/json" "$@" +} + +# Git needs the token too, for private repositories. It is handed over via +# GIT_ASKPASS rather than embedded in the URL or passed on a command line, +# so it stays out of `ps` output, out of the remote's stored config, and out +# of any error message git prints on failure. +ASKPASS=$(mktemp) +chmod 700 "$ASKPASS" +printf '#!/bin/sh\nprintf "%%s" "$FORGEJO_TOKEN"\n' > "$ASKPASS" +export GIT_ASKPASS="$ASKPASS" +export GIT_TERMINAL_PROMPT=0 +trap 'rm -f "$ASKPASS"' EXIT + +# --------------------------------------------------------------------------- +# What already exists on Forgejo (includes private repos, hence the token). +# --------------------------------------------------------------------------- +existing=$(api "$FORGEJO_URL/api/v1/user/repos?limit=100" \ + | python3 -c 'import json,sys +try: + print("\n".join(r["name"].lower() for r in json.load(sys.stdin))) +except Exception as e: + print("APIERROR", e, file=sys.stderr)') + +if [ -z "$existing" ]; then + echo "Could not list repositories — is the token valid?" >&2 + exit 1 +fi + +echo "Forgejo currently holds: $(echo "$existing" | tr '\n' ' ')" +echo + +created=0; pushed=0; skipped=0; blocked=0 + +for dir in "$REPO_ROOT"/*/; do + [ -d "$dir/.git" ] || continue + name=$(basename "$dir") + cd "$dir" || continue + + for held in "${SKIP_REPOS[@]}"; do + if [ "$name" = "$held" ]; then + echo "=== $name" + echo " HELD — in SKIP_REPOS, not created, not pushed, no remote added" + echo + continue 2 + fi + done + + branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null) + nbranches=$(git for-each-ref --format='%(refname)' refs/heads | wc -l) + ntags=$(git tag | wc -l) + dirty=$(git status --porcelain | wc -l) + + echo "=== $name ($branch, $nbranches branches, $ntags tags)" + [ "$dirty" -gt 0 ] && echo " note: $dirty uncommitted change(s) — not pushed either way" + + # Forgejo repo names are lowercased; the local directory may not be + # (LeVCS on disk is levcs there). + remote_name=$(echo "$name" | tr '[:upper:]' '[:lower:]') + + if ! echo "$existing" | grep -qx "$remote_name"; then + if ! $CREATE_MISSING; then + echo " ABSENT on Forgejo — rerun with --create-missing to create it (private)" + skipped=$((skipped+1)); echo; continue + fi + if $EXECUTE; then + code=$(api -o /dev/null -w '%{http_code}' -X POST \ + "$FORGEJO_URL/api/v1/user/repos" \ + -d "{\"name\":\"$remote_name\",\"private\":true,\"auto_init\":false}") + if [ "$code" != "201" ]; then + echo " create FAILED (HTTP $code) — skipping" + blocked=$((blocked+1)); echo; continue + fi + echo " created $remote_name (private)" + created=$((created+1)) + else + echo " would create $remote_name (private)" + created=$((created+1)) + fi + fi + + # Username in the URL, token supplied by GIT_ASKPASS. + url="https://$FORGEJO_USER@${FORGEJO_URL#https://}/$FORGEJO_USER/$remote_name.git" + + if $EXECUTE; then + git remote remove forgejo 2>/dev/null || true + git remote add forgejo "$url" + fi + + # --- divergence check ------------------------------------------------- + # Read the remote's refs without fetching. If any remote commit is not + # an object we already hold, the remote is ahead or has diverged, and a + # plain push would either fail or (with --force) destroy it. Report and + # move on; a human decides. + # A failed listing must NOT read as "nothing unknown on the remote". + # Unauthenticated ls-remote against a private repo returns empty and + # exits non-zero, which would otherwise look identical to a clean + # fast-forward and wave through a push that could be refused — or, with + # any future --force, be destructive. + if ! remote_refs=$(timeout 45 git ls-remote "$url" 'refs/heads/*' 2>/dev/null); then + echo " BLOCKED — cannot read refs from Forgejo (auth, network, or" + echo " repository missing). Not pushing blind." + blocked=$((blocked+1)); echo; continue + fi + ahead_of_us=0 + while read -r sha ref; do + [ -n "${sha:-}" ] || continue + if ! git cat-file -e "${sha}^{commit}" 2>/dev/null; then + echo " REMOTE HAS UNKNOWN COMMIT on ${ref#refs/heads/} ($(echo "$sha" | cut -c1-8))" + ahead_of_us=1 + fi + done <<< "$remote_refs" + + if [ "$ahead_of_us" = 1 ]; then + echo " BLOCKED — Forgejo holds commits this machine does not." + echo " Resolve by hand (fetch and inspect) before syncing this one." + blocked=$((blocked+1)); echo; continue + fi + + # --- push ------------------------------------------------------------- + if $EXECUTE; then + if git push --all "$url" >/dev/null 2>&1 && git push --tags "$url" >/dev/null 2>&1; then + echo " pushed all branches + tags" + pushed=$((pushed+1)) + else + echo " PUSH FAILED — rerun by hand to see git's reason:" + echo " git -C '$dir' push --all forgejo" + blocked=$((blocked+1)) + fi + else + out=$(git push --all --dry-run "$url" 2>&1 | grep -vE '^To |^$' | head -5) + if [ -z "$out" ]; then + echo " already up to date" + else + echo " would push:" + echo "$out" | sed 's/^/ /' + fi + pushed=$((pushed+1)) + fi + echo +done + +echo "----------------------------------------------------------------" +printf 'created/creatable: %d pushed/pushable: %d skipped: %d blocked: %d\n' \ + "$created" "$pushed" "$skipped" "$blocked" +$EXECUTE || echo "(dry run — nothing above actually happened)" +echo +echo "Next, only once this runs clean:" +echo " 1. Check the new repos' visibility; all were created private." +echo " 2. Configure push mirrors Forgejo → GitHub, per repo, in" +echo " Settings → Repository → Mirror Settings. A mirror force-pushes," +echo " so it is only safe now that Forgejo is not behind."