#!/usr/bin/env bash # # import-photos.sh — bulk import, for when import-photo.sh's per-file cost # stops being noise. # # The single-file tool spends roughly 0.9 s per photograph, and most of that # is process startup rather than work: exiftool is Perl and costs ~0.3 s to # launch, and it is launched twice per image. Twenty photographs cost 1.47 s # as twenty exiftool calls and 0.12 s as one. This script therefore does the # same work in a different shape — every stage that can see the whole batch # at once does: # # resize N parallel ImageMagick calls (irreducible; genuine work) # extract 1 call (was N) # strip 1 call (was N) # palette 1 call (was N) # scaffold 1 call (was N) # # so a batch costs N + 4 processes instead of ~6N. # # Input is a manifest rather than flags, because photographs gathered over # years live in scattered directories under names like IMG_4421.JPG that say # nothing. One line per photograph, tab-separated: # # /path/to/original.jpg slug Title # # Title is optional. Blank lines and lines beginning with # are ignored. # # Usage: # tools/import-photos.sh --manifest FILE [--series SLUG] [--tags a,b] # [--location "Copenhagen, Denmark"] [--jobs N] # [--execute] # # Dry run by default: it validates everything and prints the plan without # touching the repository. Nothing is written until --execute. # # Deliberately NOT handled: geo. Coordinates stay a per-photograph decision, # because a batch flag is exactly how a home address ends up on a public map. set -uo pipefail REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" MANIFEST=""; SERIES=""; TAGS=""; LOCATION=""; JOBS=4; EXECUTE=false while [ "$#" -gt 0 ]; do case "$1" in --manifest) MANIFEST="$2"; shift 2 ;; --series) SERIES="$2"; shift 2 ;; --tags) TAGS="$2"; shift 2 ;; --location) LOCATION="$2"; shift 2 ;; --jobs) JOBS="$2"; shift 2 ;; --execute) EXECUTE=true; shift ;; -h|--help) sed -n '2,40p' "$0"; exit 0 ;; *) echo "unknown argument: $1" >&2; exit 2 ;; esac done [ -n "$MANIFEST" ] || { echo "import-photos: --manifest is required" >&2; exit 2; } [ -f "$MANIFEST" ] || { echo "import-photos: no such manifest: $MANIFEST" >&2; exit 2; } if [ -n "$SERIES" ] && [[ ! "$SERIES" =~ ^[a-z0-9-]+$ ]]; then echo "import-photos: invalid series '$SERIES'" >&2; exit 2 fi for tool in magick exiftool; do command -v "$tool" >/dev/null 2>&1 || { echo "import-photos: $tool is required" >&2; exit 1; } done [ -x "$REPO_ROOT/.venv/bin/python" ] || { echo "import-photos: .venv missing — run uv sync" >&2; exit 1; } # --------------------------------------------------------------------------- # Namespace guard. # # Series and tags share one URL space: a series lives at /photography// # and a tag index at /photography//. Tagging Danish frames `denmark` while # a `denmark` series exists makes both claim the same route, and Hakyll fails # the whole build with "multiple writes for route" — which names the conflict # but not the decision that caused it, several hundred files after the fact. # # Cheaper to refuse here, where the offending tag is still on the command line. # --------------------------------------------------------------------------- collides=0 for t in $(echo "$TAGS" | tr ',' ' '); do t="$(echo "$t" | tr -d '[:space:]')" [ -z "$t" ] && continue bare="${t##*/}" if [ -d "$REPO_ROOT/content/photography/$bare" ] && [ "$bare" != "$SERIES" ]; then echo "import-photos: tag '$bare' collides with the series of the same name." >&2 echo " Both would claim /photography/$bare/ and the build would refuse." >&2 echo " Use a different tag, or rename the series." >&2 collides=1 fi done if [ -n "$SERIES" ] && grep -rqs "photography/$SERIES\b" "$REPO_ROOT/content/photography" \ --include='*.md' 2>/dev/null; then echo "import-photos: series '$SERIES' collides with an existing tag of the same name." >&2 echo " Both would claim /photography/$SERIES/ and the build would refuse." >&2 collides=1 fi [ "$collides" -eq 0 ] || exit 2 # --------------------------------------------------------------------------- # Parse and validate the whole manifest before writing anything. A batch that # fails halfway leaves a half-imported series that has to be unpicked by hand. # --------------------------------------------------------------------------- declare -a SRCS SLUGS TITLES TARGETS MDS errors=0 seen="" while IFS=$'\t' read -r src slug title || [ -n "${src:-}" ]; do case "${src:-}" in ''|'#'*) continue ;; esac src="${src%$'\r'}"; slug="${slug:-}"; title="${title:-}" if [ -z "$slug" ]; then echo " no slug for: $src" >&2; errors=$((errors+1)); continue fi if [[ ! "$slug" =~ ^[a-z0-9-]+$ ]]; then echo " invalid slug '$slug' (lowercase a-z, 0-9, hyphens)" >&2; errors=$((errors+1)); continue fi if [ ! -f "$src" ]; then echo " missing original: $src" >&2; errors=$((errors+1)); continue fi case " $seen " in *" $slug "*) echo " duplicate slug in manifest: $slug" >&2; errors=$((errors+1)); continue ;; esac seen="$seen $slug" if [ -n "$SERIES" ]; then dir="$REPO_ROOT/content/photography/$SERIES" target="$dir/$slug.jpg"; md="$dir/$slug.md" else dir="$REPO_ROOT/content/photography/$slug" target="$dir/photo.jpg"; md="$dir/index.md" fi if [ -e "$md" ] || [ -e "$target" ]; then echo " already exists: $slug" >&2; errors=$((errors+1)); continue fi SRCS+=("$src"); SLUGS+=("$slug"); TITLES+=("$title") TARGETS+=("$target"); MDS+=("$md") done < "$MANIFEST" n=${#SRCS[@]} if [ "$errors" -gt 0 ]; then echo "import-photos: $errors problem(s) in the manifest — nothing imported." >&2 exit 1 fi [ "$n" -gt 0 ] || { echo "import-photos: manifest is empty" >&2; exit 1; } echo "import-photos: $n photograph(s)${SERIES:+ into series '$SERIES'}" if ! $EXECUTE; then echo "### DRY RUN — nothing written. Pass --execute to import. ###" for ((i = 0; i < n && i < 5; i++)); do printf ' %s -> %s\n' "$(basename "${SRCS[$i]}")" "${MDS[$i]#$REPO_ROOT/}" done [ "$n" -gt 5 ] && echo " … and $((n - 5)) more" exit 0 fi mkdir -p "$(dirname "${TARGETS[0]}")" # --------------------------------------------------------------------------- # 1. Resize, in parallel. The only stage that is real per-image work. # --------------------------------------------------------------------------- # vipsthumbnail when it is available, ImageMagick otherwise. libvips asks the # JPEG decoder for a reduced-scale read instead of decoding in full and then # resampling. The win is real but modest at these sizes — 0.181 s against # 0.234 s — because 4608px to 2400px is under a factor of two, and # shrink-on-load works in halves; it would be dramatic for thumbnails. # Verified equivalent on output that matters: same dimensions, same sRGB, EXIF # intact for the extraction step that follows, and about 12% smaller files. if command -v vipsthumbnail >/dev/null 2>&1; then RESIZER=vips echo " resizing ($JOBS parallel, libvips)…" else RESIZER=magick echo " resizing ($JOBS parallel, ImageMagick)…" fi for ((i = 0; i < n; i++)); do printf '%s\0%s\0' "${SRCS[$i]}" "${TARGETS[$i]}" done | RESIZER="$RESIZER" xargs -0 -n 2 -P "$JOBS" bash -c ' if [ "$RESIZER" = vips ]; then # vipsthumbnail honours the orientation tag by default, matching # ImageMagick -auto-orient. Its module warnings on stderr concern # openslide and poppler, neither of which has anything to do with # JPEG, so they are dropped rather than alarming anyone. vipsthumbnail "$0" --size "2400x2400" -o "$1[Q=85]" 2>/dev/null \ || { echo " resize FAILED: $0" >&2; exit 0; } else magick "$0" -auto-orient -resize "2400x2400>" -colorspace sRGB -quality 85 "$1" \ || { echo " resize FAILED: $0" >&2; exit 0; } fi chmod 644 "$1" ' missing=0 for t in "${TARGETS[@]}"; do [ -f "$t" ] || missing=$((missing+1)); done if [ "$missing" -gt 0 ]; then echo "import-photos: $missing resize(s) failed — aborting before metadata." >&2 exit 1 fi # --------------------------------------------------------------------------- # 2. EXIF, one call. Reads the resized copies, which still carry their tags. # --------------------------------------------------------------------------- echo " extracting EXIF (1 call)…" ( cd "$REPO_ROOT" && .venv/bin/python tools/extract-exif.py "${TARGETS[@]}" ) >/dev/null || true # --------------------------------------------------------------------------- # 3. Strip, one call. exiftool rewrites the container without re-encoding, so # the delivered photograph is not put through a second lossy generation # merely to remove its metadata. # --------------------------------------------------------------------------- echo " stripping EXIF (1 call)…" if ! exiftool -q -all= --icc_profile:all -overwrite_original "${TARGETS[@]}"; then echo "import-photos: strip failed — removing delivery files so no EXIF-laden" >&2 echo " JPEG can be committed by the auto-snapshot in \`make build\`." >&2 rm -f -- "${TARGETS[@]}" exit 1 fi # Stripping rewrote each file, making it newer than the sidecar extracted from # it; extract-exif.py would then treat those sidecars as stale and overwrite # real metadata with width and height alone on the next build. for t in "${TARGETS[@]}"; do touch "$t.exif.yaml" 2>/dev/null || true; done # --------------------------------------------------------------------------- # 4. Palette, one call. # --------------------------------------------------------------------------- # Palette extraction is the one metadata stage that is genuinely CPU-bound — # k-means over every pixel, ~0.27 s per photograph — so batching it into a # single process saves nothing. It is split across $JOBS processes instead. # Interpreter startup is ~0.02 s, so a handful of extra ones costs nothing # against the parallelism they buy. echo " extracting palettes ($JOBS parallel)…" pal_pids=() for ((j = 0; j < JOBS; j++)); do chunk=() for ((k = j; k < n; k += JOBS)); do chunk+=("${TARGETS[$k]}"); done [ ${#chunk[@]} -eq 0 ] && continue ( cd "$REPO_ROOT" && .venv/bin/python tools/extract-palette.py "${chunk[@]}" ) >/dev/null 2>&1 & pal_pids+=("$!") done for pid in "${pal_pids[@]}"; do wait "$pid" || true; done # --------------------------------------------------------------------------- # 5. Scaffold every entry in one Python process. # --------------------------------------------------------------------------- echo " writing entries (1 call)…" plan="$(mktemp)"; trap 'rm -f "$plan"' EXIT for ((i = 0; i < n; i++)); do printf '%s\t%s\t%s\t%s\n' "${MDS[$i]}" "${TARGETS[$i]}" "${SLUGS[$i]}" "${TITLES[$i]}" done > "$plan" SERIES="$SERIES" TAGS="$TAGS" LOCATION="$LOCATION" \ "$REPO_ROOT/.venv/bin/python" "$REPO_ROOT/tools/scaffold-photos.py" "$plan" || exit 1 # --------------------------------------------------------------------------- # 6. Series landing, if this is the first import into one. # --------------------------------------------------------------------------- if [ -n "$SERIES" ]; then landing="$REPO_ROOT/content/photography/$SERIES/index.md" if [ ! -f "$landing" ]; then st="$(echo "$SERIES" | tr '-' ' ' | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}')" cat > "$landing" < TODO — what this series is, in a sentence or two. tags: [photography] --- EOF chmod 644 "$landing" echo " created series landing $landing" fi fi echo "import-photos: $n imported." echo "Next: titles and captions are placeholders — edit, then 'make dev'."