From 9e0ef60bc015e7254345d2052b6a36dcfec84fbc Mon Sep 17 00:00:00 2001
From: Colin Powell <colin@unbl.ink>
Date: Wed, 16 Sep 2026 20:05:45 -0400
Subject: [PATCH] [bin] Update nyt to scrape multiple papers

---
 bin/.local/bin/nyt | 177 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 163 insertions(+), 14 deletions(-)

diff --git a/bin/.local/bin/nyt b/bin/.local/bin/nyt
index 48a8b7c..06b2699 100755
--- a/bin/.local/bin/nyt
+++ b/bin/.local/bin/nyt
@@ -1,17 +1,166 @@
 #!/bin/bash
-BASEDIR="/media/photos/misc/nytimes"
-DATE=$(date +"%Y-%m-%d")
-TMPFILE=/tmp/nyt.pdf
-OUTFILE=$BASEDIR/$DATE.jpg
+# Download today's newspaper front pages from frontpages.com.
+# Usage:
+#   nyt                      download the default set
+#   nyt slug [slug ...]      download specific papers
+#   nyt --list               print available papers (slug -> name)
+#   nyt --names               print default set as "name slug" lines
 
-if test -f "$OUTFILE"; then
-	echo "NYT front page already found. Not downloading."
-else
-	DATEPATH=$(date +"%Y/%m/%d")
-	curl -o /tmp/nyt.pdf -L https://static01.nyt.com/images/$DATEPATH/nytfrontpage/scan.pdf
-	convert -density 150 -quality 75 $TMPFILE $OUTFILE
-	rm $BASEDIR/today.jpg
-	cp $OUTFILE $BASEDIR/today.jpg
-	curl -H "X-Title: NYT headlines today" -d "https://files.lab.unbl.ink/nytimes/today.jpg" https://ntfy.unbl.ink/news
-	rm $TMPFILE
+BASEDIR="/media/photos/misc/frontpages"
+DATE=$(date +"%Y-%m-%d")
+DATEPATH=$(date +"%Y/%m/%d")
+NTFY="https://ntfy.unbl.ink/news"
+NTFY_TITLE="NYT front pages today"
+UA="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
+
+DEFAULTS="the-new-york-times the-washington-post the-boston-globe chicago-tribune le-monde le-figaro"
+
+# slug -> display name (subset used by defaults; overridden dynamically)
+slug_to_name() {
+	case "$1" in
+		the-new-york-times)  echo "The New York Times" ;;
+		the-washington-post) echo "The Washington Post" ;;
+		the-boston-globe)    echo "The Boston Globe" ;;
+		chicago-tribune)     echo "Chicago Tribune" ;;
+		le-monde)            echo "Le Monde" ;;
+		le-figaro)           echo "Le Figaro" ;;
+		*)                   echo "$1" ;;
+	esac
+}
+
+# Fetch the slug -> name map embedded on any frontpages page.
+fetch_paper_map() {
+	html=$(curl -sS -L --max-time 30 -A "$UA" "https://www.frontpages.com/the-new-york-times/") || return 1
+	echo "$html" | rg -o '\["[^"]+","[a-z0-9-]+"\]' | sed 's/^\["//; s/","/ => /; s/"\]$//' | rg -v '^(≡|>|LIST)'
+}
+
+notify() {
+	local body="$1"
+	curl -sS -H "X-Title: $NTFY_TITLE" -d "$body" "$NTFY" >/dev/null 2>&1
+}
+
+fetch_paper() {
+	local slug="$1"
+	local name="$2"
+	local dir="$BASEDIR/$slug"
+	local tmpfile="/tmp/nyt-$slug.webp"
+	local outfile="$dir/$DATE.jpg"
+	DOWNLOADED=0
+
+	if test -f "$outfile"; then
+		echo "  $name: already downloaded"
+		return 0
+	fi
+
+	mkdir -p "$dir"
+
+	html=$(curl -sS -L --max-time 30 -A "$UA" "https://www.frontpages.com/$slug/") || {
+		echo "  $name: failed to fetch frontpages.com page"
+		return 1
+	}
+
+	token=$(echo "$html" | sed -n "s/.*atob('\([^']*\)').*/\1/p" | head -1)
+	if test -z "$token"; then
+		echo "  $name: no image token found"
+		return 1
+	fi
+
+	imgpath=$(echo "$token" | base64 -d)
+	if test -z "$imgpath" || ! echo "$imgpath" | grep -q '^/g/[0-9]\{4\}/[0-9]\{2\}/[0-9]\{2\}/'; then
+		echo "  $name: could not decode image token ($imgpath)"
+		return 1
+	fi
+
+	# The token path encodes the actual edition date; a paper's cover may lag
+	# a day behind, so trust it over today's date for the filename.
+	edate=$(echo "$imgpath" | sed 's#^/g/\([0-9]\{4\}\)/\([0-9]\{2\}\)/\([0-9]\{2\}\)/.*#\1-\2-\3#')
+	outfile="$dir/$edate.jpg"
+
+	if test -f "$outfile"; then
+		echo "  $name: already downloaded"
+		return 0
+	fi
+
+	if ! curl -sS -L --max-time 60 -A "$UA" -o "$tmpfile" "https://www.frontpages.com$imgpath"; then
+		echo "  $name: failed to download image"
+		rm -f "$tmpfile"
+		return 1
+	fi
+
+	if ! file "$tmpfile" | grep -qi 'image'; then
+		echo "  $name: downloaded file is not an image: $(file "$tmpfile")"
+		rm -f "$tmpfile"
+		return 1
+	fi
+
+	if ! magick -quality 75 "$tmpfile" "$outfile"; then
+		echo "  $name: image conversion failed"
+		rm -f "$tmpfile"
+		return 1
+	fi
+
+	ln -sf "$outfile" "$dir/today.jpg"
+	rm -f "$tmpfile"
+	DOWNLOADED=1
+	echo "  $name: saved"
+	return 0
+}
+
+if [ "$1" = "--list" ]; then
+	fetch_paper_map || {
+		echo "Failed to fetch paper list" >&2
+		exit 1
+	}
+	exit 0
 fi
+
+if [ "$1" = "--names" ]; then
+	for slug in $DEFAULTS; do
+		echo "$(slug_to_name "$slug") $slug"
+	done
+	exit 0
+fi
+
+if [ $# -eq 0 ]; then
+	papers=$DEFAULTS
+else
+	papers=$*
+fi
+
+echo "Downloading front pages for $DATE"
+success=0
+failed=0
+links=""
+for slug in $papers; do
+	name=$(slug_to_name "$slug")
+	echo "- $name ($slug)"
+	if fetch_paper "$slug" "$name"; then
+		if [ "$DOWNLOADED" = 1 ]; then
+			success=$((success + 1))
+			links+="$name — https://files.lab.unbl.ink/frontpages/$slug/today.jpg"$'\n'
+		fi
+	else
+		failed=$((failed + 1))
+	fi
+done
+
+echo "Done: $success downloaded, $failed failed"
+
+body=""
+if [ $success -gt 0 ]; then
+	body+="Front pages saved for today ($DATE):"$'\n'"$links"
+fi
+if [ $failed -gt 0 ]; then
+	if [ -n "$body" ]; then
+		body+=$'\n'
+	fi
+	body+="Failed ($failed): check the logs."
+fi
+
+if [ -n "$body" ]; then
+	notify "$body"
+else
+	echo "Nothing new to download; no notification sent."
+fi
+
+[ $failed -eq 0 ]
\ No newline at end of file