[bin] Update nyt to scrape multiple papers
This commit is contained in:
@ -1,17 +1,166 @@
|
||||
#!/bin/bash
|
||||
BASEDIR="/media/photos/misc/nytimes"
|
||||
DATE=$(date +"%Y-%m-%d")
|
||||
TMPFILE=/tmp/nyt.pdf
|
||||
OUTFILE=$BASEDIR/$DATE.jpg
|
||||
# Download today's newspaper front pages from frontpages.com.
|
||||
# Usage:
|
||||
# nyt download the default set
|
||||
# nyt slug [slug ...] download specific papers
|
||||
# nyt --list print available papers (slug -> name)
|
||||
# nyt --names print default set as "name slug" lines
|
||||
|
||||
if test -f "$OUTFILE"; then
|
||||
echo "NYT front page already found. Not downloading."
|
||||
else
|
||||
DATEPATH=$(date +"%Y/%m/%d")
|
||||
curl -o /tmp/nyt.pdf -L https://static01.nyt.com/images/$DATEPATH/nytfrontpage/scan.pdf
|
||||
convert -density 150 -quality 75 $TMPFILE $OUTFILE
|
||||
rm $BASEDIR/today.jpg
|
||||
cp $OUTFILE $BASEDIR/today.jpg
|
||||
curl -H "X-Title: NYT headlines today" -d "https://files.lab.unbl.ink/nytimes/today.jpg" https://ntfy.unbl.ink/news
|
||||
rm $TMPFILE
|
||||
BASEDIR="/media/photos/misc/frontpages"
|
||||
DATE=$(date +"%Y-%m-%d")
|
||||
DATEPATH=$(date +"%Y/%m/%d")
|
||||
NTFY="https://ntfy.unbl.ink/news"
|
||||
NTFY_TITLE="NYT front pages today"
|
||||
UA="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
|
||||
|
||||
DEFAULTS="the-new-york-times the-washington-post the-boston-globe chicago-tribune le-monde le-figaro"
|
||||
|
||||
# slug -> display name (subset used by defaults; overridden dynamically)
|
||||
slug_to_name() {
|
||||
case "$1" in
|
||||
the-new-york-times) echo "The New York Times" ;;
|
||||
the-washington-post) echo "The Washington Post" ;;
|
||||
the-boston-globe) echo "The Boston Globe" ;;
|
||||
chicago-tribune) echo "Chicago Tribune" ;;
|
||||
le-monde) echo "Le Monde" ;;
|
||||
le-figaro) echo "Le Figaro" ;;
|
||||
*) echo "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Fetch the slug -> name map embedded on any frontpages page.
|
||||
fetch_paper_map() {
|
||||
html=$(curl -sS -L --max-time 30 -A "$UA" "https://www.frontpages.com/the-new-york-times/") || return 1
|
||||
echo "$html" | rg -o '\["[^"]+","[a-z0-9-]+"\]' | sed 's/^\["//; s/","/ => /; s/"\]$//' | rg -v '^(≡|>|LIST)'
|
||||
}
|
||||
|
||||
notify() {
|
||||
local body="$1"
|
||||
curl -sS -H "X-Title: $NTFY_TITLE" -d "$body" "$NTFY" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
fetch_paper() {
|
||||
local slug="$1"
|
||||
local name="$2"
|
||||
local dir="$BASEDIR/$slug"
|
||||
local tmpfile="/tmp/nyt-$slug.webp"
|
||||
local outfile="$dir/$DATE.jpg"
|
||||
DOWNLOADED=0
|
||||
|
||||
if test -f "$outfile"; then
|
||||
echo " $name: already downloaded"
|
||||
return 0
|
||||
fi
|
||||
|
||||
mkdir -p "$dir"
|
||||
|
||||
html=$(curl -sS -L --max-time 30 -A "$UA" "https://www.frontpages.com/$slug/") || {
|
||||
echo " $name: failed to fetch frontpages.com page"
|
||||
return 1
|
||||
}
|
||||
|
||||
token=$(echo "$html" | sed -n "s/.*atob('\([^']*\)').*/\1/p" | head -1)
|
||||
if test -z "$token"; then
|
||||
echo " $name: no image token found"
|
||||
return 1
|
||||
fi
|
||||
|
||||
imgpath=$(echo "$token" | base64 -d)
|
||||
if test -z "$imgpath" || ! echo "$imgpath" | grep -q '^/g/[0-9]\{4\}/[0-9]\{2\}/[0-9]\{2\}/'; then
|
||||
echo " $name: could not decode image token ($imgpath)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# The token path encodes the actual edition date; a paper's cover may lag
|
||||
# a day behind, so trust it over today's date for the filename.
|
||||
edate=$(echo "$imgpath" | sed 's#^/g/\([0-9]\{4\}\)/\([0-9]\{2\}\)/\([0-9]\{2\}\)/.*#\1-\2-\3#')
|
||||
outfile="$dir/$edate.jpg"
|
||||
|
||||
if test -f "$outfile"; then
|
||||
echo " $name: already downloaded"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! curl -sS -L --max-time 60 -A "$UA" -o "$tmpfile" "https://www.frontpages.com$imgpath"; then
|
||||
echo " $name: failed to download image"
|
||||
rm -f "$tmpfile"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! file "$tmpfile" | grep -qi 'image'; then
|
||||
echo " $name: downloaded file is not an image: $(file "$tmpfile")"
|
||||
rm -f "$tmpfile"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! magick -quality 75 "$tmpfile" "$outfile"; then
|
||||
echo " $name: image conversion failed"
|
||||
rm -f "$tmpfile"
|
||||
return 1
|
||||
fi
|
||||
|
||||
ln -sf "$outfile" "$dir/today.jpg"
|
||||
rm -f "$tmpfile"
|
||||
DOWNLOADED=1
|
||||
echo " $name: saved"
|
||||
return 0
|
||||
}
|
||||
|
||||
if [ "$1" = "--list" ]; then
|
||||
fetch_paper_map || {
|
||||
echo "Failed to fetch paper list" >&2
|
||||
exit 1
|
||||
}
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$1" = "--names" ]; then
|
||||
for slug in $DEFAULTS; do
|
||||
echo "$(slug_to_name "$slug") $slug"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
papers=$DEFAULTS
|
||||
else
|
||||
papers=$*
|
||||
fi
|
||||
|
||||
echo "Downloading front pages for $DATE"
|
||||
success=0
|
||||
failed=0
|
||||
links=""
|
||||
for slug in $papers; do
|
||||
name=$(slug_to_name "$slug")
|
||||
echo "- $name ($slug)"
|
||||
if fetch_paper "$slug" "$name"; then
|
||||
if [ "$DOWNLOADED" = 1 ]; then
|
||||
success=$((success + 1))
|
||||
links+="$name — https://files.lab.unbl.ink/frontpages/$slug/today.jpg"$'\n'
|
||||
fi
|
||||
else
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Done: $success downloaded, $failed failed"
|
||||
|
||||
body=""
|
||||
if [ $success -gt 0 ]; then
|
||||
body+="Front pages saved for today ($DATE):"$'\n'"$links"
|
||||
fi
|
||||
if [ $failed -gt 0 ]; then
|
||||
if [ -n "$body" ]; then
|
||||
body+=$'\n'
|
||||
fi
|
||||
body+="Failed ($failed): check the logs."
|
||||
fi
|
||||
|
||||
if [ -n "$body" ]; then
|
||||
notify "$body"
|
||||
else
|
||||
echo "Nothing new to download; no notification sent."
|
||||
fi
|
||||
|
||||
[ $failed -eq 0 ]
|
||||
Reference in New Issue
Block a user