User:Lupin/histdump.sh
histdump.sh
[edit]Dump history of a page.
#!/bin/bash
# Usage: histdump.sh [-r] 'Title of wiki page'
# -r: download in chronological order instead of most recent first
# FIXME: deal with more than 500 entries
if [ "$1" = "-r" ]; then mung=tac; shift; else mung=cat; fi
page=$(echo "$1" | sed -e 's/ /_/g')
outputdir="$page"
mkdir "$outputdir"
sitebase='http://en.wikipedia.org'
histpage="${sitebase}/w/index.php?title=${page}&limit=500&action=history"
hist="$(wget -q -O- $histpage|grep '^<li>')"
numedits=$(echo "$hist"|wc -l)
i=0
echo "$hist" | "$mung" | while read line; do
url=${sitebase}$(echo $line|sed -e 's/.*a href="\([^"]*oldid=[0-9]*\)" title="[^"]*">[0-9].*/\1/' -e 's/[&]amp;/\&/g')
time=$(echo $line|sed -e 's/.*a href="\([^"]*oldid=[0-9]*\)" title="[^"]*">\([0-9][^<]*\).*/\2/'|sed -e 's/,//g')
oldid=$(echo $url|sed -e 's/.*oldid=\([0-9]*\).*/\1/')
user=$(echo $line|sed -e 's/.*class=.history-user[^>]*>[^<]*<a[^>]*>\([^<]*\).*/\1/')
comment=$(echo $line|sed -e 's/.*class=.comment.>[(]\([^<]*\)[)].*/\1/')
if [ "$comment" = "$line" ]; then comment=''; fi
outputfile="$outputdir"/"$oldid";
i=$((i+1));
echo -e "Edit ${i} of ${numedits} by ${user}, oldid ${oldid}, time ${time}"
(echo Page: "$page"; echo Editor: "$user"; echo Time: "$time"; echo Edit summary: "$comment") > "$outputfile"
wget -q -O- "${sitebase}/w/index.php?title=${page}&action=raw&oldid=${oldid}" >> "$outputfile"
touch --date="${time}" "$outputfile"
done
diffgen.sh
[edit]Generate diffs from a set of dumped revisions. Usage: diffgen.sh rev1 rev2 ...
Uses the timestamps on the revision files to sort them.
#!/bin/bash
files=$(ls -Srt "$@")
DIFF="diff -u"
oldfile=
for newfile in $files ; do
if [ ! -z "$oldfile" ]; then
$DIFF "$oldfile" "$newfile"
fi
oldfile=$newfile
done