This is the historical archive of the now-inactive 'grubstreet' list.
Discussion on OpenGuides development has now moved to OpenGuides-Dev. Discussion on The Open Guide to London now takes place on OpenGuides-London.
[grubstreet] wikihoover.pl version 1.01

[prev] [thread] [next] [lurker] [Date index for 2003/1/12]
From: Ivor Williams
Subject: [grubstreet] wikihoover.pl version 1.01
Date: 23:35 on 12 Jan 2003
------=_NextPart_000_0007_01C2BA93.49A368C0
Content-Type: multipart/alternative;
	boundary="----=_NextPart_001_0008_01C2BA93.49B431A0"


------=_NextPart_001_0008_01C2BA93.49B431A0
Content-Type: text/plain;
	charset="Windows-1252"
Content-Transfer-Encoding: quoted-printable

Wikihoover, now with added features: Ability to hoover RecentChanges, =
detection of differences, ability to merge between streams.

Enjoy.

Ivor.
------=_NextPart_001_0008_01C2BA93.49B431A0
Content-Type: text/html;
	charset="Windows-1252"
Content-Transfer-Encoding: quoted-printable

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<HTML><HEAD>
<META http-equiv=3DContent-Type content=3D"text/html; =
charset=3Dwindows-1252">
<META content=3D"MSHTML 6.00.2800.1106" name=3DGENERATOR>
<STYLE></STYLE>
</HEAD>
<BODY bgColor=3D#ffffff>
<DIV><FONT size=3D2>Wikihoover, now with added features: Ability to =
hoover=20
RecentChanges, detection of differences, ability to merge between=20
streams.</FONT></DIV>
<DIV><FONT size=3D2></FONT>&nbsp;</DIV>
<DIV><FONT size=3D2>Enjoy.</FONT></DIV>
<DIV><FONT size=3D2></FONT>&nbsp;</DIV>
<DIV><FONT size=3D2>Ivor.</FONT></DIV></BODY></HTML>

------=_NextPart_001_0008_01C2BA93.49B431A0--

------=_NextPart_000_0007_01C2BA93.49A368C0
Content-Type: text/plain;
	name="wikihoover.pl"
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment;
	filename="wikihoover.pl"

#!perl=0A=
=0A=
use strict;=0A=
use warnings;=0A=
=0A=
our $VERSION;=0A=
=0A=
my ($progname) =3D ($0 =3D~ /(\w+)(\.\w*)?$/);=0A=
=0A=
my $revision =3D '1.01';=0A=
$VERSION =3D sprintf("%d.%02d", ($revision =3D~ /(\d+)\.(\d+)/));=0A=
=0A=
use LWP::UserAgent;=0A=
use HTML::TokeParser;=0A=
use HTTP::Request::Common qw(POST);=0A=
use HTTP::Cookies;=0A=
use Getopt::Long;=0A=
use VCS::Lite;=0A=
=0A=
sub findtokens;=0A=
sub formdata;=0A=
=0A=
my $recent =3D 0;=0A=
my $super =3D 0;=0A=
my $merge =3D 0;=0A=
=0A=
GetOptions(=0A=
	'recent+' 		=3D> \$recent,=0A=
	'supersede+'	=3D> \$super,=0A=
	'merge+'		=3D> \$merge,=0A=
);=0A=
=0A=
&usage if ($super && $merge) || (@ARGV !=3D 2);=0A=
=0A=
my ($wiki_from,$wiki_to) =3D @ARGV;=0A=
=0A=
# Set up user agent=0A=
=0A=
my $ua =3D new LWP::UserAgent;=0A=
$ua->agent("$progname/$VERSION " . $ua->agent);=0A=
$ua->env_proxy;=0A=
$ua->cookie_jar(HTTP::Cookies->new(file =3D> "whcookies.txt",=0A=
		autosave =3D> 1));=0A=
									 =0A=
# Set up GET request=0A=
=0A=
my $req =3D HTTP::Request->new('GET');=0A=
$req->header('Accept' =3D> 'text/html');=0A=
=0A=
# We need a user profile on the source wiki for RecentChanges to =0A=
# be able to give all changes. This creates the name 'hoover'=0A=
# as an id, enabling "Show Minor Edits" for this user. =0A=
=0A=
# First find out if the UID has already been set. Grab a page in edit=0A=
# and see whether the link is to a user or "Preferences".=0A=
=0A=
$req->url("$wiki_from?action=3Dedit&id=3DRecentChanges"); #can be any =
page=0A=
my $rcdummy =3D $ua->request($req);=0A=
die "Error on RecentChanges: " . $rcdummy->status_line . "\n" =0A=
	unless $rcdummy->is_success;=0A=
=0A=
# Parse out the content=0A=
=0A=
my $rcd =3D $rcdummy->content;=0A=
my $rcdp =3D HTML::TokeParser->new(\$rcd);=0A=
my $wholink =3D '';	# will receive user ID=0A=
while (my $in =3D $rcdp->get_tag('input')) {=0A=
	next if $in->[1]{type} ne 'submit';	# Find the buttons=0A=
	$rcdp->get_tag('a');=0A=
	$wholink =3D $rcdp->get_text('/a');=0A=
	last;=0A=
}=0A=
die "Error determining username" unless $wholink;=0A=
=0A=
$wholink =3D~ s/Preferences/not set/;=0A=
=0A=
print "Current user ID is $wholink\n";=0A=
=0A=
# Set up the 'hoover' ID if it has not been set=0A=
=0A=
if ($wholink !~ /^hoover$/i) {=0A=
	$req->url("$wiki_from?action=3Deditprefs");=0A=
	my %fd =3D formdata($ua->request($req)->content);=0A=
	$fd{p_username} =3D 'hoover';=0A=
	$fd{p_rcshowedit} =3D 1;=0A=
	$fd{Save} =3D 'Save';=0A=
	my $ureq =3D POST($wiki_from,\%fd);=0A=
	my $upd =3D $ua->request($ureq);=0A=
	die "Failed to set user ID, " . $upd->status_line=0A=
		unless $upd->is_success;=0A=
}=0A=
=0A=
# Get the Wiki index page or RecentChanges=0A=
my $frompag =3D $recent ? 'RecentChanges' : 'action=3Dindex';=0A=
$req->url("$wiki_from?$frompag");=0A=
my $res =3D $ua->request($req);=0A=
=0A=
die "Error on index: " . $res->status_line . "\n" =0A=
	unless $res->is_success;=0A=
=0A=
# Get all the links off this page.=0A=
# Ignore RecentChanges and all ?action=3Dfoo type links=0A=
# Deduplicate list via hash %tocopy=0A=
=0A=
my $index =3D $res->content;=0A=
my $indexp =3D HTML::TokeParser->new(\$index) or die "Index: bad HTML";=0A=
my %tocopy;=0A=
while (my $link =3D $indexp->get_tag('a')) {=0A=
	my ($atag,$aatt) =3D @$link;=0A=
	next unless $aatt->{href} =3D~ /\?([^&=3D]+)$/;=0A=
	my $qs =3D $1;=0A=
	$tocopy{$qs}++ unless $qs eq 'RecentChanges';=0A=
}=0A=
=0A=
# Hoover each page=0A=
=0A=
PAGE:=0A=
for (keys %tocopy) {=0A=
=0A=
# Bring up page on source Wiki in edit mode=0A=
	print "Get $_".(' 'x40)."\r";=0A=
	$req->url("$wiki_from?action=3Dedit&id=3D$_");=0A=
	my $srcpage =3D $ua->request($req);=0A=
	next unless $srcpage->is_success;=0A=
=0A=
# Parse out the page text=0A=
	my $srcpc =3D $srcpage->content;=0A=
	my $srcp =3D HTML::TokeParser->new(\$srcpc) or (print "Problems with =
$_\n"),next;=0A=
	my $field =3D $srcp->get_tag('textarea');=0A=
	my $text =3D $srcp->get_text('/textarea');=0A=
=0A=
# Bring up edit mode on destination Wiki to get remaining fields=0A=
	$req->url("$wiki_to?action=3Dedit&id=3D$_");=0A=
=0A=
# Parse out form to get the remaining fields=0A=
	my %fd =3D formdata($ua->request($req)->content);=0A=
	next if $fd{text} eq $text;=0A=
	my $action =3D "Copied";=0A=
	print "\n";=0A=
=0A=
# Form on from and to wikis are different=0A=
=0A=
	my $lite1 =3D VCS::Lite->new([split /^/,$text]);=0A=
	my $lite2 =3D VCS::Lite->new([split /^/,$fd{text}]);=0A=
	=0A=
	DECIDE: {=0A=
		last DECIDE if $fd{text} eq "Describe the new page here.\n";=0A=
		=0A=
		$action =3D "Superseded";=0A=
		=0A=
		last DECIDE if $super;=0A=
=0A=
		if (!$merge) {	# Prompt user, as not --merge or --supersede=0A=
			print "$_ Pages are different: (Diff, Merge, Ignore, Supersede)?";=0A=
			my $ans =3D <STDIN>;=0A=
			=0A=
			next PAGE if $ans =3D~ /^I/i;=0A=
			=0A=
			if ($ans =3D~ /^D/i) {=0A=
				print $lite1->diff($lite2);=0A=
				redo DECIDE;=0A=
			}=0A=
			=0A=
			last DECIDE if $ans =3D~ /^S/i;=0A=
		}=0A=
		=0A=
# Attempt merge - get revision 1 from to wiki - a pretty good bet.=0A=
# This will (obviously) fetch the same text if there is only one revision=0A=
		$req->url("$wiki_to?action=3Dedit&id=3D$_&revision=3D1");=0A=
		my %ca =3D formdata($ua->request($req)->content);=0A=
		(print "No prior common ancestor, superseding\n"), last DECIDE=0A=
			if $ca{text} eq $fd{text};=0A=
		my $lite3 =3D VCS::Lite->new([split /^/,$ca{text}]);=0A=
		my $lite4 =3D $lite1->merge($lite2,$lite3);=0A=
		$text =3D $lite4->text;=0A=
		$action =3D "Merged";=0A=
	}=0A=
	=0A=
# Store field values=0A=
	$fd{text} =3D $text;=0A=
	$fd{summary} =3D 'Bulk Populate';=0A=
	$fd{Save} =3D 'Save';=0A=
=0A=
# Do the post	=0A=
	my $ureq =3D POST("$wiki_to",\%fd);=0A=
	my $upd =3D $ua->request($ureq);=0A=
	print "$action: $_\n";=0A=
}=0A=
=0A=
sub usage {=0A=
=0A=
print STDERR <<EOF;=0A=
=0A=
Usage: wikihoover.pl --option(s) fromsite tosite=0A=
=0A=
Options can be one of=0A=
	--recent	Use RecentChanges instead of whole index=0A=
	--supersede		Put from pages straight into to wiki=0A=
	--merge		Merge changes from both wikis using a common ancestor=0A=
=0A=
EOF=0A=
=0A=
exit 1;=0A=
=0A=
}=0A=
=0A=
sub formdata {=0A=
	my ($html,$formname) =3D @_;=0A=
	=0A=
	my $tp =3D HTML::TokeParser->new(\$html) or die "Bad HTML form";=0A=
	while (my $form =3D $tp->get_tag('form')) {=0A=
		last if !$formname || ($form->[1]{name} eq $formname);=0A=
		$tp->get_text('/form');=0A=
	}=0A=
	my @form;=0A=
	while (my $field =3D $tp->get_tag('input','select','textarea')) {=0A=
		my ($tag,$attr) =3D @$field;=0A=
		if ($tag eq 'textarea') {=0A=
			my $text =3D $tp->get_text('/textarea');=0A=
			push @form,$attr->{name},$text;=0A=
			next;=0A=
		}=0A=
		=0A=
		if ($tag eq 'select') {=0A=
			my $selected;=0A=
			while (my $tok =3D $tp->get_token) {=0A=
				last if $tok->[-1] =3D~ m(/select)i;=0A=
				my ($typ,$tag,$att) =3D @$tok;=0A=
				next unless $typ eq 'S' && $tag eq 'option';=0A=
				$selected =3D $att->{value} if exists $att->{selected};=0A=
			}=0A=
			=0A=
			push @form,$attr->{name},$selected if defined $selected;=0A=
			next;=0A=
		}=0A=
		=0A=
		if ($attr->{type} =3D~ /hidden|password|text/) {=0A=
			push @form,$attr->{name},$attr->{value};=0A=
		}=0A=
=0A=
		if ($attr->{type} =3D~ /radio|checkbox/ && =0A=
				exists $attr->{checked}) {=0A=
			push @form,$attr->{name},$attr->{value};=0A=
		}=0A=
	}=0A=
	@form;=0A=
}=0A=
=0A=
=3Dhead1 NAME=0A=
=0A=
	wikihoover.pl - copy a Usemod Wiki to another site=0A=
=0A=
=3Dhead1 SYNOPSIS=0A=
=0A=
	wikihoover.pl --option(s) fromsite tosite=0A=
	=0A=
Options can be one of=0A=
	--recent	Use RecentChanges instead of whole index=0A=
	--supersede		Put from pages straight into to wiki=0A=
	--merge		Merge changes from both wikis using a common ancestor=0A=
=0A=

------=_NextPart_000_0007_01C2BA93.49A368C0--


-- 
grubstreet mailing list
http://london.openguides.org/old-list-archives/
[grubstreet] wikihoover.pl version 1.01 Ivor Williams 23:35 on 12 Jan 2003
Content-Type: multipart/alternative;