WineHQ: service.cgi update
Dimitrie O. Paun
dimi at intelliware.ca
Fri Apr 30 14:07:42 CDT 2004
Lots of changes on this one, and all I can say is:
Perl is tricky!
Many thanks to Alexander, Brian, and Shachar for helping me out
with Perl and it's misteries. Special thanks to Brian for help
on testing and review.
ChangeLog
Dimitrie O. Paun <dpaun at rogers.com>
Add support for multiple releases per build per publisher.
Keep track of client's execution history, to figure out
what to instruct the client to execute next.
Don't use the cookies on the client side at all, use builds ids.
Cleanups and simplifications.
--
Dimi.
-------------- next part --------------
Index: winetest/service.cgi
===================================================================
RCS file: /home/wine/tools/winetest/service.cgi,v
retrieving revision 1.14
diff -u -r1.14 service.cgi
--- winetest/service.cgi 29 Apr 2004 20:23:06 -0000 1.14
+++ winetest/service.cgi 30 Apr 2004 19:03:26 -0000
@@ -5,6 +5,7 @@
# Windows clients
#
# Copyright (C) 2004 Brian Vincent
+# Copyright (C) 2004 Dimitrie O. Paun
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -22,7 +23,7 @@
#
# We expect our input to be one of three things:
# - publish a new file for clients to download
-# ?publish=winetest&url=http://myserver.com/path/to/current-winetest.exe
+# ?publish=http://myserver.com/path/to/current-winetest.exe
# - a request to download that looks like:
# ?winrash=1&cookie_name1=cookie_value1&cookie_name2=cookie_value2...
# - an error report to be emailed to wine-tests-results at winehq.org
@@ -30,27 +31,24 @@
# We rely on the server to have directories available for every
# program that can be published. For example, we need a winetest directory:
# winetest/
-# url contains url to find latest winetests at, must be
+# <name>.url contains url to find latest winetests at, must be
# writable
# url.mask contains a reg ex to match against, only needs to
# be read-only, in the above example it might be
# something like: http://myserver/com/path/to
-# cookie contains a unique identifier of a program that winrash
-# clients can download, must be writable
+# <name>.cookie contains a unique identifier of a program that
+# winrash clients can download, must be writable
use Digest::MD5;
-$hour_we_do_build = "5"; # hour, such as 2 or 17, in localtime, rounded up
-$path_to_test_directories="/home/winehq/opt/winetest/";
-$path_to_builds_txt="/home/winehq/opt/winetest/winetest.builds";
-$path_to_lynx="/usr/bin/lynx";
+$data_root="/home/winehq/opt/winetest";
+$lynx="/usr/bin/lynx";
+ at valid_programs=("winrash", "winetest");
&main;
sub main {
-
- my $found_a_cookie = 0;
- my $upgrade_sent = 0;
+ my $build, $urls, $cookies;
print "Content-type: text/plain\n\n";
%data_received = &User_Data;
@@ -64,66 +62,42 @@
exit;
}
- &get_all_cookies;
-
if ($data_received{"publish"}) {
- $cookie_current = read_cookie($data_received{"publisher"} . "/" .
- $data_received{"publish"});
- $cookie_to_save = &get_new_cookie;
-
- # Cookies should never be the same. Some day we could return an error
- if ($cookie_to_save ne $cookie_current) {
- save_new_cookie($cookie_to_save);
- &save_new_url;
- &update_builds_txt;
- }
- print "OK\n";
- exit;
- }
+ my $url, $response;
- if (convert_to_md5($data_received{"winrash"}) ne $cookie{"winrash"}) {
- send_upgrade("chris/winrash", "winrash", $cookie{"winrash"});
+ if (exists $data_received{"url"}) {
+ $url = $data_received{"url"};
+ } else {
+ $url = $data_received{"publish"};
+ }
+ $response = &releases_make($url);
+ print "$response\n";
exit;
}
-
- # I expect the cookie hash won't have many values, but we could potentially
- # run a lot of commands if we wanted.
-
- foreach $key (keys %data_received) {
- next if ($key eq "winrash" or $key eq "id" or $key eq "debug");
- $found_a_cookie = 1;
- if ($cookie{$key} ne $data_received{$key}) {
- send_upgrade(($publisher{$key} . "/" . $key), $key, $cookie{$key});
- $upgrade_sent = 1;
+
+ # for each of the programs we know about, see if they need an update
+ foreach $program (@valid_programs) {
+ ($build, $urls, $cookies) = &releases_read($program);
+ if (exists $data_received{$program}) {
+ my @history = split(/,/, $data_received{"$program" . "_history"});
+ push (@history, $data_received{$program});
+ foreach (@history) {
+ delete $$urls{$_};
+ delete $$cookies{$_};
+ }
+ $winetest_history = join ',', @history;
}
+ if (scalar(%$urls)) {
+ &send_upgrade($urls, $winetest_history);
+ $update_sent = 1;
+ last;
+ }
}
- # Special case where a new winrash client sends us nothing.
- if (!$found_a_cookie) {
- opendir(ALL_PUBLISHERS, $path_to_test_directories)
- or (debug("Can't open $path_to_test_directories.") && die);
-
- while( defined ($this_publisher = readdir ALL_PUBLISHERS) ) {
- next if $this_publisher =~ /^\.\.?$/; # skip . and ..
- opendir(PUBLISHER, ($path_to_test_directories . $this_publisher));
-
- while( defined($this_subdir = readdir PUBLISHER) ) {
- next unless $this_subdir eq "winetest";
- # we lie about $cookie{"winetest"} on purpose so it's always
- # set to the most current value
- $publisher{"winetest"} = $this_publisher;
- send_upgrade(($this_publisher . "/winetest"), "winetest",
- $cookie{"winetest"});
- $upgrade_sent = 1;
- }
- closedir(PUBLISHER);
- }
- closedir(ALL_PUBLISHERS);
+ if (!$update_sent) {
+ print "sleep 3600\n";
+ return;
}
- # We probably don't need to check for $found_a_cookie, but just to be safe.
- if (!$upgrade_sent && $found_a_cookie) {
- &send_wait;
- }
}
##########################################################################
@@ -133,59 +107,118 @@
# created by a web server admin thus ensuring this program is
# approved for testing
# 2) Download the cookie for the program from that web server
-# 3) Save that cookie to <publish key value>/cookie
+# 3) Save that cookie to <publish key value>/<name>.cookie
+# 4) Save the url to <publish key value>/<name>.url
#
##########################################################################
-sub get_new_cookie {
-
- my $path_to_mask = $path_to_test_directories . $data_received{"publisher"}
- . "/" . $data_received{"publish"} . "/url.mask";
- my $cookie_url = $data_received{"url"} . ".sig";
- my $url_mask = "";
- my $this_cookie = "";
+# removes the current release from disk
+sub releases_purge {
+ my ($project) = @_;
+ system("rm -f $data_root/*/$project/*.cookie");
+ system("rm -f $data_root/*/$project/*.url");
+}
+
+# this function reads the current release information from disk
+# and returns ($thisrelease, %url, %cookies)
+# where the two maps are keyed by the full file name of the release
+sub releases_read {
+ my ($project) = @_;
+ my %urls, %cookies, $thisrelease;
+
+ @files = split(/\n/, `ls $data_root/*/$project/*.url`);
+ foreach $file (@files) {
+ if (open(GENERIC_FH, $file)) {
+ my $cookiefile, $key, $url, $this_line;
+ $this_line = <GENERIC_FH>;
+ close(GENERIC_FH);
+ chomp $this_line;
+ $file =~ /(.*)\/(.*)\.url/;
+ $cookiefile = "$1/$2.cookie";
+ $key = $2;
+ $url = $this_line;
+ if (open(GENERIC_FH, $cookiefile)) {
+ my $program, $release, @other;
+ $this_line = <GENERIC_FH>;
+ close(GENERIC_FH);
+ chomp $this_line;
+ ($program, $release, @other) = split(/-/, $key);
+ if (scalar(%urls) == 0) {
+ $thisrelease = $release;
+ } elsif ($thisrelease ne $release) {
+ &debug("Invalid release state!") && die;
+ }
+ $urls{$key} = $url;
+ $cookies{$key} = $this_line;
+ }
+ }
+ }
+ return ($thisrelease, \%urls, \%cookies);
+}
- $url_mask = getoneline("< $path_to_mask");
+# try to add a new release to our portfolio
+sub releases_make {
+ my ($url) = @_;
+ my $name, $program, $build, $publisher;
+ my $urls, $cookies, $current_build, @other;
+ my $build_path, $cookie, $program_ok;
+
+ # parse out this release's information
+ $url =~ /.*\/(.*)/;
+ $name = $1;
+ ($program, $build, $publisher, @other) = split(/-/, $name);
+
+ # check that it's a recognized program
+ foreach $prg (@valid_programs) {
+ if ($prg eq $program) {
+ $program_ok = 1;
+ last;
+ }
+ }
+ if (!$program_ok) {
+ return "Invalid program";
+ }
- if (!($data_received{"url"} =~ $url_mask)) {
- debug("URL mask unmatched, dying.");
- die;
- }
+ # FIXME: maybe we should check here that $build is sane
- $this_cookie = getoneline("$path_to_lynx -source $cookie_url |");
- return convert_to_md5($this_cookie);
-}
+ # test if this is a valid publisher
+ if (opendir(IMD, "$data_root/$publisher/")) {
+ closedir(IMD);
+ } else {
+ return "Invalid publisher";
+ }
-sub save_new_cookie {
+ # get current release info
+ ($current_build, $urls, $cookies) = &releases_read($program);
- my $cookie = $_[0];
- my $path_to_cookie = $path_to_test_directories . $data_received{"publisher"}
- . "/" . $data_received{"publish"} . "/cookie";
-
- open(COOKIE_FH, "> $path_to_cookie")
- or ( debug("Can't open $path_to_cookie for writing.") && die );
- print COOKIE_FH "$cookie";
- close(COOKIE_FH);
-}
+ # check to see how we should handle it
+ if ($build lt $current_build) {
+ return "Build is old, current build is $current_build";
+ }
+ if ($build ne $current_build) {
+ &releases_purge($program);
+ }
-sub save_new_url {
+ # check to see that the URL has the right format
+ $base_path = "$data_root/$publisher/$program";
- $path_to_url = $path_to_test_directories . $data_received{"publisher"}
- . "/" . $data_received{"publish"} . "/url";
+ $url_mask = &read_one_line("<$base_path/url.mask");
+ if (!($url =~ $url_mask)) {
+ return "Unrecognized URL format";
+ }
- open(URL_FH, "> $path_to_url")
- or ( debug("Can't open $path_to_url for writing.") && die );
- print URL_FH "$data_received{\"url\"}";
- close(URL_FH);
-}
+ # get the cookie now
+ $cookie = &read_one_line("$lynx -source $url.sig |");
-sub convert_to_md5 {
+ # all is good, store the cookie, and URL now, this activetes the release
+ &write_file(">$base_path/$name.cookie", &md5sum($cookie));
+ &write_file(">$base_path/$name.url", $url);
+ &write_file(">>$data_root/$program.builds", $build);
- local $md5 = Digest::MD5->new;
- $md5->add($_[0]);
- return $md5->hexdigest;
+ return "OK";
}
+
##########################################################################
#
# Issue commands understood by winrash. Try to be somewhat intelligent
@@ -194,64 +227,44 @@
##########################################################################
sub send_upgrade {
-
- debug("Send upgrade received: " . $_[0] . $_[1] . $_[2] );
+ my ($urls, $history) = @_;
+ my @names, $name, $url, $program;
+ my $build, $publisher, @other;
+
+ # pick a release to send, the first will do
+ @names = keys %$urls;
+ $name = $names[0];
+
+ # parse out this release's information
+ $url = $$urls{$name};
+ ($program, $build, $publisher, @other) = split(/-/, $name);
- my $path_to_url = $path_to_test_directories . $publisher{"$_[1]"} . "/" .
- $_[1] . "/url";
- my $this_url;
- my $local_filename;
- my $this_cookie = $_[2];
- my $sleeptime = "3600";
- my $formatted_date = substr(&ret_todays_date,0,8);
-
- $this_url = getoneline("< $path_to_url");
-
- @url_broken_up = split(/\//,$this_url);
- foreach $part (@url_broken_up) {
- $local_filename = $part;
- }
+ &debug("Send upgrade received: $url");
print "error_url http://test.winehq.org/service?error=true\n";
- print "error_sleep $sleeptime\n";
+ print "error_sleep 3600\n";
- print "download $local_filename $this_url\n";
- print "download $local_filename.sig $this_url.sig\n";
- print "gpgverify $local_filename.sig\n";
+ print "download $name $url\n";
+ print "download $name.sig $url.sig\n";
+ print "gpgverify $name.sig\n";
- if ($_[1] eq "winrash") {
- print "fork $local_filename\n";
+ if ($program eq "winrash") {
+ print "fork $name\n";
return;
}
# Set cookie now in case any commands following bomb.
- print "cookie $_[1] $cookie{$_[1]}\n";
+ print "cookie $program $name\n";
+ print "cookie $program" . "_history $history\n";
- if (substr($local_filename, -4, 4) =~ ".zip") {
- print "unzip $local_filename\n";
- $local_filename =~ s/zip/exe/g;
- print "run $local_filename -u -c -t $data_received{\"id\"}\n";
- }
- else {
- print "run $local_filename -u -c -t $data_received{\"id\"}\n";
+ if (substr($name, -4, 4) =~ ".zip") {
+ print "unzip $name\n";
+ $name =~ s/zip/exe/g;
}
+ print "run $name -c -t $data_received{\"id\"} -u $url\n";
- print "sleep $sleeptime\n";
-}
-
-sub send_wait {
- $sleeptime = "3600";
- print "sleep $sleeptime\n";
-}
-
-sub update_builds_txt {
-
- my $formatted_date = &ret_todays_date;
-
- open(BUILDS_FH, ">>$path_to_builds_txt")
- or ( debug("Can't open $path_to_builds_txt for appending.") && die );
- print BUILDS_FH "$formatted_date\n";
- close(BUILDS_FH);
+ # wait just 5min, we may have other stuff to execute right away
+ print "sleep 300\n";
}
sub email_error_log {
@@ -275,34 +288,11 @@
close(MAIL);
}
-sub ret_todays_date {
-
- ($nowyear, $nowmonth, $nowday, $nowhour, $nowmin, $nowsec)
- = (localtime)[5,4,3,2,1,0];
-
- $nowmonth++;
- $nowyear += 1900;
-
- my $formatted_date = $nowyear . pad_date($nowmonth) . pad_date($nowday) .
- pad_date($nowhour) . pad_date($nowmin);
-
- return $formatted_date;
-}
-
-sub pad_date {
- if ($_[0] < 10) {
- return "0" . $_[0];
- }
- else {
- return $_[0];
- }
-}
-
##########################################################################
#
# Some convenience functions. User_Data splits our name/value pairs
# by taking GET or POST for input. Useful for running on the commandline
-# by setting QUERY_STRING. Debug gives us output if debug=1. getoneline
+# by setting QUERY_STRING. Debug gives us output if debug=1. &read_one_line
# just opens files and returns one line, with trailing new lines removed.
#
##########################################################################
@@ -337,104 +327,38 @@
return %user_data;
}
+# print a message if debugging is enabled
sub debug {
if ($data_received{"debug"}) {
print "$_[0]\n";
}
}
-sub getoneline {
+# computes the MD5 sum of the arument
+sub md5sum {
+
+ local $md5 = Digest::MD5->new;
+ $md5->add($_[0]);
+ return $md5->hexdigest;
+}
+# write a string to a file
+sub write_file {
+ my ($filename, $content) = @_;
+ open(GENERIC_FH, $filename)
+ or ( &debug("Can't open $filename for writing.") && die );
+ print GENERIC_FH $content;
+ close(URL_FH);
+}
+
+# read the content of a file
+sub read_one_line {
open(GENERIC_FH, $_[0])
- or ( debug("Can't open $_[0]."), die );
- do { $this_line = <GENERIC_FH> } until $. == 1;
+ or ( &debug("Can't open $_[0]."), die );
+ $this_line = <GENERIC_FH>;
close(GENERIC_FH);
chomp $this_line;
return $this_line;
}
-
-##########################################################################
-#
-# Read the cookie files to determine the versions of winrash and tests.
-#
-##########################################################################
-
-sub get_all_cookies {
-
- my $this_publisher;
- my $this_subdir;
- my $this_file;
-
- opendir(ALL_PUBLISHERS, $path_to_test_directories)
- or (debug("Can't open $path_to_test_directories.") && die);
-
- # traverse a series of directories looking for files named "cookie"
- # for example, starting from /home/winehq we might need to find
- # /home/winehq/paul/winetest/cookie
-
- while( defined ($this_publisher = readdir ALL_PUBLISHERS) ) {
- next if $this_publisher =~ /^\.\.?$/; # skip . and ..
- opendir(PUBLISHER, ($path_to_test_directories . $this_publisher));
-
- while( defined($this_subdir = readdir PUBLISHER) ) {
- next if $this_subdir =~ /^\.\.?$/;
- opendir(BUILDS, ($path_to_test_directories . $this_publisher .
- "/" . $this_subdir));
-
- while( defined($this_file = readdir BUILDS) ) {
- if ($this_file eq "cookie") {
- (my($this_cookie_value, $mtime))=read_cookie($this_publisher
- . "/" . $this_subdir);
- set_cookie($this_subdir, $this_cookie_value, $mtime,
- $this_publisher);
- }
- }
- closedir(BUILDS);
- }
- closedir(PUBLISHER);
- }
- closedir(ALL_PUBLISHERS);
-}
-
-# we only set a cookie if it's newer than what we have.
-
-sub set_cookie {
-
- my $this_subdir = $_[0];
- my $this_cookie_value = $_[1];
- my $this_mtime = $_[2];
- my $this_publisher = $_[3];
-
- debug("Setting cookie for " . $this_publisher . " and " . $this_subdir .
- " to " . $this_cookie_value . " with a time of $this_mtime.\n");
- if(!(defined($cookie{$this_subdir}))) {
- $cookie{$this_subdir} = $this_cookie_value;
- $mtime{$this_subdir} = $this_mtime;
- $publisher{"$this_subdir"} = $this_publisher;
- }
- else {
- if($mtime{$this_subdir} < $this_mtime) {
- $cookie{$this_subdir} = $this_cookie_value;
- $mtime{$this_subdir} = $this_mtime;
- $publisher{"$this_subdir"} = $this_publisher;
- }
- }
-}
-
-sub read_cookie {
-
- my $this_publisher;
- my $this_subdir;
-
- my $path_to_cookie = $path_to_test_directories . $_[0] . "/cookie";
- my $this_cookie_value = getoneline($path_to_cookie);
-
- ($this_publisher, $this_subdir) = split("/", $_[0]);
- ( my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size,
- $atime, $mtime, $ctime, $blksize, $blocks ) ) = stat($path_to_cookie);
- debug($this_publisher . " and " . $this_subdir . " is $mtime");
-
- return $this_cookie_value, $mtime;
-}
More information about the wine-patches
mailing list