update to regression testing framework
Paul Millar
paulm at astro.gla.ac.uk
Wed Apr 10 11:37:15 CDT 2002
Hi,
The attached patch changes the way the regression testing works slightly.
The default behaviour should be the same, but there's some optional
"enhancements":
o If you set WINETEST_REPORT_SUCCESS environment variable to 1,
successful tests are reported as well as unsuccessful ones.
o If you set WINETEST_CONTINUE_ON_FAIL environment variable to 1, "make
test" will run all tests from all modules, rather than failing on the
first failing module.
o The commands "named_ok", "set_name" and "unset_name" now exist (in C
and Perl). You can use these to specify a name for tests either
individually (via named_ok) or for a region (via set_name and
unset_name)
The patch changes both the C and Perl frameworks. I'm pretty sure the Perl
is right but someone who actually knows Perl ought to check it :)
The second patch illustrates these by changing dlls/kernel/tests/atom.pl
and dlls/kernel/tests/directory.c (apologies to Alexandre & Dmitry if I
got the test semantics wrong).
HTH
----
Paul Millar
ChangeLog:
Adding test names and some optional behaviour.
-------------- next part --------------
? dlls/kernel/tests/new
Index: include/wine/test.h
===================================================================
RCS file: /home/wine/wine/include/wine/test.h,v
retrieving revision 1.3
diff -u -r1.3 test.h
--- include/wine/test.h 22 Mar 2002 00:58:00 -0000 1.3
+++ include/wine/test.h 10 Apr 2002 16:03:43 -0000
@@ -35,6 +35,9 @@
extern winetest_ok_funcptr winetest_set_ok_location( const char* file, int line );
extern winetest_trace_funcptr winetest_set_trace_location( const char* file, int line );
+extern int winetest_named_ok( const char* name, int condition, const char* msg, ...);
+extern void winetest_set_name( const char* name);
+extern void winetest_unset_name( void);
extern void winetest_start_todo( const char* platform );
extern int winetest_loop_todo(void);
extern void winetest_end_todo( const char* platform );
@@ -43,6 +46,9 @@
#define ok (*winetest_set_ok_location(__FILE__, __LINE__))
#define trace (*winetest_set_trace_location(__FILE__, __LINE__))
+#define named_ok winetest_named_ok
+#define set_name winetest_set_name
+#define unset_name winetest_unset_name
#define todo(platform) for (winetest_start_todo(platform); \
winetest_loop_todo(); \
Index: programs/winetest/runtest
===================================================================
RCS file: /home/wine/wine/programs/winetest/runtest,v
retrieving revision 1.7
diff -u -r1.7 runtest
--- programs/winetest/runtest 28 Mar 2002 18:21:06 -0000 1.7
+++ programs/winetest/runtest 10 Apr 2002 16:03:43 -0000
@@ -32,6 +32,8 @@
Options:
-q quiet mode
-v verbose mode (can be specified multiple times)
+ -s announce successful tests
+ -c always return 0 even if tests fail
-p prog name of the program to run for C tests
-I dir prepend dir to Perl include path
-P name set the current platform name
@@ -45,6 +47,8 @@
# default values
my $platform = $ENV{WINETEST_PLATFORM};
$ENV{WINETEST_DEBUG} ||= 1;
+$ENV{WINETEST_CONTINUE_ON_FAIL} ||=0;
+$ENV{WINETEST_REPORT_SUCCESS} ||=0;
my $topobjdir;
my $infile;
@@ -60,6 +64,8 @@
if ($arg eq "-p") { $program = shift @ARGV; next; }
if ($arg eq "-q") { $ENV{WINETEST_DEBUG} = 0; next; }
if ($arg eq "-v") { $ENV{WINETEST_DEBUG}++; next; }
+ if ($arg eq "-s") { $ENV{WINETEST_REPORT_SUCCESS} = 1; next;}
+ if ($arg eq "-c") { $ENV{WINETEST_CONTINUE_ON_FAIL} = 1; next;}
if ($arg eq "-P") { $platform = shift @ARGV; next; }
if ($arg eq "-M") { push @modules, split /,/, shift @ARGV; next; }
if ($arg eq "-I") { push @include_dirs, shift @ARGV; next; }
Index: programs/winetest/wtmain.c
===================================================================
RCS file: /home/wine/wine/programs/winetest/wtmain.c,v
retrieving revision 1.5
diff -u -r1.5 wtmain.c
--- programs/winetest/wtmain.c 2 Apr 2002 00:44:17 -0000 1.5
+++ programs/winetest/wtmain.c 10 Apr 2002 16:03:43 -0000
@@ -26,12 +26,33 @@
#include "wine/test.h"
#include "winbase.h"
+#define WINETEST_OUTPUT_LINE( DATA, STRING) \
+ {if( name || data->current_name) \
+ fprintf( stderr, "%s:%s: ", data->current_file, \
+ name ? name : data->current_name); \
+ else \
+ fprintf( stderr, "%s:%d: ", data->current_file, data->current_line); \
+ fprintf( stderr, "%s", STRING);}
+
+#define WINETEST_OUTPUT_TRACE( DATA, STRING) \
+ {if( data->current_name) \
+ fprintf( stderr, "%s ", data->current_name); \
+ else \
+ fprintf( stderr, "%s:%d: ", data->current_file, data->current_line); \
+ fprintf( stderr, "%s", STRING);}
+
/* debug level */
int winetest_debug = 1;
/* current platform */
const char *winetest_platform = "windows";
+/* continue after an unsuccessful test (BOOL) */
+int winetest_continue_on_fail = 0;
+
+/* report successful tests (BOOL) */
+int winetest_report_success = 0;
+
struct test
{
const char *name;
@@ -51,11 +72,16 @@
{
const char* current_file; /* file of current check */
int current_line; /* line of current check */
+ const char* current_name; /* name of current check (or NULL) */
int todo_level; /* current todo nesting level */
int todo_do_loop;
} tls_data;
static DWORD tls_index;
+/* Private prototype: backend for winetest_ok() and winetest_named_ok() */
+int winetest_generic_ok( const char *, int, const char *, va_list *);
+
+
static tls_data* get_tls_data(void)
{
tls_data* data;
@@ -72,33 +98,53 @@
return data;
}
+
/*
* Checks condition.
* Parameters:
* - condition - condition to check;
* - msg test description;
- * - file - test application source code file name of the check
- * - line - test application source code file line number of the check
* Return:
* 0 if condition does not have the expected value, 1 otherwise
*/
int winetest_ok( int condition, const char *msg, ... )
{
+ int status;
va_list valist;
+
+ va_start( valist, msg);
+ status = winetest_generic_ok( NULL, condition, msg, &valist);
+ va_end( valist);
+
+ return status;
+}
+
+int winetest_named_ok( const char *name, int condition, const char *msg, ... )
+{
+ int status;
+ va_list valist;
+
+ va_start( valist, msg);
+ status = winetest_generic_ok( name, condition, msg, &valist);
+ va_end( valist);
+
+ return status;
+}
+
+int winetest_generic_ok( const char *name, int condition, const char *msg, va_list *valist )
+{
tls_data* data=get_tls_data();
if (data->todo_level)
{
if (condition)
{
- fprintf( stderr, "%s:%d: Test succeeded inside todo block",
- data->current_file, data->current_line );
+ WINETEST_OUTPUT_LINE( data, "Test succeeded inside todo block")
+
if (msg && msg[0])
{
- va_start(valist, msg);
fprintf(stderr,": ");
- vfprintf(stderr, msg, valist);
- va_end(valist);
+ vfprintf(stderr, msg, *valist);
}
fputc( '\n', stderr );
InterlockedIncrement(&todo_failures);
@@ -110,20 +156,22 @@
{
if (!condition)
{
- fprintf( stderr, "%s:%d: Test failed",
- data->current_file, data->current_line );
+ WINETEST_OUTPUT_LINE( data, "Test failed")
if (msg && msg[0])
{
- va_start(valist, msg);
fprintf( stderr,": ");
- vfprintf(stderr, msg, valist);
- va_end(valist);
+ vfprintf(stderr, msg, *valist);
}
fputc( '\n', stderr );
InterlockedIncrement(&failures);
return 0;
}
- else InterlockedIncrement(&successes);
+ else
+ {
+ if( winetest_report_success)
+ WINETEST_OUTPUT_LINE( data, "Test succeeded\n")
+ InterlockedIncrement(&successes);
+ }
}
return 1;
}
@@ -136,6 +184,19 @@
return &winetest_ok;
}
+extern void winetest_set_name( const char* name)
+{
+ tls_data* data=get_tls_data();
+ data->current_name=name;
+}
+
+extern void winetest_unset_name( void)
+{
+ tls_data* data=get_tls_data();
+ data->current_name=NULL;
+}
+
+
void winetest_trace( const char *msg, ... )
{
va_list valist;
@@ -143,7 +204,7 @@
if (winetest_debug > 0)
{
- fprintf( stderr, "%s:%d:", data->current_file, data->current_line );
+ WINETEST_OUTPUT_TRACE( data, "")
va_start(valist, msg);
vfprintf(stderr, msg, valist);
va_end(valist);
@@ -227,7 +288,7 @@
(failures + todo_failures != 1) ? "failures" : "failure" );
}
status = (failures + todo_failures < 255) ? failures + todo_failures : 255;
- return status;
+ return winetest_continue_on_fail ? 0 : status;
}
@@ -238,6 +299,10 @@
if ((p = getenv( "WINETEST_PLATFORM" ))) winetest_platform = p;
if ((p = getenv( "WINETEST_DEBUG" ))) winetest_debug = atoi(p);
+ if ((p = getenv( "WINETEST_CONTINUE_ON_FAIL"))) winetest_continue_on_fail=\
+ atoi(p);
+ if ((p = getenv( "WINETEST_REPORT_SUCCESS"))) winetest_report_success = \
+ atoi(p);
if (!argv[1])
{
fprintf( stderr, "Usage: %s test_name\n", argv[0] );
Index: programs/winetest/include/wine.pm
===================================================================
RCS file: /home/wine/wine/programs/winetest/include/wine.pm,v
retrieving revision 1.4
diff -u -r1.4 wine.pm
--- programs/winetest/include/wine.pm 10 Mar 2002 00:21:20 -0000 1.4
+++ programs/winetest/include/wine.pm 10 Apr 2002 16:03:43 -0000
@@ -26,6 +26,8 @@
use strict;
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK $AUTOLOAD $todo_level
$successes $failures $todo_successes $todo_failures
+ $winetest_continue_on_fail $winetest_report_success
+ $name
%return_types %prototypes %loaded_modules);
require Exporter;
@@ -41,9 +43,12 @@
assert
hd
ok
+ named_ok
todo
todo_wine
trace
+ set_name
+ unset_name
wc
wclen
);
@@ -63,6 +68,9 @@
$todo_successes = 0;
$todo_failures = 0;
%loaded_modules = ();
+$name = "";
+$winetest_report_success = defined($ENV{WINETEST_REPORT_SUCCESS}) ? $ENV{WINETEST_REPORT_SUCCESS} : 0;
+$winetest_continue_on_fail = defined($ENV{WINETEST_CONTINUE_ON_FAIL}) ? $ENV{WINETEST_CONTINUE_ON_FAIL} : 0;
# --------------------------------------------------------------
# | Return-type constants |
@@ -448,28 +456,123 @@
my $assertion = shift;
my $description = shift;
my ($filename, $line) = (caller (0))[1,2];
+ generic_ok( $name, $assertion, $description, $filename, $line);
+}
+
+# ----------------------------------------------------------------------
+# Subroutine: named_ok
+#
+# Purpose: Print a specific label with warning if something fails
+#
+# Usage: named_ok NAME CONDITION [DESCRIPTION]
+#
+# Returns: (none)
+# ----------------------------------------------------------------------
+sub named_ok($;$;$)
+{
+ my $name = shift;
+ my $assertion = shift;
+ my $description = shift;
+ my ($filename, $line) = (caller (0))[1,2];
+ generic_ok( $name, $assertion, $description, $filename, $line);
+}
+
+# ----------------------------------------------------------------------
+# Subroutine: winetest_output_line
+#
+# Purpose: Print a generic formatted line to STDERR
+#
+# Usage: winetest_output_line filename line_number message [name
+# [description] ]
+#
+# Returns: (none)
+# ----------------------------------------------------------------------
+sub winetest_output_line($;$;$;$;$)
+{
+ my $filename = shift;
+ my $line = shift;
+ my $msg = shift;
+ my $name = shift;
+ my $description = shift;
+
+
+ $line=$name if ($name ne "");
+
+ print STDERR ("$filename:$line: $msg" .
+ ($description ? ": $description" : "") . "\n");
+}
+
+# ----------------------------------------------------------------------
+# Subroutine: generic_ok
+#
+# Purpose: Test an assertion and display appropriate message
+#
+# Usage: generic_ok test_name assertion description filename line_no
+#
+# Returns: (none)
+# ----------------------------------------------------------------------
+sub generic_ok($;$;$;$;$)
+{
+ my $name = shift;
+ my $assertion = shift;
+ my $description = shift;
+ my $filename = shift;
+ my $line = shift;
+
if ($todo_level)
{
if ($assertion)
{
- print STDERR ("$filename:$line: Test succeeded inside todo block" .
- ($description ? ": $description" : "") . "\n");
+ winetest_output_line( $filename, $line, "Test succeeded inside todo block", $name, $description);
$todo_failures++;
}
- else { $todo_successes++; }
+ else
+ {
+ $todo_successes++;
+ }
}
else
{
if (!$assertion)
{
- print STDERR ("$filename:$line: Test failed" .
- ($description ? ": $description" : "") . "\n");
+ winetest_output_line( $filename, $line, "Test failed", $name, $description);
$failures++;
}
- else { $successes++; }
+ else {
+ winetest_output_line( $filename, $line, "Test succeeded", $name, $description) if ($winetest_report_success);
+ $successes++;
+ }
}
}
+# ----------------------------------------------------------------------
+# Subroutine: set_name
+#
+# Purpose: Specify the name for next series of tests
+#
+# Usage: set_name NAME
+#
+# Returns: (none)
+# ----------------------------------------------------------------------
+sub set_name($)
+{
+ $name = shift;
+}
+
+# ----------------------------------------------------------------------
+# Subroutine: unset_name
+#
+# Purpose: Return to default naming scheme
+#
+# Usage: unset_name
+#
+# Returns: (none)
+# ----------------------------------------------------------------------
+sub unset_name()
+{
+ $name = "";
+}
+
# ----------------------------------------------------------------------
# Subroutine: assert
@@ -562,6 +665,7 @@
($failures + $todo_failures != 1) ? "failures" : "failure" );
}
$? = ($failures + $todo_failures < 255) ? $failures + $todo_failures : 255;
+ $? = 0 if ($winetest_continue_on_fail);
}
-------------- next part --------------
Index: dlls/kernel/tests/atom.pl
===================================================================
RCS file: /home/wine/wine/dlls/kernel/tests/atom.pl,v
retrieving revision 1.3
diff -u -r1.3 atom.pl
--- dlls/kernel/tests/atom.pl 1 Apr 2002 21:00:42 -0000 1.3
+++ dlls/kernel/tests/atom.pl 10 Apr 2002 16:22:40 -0000
@@ -29,12 +29,12 @@
# Add an atom
my $atom = GlobalAddAtomA( $name );
-ok( ($atom >= 0xc000) && ($atom <= 0xffff) && !defined($wine::err) );
+named_ok( "Adding atom \"$name\" via GlobalAddAtomA()", ($atom >= 0xc000) && ($atom <= 0xffff) && !defined($wine::err) );
# Verify that it can be found (or not) appropriately
-ok( GlobalFindAtomA($name) == $atom );
+named_ok( "Testing added atom \"".$name."\"", GlobalFindAtomA($name) == $atom );
ok( GlobalFindAtomA(uc $name) == $atom );
-ok( !GlobalFindAtomA( "_" . $name ) );
+named_ok( "Checking can't find atom \"_$name\"", !GlobalFindAtomA( "_" . $name ) );
# Add the same atom, specifying string as unicode; should
# find the first one, not add a new one
@@ -68,16 +68,19 @@
ok( !GlobalAddAtomW(0xffff) && ($wine::err == ERROR_INVALID_PARAMETER) );
# test some in between
+set_name( "Checking GlobalAddAtom[AW](i) i < 0xbffff");
for ($i = 0x0001; ($i <= 0xbfff); $i += 27)
{
ok( GlobalAddAtomA($i) && !defined($wine::err) );
ok( GlobalAddAtomW($i) && !defined($wine::err) );
}
+set_name( "Checking GlobalAddAtom[AW](i) 0xc000 < i < 0xfffff");
for ($i = 0xc000; ($i <= 0xffff); $i += 29)
{
ok( !GlobalAddAtomA($i) && ($wine::err == ERROR_INVALID_PARAMETER) );
ok( !GlobalAddAtomW($i) && ($wine::err == ERROR_INVALID_PARAMETER) );
}
+unset_name();
################################################################
Index: dlls/kernel/tests/directory.c
===================================================================
RCS file: /home/wine/wine/dlls/kernel/tests/directory.c,v
retrieving revision 1.2
diff -u -r1.2 directory.c
--- dlls/kernel/tests/directory.c 1 Apr 2002 21:00:26 -0000 1.2
+++ dlls/kernel/tests/directory.c 10 Apr 2002 16:22:40 -0000
@@ -32,8 +32,10 @@
lstrcpyA(buf, "foo");
len_with_null = GetWindowsDirectoryA(buf, 1);
- ok(lstrcmpA(buf, "foo") == 0, "should not touch the buffer");
- ok(len_with_null <= MAX_PATH, "should fit into MAX_PATH");
+ named_ok("Testing GetWindowsDirectoryA() doesn't touch buffer",
+ lstrcmpA(buf, "foo") == 0, "should not touch the buffer");
+ named_ok("Testing GetWindowsDirectoryA() returns <= MAX_PATH",
+ len_with_null <= MAX_PATH, "should fit into MAX_PATH");
lstrcpyA(buf, "foo");
len = GetWindowsDirectoryA(buf, len_with_null - 1);
@@ -41,10 +43,12 @@
ok(len == len_with_null, "should return length with terminating 0");
lstrcpyA(buf, "foo");
+ set_name( "Testing GetWindowsDirectoryA");
len = GetWindowsDirectoryA(buf, len_with_null);
ok(lstrcmpA(buf, "foo") != 0, "should touch the buffer");
ok(len == lstrlenA(buf), "returned length should be equal to the length of string");
ok(len == (len_with_null - 1), "should return length without terminating 0");
+ unset_name();
}
static void test_GetWindowsDirectoryW(void)
More information about the wine-devel
mailing list