mirror of https://github.com/python/cpython.git
New regression testing harness.
(testall.out): obsolete. see individual test output files in the Output directory. (testall.py): now contains only the list of all tests (autotest.py): The new testing harness. Supports -g option to generate output files instead of verifying; -w option to warn about non-tested optional modules; additional command line arguments to generate or test individual tests (useful for generating output file of new tests).
This commit is contained in:
Barry Warsaw
parent
0aa9ee65ab
commit
272c00b634
|
|
@ -1,34 +1,58 @@
|
|||
# Automatic Python regression test.
|
||||
#
|
||||
# Some essential parts of the Python interpreter are tested by the module
|
||||
# 'testall'. (Despite its name, it doesn't test everything -- that would
|
||||
# be a truly Herculean task!) When a test fails, 'testall' raises an
|
||||
# exception. When all tests succeed, it produces quite a lot of output.
|
||||
#
|
||||
# For a normal regression test, this output is never looked at unless
|
||||
# something goes wrong. Thus, it would be wise to suppress the output
|
||||
# normally. This module does that, but it doesn't just throw the output
|
||||
# from 'testall' away -- it compares it with the output from a previous
|
||||
# run. If a difference is noticed it raises an exception; if all is well,
|
||||
# it prints nothing except 'All tests OK.' at the very end.
|
||||
#
|
||||
# The output from a previous run is supposed to be in a file 'testall.out'
|
||||
# somewhere on the search path for modules (sys.path, initialized from
|
||||
# $PYTHONPATH plus some default places).
|
||||
#
|
||||
# Of course, if the normal output of the tests is changed because the
|
||||
# tests have been changed (rather than a test producing the wrong output),
|
||||
# 'autotest' will fail as well. In this case, run 'testall' manually
|
||||
# and direct its output to 'testall.out'.
|
||||
#
|
||||
# The comparison uses (and demonstrates!) a rather new Python feature:
|
||||
# program output that normally goes to stdout (by 'print' statements
|
||||
# or by writing directly to sys.stdout) can be redirected to an
|
||||
# arbitrary class instance as long as it has a 'write' method.
|
||||
"""
|
||||
Automatic Python regression test.
|
||||
|
||||
The list of individual tests is contained in the `testall' module.
|
||||
These test some (but not all) essential parts of the Python
|
||||
interpreter and built-in modules. When a test fails, an exception is
|
||||
raised and testing halts. When a test succeeds, it can produce quite
|
||||
a lot of output, which is compared against the output from a previous
|
||||
run. If a difference is noticed it raises an exception; if all is
|
||||
well, it prints nothing except 'All tests OK.' at the very end.
|
||||
|
||||
The output from a previous run is supposed to be contained in separate
|
||||
files (one per test) in the `Output' subdirectory somewhere on the
|
||||
search path for modules (sys.path, initialized from $PYTHONPATH plus
|
||||
some default places).
|
||||
|
||||
Of course, if the normal output of the tests is changed because the
|
||||
tests have been changed (rather than a test producing the wrong
|
||||
output), 'autotest' will fail as well. In this case, run 'autotest'
|
||||
with the -g option.
|
||||
|
||||
Usage:
|
||||
|
||||
%s [-g] [-w] [-h] [test1 [test2 ...]]
|
||||
|
||||
Options:
|
||||
|
||||
-g, --generate : generate the output files instead of verifying
|
||||
the results
|
||||
|
||||
-w, --warn : warn about un-importable tests
|
||||
|
||||
-h, --help : print this message
|
||||
|
||||
If individual tests are provided on the command line, only those tests
|
||||
will be performed or generated. Otherwise, all tests (as contained in
|
||||
testall.py) will be performed.
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import getopt
|
||||
import traceback
|
||||
from test_support import *
|
||||
|
||||
# Exception raised when the test failed (not the same as in test_support)
|
||||
TestFailed = 'autotest.TestFailed'
|
||||
|
||||
# defaults
|
||||
generate = 0
|
||||
warn = 0
|
||||
|
||||
|
||||
|
||||
# Function to find a file somewhere on sys.path
|
||||
def findfile(filename):
|
||||
for dirname in sys.path:
|
||||
|
|
@ -37,9 +61,8 @@ def findfile(filename):
|
|||
return fullname
|
||||
return filename # Will cause exception later
|
||||
|
||||
# Exception raised when the test failed (not the same as in test_support)
|
||||
TestFailed = 'autotest.TestFailed'
|
||||
|
||||
|
||||
# Class substituted for sys.stdout, to compare it with the given file
|
||||
class Compare:
|
||||
def __init__(self, filename):
|
||||
|
|
@ -50,18 +73,73 @@ def write(self, data):
|
|||
raise TestFailed, \
|
||||
'Writing: '+`data`+', expected: '+`expected`
|
||||
def close(self):
|
||||
leftover = self.fp.read()
|
||||
if leftover:
|
||||
raise TestFailed, 'Unread: '+`leftover`
|
||||
self.fp.close()
|
||||
|
||||
|
||||
# The main program
|
||||
def main():
|
||||
import sys
|
||||
filename = findfile('testall.out')
|
||||
def usage(status):
|
||||
print __doc__ % sys.argv[0]
|
||||
sys.exit(status)
|
||||
|
||||
|
||||
|
||||
def do_one_test(t, outdir):
|
||||
filename = os.path.join(outdir, t)
|
||||
real_stdout = sys.stdout
|
||||
try:
|
||||
sys.stdout = Compare(filename)
|
||||
import testall
|
||||
if generate:
|
||||
print 'Generating:', filename
|
||||
sys.stdout = open(filename, 'w')
|
||||
else:
|
||||
sys.stdout = Compare(filename)
|
||||
print t
|
||||
unload(t)
|
||||
try:
|
||||
__import__(t, globals(), locals())
|
||||
except ImportError, msg:
|
||||
if warn:
|
||||
sys.stderr.write(msg+': Un-installed'
|
||||
' optional module?\n')
|
||||
finally:
|
||||
sys.stdout.close()
|
||||
sys.stdout = real_stdout
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
global generate
|
||||
global warn
|
||||
try:
|
||||
opts, args = getopt.getopt(
|
||||
sys.argv[1:], 'ghw',
|
||||
['generate', 'help', 'warn'])
|
||||
except getopt.error, msg:
|
||||
print msg
|
||||
usage(1)
|
||||
for opt, val in opts:
|
||||
if opt in ['-h', '--help']:
|
||||
usage(0)
|
||||
elif opt in ['-g', '--generate']:
|
||||
generate = 1
|
||||
elif opt in ['-w', '--warn']:
|
||||
warn = 1
|
||||
|
||||
# find the output directory
|
||||
outdir = findfile('Output')
|
||||
if args:
|
||||
tests = args
|
||||
else:
|
||||
import testall
|
||||
tests = testall.tests
|
||||
for test in tests:
|
||||
try:
|
||||
do_one_test(test, outdir)
|
||||
except TestFailed, msg:
|
||||
print 'Failure of test:', test
|
||||
traceback.print_exc()
|
||||
print 'All tests OK.'
|
||||
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,184 +0,0 @@
|
|||
test_grammar
|
||||
1. Parser
|
||||
1.1 Tokens
|
||||
1.1.1 Backslashes
|
||||
1.1.2 Numeric literals
|
||||
1.1.2.1 Plain integers
|
||||
1.1.2.2 Long integers
|
||||
1.1.2.3 Floating point
|
||||
1.1.3 String literals
|
||||
1.2 Grammar
|
||||
single_input
|
||||
file_input
|
||||
expr_input
|
||||
eval_input
|
||||
funcdef
|
||||
simple_stmt
|
||||
expr_stmt
|
||||
print_stmt
|
||||
1 2 3
|
||||
1 2 3
|
||||
1 1 1
|
||||
del_stmt
|
||||
pass_stmt
|
||||
flow_stmt
|
||||
break_stmt
|
||||
continue_stmt
|
||||
return_stmt
|
||||
raise_stmt
|
||||
import_stmt
|
||||
global_stmt
|
||||
exec_stmt
|
||||
if_stmt
|
||||
while_stmt
|
||||
for_stmt
|
||||
try_stmt
|
||||
suite
|
||||
test
|
||||
comparison
|
||||
binary mask ops
|
||||
shift ops
|
||||
additive ops
|
||||
multiplicative ops
|
||||
unary ops
|
||||
selectors
|
||||
atoms
|
||||
classdef
|
||||
test_opcodes
|
||||
2. Opcodes
|
||||
XXX Not yet fully implemented
|
||||
2.1 try inside for loop
|
||||
2.2 raise class exceptions
|
||||
test_operations
|
||||
3. Operations
|
||||
XXX Not yet implemented
|
||||
test_builtin
|
||||
4. Built-in functions
|
||||
test_b1
|
||||
__import__
|
||||
abs
|
||||
apply
|
||||
callable
|
||||
chr
|
||||
cmp
|
||||
coerce
|
||||
compile
|
||||
delattr
|
||||
dir
|
||||
divmod
|
||||
eval
|
||||
execfile
|
||||
filter
|
||||
float
|
||||
getattr
|
||||
hasattr
|
||||
hash
|
||||
hex
|
||||
id
|
||||
int
|
||||
len
|
||||
long
|
||||
map
|
||||
max
|
||||
min
|
||||
test_b2
|
||||
oct
|
||||
open
|
||||
ord
|
||||
pow
|
||||
range
|
||||
input and raw_input
|
||||
testing
|
||||
testing
|
||||
reduce
|
||||
reload
|
||||
repr
|
||||
round
|
||||
setattr
|
||||
str
|
||||
tuple
|
||||
type
|
||||
vars
|
||||
xrange
|
||||
test_exceptions
|
||||
5. Built-in exceptions
|
||||
AttributeError
|
||||
EOFError
|
||||
IOError
|
||||
ImportError
|
||||
IndexError
|
||||
KeyError
|
||||
KeyboardInterrupt
|
||||
(not testable in a script)
|
||||
MemoryError
|
||||
(not safe to test)
|
||||
NameError
|
||||
OverflowError
|
||||
RuntimeError
|
||||
(not used any more?)
|
||||
SyntaxError
|
||||
SystemError
|
||||
(hard to reproduce)
|
||||
SystemExit
|
||||
TypeError
|
||||
ValueError
|
||||
ZeroDivisionError
|
||||
test_types
|
||||
6. Built-in types
|
||||
6.1 Truth value testing
|
||||
6.2 Boolean operations
|
||||
6.3 Comparisons
|
||||
6.4 Numeric types (mostly conversions)
|
||||
6.4.1 32-bit integers
|
||||
6.4.2 Long integers
|
||||
6.4.3 Floating point numbers
|
||||
6.5 Sequence types
|
||||
6.5.1 Strings
|
||||
6.5.2 Tuples
|
||||
6.5.3 Lists
|
||||
6.5.3a Additional list operations
|
||||
6.6 Mappings == Dictionaries
|
||||
test_math
|
||||
math module, testing with eps 1e-05
|
||||
constants
|
||||
acos
|
||||
asin
|
||||
atan
|
||||
atan2
|
||||
ceil
|
||||
cos
|
||||
cosh
|
||||
exp
|
||||
fabs
|
||||
floor
|
||||
fmod
|
||||
frexp
|
||||
hypot
|
||||
ldexp
|
||||
log
|
||||
log10
|
||||
modf
|
||||
pow
|
||||
sin
|
||||
sinh
|
||||
sqrt
|
||||
tan
|
||||
tanh
|
||||
test_time
|
||||
test_array
|
||||
test_strop
|
||||
test_md5
|
||||
MD5 test suite:
|
||||
MD5 ("") = d41d8cd98f00b204e9800998ecf8427e
|
||||
MD5 ("a") = 0cc175b9c0f1b6a831c399e269772661
|
||||
MD5 ("abc") = 900150983cd24fb0d6963f7d28e17f72
|
||||
MD5 ("message digest") = f96b697d7cb7938d525a2f31aaf161d0
|
||||
MD5 ("abcdefghijklmnopqrstuvwxyz") = c3fcd3d76192e4007dfb496cca67e13b
|
||||
MD5 ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") = d174ab98d277d9f5a5611c2c9f419d9f
|
||||
MD5 ("12345678901234567890123456789012345678901234567890123456789012345678901234567890") = 57edf4a22be3c955ac49da2e2107b67a
|
||||
test_cmath
|
||||
test_crypt
|
||||
Test encryption: abeTm2pJKypwA
|
||||
test_dbm
|
||||
test_new
|
||||
Passed all tests.
|
||||
|
|
@ -1,25 +1,19 @@
|
|||
# testall.py -- a regression test for the Python interpreter.
|
||||
# To run the tests, execute "import testall" in a clean interpreter.
|
||||
# It is a good idea to do this whenever you build a new interpreter.
|
||||
# Remember to add new tests when new features are added!
|
||||
# This file now contains only the list of separate regression tests.
|
||||
# All of the testing harness is now contained in autotest.py.
|
||||
|
||||
import sys
|
||||
from test_support import *
|
||||
|
||||
print 'test_grammar'
|
||||
forget('test_grammar')
|
||||
import test_grammar
|
||||
|
||||
for t in ['test_opcodes', 'test_operations', 'test_builtin',
|
||||
'test_exceptions', 'test_types', 'test_math', 'test_time',
|
||||
'test_array', 'test_strop', 'test_md5', 'test_cmath',
|
||||
'test_crypt', 'test_dbm', 'test_new',
|
||||
]:
|
||||
print t
|
||||
unload(t)
|
||||
try:
|
||||
__import__(t, globals(), locals())
|
||||
except ImportError, msg:
|
||||
sys.stderr.write('%s. Uninstalled optional module?\n' % msg)
|
||||
|
||||
print 'Passed all tests.'
|
||||
tests = ['test_grammar',
|
||||
'test_opcodes',
|
||||
'test_operations',
|
||||
'test_builtin',
|
||||
'test_exceptions',
|
||||
'test_types',
|
||||
'test_math',
|
||||
'test_time',
|
||||
'test_array',
|
||||
'test_strop',
|
||||
'test_md5',
|
||||
'test_cmath',
|
||||
'test_crypt',
|
||||
'test_dbm',
|
||||
'test_new',
|
||||
]
|
||||
|
|
|
|||
Loading…
Reference in New Issue