#!/usr/bin/env python # Copyright (c) 2013, Arvid Norberg # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # This is meant to be run from the root directory of the repo. It will # look for the .regression.yml file and expect a regression_tests directory # with results from test runs previously produced by run_tests.py import os import sys import glob import json # TODO: different parsers could be run on output from different actions # if we would use the xml output instead of stdout/stderr def style_output(logfile, outfile): subtle = False for l in logfile.split('\n'): l = l.encode('utf-8') l = l.replace('<', '<') l = l.replace('>', '>') if 'TEST_CHECK' in l or \ 'TEST_EQUAL_ERROR' in l or \ '"ERROR: "' in l or \ l.startswith('EXIT STATUS: ') or \ ' second time limit exceeded' in l or l.startswith('signal: SIG') or \ 'jump or move depends on uninitialised value(s)' in l or \ 'Invalid read of size' in l or \ 'Invalid write of size' in l or \ 'Use of uninitialised value of size' in l or \ 'Uninitialised byte(s) found during' in l or \ 'Terminated with exception: ' in l or \ 'TEST(S) FAILED' in l or \ 'points to uninitialised byte(s)' in l: print >>outfile, '%s' % l elif '**passed**' in l: print >>outfile, '%s' % l elif ': error: ' in l or \ ';1;31merror: ' in l or \ ': fatal error: ' in l or \ ' : fatal error ' in l or \ 'failed to write output file' in l or \ ') : error C' in l or \ ' : error LNK' in l or \ ': undefined reference to ' in l: print >>outfile, '%s' % l elif ': warning: ' in l or \ ') : warning C' in l or \ '0;1;35mwarning: ' in l or \ 'Uninitialised value was created by a' in l or \ 'bytes after a block of size' in l or \ 'bytes inside a block of size' in l: print >>outfile, '%s' % l.strip() elif l == '====== END OUTPUT ======' and not subtle: print >>outfile, '%s' % l subtle = True else: print >>outfile, '%s' % l if subtle: print >>outfile, '' def modification_time(file): mtime = 0 try: mtime = os.stat(file).st_mtime except: pass return mtime def save_log_file(log_name, project_name, branch_name, test_name, timestamp, data): if not os.path.exists(os.path.split(log_name)[0]): os.mkdir(os.path.split(log_name)[0]) try: # if the log file already exists, and it's newer than # the source, no need to re-parse it mtime = os.stat(log_name).st_mtime if mtime >= timestamp: return except: pass html = open(log_name, 'w+') print >>html, '''%s - %s

%s - %s

''' % (project_name, branch_name, project_name, branch_name) print >>html, '

%s

' % test_name.encode('utf-8')
	style_output(data, html)

	print >>html, '
' html.close() sys.stdout.write('.') sys.stdout.flush() def parse_tests(rev_dir): # this contains mappings from platforms to # the next layer of dictionaries. The next # layer contains a mapping of toolsets to # dictionaries the next layer of dictionaries. # those dictionaries contain a mapping from # feature-sets to the next layer of dictionaries. # the next layer contains a mapping from # tests to information about those tests, such # as whether it passed and the output from the # command # example: # { # darwin: { # clang-4.2.1: { # ipv6=off: { # test_primitives: { # output: ... # status: 1 # } # } # } # } # } platforms = {} tests = {} for f in glob.glob(os.path.join(rev_dir, '*.json')): platform_toolset = os.path.split(f)[1].split('.json')[0].split('#') try: j = json.loads(open(f, 'rb').read()) timestamp = os.stat(f).st_mtime except Exception, e: print '\nFAILED TO LOAD "%s": %s\n' % (f, e) continue platform = platform_toolset[0] toolset = platform_toolset[1] for cfg in j: test_name = cfg.split('|')[0] features = cfg.split('|')[1] if not features in tests: tests[features] = set() tests[features].add(test_name) if not platform in platforms: platforms[platform] = {} if not toolset in platforms[platform]: platforms[platform][toolset] = {} if not features in platforms[platform][toolset]: platforms[platform][toolset][features] = {} platforms[platform][toolset][features][test_name] = j[cfg] platforms[platform][toolset][features][test_name]['timestamp'] = timestamp return (platforms, tests) # TODO: remove this dependency by encoding it in the output files # this script should work from outside of the repo, just having # access to the shared folder project_name = 'libtorrent' # maps branch name to latest rev revs = {} input_dir = os.path.abspath('regression_tests') for rev in os.listdir(input_dir): try: branch = rev.split('-')[0] if branch == 'logs': continue r = int(rev.split('-')[1]) if not branch in revs: revs[branch] = r else: if r > revs[branch]: revs[branch] = r except: print 'ignoring %s' % rev if revs == {}: print 'no test files found' sys.exit(1) print 'latest versions' for b in revs: print '%s\t%d' % (b, revs[b]) try: os.mkdir('regression_test_report') except: pass os.chdir('regression_test_report') for branch_name in revs: latest_rev = revs[branch_name] html_file = '%s.html' % branch_name html = open(html_file, 'w+') print >>html, '''regression tests, %s

%s - %s

''' % (project_name, project_name, branch_name) print >>html, '' num_printed_revs = 0; for r in range(latest_rev, latest_rev - 40, -1): sys.stdout.write('.') sys.stdout.flush() rev_dir = os.path.join(input_dir, '%s-%d' % (branch_name, r)) (platforms, tests) = parse_tests(rev_dir) if len(tests) + len(platforms) == 0: continue print >>html, '' % r features = tests.keys() features = sorted(features, key=lambda x: len(tests[x])) for f in features: title = f if len(tests[f]) < 10: title = '#' print >>html, '' % (len(tests[f]), len(tests[f])*6 - 5, title) print >>html, '' for p in platforms: print >>html, '' % (len(platforms[p]), p) idx = 0 for toolset in platforms[p]: if idx > 0: print >>html, '' log_dir = 'logs-%s-%d' % (branch_name, r) if not os.path.exists(log_dir): os.mkdir(log_dir) details_name = os.path.join(log_dir, '%s-%s.html' % (p, toolset)) details_file = open(details_name, 'w+') print >>details_file, '''%s %s [%s]''' % (p, toolset, branch_name) print >>html, '' % (details_name, toolset) deferred_end_table = False for f in features: title = f if len(tests[f]) < 10: title = '#' if title != '#': if deferred_end_table: print >>details_file, '
revision %d%s
%s
%s
' print >>details_file, '' % title deferred_end_table = False else: print >>details_file, '
%s
' print >>details_file, '' % title elif not deferred_end_table: print >>details_file, '
%s
' print >>details_file, '' % title if not f in platforms[p][toolset]: for i in range(len(tests[f])): print >>html, '' % (f) continue for t in platforms[p][toolset][f]: details = platforms[p][toolset][f][t] exitcode = details['status'] if exitcode == 0: error_state = 'passed' c = 'passed' elif exitcode == 222: error_state = 'valgrind error' c = 'valgrind-error' elif exitcode == 139 or \ exitcode == 138: error_state = 'crash' c = 'crash' elif exitcode == -1073740777: error_state = 'timeout' c = 'timeout' elif exitcode == 333 or \ exitcode == 77: error_code = 'test-failed' c = 'failed' else: error_state = 'compile-failed (%d)' % exitcode c = 'compile-failed' log_name = os.path.join('logs-%s-%d' % (branch_name, r), p + '~' + toolset + '~' + t + '~' + f.replace(' ', '.') + '.html') print >>html, '' % (t, f, c, log_name) print >>details_file, '' % (c, os.path.split(log_name)[1], t, error_state) save_log_file(log_name, project_name, branch_name, '%s - %s' % (t, f), int(details['timestamp']), details['output']) if title != '#': print >>details_file, '
%s
%s [%s]
' deferred_end_table = False else: deferred_end_table = True if deferred_end_table: print >>details_file, '' print >>html, '' idx += 1 print >>details_file, '' details_file.close() num_printed_revs += 1 if num_printed_revs >= 20: break print >>html, '' html.close() print ''