#!/bin/python # Copyright (c) 2013, Arvid Norberg # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # This is meant to be run from the root directory of the repo. It will # look for the .regression.yml file and expect a regression_tests directory # with results from test runs previously produced by run_tests.py import os import sys import glob import json import yaml def style_output(o): ret = '' subtle = False for l in o.split('\n'): if 'TEST_CHECK' in l or 'TEST_EQUAL_ERROR' in l or l.startswith('EXIT STATUS: '): ret += '%s\n' % l elif '**passed**' in l: ret += '%s\n' % l elif ': error: ' in l or ': fatal error: ' in l or ' : fatal error ' in l or \ 'failed to write output file' in l: ret += '%s\n' % l elif ': warning: ' in l or ') : warning C' in l: ret += '%s\n' % l elif l == '====== END OUTPUT ======' and not subtle: ret += '%s\n' % l subtle = True else: ret += '%s\n' % l if subtle: ret += '' return ret project_name = '' try: cfg = open('.regression.yml', 'r') except: print '.regression.yml not found in current directory' sys.exit(1) cfg = yaml.load(cfg.read()) if 'project' in cfg: project_name = cfg['project'] branch_name = 'trunk' if 'branch' in cfg: branch_name = cfg['branch'] os.chdir('regression_tests') def modification_time(file): mtime = 0 try: mtime = os.stat(file).st_mtime except: pass return mtime index_mtime = modification_time('index.html') latest_rev = 0 for rev in os.listdir('.'): try: if not rev.startswith('%s-', branch_name): continue r = int(rev[len(branch_name)+1:]) if r > latest_rev: latest_rev = r except: pass if latest_rev == 0: print 'no test files found' sys.exit(1) rev_dir = '%s-%d' % (branch_name, latest_rev) need_refresh = False for f in glob.glob(os.path.join(rev_dir, '*.json')): mtime = modification_time(f) if mtime > index_mtime: need_refresh = True break if not need_refresh: print 'all up to date' sys.exit(0) # this contains mappings from platforms to # the next layer of dictionaries. The next # layer contains a mapping of toolsets to # dictionaries the next layer of dictionaries. # those dictionaries contain a mapping from # feature-sets to the next layer of dictionaries. # the next layer contains a mapping from # tests to information about those tests, such # as whether it passed and the output from the # command # example: # { # darwin: { # clang-4.2.1: { # ipv6=off: { # test_primitives: { # output: ... # status: 1 # } # } # } # } # } platforms = {} tests = {} for f in glob.glob(os.path.join(rev_dir, '*.json')): platform_toolset = os.path.split(f)[1].split('.json')[0].split('#') j = json.loads(open(f, 'rb').read()) platform = platform_toolset[0] toolset = platform_toolset[1] for cfg in j: test_name = cfg.split('|')[0] features = cfg.split('|')[1] if not features in tests: tests[features] = set() tests[features].add(test_name) if not platform in platforms: platforms[platform] = {} if not toolset in platforms[platform]: platforms[platform][toolset] = {} if not features in platforms[platform][toolset]: platforms[platform][toolset][features] = {} platforms[platform][toolset][features][test_name] = j[cfg] html = open('index.html', 'w') print >>html, '''regression tests, %s revision %d''' % (project_name, latest_rev) print >>html, '

%s revision %d

' % (project_name, latest_rev) print >>html, '' for f in tests: print >>html, '' % (len(tests[f]), f) print >>html, '' details_id = 0 details = [] for p in platforms: print >>html, '' % (len(platforms[p]), p) idx = 0 for toolset in platforms[p]: if idx > 0: print >>html, '' print >>html, '' % toolset for f in platforms[p][toolset]: for t in platforms[p][toolset][f]: if platforms[p][toolset][f][t][u'status'] == 0: c = 'passed' else: c = 'failed' print >>html, '' % ('%s %s' % (t, f), c, details_id) platforms[p][toolset][f][t]['name'] = t platforms[p][toolset][f][t]['features'] = f details.append(platforms[p][toolset][f][t]) details_id += 1 print >>html, '' idx += 1 print >>html, '
%s
%s
%s
' details_id = 0 for d in details: print >>html, '' % \ (details_id, d['name'].encode('utf8'), d['features'].encode('utf-8'), style_output(d['output']).encode('utf-8')) details_id += 1 print >>html, '' html.close()