wornvirtue
/
premiere-libtorrent
forked from premiere/premiere-libtorrent
1
0
Fork 0
Code Pull Requests Activity
premiere-libtorrent/tools/parse_test_results.py

264 lines
8.3 KiB
Python
Raw Normal View History

some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
#!/bin/python
# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the distribution.
# * Neither the name of the author nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# This is meant to be run from the root directory of the repo. It will
# look for the .regression.yml file and expect a regression_tests directory
# with results from test runs previously produced by run_tests.py
import os
import sys
import glob
import json
improve regression test reports
2013-06-18 05:58:06 +02:00
# TODO: different parsers could be run on output from different actions
# if we would use the xml output in stead of stdout/stderr
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
def style_output(o):
ret = ''
subtle = False
for l in o.split('\n'):
test script fixes
2013-06-23 03:34:19 +02:00
l = l.replace('<', '&lt;')
l = l.replace('>', '&gt;')
one more getpid() fix for windows
2013-06-18 10:48:19 +02:00
if 'TEST_CHECK' in l or 'TEST_EQUAL_ERROR' in l or l.startswith('EXIT STATUS: ') or \
improve parse_test_results
2013-08-04 05:48:18 +02:00
' second time limit exceeded' in l or l.startswith('signal: SIG'):
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
ret += '<span class="test-error">%s</span>\n' % l
elif '**passed**' in l:
ret += '<span class="test-pass">%s</span>\n' % l
first stab at script to atomatically run unit tests for new check-ins
2013-06-16 22:37:54 +02:00
elif ': error: ' in l or ': fatal error: ' in l or ' : fatal error ' in l or \
parse msvc link errors
2013-06-17 17:49:45 +02:00
'failed to write output file' in l or ') : error C' in l or \
fix msvc link error parsing
2013-06-23 10:08:07 +02:00
' : error LNK' in l or ': undefined reference to ' in l:
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
ret += '<span class="compile-error">%s</span>\n' % l
first stab at script to atomatically run unit tests for new check-ins
2013-06-16 22:37:54 +02:00
elif ': warning: ' in l or ') : warning C' in l:
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
ret += '<span class="compile-warning">%s</span>\n' % l
elif l == '====== END OUTPUT ======' and not subtle:
ret += '<span class="subtle">%s\n' % l
subtle = True
else:
ret += '%s\n' % l
if subtle: ret += '</span>'
return ret
fix test output parser
2013-06-17 06:10:40 +02:00
def modification_time(file):
mtime = 0
try:
mtime = os.stat(file).st_mtime
except: pass
return mtime
make parse_test_results.py more efficient parsing and writing logs. It won't re-parse everything every time
2013-06-20 02:48:17 +02:00
def save_log_file(log_name, project_name, branch_name, test_name, timestamp, data):
if not os.path.exists(os.path.split(log_name)[0]):
os.mkdir(os.path.split(log_name)[0])
try:
# if the log file already exists, and it's newer than
# the source, no need to re-parse it
mtime = os.stat(log_name).st_mtime
if mtime >= timestamp: return
except: pass
html = open(log_name, 'w+')
print >>html, '''<html><head><title>%s - %s</title><style type="text/css">
.compile-error { color: #f13; font-weight: bold; }
.compile-warning { font-weight: bold; color: black; }
.test-error { color: #f13; font-weight: bold; }
.test-pass { color: #1c2; font-weight: bold; }
polish parse test output script
2013-07-22 02:16:51 +02:00
.subtle { color: #ddd; }
improve test result html
2013-06-21 06:17:46 +02:00
pre { color: #999; white-space: pre-wrap; word-wrap: break-word; }
make parse_test_results.py more efficient parsing and writing logs. It won't re-parse everything every time
2013-06-20 02:48:17 +02:00
</style>
</head><body><h1>%s - %s</h1>''' % (project_name, branch_name, project_name, branch_name)
print >>html, '<h3>%s</h3><pre>%s</pre>' % \
(test_name.encode('utf-8'), style_output(data).encode('utf-8'))
print >>html, '</body></html>'
html.close()
sys.stdout.write('.')
sys.stdout.flush()
improve regression test reports
2013-06-18 05:58:06 +02:00
def parse_tests(rev_dir):
# this contains mappings from platforms to
# the next layer of dictionaries. The next
# layer contains a mapping of toolsets to
# dictionaries the next layer of dictionaries.
# those dictionaries contain a mapping from
# feature-sets to the next layer of dictionaries.
# the next layer contains a mapping from
# tests to information about those tests, such
# as whether it passed and the output from the
# command
# example:
# {
# darwin: {
# clang-4.2.1: {
# ipv6=off: {
# test_primitives: {
# output: ...
# status: 1
# }
# }
# }
# }
# }
platforms = {}
tests = {}
for f in glob.glob(os.path.join(rev_dir, '*.json')):
platform_toolset = os.path.split(f)[1].split('.json')[0].split('#')
tolerate invalid json files
2013-06-19 06:22:57 +02:00
try:
j = json.loads(open(f, 'rb').read())
make parse_test_results.py more efficient parsing and writing logs. It won't re-parse everything every time
2013-06-20 02:48:17 +02:00
timestamp = os.stat(f).st_mtime
tolerate invalid json files
2013-06-19 06:22:57 +02:00
except:
improve run_tests.py robustness as well as support cleaning of test directory after each full run-through of the tests
2013-06-20 05:20:02 +02:00
print '\nFAILED TO LOAD "%s"\n' %f
tolerate invalid json files
2013-06-19 06:22:57 +02:00
continue
improve regression test reports
2013-06-18 05:58:06 +02:00
platform = platform_toolset[0]
toolset = platform_toolset[1]
for cfg in j:
test_name = cfg.split('|')[0]
features = cfg.split('|')[1]
if not features in tests:
tests[features] = set()
tests[features].add(test_name)
if not platform in platforms:
platforms[platform] = {}
if not toolset in platforms[platform]:
platforms[platform][toolset] = {}
if not features in platforms[platform][toolset]:
platforms[platform][toolset][features] = {}
platforms[platform][toolset][features][test_name] = j[cfg]
make parse_test_results.py more efficient parsing and writing logs. It won't re-parse everything every time
2013-06-20 02:48:17 +02:00
platforms[platform][toolset][features][test_name]['timestamp'] = timestamp
improve regression test reports
2013-06-18 05:58:06 +02:00
return (platforms, tests)
minor test report polishing
2013-06-16 10:15:38 +02:00
improve regression test reports
2013-06-18 05:58:06 +02:00
# TODO: remove this dependency by encoding it in the output files
make parse_test_results.py more efficient parsing and writing logs. It won't re-parse everything every time
2013-06-20 02:48:17 +02:00
# this script should work from outside of the repo, just having
# access to the shared folder
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
project_name = 'libtorrent'
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
# maps branch name to latest rev
revs = {}
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
os.chdir('regression_tests')
support running tests for different branches. assume regression_tests is a shared folder
2013-06-17 03:30:45 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
for rev in os.listdir('.'):
try:
branch = rev.split('-')[0]
if branch == 'logs': continue
r = int(rev.split('-')[1])
if not branch in revs:
revs[branch] = r
else:
if r > revs[branch]:
revs[branch] = r
except:
print 'ignoring %s' % rev
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
if revs == {}:
print 'no test files found'
sys.exit(1)
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
print 'latest versions'
for b in revs:
print '%s\t%d' % (b, revs[b])
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
for branch_name in revs:
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
latest_rev = revs[branch_name]
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
html_file = '%s.html' % branch_name
fix test output parser
2013-06-17 06:10:40 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
html = open(html_file, 'w+')
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
print >>html, '''<html><head><title>regression tests, %s</title><style type="text/css">
.passed { display: block; width: 8px; height: 1em; background-color: #6f8 }
.failed { display: block; width: 8px; height: 1em; background-color: #f68 }
table { border: 0; border-collapse: collapse; }
h1 { font-size: 15pt; }
th { font-size: 8pt; }
improve parse_test_results
2013-08-04 05:48:18 +02:00
td { border: 0; border-spacing: 0px; padding: 1px 0px 0px 1px; }
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
.left-head { white-space: nowrap; }
</style>
</head><body><h1>%s - %s</h1>''' % (project_name, project_name, branch_name)
minor improvement to test result parser
2013-06-17 06:39:20 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
print >>html, '<table border="1">'
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
improve parse_test_results
2013-08-04 05:48:18 +02:00
num_printed_revs = 0;
for r in range(latest_rev, latest_rev - 40, -1):
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
sys.stdout.write('.')
sys.stdout.flush()
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
rev_dir = '%s-%d' % (branch_name, r)
(platforms, tests) = parse_tests(rev_dir)
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
improve parse_test_results
2013-08-04 05:48:18 +02:00
if len(tests) + len(platforms) == 0: continue
print >>html, '<tr><th colspan="2" style="border:0;">revision %d</th>' % r
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
for f in tests:
print >>html, '<th colspan="%d" style="width: %dpx;">%s</th>' % (len(tests[f]), len(tests[f])*9 - 5, f)
print >>html, '</tr>'
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
for p in platforms:
print >>html, '<tr><th class="left-head" rowspan="%d">%s</th>' % (len(platforms[p]), p)
idx = 0
for toolset in platforms[p]:
if idx > 0: print >>html, '<tr>'
print >>html, '<th class="left-head">%s</th>' % toolset
for f in platforms[p][toolset]:
for t in platforms[p][toolset][f]:
details = platforms[p][toolset][f][t]
if details['status'] == 0: c = 'passed'
else: c = 'failed'
log_name = os.path.join('logs-%s-%d' % (branch_name, r), p + '~' + toolset + '~' + t + '~' + f.replace(' ', '.') + '.html')
print >>html, '<td title="%s %s"><a class="%s" href="%s"></a></td>' % (t, f, c, log_name)
save_log_file(log_name, project_name, branch_name, '%s - %s' % (t, f), int(details['timestamp']), details['output'])
improve regression test reports
2013-06-18 05:58:06 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
print >>html, '</tr>'
idx += 1
improve parse_test_results
2013-08-04 05:48:18 +02:00
num_printed_revs += 1
if num_printed_revs >= 20: break
some helper scripts for running and collecting summary report on unit tests, can be expanded to automatically run regression tests
2013-06-16 01:25:14 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
print >>html, '</table></body></html>'
html.close()
improve regression test reports
2013-06-18 05:58:06 +02:00
drop yaml dependency in parse_test_results.py and don't require it to be run from a repository
2013-07-02 07:24:10 +02:00
print ''
fix build of test_storage without deprecated functions
2013-06-18 09:33:49 +02:00