trixie-backports/binutils
1
0
Fork 0
Code Activity

initial import from Debian

Browse Source 
version: 2.45-4 (UNRELEASED)
commit: bf4f75f17a4f370adc9bf9feca09710ce76ecc63
This commit is contained in:
Konstantin Demin
2025-08-11 12:46:32 +03:00
commit 8c29d0a2c2
75 changed files with 19903 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

259
debian/test-suite-compare.py vendored Normal file
View File

@@ -0,0 +1,259 @@
#!/usr/bin/python3
# Quick'n'dirty regression check for dejagnu testsuites
# Copyright (C) 2003, 2004, 2005, 2006, 2007 James Troup <james@nocrew.org>
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU;5B General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
################################################################################
import optparse
import os
import sys
################################################################################
ignored_regressions = {
# 'gprofng.display/gp-collect-app_F.exp': ('tmpdir/gp-collect-app_F',),
'gprofng.display/display.exp': ('mttest', 'synprog',),
# 'ld-bootstrap/bootstrap.exp': ('*',),
# 'ld-ifunc/ifunc.exp': ('Run pr18841 with libpr18841c.so', 'Run pr18841 with libpr18841cn.so (-z now)'),
# 'ld-loongarch-elf/pic.exp': ('nopic link exec test',),
# 'ld-x86-64/x86-64.exp': ('*',),
}
#ignored_regressions = {}
def ignore_regression(section, test):
ign_tests = ignored_regressions.get(section)
if ign_tests == None:
return False
return ign_tests[0] == '*' or test in ign_tests
################################################################################
def fubar(msg, exit_code=1):
sys.stderr.write("E: %s\n" % (msg))
sys.exit(exit_code)
def warn(msg):
sys.stderr.write("W: %s\n" % (msg))
def info(msg):
sys.stderr.write("I: %s\n" % (msg))
################################################################################
def read_testsummary(filename):
results = {}
file = open(filename)
for line in file.readlines():
if not line:
continue
if line.startswith("Running"):
s = line.split()
if "/" in s[1]:
x = s[1]
if x.find("/testsuite/") == -1:
fubar("Can't find /testsuite/ in '%s'." % (x))
# 'Running /home/james/debian/packages/binutils/binutils-2.14.90.0.7/gas/testsuite/gas/hppa/unsorted/unsorted.exp ...' -> 'gas/hppa/unsorted/unsorted.exp'
# ... since using basename() isn't dupe safe.
section = x[x.find("/testsuite/"):].replace("/testsuite/","").split()[0]
# Tests can be duplicated, e.g. hppa/basic/basic.exp
# is run twice, once for hppa-linux and once for
# hppa64-linux. This is of course a horrible bodge,
# but I can't think of anything trivial and better off
# hand.
if section in results:
extra = 1
too_many = 10
while section in results and extra < too_many:
section = "%s.%s" % (section, extra)
extra += 1
if extra >= too_many:
fubar("gave up trying to unduplicate %s." % (section))
results[section] = {}
continue
got_state = 0
for state in [ "PASS", "XPASS", "FAIL", "XFAIL", "UNRESOLVED",
"UNTESTED", "UNSUPPORTED" ]:
if line.startswith(state):
s = line.split(':')
state = s[0]
test = ':'.join(s[1:]).strip()
if test in results:
warn("%s/%s is duplicated." % (section, test))
results[section][test] = state
got_state = 1
break
if got_state:
continue
return results
################################################################################
def compare_results(old, new):
total_num = 0
pass_count = 0
fail_count = 0
xfail_count = 0
untested_count = 0
regression_count = 0
ign_regression_count = 0
progression_count = 0
change_count = 0
for section in list(new.keys()):
for test in list(new[section].keys()):
state = new[section][test]
# Stats pr0n
total_num += 1
if state == "PASS" or state == "XPASS":
pass_count += 1
elif state == "FAIL" or state == "UNRESOLVED":
fail_count += 1
elif state == "XFAIL":
xfail_count += 1
elif state == "UNTESTED":
untested_count += 1
# Compare to old
if section not in old:
continue
if test not in old[section]:
continue
old_state = old[section][test]
if state == "PASS":
if old_state != "PASS":
progression_count += 1
info("[%s] progression (%s -> %s): %s" % (section, old_state, state, test))
elif state == "XPASS":
if old_state != "XPASS" and old_state != "PASS":
progression_count += 1
warn("[%s] %s: %s" % (section, state, test))
elif state == "FAIL":
if old_state != "FAIL":
if ignore_regression(section, test):
ign_regression_count += 1
warn("[%s] IGNORED REGRESSION (%s -> %s): %s" % (section, old_state, state, test))
else:
regression_count += 1
warn("[%s] REGRESSION (%s -> %s): %s" % (section, old_state, state, test))
elif state == "XFAIL":
if old_state != "XFAIL":
change_count += 1
info("[%s] change (%s -> %s): %s" % (section, old_state, state, test))
elif state == "UNRESOLVED":
if old_state != "UNRESOLVED" and old_state != "FAIL":
if ignore_regression(section, test):
ign_regression_count += 1
warn("[%s] IGNORED REGRESSION (%s -> %s): %s" % (section, old_state, state, test))
else:
regression_count += 1
warn("[%s] REGRESSION (%s -> %s): %s" % (section, old_state, state, test))
if old_state == "FAIL":
change_count += 1
info("[%s] change (%s -> %s): %s" % (section, old_state, state, test))
elif state == "UNTESTED":
if old_state != "UNTESTED":
change_count += 1
warn("[%s] REGRESSION (%s -> %s): %s" % (section, old_state, state, test))
if ign_regression_count:
print("%d IGNORED REGRESSIONS (%.2f%%)." % (ign_regression_count, (float(ign_regression_count)/total_num)*100))
if regression_count:
print("%d REGRESSIONS (%.2f%%)." % (regression_count, (float(regression_count)/total_num)*100))
if progression_count:
print("%d progressions (%.2f%%)." % (progression_count, (float(progression_count)/total_num)*100))
if change_count:
print("%d changes (%.2f%%)." % (change_count, (float(change_count)/total_num)*100))
print("%d tests: %d pass (%.2f%%), %d fail (%.2f%%), %d xfail (%.2f%%) %d untested (%.2f%%)." \
% (total_num, pass_count, (float(pass_count)/total_num)*100,
fail_count, (float(fail_count)/total_num)*100,
xfail_count, (float(xfail_count)/total_num)*100,
untested_count, (float(untested_count)/total_num)*100))
if regression_count:
sys.exit(1)
################################################################################
def compare_multiple(directory, first_version, second_version):
architectures = [ "alpha", "arm", "hppa", "i386", "ia64", "mips",
"m68k", "mipsel", "powerpc", "s390", "sparc" ]
for arch in architectures:
print("*********************************** %s ******************************" % (arch))
second_filename = "%s/%s_%s" % (directory, second_version, arch)
if not os.path.exists(second_filename):
print(" -- NOT AVAILABLE --")
continue
new = read_testsummary(second_filename)
first_filename = "%s/%s_%s" % (directory, first_version, arch)
old = read_testsummary(first_filename)
compare_results(old, new)
################################################################################
def init():
"""Initalization, including parsing of options."""
usage = """usage: %prog [OPTIONS] <OLD> <NEW>
compare (binutils) dejagnu testsuite results.
Example usage:
test-suite-compare.py binutils-2.17/test-summary binutils-2.18/test-summary
Or to compare across all architectures (with test results stored in a
'test-summary' directory):
test-suite-compare.py -mtest-summary 2.17-3 2.18-1"""
parser = optparse.OptionParser(usage)
parser.add_option("-m", "--multiple", dest="multiple",
nargs=1, type="string",
help="compare multiple architectures")
(options, args) = parser.parse_args()
if len(args) > 2 or len(args) < 2:
parser.error("takes 2 arguments (old and new)")
(old_version, new_version) = args
return options, old_version, new_version
################################################################################
def main():
(options, old_version, new_version) = init()
if options.multiple:
compare_multiple(options.multiple, old_version, new_version)
else:
old = read_testsummary(old_version)
new = read_testsummary(new_version)
compare_results(old, new)
################################################################################
if __name__ == '__main__':
main()