diff options
Diffstat (limited to '3rdparty/catch/scripts/benchmarkRunner.py')
-rw-r--r-- | 3rdparty/catch/scripts/benchmarkRunner.py | 56 |
1 files changed, 56 insertions, 0 deletions
diff --git a/3rdparty/catch/scripts/benchmarkRunner.py b/3rdparty/catch/scripts/benchmarkRunner.py new file mode 100644 index 00000000000..dc753cf0050 --- /dev/null +++ b/3rdparty/catch/scripts/benchmarkRunner.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import subprocess, os, sys +import xml.etree.ElementTree as ET +from collections import defaultdict +from statistics import median, stdev +from datetime import datetime + +def get_commit_hash(): + res = subprocess.run('git rev-parse HEAD'.split(), check=True, stdout=subprocess.PIPE, universal_newlines=True) + return res.stdout.strip() + +if len(sys.argv) < 2: + print('Usage: {} benchmark-binary'.format(sys.argv[0])) + exit(1) + + +num_runs = 10 +data = defaultdict(list) + + +def parse_file(file): + + def recursive_search(node): + if node.tag == 'TestCase': + results = node.find('OverallResult') + time = results.get('durationInSeconds') + data[node.get('name')].append(float(time)) + elif node.tag in ('Group', 'Catch'): + for child in node: + recursive_search(child) + + tree = ET.parse(file) + recursive_search(tree.getroot()) + +def run_benchmarks(binary): + call = [binary] + '-d yes -r xml -o'.split() + for i in range(num_runs): + file = 'temp{}.xml'.format(i) + print('Run number {}'.format(i)) + subprocess.run(call + [file]) + parse_file(file) + # Remove file right after parsing, because benchmark output can be big + os.remove(file) + + +# Run benchmarks +run_benchmarks(sys.argv[1]) + +result_file = '{:%Y-%m-%dT%H-%M-%S}-{}.result'.format(datetime.now(), get_commit_hash()) + + +print('Writing results to {}'.format(result_file)) +with open(result_file, 'w') as file: + for k in sorted(data): + file.write('{}: median: {} (s), stddev: {} (s)\n'.format(k, median(data[k]), stdev(data[k]))) |