summaryrefslogtreecommitdiffstatshomepage
path: root/3rdparty/catch/scripts/approvalTests.py
diff options
context:
space:
mode:
author Miodrag Milanovic <mmicko@gmail.com>2017-02-05 15:45:26 +0100
committer Miodrag Milanovic <mmicko@gmail.com>2017-02-05 15:46:07 +0100
commit02c97cd0f7c4d630f1a90f9f2626746e47a28c09 (patch)
tree2b48d051a55f2b65525824546cd9fcf9e05907cf /3rdparty/catch/scripts/approvalTests.py
parentd20b4ba6864b45d0c068d98d43e63c8bc5014a3b (diff)
Updated Catch to latest (nw)
Diffstat (limited to '3rdparty/catch/scripts/approvalTests.py')
-rw-r--r--3rdparty/catch/scripts/approvalTests.py221
1 files changed, 129 insertions, 92 deletions
diff --git a/3rdparty/catch/scripts/approvalTests.py b/3rdparty/catch/scripts/approvalTests.py
index 618812f888d..6c51aa4eff2 100644
--- a/3rdparty/catch/scripts/approvalTests.py
+++ b/3rdparty/catch/scripts/approvalTests.py
@@ -1,115 +1,152 @@
-from __future__ import print_function
+#!/usr/bin/env python
+
+from __future__ import print_function
import os
import sys
import subprocess
import re
+import difflib
+import scriptCommon
from scriptCommon import catchPath
-rootPath = os.path.join( catchPath, 'projects/SelfTest/Baselines' )
+rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines')
+
-filenameParser = re.compile( r'(.*)/(.*\..pp:)(.*)' )
-filelineParser = re.compile( r'(.*\..pp:)([0-9]*)(.*)' )
-pathParser = re.compile( r'(.*?)/(.*\..pp)(.*)' )
-lineNumberParser = re.compile( r'(.*)line="[0-9]*"(.*)' )
-hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' )
-durationsParser = re.compile( r'(.*)time="[0-9]*\.[0-9]*"(.*)' )
-versionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)' )
-devVersionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)' )
+filelocParser = re.compile(r'''
+ .*/
+ (.+\.[ch]pp) # filename
+ (?::|\() # : is starting separator between filename and line number on Linux, ( on Windows
+ ([0-9]*) # line number
+ \)? # Windows also has an ending separator, )
+''', re.VERBOSE)
+lineNumberParser = re.compile(r' line="[0-9]*"')
+hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b')
+durationsParser = re.compile(r' time="[0-9]*\.[0-9]*"')
+timestampsParser = re.compile(r' timestamp="\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}Z"')
+versionParser = re.compile(r'Catch v[0-9]+\.[0-9]+\.[0-9]+(-develop\.[0-9]+)?')
+nullParser = re.compile(r'\b(__null|nullptr)\b')
+exeNameParser = re.compile(r'''
+ \b
+ (CatchSelfTest|SelfTest) # Expected executable name
+ (?:.exe)? # Executable name contains .exe on Windows.
+ \b
+''', re.VERBOSE)
+# This is a hack until something more reasonable is figured out
+specialCaseParser = re.compile(r'file\((\d+)\)')
if len(sys.argv) == 2:
- cmdPath = sys.argv[1]
+ cmdPath = sys.argv[1]
else:
- cmdPath = os.path.join( catchPath, 'projects/XCode/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' )
+ cmdPath = os.path.join(catchPath, scriptCommon.getBuildExecutable())
overallResult = 0
-def filterLine( line ):
- m = filenameParser.match( line )
- if m:
- line = m.group(2) + m.group(3)
- m2 = filelineParser.match( line )
- if m2:
- line = m2.group(1) + "<line number>" + m2.group(3)
- else:
- m2 = lineNumberParser.match( line )
- if m2:
- line = m2.group(1) + m2.group(2)
- m = pathParser.match( line )
- if m:
- path = "/" + m.group(2)
- if path.startswith( catchPath ):
- path = path[1+len(catchPath):]
- line = m.group(1) + path + m.group(3)
- m = devVersionParser.match( line )
- if m:
- line = m.group(1) + "<version>" + m.group(2)
- else:
- m = versionParser.match( line )
- if m:
- line = m.group(1) + "<version>" + m.group(2)
-
- while True:
- m = hexParser.match( line )
- if m:
- line = m.group(1) + "0x<hex digits>" + m.group(3)
- else:
- break
- m = durationsParser.match( line )
- if m:
- line = m.group(1) + 'time="{duration}"' + m.group(2)
- return line
-
-def approve( baseName, args ):
- global overallResult
- args[0:0] = [cmdPath]
- if not os.path.exists( cmdPath ):
- raise Exception( "Executable doesn't exist at " + cmdPath )
- baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) )
- rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) )
- filteredResultsPath = os.path.join( rootPath, '{0}.unapproved.txt'.format( baseName ) )
-
- f = open( rawResultsPath, 'w' )
- subprocess.call( args, stdout=f, stderr=f )
- f.close()
-
- rawFile = open( rawResultsPath, 'r' )
- filteredFile = open( filteredResultsPath, 'w' )
- for line in rawFile:
- filteredFile.write( filterLine( line ).rstrip() + "\n" )
- filteredFile.close()
- rawFile.close()
-
- os.remove( rawResultsPath )
- print()
- print( baseName + ":" )
- if os.path.exists( baselinesPath ):
- diffResult = subprocess.call([ "diff", baselinesPath, filteredResultsPath ] )
- if diffResult == 0:
- os.remove( filteredResultsPath )
- print( " \033[92mResults matched" )
- else:
- print( " \n****************************\n \033[91mResults differed" )
- if diffResult > overallResult:
- overallResult = diffResult
- print( "\033[0m" )
- else:
- print( " first approval" )
- if overallResult == 0:
- overallResult = 1
+def diffFiles(fileA, fileB):
+ with open(fileA, 'r') as file:
+ aLines = file.readlines()
+ with open(fileB, 'r') as file:
+ bLines = file.readlines()
+
+ shortenedFilenameA = fileA.rsplit(os.sep, 1)[-1]
+ shortenedFilenameB = fileB.rsplit(os.sep, 1)[-1]
+
+ diff = difflib.unified_diff(aLines, bLines, fromfile=shortenedFilenameA, tofile=shortenedFilenameB, n=0)
+ return [line for line in diff if line[0] in ('+', '-')]
+
+
+def filterLine(line):
+ if catchPath in line:
+ # make paths relative to Catch root
+ line = line.replace(catchPath + os.sep, '')
+ # go from \ in windows paths to /
+ line = line.replace('\\', '/')
+
+
+ # strip source line numbers
+ m = filelocParser.match(line)
+ if m:
+ # note that this also strips directories, leaving only the filename
+ filename, lnum = m.groups()
+ lnum = ":<line number>" if lnum else ""
+ line = filename + lnum + line[m.end():]
+ else:
+ line = lineNumberParser.sub(" ", line)
+
+ # strip Catch version number
+ line = versionParser.sub("<version>", line)
+
+ # replace *null* with 0
+ line = nullParser.sub("0", line)
+
+ # strip executable name
+ line = exeNameParser.sub("<exe-name>", line)
+
+ # strip hexadecimal numbers (presumably pointer values)
+ line = hexParser.sub("0x<hex digits>", line)
+
+ # strip durations and timestamps
+ line = durationsParser.sub(' time="{duration}"', line)
+ line = timestampsParser.sub(' timestamp="{iso8601-timestamp}"', line)
+ line = specialCaseParser.sub('file:\g<1>', line)
+ return line
+
+
+def approve(baseName, args):
+ global overallResult
+ args[0:0] = [cmdPath]
+ if not os.path.exists(cmdPath):
+ raise Exception("Executable doesn't exist at " + cmdPath)
+ baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
+ rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
+ filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
+
+ f = open(rawResultsPath, 'w')
+ subprocess.call(args, stdout=f, stderr=f)
+ f.close()
+
+ rawFile = open(rawResultsPath, 'r')
+ filteredFile = open(filteredResultsPath, 'w')
+ for line in rawFile:
+ filteredFile.write(filterLine(line).rstrip() + "\n")
+ filteredFile.close()
+ rawFile.close()
+
+ os.remove(rawResultsPath)
+ print()
+ print(baseName + ":")
+ if os.path.exists(baselinesPath):
+ diffResult = diffFiles(baselinesPath, filteredResultsPath)
+ if diffResult:
+ print(''.join(diffResult))
+ print(" \n****************************\n \033[91mResults differed")
+ if len(diffResult) > overallResult:
+ overallResult = len(diffResult)
+ else:
+ os.remove(filteredResultsPath)
+ print(" \033[92mResults matched")
+ print("\033[0m")
+ else:
+ print(" first approval")
+ if overallResult == 0:
+ overallResult = 1
+
+
+print("Running approvals against executable:")
+print(" " + cmdPath)
# Standard console reporter
-approve( "console.std", ["~_"] )
+approve("console.std", ["~[c++11]~[!nonportable]", "--order", "lex"])
# console reporter, include passes, warn about No Assertions
-approve( "console.sw", ["~_", "-s", "-w", "NoAssertions"] )
+approve("console.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"])
# console reporter, include passes, warn about No Assertions, limit failures to first 4
-approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4"] )
+approve("console.swa4", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
# junit reporter, include passes, warn about No Assertions
-approve( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit"] )
+approve("junit.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
# xml reporter, include passes, warn about No Assertions
-approve( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml"] )
+approve("xml.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
if overallResult != 0:
- print( "run approve.py to approve new baselines" )
-exit( overallResult)
+ print("If these differenecs are expected run approve.py to approve new baselines")
+exit(overallResult)