237 lines
10 KiB
Python
237 lines
10 KiB
Python
|
# pylint: disable=C0301,C0103,C0111
|
||
|
|
from sys import platform
|
||
|
|
from platform import machine
|
||
|
|
import os
|
||
|
|
import signal
|
||
|
|
import psutil
|
||
|
|
import subprocess
|
||
|
|
from time import monotonic
|
||
|
|
from typing import Dict, List
|
||
|
|
|
||
|
|
import xml.etree.ElementTree as ET
|
||
|
|
import glob
|
||
|
|
|
||
|
|
from tests_engine import TestResult
|
||
|
|
|
||
|
|
|
||
|
|
THIS_PATH = os.path.abspath(os.path.dirname(__file__))
|
||
|
|
|
||
|
|
|
||
|
|
def install_cli_arguments(parser):
|
||
|
|
parser.add_argument("--properties-file", action="store", help="Location of properties file.")
|
||
|
|
parser.add_argument("--skip-building", action="store_true", help="Do not build tests before run.")
|
||
|
|
parser.add_argument("--force-net-framework-version", action="store", dest="framework_ver_override", help="Override target .NET Framework version when building tests.")
|
||
|
|
|
||
|
|
|
||
|
|
class NUnitTestSuite(object):
|
||
|
|
nunit_path = os.path.join(THIS_PATH, './../lib/resources/tools/nunit3/nunit3-console.exe')
|
||
|
|
|
||
|
|
def __init__(self, path):
|
||
|
|
self.path = path
|
||
|
|
|
||
|
|
def check(self, options, number_of_runs): # API requires this method
|
||
|
|
pass
|
||
|
|
|
||
|
|
def get_output_dir(self, options, iteration_index, suite_retry_index):
|
||
|
|
# Unused mechanism, this exists to keep a uniform interface with
|
||
|
|
# robot_tests_provider.py.
|
||
|
|
return options.results_directory
|
||
|
|
|
||
|
|
# NOTE: if we switch to using msbuild on all platforms, we can get rid of this function and only use the '-' prefix
|
||
|
|
def build_params(self, *params):
|
||
|
|
def __decorate_build_param(p):
|
||
|
|
if self.builder == 'xbuild':
|
||
|
|
return '/' + p
|
||
|
|
else:
|
||
|
|
return '-' + p
|
||
|
|
|
||
|
|
ret = []
|
||
|
|
for i in params:
|
||
|
|
ret += [__decorate_build_param(i)]
|
||
|
|
return ret
|
||
|
|
|
||
|
|
def prepare(self, options):
|
||
|
|
if not options.skip_building:
|
||
|
|
self._adjust_path(options)
|
||
|
|
print("Building {0}".format(self.path))
|
||
|
|
arch = 'arm' if machine() in ['aarch64', 'arm64'] else 'i386'
|
||
|
|
if options.runner == 'dotnet':
|
||
|
|
self.builder = 'dotnet'
|
||
|
|
params = ['build', '--verbosity', 'quiet', '--configuration', options.configuration, '/p:NET=true', f'/p:Architecture={arch}']
|
||
|
|
else:
|
||
|
|
if platform == "win32":
|
||
|
|
self.builder = 'MSBuild.exe'
|
||
|
|
else:
|
||
|
|
self.builder = 'xbuild'
|
||
|
|
params = self.build_params(
|
||
|
|
f'p:PropertiesLocation={options.properties_file}',
|
||
|
|
f'p:OutputPath={options.results_directory}',
|
||
|
|
'nologo',
|
||
|
|
'verbosity:quiet',
|
||
|
|
'p:OutputDir=tests_output',
|
||
|
|
f'p:Configuration={options.configuration}',
|
||
|
|
f'p:Architecture={arch}')
|
||
|
|
if options.framework_ver_override:
|
||
|
|
params += self.build_params(f'p:TargetFrameworkVersion=v{options.framework_ver_override}')
|
||
|
|
result = subprocess.call([self.builder, *params, self.path])
|
||
|
|
if result != 0:
|
||
|
|
print("Building project `{}` failed with error code: {}".format(self.path, result))
|
||
|
|
return result
|
||
|
|
else:
|
||
|
|
print('Skipping the build')
|
||
|
|
|
||
|
|
return 0
|
||
|
|
|
||
|
|
def _adjust_path(self, options):
|
||
|
|
path, proj = os.path.split(self.path)
|
||
|
|
_match = (options.runner == 'dotnet', proj.endswith("_NET.csproj"))
|
||
|
|
if _match == (True, False):
|
||
|
|
proj_alt = proj[:-7] + "_NET.csproj"
|
||
|
|
elif _match == (False, True):
|
||
|
|
proj_alt = proj[:-11] + ".csproj"
|
||
|
|
else:
|
||
|
|
proj_alt = proj
|
||
|
|
if proj != proj_alt and os.path.exists(path_alt := os.path.join(path, proj_alt)):
|
||
|
|
print(f"{options.runner} runner detected, switching: {proj} -> {proj_alt}")
|
||
|
|
self.path = path_alt
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def _cleanup_dangling(self, process, proc_name, test_agent_name):
|
||
|
|
for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
|
||
|
|
if proc_name in (proc.info['name'] or ''):
|
||
|
|
flat_cmdline = ' '.join(proc.info['cmdline'] or [])
|
||
|
|
if test_agent_name in flat_cmdline and '--pid={}'.format(process.pid) in flat_cmdline:
|
||
|
|
# let's kill it
|
||
|
|
print('KILLING A DANGLING {} test process {}'.format(test_agent_name, proc.info['pid']))
|
||
|
|
os.kill(proc.info['pid'], signal.SIGTERM)
|
||
|
|
|
||
|
|
def run(self, options, iteration_index=1, suite_retry_index=0):
|
||
|
|
# The iteration_index and suite_retry_index arguments are not implemented.
|
||
|
|
# They exist for the sake of a uniform interface with robot_tests_provider.
|
||
|
|
print('Running ' + self.path)
|
||
|
|
|
||
|
|
project_file = os.path.split(self.path)[1]
|
||
|
|
output_file = os.path.join(options.results_directory, 'results-{}.xml'.format(project_file))
|
||
|
|
|
||
|
|
if options.runner == 'dotnet':
|
||
|
|
print('Using native dotnet test runner -' + self.path, flush=True)
|
||
|
|
# we don't build here - we had problems with concurrently occurring builds when copying files to one output directory
|
||
|
|
# so we run test with --no-build and build tests in previous stage
|
||
|
|
args = ['dotnet', 'test', "--no-build", "--logger", "console;verbosity=detailed", "--logger", "trx;LogFileName={}".format(output_file), '--configuration', options.configuration, self.path]
|
||
|
|
else:
|
||
|
|
args = [NUnitTestSuite.nunit_path, '--domain=None', '--noheader', '--labels=Before', '--result={}'.format(output_file), project_file.replace("csproj", "dll")]
|
||
|
|
|
||
|
|
# Unfortunately, debugging like this won't work on .NET, see: https://github.com/dotnet/sdk/issues/4994
|
||
|
|
# The easiest workaround is to set VSTEST_HOST_DEBUG=1 in your environment
|
||
|
|
if options.stop_on_error:
|
||
|
|
args.append('--stoponerror')
|
||
|
|
if (platform.startswith("linux") or platform == "darwin") and options.runner != 'dotnet':
|
||
|
|
args.insert(0, 'mono')
|
||
|
|
if options.port is not None:
|
||
|
|
if options.suspend:
|
||
|
|
print('Waiting for a debugger at port: {}'.format(options.port))
|
||
|
|
args.insert(1, '--debug')
|
||
|
|
args.insert(2, '--debugger-agent=transport=dt_socket,server=y,suspend={0},address=127.0.0.1:{1}'.format('y' if options.suspend else 'n', options.port))
|
||
|
|
elif options.debug_mode:
|
||
|
|
args.insert(1, '--debug')
|
||
|
|
|
||
|
|
where_conditions = []
|
||
|
|
if options.fixture:
|
||
|
|
if options.runner == 'dotnet':
|
||
|
|
where_conditions.append(options.fixture)
|
||
|
|
else:
|
||
|
|
where_conditions.append('test =~ .*{}.*'.format(options.fixture))
|
||
|
|
|
||
|
|
cat = 'TestCategory' if options.runner == 'dotnet' else 'cat'
|
||
|
|
equals = '=' if options.runner == 'dotnet' else '=='
|
||
|
|
if options.exclude:
|
||
|
|
for category in options.exclude:
|
||
|
|
where_conditions.append('{} != {}'.format(cat, category))
|
||
|
|
if options.include:
|
||
|
|
for category in options.include:
|
||
|
|
where_conditions.append('{} {} {}'.format(cat, equals, category))
|
||
|
|
|
||
|
|
if where_conditions:
|
||
|
|
if options.runner == 'dotnet':
|
||
|
|
args.append('--filter')
|
||
|
|
args.append(' & '.join('({})'.format(x) for x in where_conditions))
|
||
|
|
else:
|
||
|
|
args.append('--where= ' + ' and '.join(['({})'.format(x) for x in where_conditions]))
|
||
|
|
|
||
|
|
if options.run_gdb:
|
||
|
|
if options.runner == 'dotnet':
|
||
|
|
signals_to_handle = 'SIG34'
|
||
|
|
else:
|
||
|
|
signals_to_handle = 'SIGXCPU SIG33 SIG35 SIG36 SIGPWR'
|
||
|
|
command = ['gdb', '-nx', '-ex', 'handle ' + signals_to_handle + ' nostop noprint', '--args'] + args
|
||
|
|
|
||
|
|
startTimestamp = monotonic()
|
||
|
|
if options.runner == 'dotnet':
|
||
|
|
args += ['--', 'NUnit.DisplayName=FullName']
|
||
|
|
process = subprocess.Popen(args)
|
||
|
|
print('dotnet test runner PID is {}'.format(process.pid), flush=True)
|
||
|
|
else:
|
||
|
|
if platform != "win32":
|
||
|
|
# This is alias for '--process=Single' - means no TCP connection at all so that we can see what happens underneath
|
||
|
|
# This causes failure on some Windows setups
|
||
|
|
args.append("--inprocess")
|
||
|
|
process = subprocess.Popen(args, cwd=options.results_directory)
|
||
|
|
print('NUnit3 runner PID is {}'.format(process.pid), flush=True)
|
||
|
|
|
||
|
|
process.wait()
|
||
|
|
if options.runner == 'dotnet':
|
||
|
|
self._cleanup_dangling(process, 'dotnet', 'dotnet test')
|
||
|
|
else:
|
||
|
|
self._cleanup_dangling(process, 'mono', 'nunit-agent.exe')
|
||
|
|
|
||
|
|
result = process.returncode == 0
|
||
|
|
endTimestamp = monotonic()
|
||
|
|
print('Suite ' + self.path + (' finished successfully!' if result else ' failed!') + ' in ' + str(round(endTimestamp - startTimestamp, 2)) + ' seconds.', flush=True)
|
||
|
|
return TestResult(result, [output_file])
|
||
|
|
|
||
|
|
def cleanup(self, options):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def should_retry_suite(self, options, iteration_index, suite_retry_index):
|
||
|
|
# Unused mechanism, this exists to keep a uniform interface with
|
||
|
|
# robot_tests_provider.py.
|
||
|
|
return False
|
||
|
|
|
||
|
|
def tests_failed_due_to_renode_crash(self) -> bool:
|
||
|
|
# Unused mechanism, this exists to keep a uniform interface with
|
||
|
|
# robot_tests_provider.py.
|
||
|
|
return False
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def find_failed_tests(path, files_pattern='*.csproj.xml'):
|
||
|
|
test_files = glob.glob(os.path.join(path, files_pattern))
|
||
|
|
ret = {'mandatory': []}
|
||
|
|
for test_file in test_files:
|
||
|
|
tree = ET.parse(test_file)
|
||
|
|
root = tree.getroot()
|
||
|
|
|
||
|
|
# we analyze both types of output files (nunit and dotnet test) to avoid passing options as parameter
|
||
|
|
# the cost should be negligible in the context of compiling and running test suites
|
||
|
|
|
||
|
|
# nunit runner
|
||
|
|
for test in root.iter('test-case'):
|
||
|
|
if test.attrib['result'] == 'Failed':
|
||
|
|
ret['mandatory'].append(test.attrib['fullname'])
|
||
|
|
|
||
|
|
# dotnet runner
|
||
|
|
xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010"
|
||
|
|
for test in root.iter(f"{{{xmlns}}}UnitTestResult"):
|
||
|
|
if test.attrib['outcome'] == 'Failed':
|
||
|
|
ret['mandatory'].append(test.attrib['testName'])
|
||
|
|
|
||
|
|
if not ret['mandatory']:
|
||
|
|
return None
|
||
|
|
return ret
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def find_rerun_tests(path):
|
||
|
|
# Unused mechanism, this exists to keep a uniform interface with
|
||
|
|
# robot_tests_provider.py.
|
||
|
|
return None
|