1180 lines
53 KiB
Python
1180 lines
53 KiB
Python
|
# pylint: disable=C0301,C0103,C0111
|
||
|
|
from __future__ import print_function
|
||
|
|
from sys import platform
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import socket
|
||
|
|
import fnmatch
|
||
|
|
import subprocess
|
||
|
|
import psutil
|
||
|
|
import shutil
|
||
|
|
import tempfile
|
||
|
|
import uuid
|
||
|
|
import re
|
||
|
|
import signal
|
||
|
|
from collections import OrderedDict, defaultdict
|
||
|
|
from time import monotonic, sleep
|
||
|
|
from typing import List, Dict, Tuple, Set
|
||
|
|
from argparse import Namespace
|
||
|
|
|
||
|
|
import robot, robot.result, robot.running
|
||
|
|
from robot.libraries.BuiltIn import BuiltIn
|
||
|
|
from robot.libraries.DateTime import Time
|
||
|
|
|
||
|
|
import xml.etree.ElementTree as ET
|
||
|
|
|
||
|
|
from tests_engine import TestResult
|
||
|
|
|
||
|
|
this_path = os.path.abspath(os.path.dirname(__file__))
|
||
|
|
|
||
|
|
|
||
|
|
class Timeout:
|
||
|
|
def __init__(self, value: str):
|
||
|
|
self.seconds = Time(value).seconds
|
||
|
|
self.value = value
|
||
|
|
|
||
|
|
def __repr__(self):
|
||
|
|
return f"{self.value} ({self.seconds}s)"
|
||
|
|
|
||
|
|
|
||
|
|
def install_cli_arguments(parser):
|
||
|
|
group = parser.add_mutually_exclusive_group()
|
||
|
|
|
||
|
|
group.add_argument("--robot-framework-remote-server-full-directory",
|
||
|
|
dest="remote_server_full_directory",
|
||
|
|
action="store",
|
||
|
|
help="Full location of robot framework remote server binary.")
|
||
|
|
|
||
|
|
group.add_argument("--robot-framework-remote-server-directory-prefix",
|
||
|
|
dest="remote_server_directory_prefix",
|
||
|
|
action="store",
|
||
|
|
default=os.path.join(this_path, '../output/bin'),
|
||
|
|
help="Directory of robot framework remote server binary. This is concatenated with current configuration to create full path.")
|
||
|
|
|
||
|
|
parser.add_argument("--robot-framework-remote-server-name",
|
||
|
|
dest="remote_server_name",
|
||
|
|
action="store",
|
||
|
|
default="Renode.exe",
|
||
|
|
help="Name of robot framework remote server binary.")
|
||
|
|
|
||
|
|
parser.add_argument("--robot-framework-remote-server-port", "-P",
|
||
|
|
dest="remote_server_port",
|
||
|
|
action="store",
|
||
|
|
default=0,
|
||
|
|
type=int,
|
||
|
|
help="Port of robot framework remote server binary. Use '0' to automatically select any unused private port.")
|
||
|
|
|
||
|
|
parser.add_argument("--enable-xwt",
|
||
|
|
dest="enable_xwt",
|
||
|
|
action="store_true",
|
||
|
|
default=False,
|
||
|
|
help="Enables support for XWT.")
|
||
|
|
|
||
|
|
parser.add_argument("--show-log",
|
||
|
|
dest="show_log",
|
||
|
|
action="store_true",
|
||
|
|
default=False,
|
||
|
|
help="Display log messages in console (might corrupt robot summary output).")
|
||
|
|
|
||
|
|
parser.add_argument("--keep-renode-output",
|
||
|
|
dest="keep_renode_output",
|
||
|
|
action="store_true",
|
||
|
|
default=False,
|
||
|
|
help=" ".join([
|
||
|
|
"Redirect Renode stdout and stderr to log files.",
|
||
|
|
"Only non-empty log files are kept (i.e. up to 2 per suite).",
|
||
|
|
"This is separate from the usual logs generated by RobotFramework.",
|
||
|
|
"Implies --show-log (output is redirected and does not appear in console).",
|
||
|
|
]))
|
||
|
|
|
||
|
|
parser.add_argument("--verbose",
|
||
|
|
dest="verbose",
|
||
|
|
action="store_true",
|
||
|
|
default=False,
|
||
|
|
help="Print verbose info from Robot Framework.")
|
||
|
|
|
||
|
|
parser.add_argument("--hot-spot",
|
||
|
|
dest="hotspot",
|
||
|
|
action="store",
|
||
|
|
default=None,
|
||
|
|
help="Test given hot spot action.")
|
||
|
|
|
||
|
|
parser.add_argument("--variable",
|
||
|
|
dest="variables",
|
||
|
|
action="append",
|
||
|
|
default=None,
|
||
|
|
help="Variable to pass to Robot.")
|
||
|
|
|
||
|
|
parser.add_argument("--css-file",
|
||
|
|
dest="css_file",
|
||
|
|
action="store",
|
||
|
|
default=os.path.join(this_path, '../lib/resources/styles/robot.css'),
|
||
|
|
help="Custom CSS style for the result files.")
|
||
|
|
|
||
|
|
parser.add_argument("--debug-on-error",
|
||
|
|
dest="debug_on_error",
|
||
|
|
action="store_true",
|
||
|
|
default=False,
|
||
|
|
help="Enables the Renode User Interface when test fails.")
|
||
|
|
|
||
|
|
parser.add_argument("--cleanup-timeout",
|
||
|
|
dest="cleanup_timeout",
|
||
|
|
action="store",
|
||
|
|
default=3,
|
||
|
|
type=int,
|
||
|
|
help="Robot frontend process cleanup timeout in seconds.")
|
||
|
|
|
||
|
|
parser.add_argument("--kill-timeout",
|
||
|
|
dest="kill_timeout",
|
||
|
|
action="store",
|
||
|
|
default=15,
|
||
|
|
type=int,
|
||
|
|
help="Robot frontend process kill timeout in seconds if cleanup attempt fails.")
|
||
|
|
|
||
|
|
parser.add_argument("--listener",
|
||
|
|
action="append",
|
||
|
|
help="Path to additional progress listener (can be provided many times).")
|
||
|
|
|
||
|
|
parser.add_argument("--renode-config",
|
||
|
|
dest="renode_config",
|
||
|
|
action="store",
|
||
|
|
default=None,
|
||
|
|
help="Path to the Renode config file.")
|
||
|
|
|
||
|
|
parser.add_argument("--kill-stale-renode-instances",
|
||
|
|
dest="autokill_renode",
|
||
|
|
action="store_true",
|
||
|
|
default=False,
|
||
|
|
help="Automatically kill stale Renode instances without asking.")
|
||
|
|
|
||
|
|
parser.add_argument("--gather-execution-metrics",
|
||
|
|
dest="execution_metrics",
|
||
|
|
action="store_true",
|
||
|
|
default=False,
|
||
|
|
help="Gather execution metrics for each suite.")
|
||
|
|
|
||
|
|
parser.add_argument("--test-timeout",
|
||
|
|
dest="timeout",
|
||
|
|
action="store",
|
||
|
|
default=None,
|
||
|
|
type=Timeout,
|
||
|
|
help=" ".join([
|
||
|
|
"Default test case timeout after which Renode keywords will be interrupted.",
|
||
|
|
"It's parsed by Robot Framework's DateTime library so all its time formats are supported.",
|
||
|
|
]))
|
||
|
|
|
||
|
|
|
||
|
|
def verify_cli_arguments(options):
|
||
|
|
# port is not available on Windows
|
||
|
|
if platform != "win32":
|
||
|
|
if options.port == str(options.remote_server_port):
|
||
|
|
raise Exception('Port {} is reserved for Robot Framework remote server and cannot be used for remote debugging.'.format(options.remote_server_port))
|
||
|
|
if options.port is not None and options.jobs != 1:
|
||
|
|
raise Exception("Debug port cannot be used in parallel runs")
|
||
|
|
|
||
|
|
if options.css_file:
|
||
|
|
if not os.path.isabs(options.css_file):
|
||
|
|
options.css_file = os.path.join(this_path, options.css_file)
|
||
|
|
|
||
|
|
if not os.path.isfile(options.css_file):
|
||
|
|
raise Exception("Unable to find provided CSS file: {0}.".format(options.css_file))
|
||
|
|
|
||
|
|
if options.remote_server_port != 0 and options.jobs != 1:
|
||
|
|
raise Exception("Parallel execution and fixed Robot port number options cannot be used together")
|
||
|
|
|
||
|
|
def is_process_running(pid):
|
||
|
|
if not psutil.pid_exists(pid):
|
||
|
|
return False
|
||
|
|
proc = psutil.Process(pid)
|
||
|
|
# docs note: is_running() will return True also if the process is a zombie (p.status() == psutil.STATUS_ZOMBIE)
|
||
|
|
return proc.is_running() and proc.status() != psutil.STATUS_ZOMBIE
|
||
|
|
|
||
|
|
|
||
|
|
def is_port_available(port, autokill):
|
||
|
|
port_handle = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||
|
|
available = False
|
||
|
|
try:
|
||
|
|
port_handle.bind(("localhost", port))
|
||
|
|
port_handle.close()
|
||
|
|
available = True
|
||
|
|
except:
|
||
|
|
available = can_be_freed_by_killing_other_job(port, autokill)
|
||
|
|
return available
|
||
|
|
|
||
|
|
|
||
|
|
def can_be_freed_by_killing_other_job(port, autokill):
|
||
|
|
if not sys.stdin.isatty():
|
||
|
|
return
|
||
|
|
try:
|
||
|
|
for proc in [psutil.Process(pid) for pid in psutil.pids()]:
|
||
|
|
if '--robot-server-port' in proc.cmdline() and str(port) in proc.cmdline():
|
||
|
|
if not is_process_running(proc.pid):
|
||
|
|
# process is zombie
|
||
|
|
continue
|
||
|
|
|
||
|
|
if autokill:
|
||
|
|
result = 'y'
|
||
|
|
else:
|
||
|
|
print('It seems that Renode process (pid {}, name {}) is currently running on port {}'.format(proc.pid, proc.name(), port))
|
||
|
|
result = input('Do you want me to kill it? [y/N] ')
|
||
|
|
|
||
|
|
if result in ['Y', 'y']:
|
||
|
|
proc.kill()
|
||
|
|
return True
|
||
|
|
break
|
||
|
|
except Exception:
|
||
|
|
# do nothing here
|
||
|
|
pass
|
||
|
|
return False
|
||
|
|
|
||
|
|
def report_dead_subprocess(process):
|
||
|
|
exception_message = "\n".join([
|
||
|
|
f"{process.executable} process died(pid == -1)?? Here are some details",
|
||
|
|
f"\t stdout = {process.stdout}",
|
||
|
|
f"\t stderr = {process.stderr}",
|
||
|
|
f"\t return code = {process.returncode}"])
|
||
|
|
raise RuntimeError(exception_message)
|
||
|
|
|
||
|
|
|
||
|
|
class KeywordsFinder(robot.model.SuiteVisitor):
|
||
|
|
def __init__(self, keyword):
|
||
|
|
self.keyword = keyword
|
||
|
|
self.occurences = 0
|
||
|
|
self.arguments = []
|
||
|
|
|
||
|
|
|
||
|
|
def visit_keyword(self, keyword):
|
||
|
|
if keyword.name == self.keyword:
|
||
|
|
self.occurences += 1
|
||
|
|
arguments = keyword.args
|
||
|
|
self.arguments.append(arguments)
|
||
|
|
|
||
|
|
|
||
|
|
def got_results(self):
|
||
|
|
return self.occurences > 0
|
||
|
|
|
||
|
|
|
||
|
|
class TestsFinder(robot.model.SuiteVisitor):
|
||
|
|
def __init__(self, keyword):
|
||
|
|
self.keyword = keyword
|
||
|
|
self.tests_matching = []
|
||
|
|
self.tests_not_matching = []
|
||
|
|
|
||
|
|
|
||
|
|
def isMatching(self, test):
|
||
|
|
finder = KeywordsFinder(self.keyword)
|
||
|
|
test.visit(finder)
|
||
|
|
return finder.got_results()
|
||
|
|
|
||
|
|
|
||
|
|
def visit_test(self, test):
|
||
|
|
if self.isMatching(test):
|
||
|
|
self.tests_matching.append(test)
|
||
|
|
else:
|
||
|
|
self.tests_not_matching.append(test)
|
||
|
|
|
||
|
|
|
||
|
|
class RobotTestSuite(object):
|
||
|
|
after_timeout_message_suffix = ' '.join([
|
||
|
|
"Failed on Renode restarted after timeout,",
|
||
|
|
"will be retried if `-N/--retry` option was used."
|
||
|
|
])
|
||
|
|
instances_count = 0
|
||
|
|
robot_frontend_process = None
|
||
|
|
renode_pid = -1 # It's not always robot_frontend_process.pid, e.g., with `--run-gdb` option.
|
||
|
|
hotspot_action = ['None', 'Pause', 'Serialize']
|
||
|
|
# Used to share the port between all suites when running sequentially
|
||
|
|
remote_server_port = -1
|
||
|
|
retry_test_regex = re.compile(r"\[RETRY\] (PASS|FAIL) on (\d+)\. retry\.")
|
||
|
|
retry_suite_regex = re.compile(r"|".join((
|
||
|
|
r"\[Errno \d+\] Connection refused",
|
||
|
|
r"Connection to remote server broken: \[WinError \d+\]",
|
||
|
|
r"Connecting remote server at [^ ]+ failed",
|
||
|
|
"Getting keyword names from library 'Remote' failed",
|
||
|
|
after_timeout_message_suffix,
|
||
|
|
)))
|
||
|
|
timeout_expected_tag = 'timeout_expected'
|
||
|
|
|
||
|
|
def __init__(self, path):
|
||
|
|
self.path = path
|
||
|
|
self._dependencies_met = set()
|
||
|
|
self.remote_server_directory = None
|
||
|
|
# Subset of RobotTestSuite.log_files which are "owned" by the running instance
|
||
|
|
self.suite_log_files = None
|
||
|
|
|
||
|
|
self.tests_with_hotspots = []
|
||
|
|
self.tests_without_hotspots = []
|
||
|
|
|
||
|
|
|
||
|
|
def get_output_dir(self, options, iteration_index, suite_retry_index):
|
||
|
|
return os.path.join(
|
||
|
|
options.results_directory,
|
||
|
|
f"iteration{iteration_index}" if options.iteration_count > 1 else "",
|
||
|
|
f"retry{suite_retry_index}" if options.retry_count > 1 else "",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def prepare(self, options):
|
||
|
|
RobotTestSuite.instances_count += 1
|
||
|
|
|
||
|
|
hotSpotTestFinder = TestsFinder(keyword="Handle Hot Spot")
|
||
|
|
suiteBuilder = robot.running.builder.TestSuiteBuilder()
|
||
|
|
suite = suiteBuilder.build(self.path)
|
||
|
|
suite.visit(hotSpotTestFinder)
|
||
|
|
|
||
|
|
self.tests_with_hotspots = [test.name for test in hotSpotTestFinder.tests_matching]
|
||
|
|
self.tests_without_hotspots = [test.name for test in hotSpotTestFinder.tests_not_matching]
|
||
|
|
|
||
|
|
# In parallel runs, Renode is started for each suite.
|
||
|
|
# The same is done in sequential runs with --keep-renode-output.
|
||
|
|
# see: run
|
||
|
|
if options.jobs == 1 and not options.keep_renode_output:
|
||
|
|
if not RobotTestSuite._is_frontend_running():
|
||
|
|
RobotTestSuite.robot_frontend_process = self._run_remote_server(options)
|
||
|
|
# Save port to reuse when running sequentially
|
||
|
|
RobotTestSuite.remote_server_port = self.remote_server_port
|
||
|
|
else:
|
||
|
|
# Restore port allocated by a previous suite
|
||
|
|
self.remote_server_port = RobotTestSuite.remote_server_port
|
||
|
|
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def _is_frontend_running(cls):
|
||
|
|
return cls.robot_frontend_process is not None and is_process_running(cls.robot_frontend_process.pid)
|
||
|
|
|
||
|
|
|
||
|
|
def _run_remote_server(self, options, iteration_index=1, suite_retry_index=0, remote_server_port=None):
|
||
|
|
# Let's reset PID and check it's set before returning to prevent keeping old PID.
|
||
|
|
self.renode_pid = -1
|
||
|
|
|
||
|
|
if options.runner == 'dotnet':
|
||
|
|
remote_server_name = "Renode.dll"
|
||
|
|
else:
|
||
|
|
remote_server_name = options.remote_server_name
|
||
|
|
|
||
|
|
self.remote_server_directory = options.remote_server_full_directory
|
||
|
|
remote_server_binary = os.path.join(self.remote_server_directory, remote_server_name)
|
||
|
|
|
||
|
|
if not os.path.isfile(remote_server_binary):
|
||
|
|
raise Exception("Robot framework remote server binary not found: '{}'! Did you forget to build?".format(remote_server_binary))
|
||
|
|
|
||
|
|
if remote_server_port is None:
|
||
|
|
remote_server_port = options.remote_server_port
|
||
|
|
|
||
|
|
if remote_server_port != 0 and not is_port_available(remote_server_port, options.autokill_renode):
|
||
|
|
raise Exception("The selected port {} is not available".format(remote_server_port))
|
||
|
|
|
||
|
|
command = [remote_server_binary, '--robot-server-port', str(remote_server_port)]
|
||
|
|
if not options.show_log and not options.keep_renode_output:
|
||
|
|
command.append('--hide-log')
|
||
|
|
if not options.enable_xwt:
|
||
|
|
command.append('--disable-gui')
|
||
|
|
if options.debug_on_error:
|
||
|
|
command.append('--robot-debug-on-error')
|
||
|
|
if options.keep_temps:
|
||
|
|
command.append('--keep-temporary-files')
|
||
|
|
if options.renode_config:
|
||
|
|
command.append('--config')
|
||
|
|
command.append(options.renode_config)
|
||
|
|
|
||
|
|
if options.runner == 'mono':
|
||
|
|
command.insert(0, 'mono')
|
||
|
|
if options.port is not None:
|
||
|
|
if options.suspend:
|
||
|
|
print('Waiting for a debugger at port: {}'.format(options.port))
|
||
|
|
command.insert(1, '--debug')
|
||
|
|
command.insert(2, '--debugger-agent=transport=dt_socket,server=y,suspend={0},address=127.0.0.1:{1}'.format('y' if options.suspend else 'n', options.port))
|
||
|
|
elif options.debug_mode:
|
||
|
|
command.insert(1, '--debug')
|
||
|
|
options.exclude.append('skip_mono')
|
||
|
|
elif options.runner == 'dotnet':
|
||
|
|
command.insert(0, 'dotnet')
|
||
|
|
options.exclude.append('skip_dotnet')
|
||
|
|
|
||
|
|
renode_command = command
|
||
|
|
stdout_path, stderr_path = None, None
|
||
|
|
|
||
|
|
# if we started GDB, wait for the user to start Renode as a child process
|
||
|
|
if options.run_gdb:
|
||
|
|
if options.runner == 'dotnet':
|
||
|
|
signals_to_handle = 'SIG34'
|
||
|
|
else:
|
||
|
|
signals_to_handle = 'SIGXCPU SIG33 SIG35 SIG36 SIGPWR'
|
||
|
|
command = ['gdb', '-nx', '-ex', 'handle ' + signals_to_handle + ' nostop noprint', '--args'] + command
|
||
|
|
process = psutil.Popen(command, cwd=self.remote_server_directory, bufsize=1)
|
||
|
|
|
||
|
|
if options.keep_renode_output:
|
||
|
|
print("Note: --keep-renode-output is not supported when using --run-gdb")
|
||
|
|
|
||
|
|
if process.pid == -1:
|
||
|
|
report_dead_subprocess(process)
|
||
|
|
|
||
|
|
print("Waiting for Renode process to start")
|
||
|
|
while True:
|
||
|
|
# We strip argv[0] because if we pass just `mono` to GDB it will resolve
|
||
|
|
# it to a full path to mono on the PATH, for example /bin/mono
|
||
|
|
renode_child = next((c for c in process.children() if c.cmdline()[1:] == renode_command[1:]), None)
|
||
|
|
if renode_child:
|
||
|
|
break
|
||
|
|
sleep(0.5)
|
||
|
|
self.renode_pid = renode_child.pid
|
||
|
|
elif options.perf_output_path:
|
||
|
|
pid_file_uuid = uuid.uuid4()
|
||
|
|
pid_filename = f'pid_file_{pid_file_uuid}'
|
||
|
|
|
||
|
|
command = ['perf', 'record', '-q', '-g', '-F', 'max'] + command + ['--pid-file', pid_filename]
|
||
|
|
|
||
|
|
stdout_path = "perf_stdout_stderr"
|
||
|
|
|
||
|
|
if options.keep_renode_output:
|
||
|
|
print("Note: --keep-renode-output is not supported when using --perf-output-path")
|
||
|
|
|
||
|
|
print(f"WARNING: perf stdout and stderr is being redirected to {stdout_path}")
|
||
|
|
|
||
|
|
perf_stdout_stderr_file = open(stdout_path, "w")
|
||
|
|
process = subprocess.Popen(command, cwd=self.remote_server_directory, bufsize=1, stdout=perf_stdout_stderr_file, stderr=perf_stdout_stderr_file)
|
||
|
|
|
||
|
|
pid_file_path = os.path.join(self.remote_server_directory, pid_filename)
|
||
|
|
perf_renode_timeout = 10
|
||
|
|
|
||
|
|
if process.pid == -1:
|
||
|
|
report_dead_subprocess(process)
|
||
|
|
|
||
|
|
while not os.path.exists(pid_file_path) and perf_renode_timeout > 0:
|
||
|
|
sleep(0.5)
|
||
|
|
perf_renode_timeout -= 1
|
||
|
|
|
||
|
|
if perf_renode_timeout <= 0:
|
||
|
|
raise RuntimeError("Renode pid file could not be found, can't attach perf")
|
||
|
|
|
||
|
|
with open(pid_file_path, 'r') as pid_file:
|
||
|
|
self.renode_pid = pid_file.read()
|
||
|
|
else:
|
||
|
|
# Start Renode
|
||
|
|
file_name = os.path.splitext(os.path.basename(self.path))[0]
|
||
|
|
os.environ["GCOV_PREFIX"] = os.path.join(os.getcwd(), f"output/coverage/{file_name}")
|
||
|
|
if options.keep_renode_output:
|
||
|
|
output_dir = self.get_output_dir(options, iteration_index, suite_retry_index)
|
||
|
|
logs_dir = os.path.join(output_dir, 'logs')
|
||
|
|
os.makedirs(logs_dir, exist_ok=True)
|
||
|
|
suite_name = RobotTestSuite._create_suite_name(file_name, None)
|
||
|
|
stdout_path = os.path.join(logs_dir, f"{suite_name}.renode_stdout.log")
|
||
|
|
stderr_path = os.path.join(logs_dir, f"{suite_name}.renode_stderr.log")
|
||
|
|
fout = open(stdout_path, "wb", buffering=0)
|
||
|
|
ferr = open(stderr_path, "wb", buffering=0)
|
||
|
|
process = subprocess.Popen(command, cwd=self.remote_server_directory, bufsize=1, stdout=fout, stderr=ferr)
|
||
|
|
if process.pid == -1:
|
||
|
|
report_dead_subprocess(process)
|
||
|
|
|
||
|
|
self.renode_pid = process.pid
|
||
|
|
else:
|
||
|
|
process = subprocess.Popen(command, cwd=self.remote_server_directory, bufsize=1)
|
||
|
|
if process.pid == -1:
|
||
|
|
report_dead_subprocess(process)
|
||
|
|
self.renode_pid = process.pid
|
||
|
|
|
||
|
|
assert self.renode_pid != -1, "Renode PID has to be set before trying to acces the port file"
|
||
|
|
|
||
|
|
timeout_s = 180
|
||
|
|
countdown = float(timeout_s)
|
||
|
|
temp_dir = tempfile.gettempdir()
|
||
|
|
renode_port_file = os.path.join(temp_dir, f'renode-{self.renode_pid}', 'robot_port')
|
||
|
|
while countdown > 0:
|
||
|
|
try:
|
||
|
|
with open(renode_port_file, 'r') as f:
|
||
|
|
port_num = f.readline()
|
||
|
|
if port_num == '':
|
||
|
|
continue
|
||
|
|
self.remote_server_port = int(port_num)
|
||
|
|
break
|
||
|
|
except (FileNotFoundError, PermissionError):
|
||
|
|
sleep(0.5)
|
||
|
|
countdown -= 0.5
|
||
|
|
else:
|
||
|
|
# PID is preserved for error message as the field gets reset in `self._close_remote_server`.
|
||
|
|
process_pid = self.renode_pid
|
||
|
|
self.log_process_data(process, stdout_path, stderr_path)
|
||
|
|
self._close_remote_server(process, options)
|
||
|
|
raise TimeoutError(f"Couldn't access port file for Renode instance pid {process_pid}; timed out after {timeout_s}s")
|
||
|
|
|
||
|
|
# If a certain port was expected, let's make sure Renode uses it.
|
||
|
|
if remote_server_port and remote_server_port != self.remote_server_port:
|
||
|
|
self._close_remote_server(process, options)
|
||
|
|
raise RuntimeError(f"Renode was expected to use port {remote_server_port} but {self.remote_server_port} port is used instead!")
|
||
|
|
|
||
|
|
return process
|
||
|
|
|
||
|
|
def log_process_data(self, process, stdout_path, stderr_path):
|
||
|
|
process.poll()
|
||
|
|
|
||
|
|
separator = "=" * 80
|
||
|
|
process_summary = "\n".join(
|
||
|
|
[
|
||
|
|
separator,
|
||
|
|
f"Process `{' '.join(process.args)}` [pid: {process.pid}] state:",
|
||
|
|
f"return code: {process.returncode}",
|
||
|
|
]
|
||
|
|
)
|
||
|
|
if stdout_path:
|
||
|
|
with open(stdout_path, 'rt') as stdout:
|
||
|
|
process_summary += f"\nstdout: \n{stdout.read()}"
|
||
|
|
if stderr_path:
|
||
|
|
with open(stderr_path, 'rt') as stderr:
|
||
|
|
process_summary += f"\nstderr: \n{stderr.read()}"
|
||
|
|
process_summary += separator
|
||
|
|
print(process_summary)
|
||
|
|
|
||
|
|
def __move_perf_data(self, options):
|
||
|
|
perf_data_path = os.path.join(self.remote_server_directory, "perf.data")
|
||
|
|
|
||
|
|
if not perf_data_path:
|
||
|
|
raise RuntimeError("perf.data file was not generated succesfully")
|
||
|
|
|
||
|
|
if not os.path.isdir(options.perf_output_path):
|
||
|
|
raise RuntimeError(f"{options.perf_output_path} is not a valid directory path")
|
||
|
|
|
||
|
|
shutil.move(perf_data_path, options.perf_output_path)
|
||
|
|
|
||
|
|
def _close_remote_server(self, proc, options):
|
||
|
|
if not proc:
|
||
|
|
return
|
||
|
|
|
||
|
|
renode = f"Renode pid {proc.pid}"
|
||
|
|
renode_killed = False
|
||
|
|
|
||
|
|
# Let's prevent using these after the server is closed.
|
||
|
|
self.robot_frontend_process = None
|
||
|
|
self.renode_pid = -1
|
||
|
|
|
||
|
|
# None of the previously provided states will be available.
|
||
|
|
self._dependencies_met = set()
|
||
|
|
|
||
|
|
# poll returns exit code if process exited which means it doesn't need to be closed.
|
||
|
|
if (exit_code := proc.poll()) is None:
|
||
|
|
print(f"Closing {renode}")
|
||
|
|
try:
|
||
|
|
process = psutil.Process(proc.pid)
|
||
|
|
os.kill(proc.pid, signal.SIGINT)
|
||
|
|
process.wait(timeout=options.cleanup_timeout)
|
||
|
|
|
||
|
|
if options.perf_output_path:
|
||
|
|
self.__move_perf_data(options)
|
||
|
|
except psutil.TimeoutExpired:
|
||
|
|
print(f"{renode} didn't close in {options.cleanup_timeout}s, killing it")
|
||
|
|
process.kill()
|
||
|
|
|
||
|
|
try:
|
||
|
|
process.wait(timeout=options.kill_timeout)
|
||
|
|
renode_killed = True
|
||
|
|
except psutil.TimeoutExpired:
|
||
|
|
raise RuntimeError(f"{renode} didn't react to kill in {options.kill_timeout}s")
|
||
|
|
|
||
|
|
if options.perf_output_path and proc.stdout:
|
||
|
|
proc.stdout.close()
|
||
|
|
|
||
|
|
if exit_code:
|
||
|
|
print(f"{renode} exited before close request, exit code: {exit_code}")
|
||
|
|
elif renode_killed:
|
||
|
|
print(f"{renode} killed")
|
||
|
|
else:
|
||
|
|
print(f"{renode} closed")
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def _has_renode_crashed(cls, test: ET.Element) -> bool:
|
||
|
|
# only finds immediate children - required because `status`
|
||
|
|
# nodes are also present lower in the tree for example
|
||
|
|
# for every keyword but we only need the status of the test
|
||
|
|
status: ET.Element = test.find('status')
|
||
|
|
if status.text is not None and cls.retry_suite_regex.search(status.text):
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
return any(cls.retry_suite_regex.search(msg.text) for msg in test.iter("msg"))
|
||
|
|
|
||
|
|
|
||
|
|
def run(self, options, iteration_index=1, suite_retry_index=0):
|
||
|
|
if self.path.endswith('renode-keywords.robot'):
|
||
|
|
print('Ignoring helper file: {}'.format(self.path))
|
||
|
|
return True
|
||
|
|
|
||
|
|
# The list is cleared only on the first run attempt in each iteration so
|
||
|
|
# that tests that time out aren't retried in the given iteration but are
|
||
|
|
# started as usual in subsequent iterations.
|
||
|
|
if suite_retry_index == 0:
|
||
|
|
self.tests_with_unexpected_timeouts = []
|
||
|
|
|
||
|
|
# in non-parallel runs there is only one Renode process for all runs,
|
||
|
|
# unless --keep-renode-output is enabled, in which case a new process
|
||
|
|
# is spawned for every suite to ensure logs are separate files.
|
||
|
|
# see: prepare
|
||
|
|
if options.jobs != 1 or options.keep_renode_output:
|
||
|
|
# Parallel groups run in separate processes so these aren't really
|
||
|
|
# shared, they're only needed to restart Renode in timeout handler.
|
||
|
|
RobotTestSuite.robot_frontend_process = self._run_remote_server(options, iteration_index, suite_retry_index)
|
||
|
|
RobotTestSuite.remote_server_port = self.remote_server_port
|
||
|
|
|
||
|
|
print(f'Running suite on Renode pid {self.renode_pid} using port {self.remote_server_port}: {self.path}')
|
||
|
|
|
||
|
|
result = None
|
||
|
|
def get_result():
|
||
|
|
return result if result is not None else TestResult(True, None)
|
||
|
|
|
||
|
|
start_timestamp = monotonic()
|
||
|
|
|
||
|
|
if any(self.tests_without_hotspots):
|
||
|
|
result = get_result().ok and self._run_inner(options.fixture,
|
||
|
|
None,
|
||
|
|
self.tests_without_hotspots,
|
||
|
|
options,
|
||
|
|
iteration_index,
|
||
|
|
suite_retry_index)
|
||
|
|
if any(self.tests_with_hotspots):
|
||
|
|
for hotspot in RobotTestSuite.hotspot_action:
|
||
|
|
if options.hotspot and options.hotspot != hotspot:
|
||
|
|
continue
|
||
|
|
result = get_result().ok and self._run_inner(options.fixture,
|
||
|
|
hotspot,
|
||
|
|
self.tests_with_hotspots,
|
||
|
|
options,
|
||
|
|
iteration_index,
|
||
|
|
suite_retry_index)
|
||
|
|
|
||
|
|
end_timestamp = monotonic()
|
||
|
|
|
||
|
|
if result is None:
|
||
|
|
print(f'No tests executed for suite {self.path}', flush=True)
|
||
|
|
else:
|
||
|
|
status = 'finished successfully' if result.ok else 'failed'
|
||
|
|
exec_time = round(end_timestamp - start_timestamp, 2)
|
||
|
|
print(f'Suite {self.path} {status} in {exec_time} seconds.', flush=True)
|
||
|
|
|
||
|
|
if options.jobs != 1 or options.keep_renode_output:
|
||
|
|
self._close_remote_server(RobotTestSuite.robot_frontend_process, options)
|
||
|
|
|
||
|
|
# make sure renode is still alive when a non-parallel run depends on it
|
||
|
|
if options.jobs == 1 and not options.keep_renode_output:
|
||
|
|
if not self._is_frontend_running():
|
||
|
|
print("Renode has unexpectedly died when running sequentially! Trying to respawn before continuing...")
|
||
|
|
RobotTestSuite.robot_frontend_process = self._run_remote_server(options, iteration_index, suite_retry_index)
|
||
|
|
# Save port to reuse when running sequentially
|
||
|
|
RobotTestSuite.remote_server_port = self.remote_server_port
|
||
|
|
|
||
|
|
return get_result()
|
||
|
|
|
||
|
|
|
||
|
|
def _get_dependencies(self, test_case):
|
||
|
|
|
||
|
|
suiteBuilder = robot.running.builder.TestSuiteBuilder()
|
||
|
|
suite = suiteBuilder.build(self.path)
|
||
|
|
test = next(t for t in suite.tests if hasattr(t, 'name') and t.name == test_case)
|
||
|
|
requirements = [s.args[0] for s in test.body if hasattr(s, 'name') and s.name == 'Requires']
|
||
|
|
if len(requirements) == 0:
|
||
|
|
return set()
|
||
|
|
if len(requirements) > 1:
|
||
|
|
raise Exception('Too many requirements for a single test. At most one is allowed.')
|
||
|
|
providers = [t for t in suite.tests if any(hasattr(s, 'name') and s.name == 'Provides' and s.args[0] == requirements[0] for s in t.body)]
|
||
|
|
if len(providers) > 1:
|
||
|
|
raise Exception('Too many providers for state {0} found: {1}'.format(requirements[0], ', '.join(providers.name)))
|
||
|
|
if len(providers) == 0:
|
||
|
|
raise Exception('No provider for state {0} found'.format(requirements[0]))
|
||
|
|
res = self._get_dependencies(providers[0].name)
|
||
|
|
res.add(providers[0].name)
|
||
|
|
return res
|
||
|
|
|
||
|
|
|
||
|
|
def cleanup(self, options):
|
||
|
|
assert hasattr(RobotTestSuite, "log_files"), "tests_engine.py did not assign RobotTestSuite.log_files"
|
||
|
|
RobotTestSuite.instances_count -= 1
|
||
|
|
if RobotTestSuite.instances_count == 0:
|
||
|
|
self._close_remote_server(RobotTestSuite.robot_frontend_process, options)
|
||
|
|
print("Aggregating all robot results")
|
||
|
|
grouped_log_files = self.group_log_paths(RobotTestSuite.log_files)
|
||
|
|
for iteration in range(1, options.iteration_count + 1):
|
||
|
|
for retry in range(options.retry_count):
|
||
|
|
output_dir = self.get_output_dir(options, iteration, retry)
|
||
|
|
log_files = grouped_log_files[(iteration, retry)]
|
||
|
|
|
||
|
|
# An output_dir can be missing for suite retries that were never "used"
|
||
|
|
if not os.path.isdir(output_dir) or not log_files:
|
||
|
|
continue
|
||
|
|
|
||
|
|
robot.rebot(
|
||
|
|
*log_files,
|
||
|
|
processemptysuite=True,
|
||
|
|
name='Test Suite',
|
||
|
|
loglevel="TRACE:INFO",
|
||
|
|
outputdir=output_dir,
|
||
|
|
output='robot_output.xml'
|
||
|
|
)
|
||
|
|
for file in set(log_files):
|
||
|
|
os.remove(file)
|
||
|
|
if options.css_file:
|
||
|
|
with open(options.css_file) as style:
|
||
|
|
style_content = style.read()
|
||
|
|
for report_name in ("report.html", "log.html"):
|
||
|
|
with open(os.path.join(output_dir, report_name), "a") as report:
|
||
|
|
report.write("<style media=\"all\" type=\"text/css\">")
|
||
|
|
report.write(style_content)
|
||
|
|
report.write("</style>")
|
||
|
|
|
||
|
|
|
||
|
|
if options.keep_renode_output:
|
||
|
|
logs_pattern = re.compile(r"(?P<suite_name>.*)\.renode_std(out|err)\.log")
|
||
|
|
for dirpath, _, fnames in os.walk(options.results_directory):
|
||
|
|
if os.path.basename(dirpath.rstrip("/")) != "logs":
|
||
|
|
continue
|
||
|
|
|
||
|
|
failed_suites = self.find_suites_with_fails(os.path.dirname(dirpath))
|
||
|
|
for fname in fnames:
|
||
|
|
fpath = os.path.join(dirpath, fname)
|
||
|
|
m = logs_pattern.match(fname)
|
||
|
|
if m:
|
||
|
|
# Remove empty logs
|
||
|
|
if os.path.getsize(fpath) == 0:
|
||
|
|
os.remove(fpath)
|
||
|
|
continue
|
||
|
|
|
||
|
|
if options.save_logs == "onfail":
|
||
|
|
# Remove logs which weren't failures
|
||
|
|
suite_name = m.group("suite_name")
|
||
|
|
if suite_name not in failed_suites:
|
||
|
|
os.remove(fpath)
|
||
|
|
|
||
|
|
# If the logs directory is empty, delete it
|
||
|
|
try:
|
||
|
|
os.rmdir(dirpath)
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
def should_retry_suite(self, options, iteration_index, suite_retry_index):
|
||
|
|
tree = None
|
||
|
|
assert self.suite_log_files is not None, "The suite has not yet been run."
|
||
|
|
output_dir = self.get_output_dir(options, iteration_index, suite_retry_index)
|
||
|
|
for log_file in self.suite_log_files:
|
||
|
|
try:
|
||
|
|
tree = ET.parse(os.path.join(output_dir, log_file))
|
||
|
|
except FileNotFoundError as e:
|
||
|
|
raise e
|
||
|
|
|
||
|
|
root = tree.getroot()
|
||
|
|
for suite in root.iter('suite'):
|
||
|
|
if not suite.get('source', False):
|
||
|
|
continue # it is a tag used to group other suites without meaning on its own
|
||
|
|
|
||
|
|
# Always retry if our Setup failed.
|
||
|
|
for kw in suite.iter('kw'):
|
||
|
|
if kw.get('name') == 'Setup' and kw.get('library') == 'renode-keywords':
|
||
|
|
if kw.find('status').get('status') != 'PASS':
|
||
|
|
print('Renode Setup failure detected!')
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
break
|
||
|
|
|
||
|
|
# Look for regular expressions signifying a crash.
|
||
|
|
# Suite Setup and Suite Teardown aren't checked here cause they're in the `kw` tags.
|
||
|
|
for test in suite.iter('test'):
|
||
|
|
if self._has_renode_crashed(test):
|
||
|
|
return True
|
||
|
|
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _create_suite_name(test_name, hotspot):
|
||
|
|
return test_name + (' [HotSpot action: {0}]'.format(hotspot) if hotspot else '')
|
||
|
|
|
||
|
|
|
||
|
|
def _run_dependencies(self, test_cases_names, options, iteration_index=1, suite_retry_index=0):
|
||
|
|
test_cases_names.difference_update(self._dependencies_met)
|
||
|
|
if not any(test_cases_names):
|
||
|
|
return True
|
||
|
|
self._dependencies_met.update(test_cases_names)
|
||
|
|
return self._run_inner(None, None, test_cases_names, options, iteration_index, suite_retry_index)
|
||
|
|
|
||
|
|
|
||
|
|
def _run_inner(self, fixture, hotspot, test_cases_names, options, iteration_index=1, suite_retry_index=0):
|
||
|
|
file_name = os.path.splitext(os.path.basename(self.path))[0]
|
||
|
|
suite_name = RobotTestSuite._create_suite_name(file_name, hotspot)
|
||
|
|
|
||
|
|
output_dir = self.get_output_dir(options, iteration_index, suite_retry_index)
|
||
|
|
variables = [
|
||
|
|
'SKIP_RUNNING_SERVER:True',
|
||
|
|
'DIRECTORY:{}'.format(self.remote_server_directory),
|
||
|
|
'PORT_NUMBER:{}'.format(self.remote_server_port),
|
||
|
|
'RESULTS_DIRECTORY:{}'.format(output_dir),
|
||
|
|
]
|
||
|
|
if hotspot:
|
||
|
|
variables.append('HOTSPOT_ACTION:' + hotspot)
|
||
|
|
if options.debug_mode:
|
||
|
|
variables.append('CONFIGURATION:Debug')
|
||
|
|
if options.debug_on_error:
|
||
|
|
variables.append('HOLD_ON_ERROR:True')
|
||
|
|
if options.execution_metrics:
|
||
|
|
variables.append('CREATE_EXECUTION_METRICS:True')
|
||
|
|
if options.save_logs == "always":
|
||
|
|
variables.append('SAVE_LOGS_WHEN:Always')
|
||
|
|
if options.runner == 'dotnet':
|
||
|
|
variables.append('BINARY_NAME:Renode.dll')
|
||
|
|
variables.append('RENODE_PID:{}'.format(self.renode_pid))
|
||
|
|
variables.append('NET_PLATFORM:True')
|
||
|
|
else:
|
||
|
|
options.exclude.append('profiling')
|
||
|
|
|
||
|
|
if options.variables:
|
||
|
|
variables += options.variables
|
||
|
|
|
||
|
|
test_cases = [(test_name, '{0}.{1}'.format(suite_name, test_name)) for test_name in test_cases_names]
|
||
|
|
if fixture:
|
||
|
|
test_cases = [x for x in test_cases if fnmatch.fnmatch(x[1], '*' + fixture + '*')]
|
||
|
|
if len(test_cases) == 0:
|
||
|
|
return None
|
||
|
|
deps = set()
|
||
|
|
for test_name in (t[0] for t in test_cases):
|
||
|
|
deps.update(self._get_dependencies(test_name))
|
||
|
|
if not self._run_dependencies(deps, options, iteration_index, suite_retry_index):
|
||
|
|
return False
|
||
|
|
|
||
|
|
# Listeners are called in the exact order as in `listeners` list for both `start_test` and `end_test`.
|
||
|
|
output_formatter = 'robot_output_formatter_verbose.py' if options.verbose else 'robot_output_formatter.py'
|
||
|
|
listeners = [
|
||
|
|
os.path.join(this_path, f'retry_and_timeout_listener.py:{options.retry_count}'),
|
||
|
|
# Has to be the last one to print final state, message etc. after all the changes made by other listeners.
|
||
|
|
os.path.join(this_path, output_formatter),
|
||
|
|
]
|
||
|
|
if options.listener:
|
||
|
|
listeners += options.listener
|
||
|
|
|
||
|
|
metadata = {"HotSpot_Action": hotspot if hotspot else '-'}
|
||
|
|
log_file = os.path.join(output_dir, 'results-{0}{1}.robot.xml'.format(file_name, '_' + hotspot if hotspot else ''))
|
||
|
|
|
||
|
|
keywords_path = os.path.abspath(os.path.join(this_path, "renode-keywords.robot"))
|
||
|
|
keywords_path = keywords_path.replace(os.path.sep, "/") # Robot wants forward slashes even on Windows
|
||
|
|
# This variable is provided for compatibility with Robot files that use Resource ${RENODEKEYWORDS}
|
||
|
|
variables.append('RENODEKEYWORDS:{}'.format(keywords_path))
|
||
|
|
tools_path = os.path.join(os.path.dirname(this_path), "tools")
|
||
|
|
tools_path = tools_path.replace(os.path.sep, "/")
|
||
|
|
variables.append('RENODETOOLS:{}'.format(tools_path))
|
||
|
|
suite_builder = robot.running.builder.TestSuiteBuilder()
|
||
|
|
suite = suite_builder.build(self.path)
|
||
|
|
suite.resource.imports.create(type="Resource", name=keywords_path)
|
||
|
|
|
||
|
|
suite.configure(include_tags=options.include, exclude_tags=options.exclude,
|
||
|
|
include_tests=[t[1] for t in test_cases], metadata=metadata,
|
||
|
|
name=suite_name, empty_suite_ok=True)
|
||
|
|
|
||
|
|
# Provide default values for {Suite,Test}{Setup,Teardown}
|
||
|
|
if not suite.setup:
|
||
|
|
suite.setup.config(name="Setup")
|
||
|
|
if not suite.teardown:
|
||
|
|
suite.teardown.config(name="Teardown")
|
||
|
|
|
||
|
|
for test in suite.tests:
|
||
|
|
if not test.setup:
|
||
|
|
test.setup.config(name="Reset Emulation")
|
||
|
|
if not test.teardown:
|
||
|
|
test.teardown.config(name="Test Teardown")
|
||
|
|
|
||
|
|
# Let's just fail tests which previously unexpectedly timed out.
|
||
|
|
if test.name in self.tests_with_unexpected_timeouts:
|
||
|
|
test.config(setup=None, teardown=None)
|
||
|
|
test.body.clear()
|
||
|
|
test.body.create_keyword('Fail', ["Test timed out in a previous run and won't be retried."])
|
||
|
|
|
||
|
|
# This tag tells `RetryFailed` max retries for this test.
|
||
|
|
if 'test:retry' in test.tags:
|
||
|
|
test.tags.remove('test:retry')
|
||
|
|
test.tags.add('test:retry(0)')
|
||
|
|
|
||
|
|
# Timeout tests with `self.timeout_expected_tag` will be set as passed in the listener
|
||
|
|
# during timeout handling. Their timeout won't be influenced by the global timeout option.
|
||
|
|
if self.timeout_expected_tag in test.tags:
|
||
|
|
if not test.timeout:
|
||
|
|
raise Exception(f"!!!!! Test with a `{self.timeout_expected_tag}` tag must have `[Timeout]` set: {test.longname}")
|
||
|
|
elif options.timeout:
|
||
|
|
# Timeout from tags is used if it's shorter than the global timeout.
|
||
|
|
if not test.timeout or Time(test.timeout).seconds >= options.timeout.seconds:
|
||
|
|
test.timeout = options.timeout.value
|
||
|
|
|
||
|
|
# Timeout handler is used in `retry_and_timeout_listener.py` and, to be able to call it,
|
||
|
|
# `self` is smuggled in suite's `parent` which is typically None for the main suite.
|
||
|
|
# Listener grabs it on suite start and resets to original value.
|
||
|
|
suite.parent = (self, suite.parent)
|
||
|
|
self.timeout_handler = self._create_timeout_handler(options, iteration_index, suite_retry_index)
|
||
|
|
|
||
|
|
result = suite.run(console='none', listener=listeners, exitonfailure=options.stop_on_error, output=log_file, log=None, loglevel='TRACE', report=None, variable=variables, skiponfailure=['non_critical', 'skipped'])
|
||
|
|
|
||
|
|
self.suite_log_files = []
|
||
|
|
file_name = os.path.splitext(os.path.basename(self.path))[0]
|
||
|
|
output_dir = self.get_output_dir(options, iteration_index, suite_retry_index)
|
||
|
|
if any(self.tests_without_hotspots):
|
||
|
|
log_file = os.path.join(output_dir, 'results-{0}.robot.xml'.format(file_name))
|
||
|
|
if os.path.isfile(log_file):
|
||
|
|
self.suite_log_files.append(log_file)
|
||
|
|
if any(self.tests_with_hotspots):
|
||
|
|
for hotspot in RobotTestSuite.hotspot_action:
|
||
|
|
if options.hotspot and options.hotspot != hotspot:
|
||
|
|
continue
|
||
|
|
log_file = os.path.join(output_dir, 'results-{0}{1}.robot.xml'.format(file_name, '_' + hotspot if hotspot else ''))
|
||
|
|
if os.path.isfile(log_file):
|
||
|
|
self.suite_log_files.append(log_file)
|
||
|
|
|
||
|
|
if options.runner == "mono":
|
||
|
|
self.copy_mono_logs(options, iteration_index, suite_retry_index)
|
||
|
|
|
||
|
|
return TestResult(result.return_code == 0, self.suite_log_files)
|
||
|
|
|
||
|
|
def _create_timeout_handler(self, options, iteration_index, suite_retry_index):
|
||
|
|
def _timeout_handler(test: robot.running.TestCase, result: robot.result.TestCase):
|
||
|
|
if self.timeout_expected_tag in test.tags:
|
||
|
|
message_start = '----- Test timed out, as expected,'
|
||
|
|
else:
|
||
|
|
# Let's make the first message stand out if the timeout wasn't expected.
|
||
|
|
message_start = '!!!!! Test timed out'
|
||
|
|
|
||
|
|
# Tests with unexpected timeouts won't be retried.
|
||
|
|
self.tests_with_unexpected_timeouts = test.name
|
||
|
|
print(f"{message_start} after {Time(test.timeout).seconds}s: {test.parent.name}.{test.name}")
|
||
|
|
print(f"----- Skipped flushing emulation log and saving state due to the timeout, restarting Renode...")
|
||
|
|
|
||
|
|
self._close_remote_server(RobotTestSuite.robot_frontend_process, options)
|
||
|
|
RobotTestSuite.robot_frontend_process = self._run_remote_server(options, iteration_index, suite_retry_index, self.remote_server_port)
|
||
|
|
|
||
|
|
# It's typically used in suite setup (renode-keywords.robot:Setup) but we don't need to
|
||
|
|
# call full setup which imports library etc. We only need to resend settings to Renode.
|
||
|
|
BuiltIn().run_keyword("Setup Renode")
|
||
|
|
print(f"----- ...done, running remaining tests on Renode pid {self.renode_pid} using the same port {self.remote_server_port}")
|
||
|
|
return _timeout_handler
|
||
|
|
|
||
|
|
|
||
|
|
def copy_mono_logs(self, options: Namespace, iteration_index: int, suite_retry_index: int) -> None:
|
||
|
|
"""Copies 'mono_crash.*.json' files into the suite's logs directory.
|
||
|
|
|
||
|
|
These files are occasionally created when mono crashes. There are also 'mono_crash.*.blob'
|
||
|
|
files, but they contain heavier memory dumps and have questionable usefulness."""
|
||
|
|
output_dir = self.get_output_dir(options, iteration_index, suite_retry_index)
|
||
|
|
logs_dir = os.path.join(output_dir, "logs")
|
||
|
|
for dirpath, dirnames, fnames in os.walk(os.getcwd()):
|
||
|
|
# Do not descend into "logs" directories, to prevent later invocations from
|
||
|
|
# stealing files already moved by earlier invocations
|
||
|
|
logs_indices = [x for x in range(len(dirnames)) if dirnames[x] == "logs"]
|
||
|
|
logs_indices.sort(reverse=True)
|
||
|
|
for logs_idx in logs_indices:
|
||
|
|
del dirnames[logs_idx]
|
||
|
|
|
||
|
|
for fname in filter(lambda x: x.startswith("mono_crash.") and x.endswith(".json"), fnames):
|
||
|
|
os.makedirs(logs_dir, exist_ok=True)
|
||
|
|
src_fpath = os.path.join(dirpath, fname)
|
||
|
|
dest_fpath = os.path.join(logs_dir, fname)
|
||
|
|
print(f"Moving mono_crash file: '{src_fpath}' -> '{dest_fpath}'")
|
||
|
|
os.rename(src_fpath, dest_fpath)
|
||
|
|
|
||
|
|
|
||
|
|
def tests_failed_due_to_renode_crash(self) -> bool:
|
||
|
|
# Return false if the test has not yet run
|
||
|
|
if self.suite_log_files is None:
|
||
|
|
return
|
||
|
|
for file in self.suite_log_files:
|
||
|
|
try:
|
||
|
|
tree = ET.parse(file)
|
||
|
|
except FileNotFoundError:
|
||
|
|
continue
|
||
|
|
|
||
|
|
root = tree.getroot()
|
||
|
|
|
||
|
|
for suite in root.iter('suite'):
|
||
|
|
if not suite.get('source', False):
|
||
|
|
continue # it is a tag used to group other suites without meaning on its own
|
||
|
|
for test in suite.iter('test'):
|
||
|
|
# do not check skipped tests
|
||
|
|
if test.find("./tags/[tag='skipped']"):
|
||
|
|
continue
|
||
|
|
|
||
|
|
# only finds immediate children - required because `status`
|
||
|
|
# nodes are also present lower in the tree for example
|
||
|
|
# for every keyword but we only need the status of the test
|
||
|
|
status = test.find('status')
|
||
|
|
if status.attrib["status"] != "FAIL":
|
||
|
|
continue # passed tests should not be checked for crashes
|
||
|
|
|
||
|
|
# check whether renode crashed during this test
|
||
|
|
if self._has_renode_crashed(test):
|
||
|
|
return True
|
||
|
|
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def find_failed_tests(path, file="robot_output.xml"):
|
||
|
|
ret = {'mandatory': set(), 'non_critical': set()}
|
||
|
|
|
||
|
|
# Aggregate failed tests from all report files (can be multiple if iterations or retries were used)
|
||
|
|
for dirpath, _, fnames in os.walk(path):
|
||
|
|
for fname in filter(lambda x: x == file, fnames):
|
||
|
|
tree = ET.parse(os.path.join(dirpath, fname))
|
||
|
|
root = tree.getroot()
|
||
|
|
for suite in root.iter('suite'):
|
||
|
|
if not suite.get('source', False):
|
||
|
|
continue # it is a tag used to group other suites without meaning on its own
|
||
|
|
for test in suite.iter('test'):
|
||
|
|
status = test.find('status') # only finds immediate children - important requirement
|
||
|
|
if status.attrib['status'] == 'FAIL':
|
||
|
|
test_name = test.attrib['name']
|
||
|
|
suite_name = suite.attrib['name']
|
||
|
|
if suite_name == "Test Suite":
|
||
|
|
# If rebot is invoked with only 1 suite, it renames that suite to Test Suite
|
||
|
|
# instead of wrapping in a new top-level Test Suite. A workaround is to extract
|
||
|
|
# the suite name from the *.robot file name.
|
||
|
|
suite_name = os.path.basename(suite.attrib["source"]).rsplit(".", 1)[0]
|
||
|
|
if test.find("./tags/[tag='skipped']"):
|
||
|
|
continue # skipped test should not be classified as fail
|
||
|
|
if test.find("./tags/[tag='non_critical']"):
|
||
|
|
ret['non_critical'].add(f"{suite_name}.{test_name}")
|
||
|
|
else:
|
||
|
|
ret['mandatory'].add(f"{suite_name}.{test_name}")
|
||
|
|
|
||
|
|
if not ret['mandatory'] and not ret['non_critical']:
|
||
|
|
return None
|
||
|
|
return ret
|
||
|
|
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def find_suites_with_fails(cls, path, file="robot_output.xml"):
|
||
|
|
"""Finds suites which contain at least one test case failure.
|
||
|
|
|
||
|
|
A suite may be successful and still contain failures, e.g. if the --retry option
|
||
|
|
was used and a test passed on a later attempt."""
|
||
|
|
ret = set()
|
||
|
|
|
||
|
|
for dirpath, _, fnames in os.walk(path):
|
||
|
|
for fname in filter(lambda x: x == file, fnames):
|
||
|
|
tree = ET.parse(os.path.join(dirpath, fname))
|
||
|
|
root = tree.getroot()
|
||
|
|
for suite in root.iter('suite'):
|
||
|
|
if not suite.get('source', False):
|
||
|
|
continue # it is a tag used to group other suites without meaning on its own
|
||
|
|
|
||
|
|
suite_name = suite.attrib['name']
|
||
|
|
if suite_name == "Test Suite":
|
||
|
|
# If rebot is invoked with only 1 suite, it renames that suite to Test Suite
|
||
|
|
# instead of wrapping in a new top-level Test Suite. A workaround is to extract
|
||
|
|
# the suite name from the *.robot file name.
|
||
|
|
suite_name = os.path.basename(suite.attrib["source"]).rsplit(".", 1)[0]
|
||
|
|
|
||
|
|
for test in suite.iter('test'):
|
||
|
|
if test.find("./tags/[tag='skipped']"):
|
||
|
|
continue # skipped test should not be classified as fail
|
||
|
|
status = test.find('status') # only finds immediate children - important requirement
|
||
|
|
if status.attrib["status"] == "FAIL":
|
||
|
|
ret.add(suite_name)
|
||
|
|
break
|
||
|
|
if status.text is not None and cls.retry_test_regex.search(status.text):
|
||
|
|
# Retried test cases still count as fails
|
||
|
|
ret.add(suite_name)
|
||
|
|
break
|
||
|
|
|
||
|
|
return ret
|
||
|
|
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def group_log_paths(paths: List[str]) -> Dict[Tuple[int, int], Set[str]]:
|
||
|
|
"""Breaks a list of log paths into subsets grouped by (iteration, suite_retry) pairs."""
|
||
|
|
re_path_indices_patterns = (
|
||
|
|
re.compile(r"\biteration(?P<iteration>\d+)/retry(?P<suite_retry>\d+)/"),
|
||
|
|
re.compile(r"\bretry(?P<suite_retry>\d+)/"),
|
||
|
|
re.compile(r"\biteration(?P<iteration>\d+)/"),
|
||
|
|
)
|
||
|
|
ret = defaultdict(lambda: set())
|
||
|
|
for path in paths:
|
||
|
|
iteration = 1
|
||
|
|
suite_retry = 0
|
||
|
|
for pattern in re_path_indices_patterns:
|
||
|
|
match = pattern.search(path)
|
||
|
|
if match is None:
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
iteration = int(match.group("iteration"))
|
||
|
|
except IndexError:
|
||
|
|
pass
|
||
|
|
try:
|
||
|
|
suite_retry = int(match.group("suite_retry"))
|
||
|
|
except IndexError:
|
||
|
|
pass
|
||
|
|
ret[(iteration, suite_retry)].add(path)
|
||
|
|
return ret
|
||
|
|
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def find_rerun_tests(cls, path):
|
||
|
|
|
||
|
|
def analyze_xml(label, retry_dir, file="robot_output.xml"):
|
||
|
|
try:
|
||
|
|
tree = ET.parse(os.path.join(retry_dir, file))
|
||
|
|
except FileNotFoundError:
|
||
|
|
return
|
||
|
|
root = tree.getroot()
|
||
|
|
for suite in root.iter('suite'):
|
||
|
|
if not suite.get('source', False):
|
||
|
|
continue # it is a tag used to group other suites without meaning on its own
|
||
|
|
suite_name = suite.attrib['name']
|
||
|
|
if suite_name == "Test Suite":
|
||
|
|
# If rebot is invoked with only 1 suite, it renames that suite to Test Suite
|
||
|
|
# instead of wrapping in a new top-level Test Suite. A workaround is to extract
|
||
|
|
# the suite name from the *.robot file name.
|
||
|
|
suite_name = os.path.basename(suite.attrib["source"]).rsplit(".", 1)[0]
|
||
|
|
for test in suite.iter('test'):
|
||
|
|
test_name = test.attrib['name']
|
||
|
|
tags = []
|
||
|
|
if test.find("./tags/[tag='skipped']"):
|
||
|
|
continue # skipped test should not be classified as fail
|
||
|
|
if test.find("./tags/[tag='non_critical']"):
|
||
|
|
tags.append("non_critical")
|
||
|
|
status = test.find('status') # only finds immediate children - important requirement
|
||
|
|
m = cls.retry_test_regex.search(status.text) if status.text is not None else None
|
||
|
|
|
||
|
|
# Check whether renode crashed during this test
|
||
|
|
has_renode_crashed = cls._has_renode_crashed(test)
|
||
|
|
|
||
|
|
status_str = status.attrib["status"]
|
||
|
|
nth = (1 + int(m.group(2))) if m else 1
|
||
|
|
key = f"{suite_name}.{test_name}"
|
||
|
|
if key not in data:
|
||
|
|
data[key] = []
|
||
|
|
data[key].append({
|
||
|
|
"label": label, # e.g. "retry0", "retry1", "iteration1/retry2", ...
|
||
|
|
"status": status_str, # e.g. "PASS", "FAIL", "SKIP", ...
|
||
|
|
"nth": nth, # The number of test case attempts that led to the above status
|
||
|
|
"tags": tags, # e.g. ["non_critical"], [], ...
|
||
|
|
"crash": has_renode_crashed,
|
||
|
|
})
|
||
|
|
|
||
|
|
def analyze_iteration(iteration_dir):
|
||
|
|
iteration_dirname = os.path.basename(iteration_dir)
|
||
|
|
report_fpath = os.path.join(iteration_dir, "robot_output.xml")
|
||
|
|
if os.path.isfile(report_fpath):
|
||
|
|
analyze_xml(iteration_dirname, iteration_dir)
|
||
|
|
return
|
||
|
|
i = -1
|
||
|
|
while True:
|
||
|
|
i += 1
|
||
|
|
retry_dirpath = os.path.join(iteration_dir, f"retry{i}")
|
||
|
|
if os.path.isdir(retry_dirpath):
|
||
|
|
analyze_xml(os.path.join(iteration_dirname, f"retry{i}"), retry_dirpath)
|
||
|
|
continue
|
||
|
|
break
|
||
|
|
|
||
|
|
data = OrderedDict()
|
||
|
|
i = -1
|
||
|
|
while True:
|
||
|
|
i += 1
|
||
|
|
iteration_dirpath = os.path.join(path, f"iteration{i + 1}")
|
||
|
|
retry_dirpath = os.path.join(path, f"retry{i}")
|
||
|
|
if os.path.isdir(iteration_dirpath):
|
||
|
|
analyze_iteration(iteration_dirpath)
|
||
|
|
continue
|
||
|
|
elif os.path.isdir(retry_dirpath):
|
||
|
|
analyze_xml(f"retry{i}", retry_dirpath)
|
||
|
|
continue
|
||
|
|
break
|
||
|
|
|
||
|
|
return data
|