From 93a175bdf84370acd35ab6e857a3d688321c3af5 Mon Sep 17 00:00:00 2001 From: gerioldman Date: Wed, 4 Sep 2024 15:22:04 +0200 Subject: [PATCH] Deprecate should_error, renamed to expected_error and add success_returncode for exitcode tests --- docs/markdown/Unit-tests.md | 7 ++++++- docs/yaml/functions/benchmark.yaml | 18 ++++++++++++++++++ mesonbuild/backend/backends.py | 5 +++-- mesonbuild/interpreter/interpreter.py | 11 ++++++++++- mesonbuild/interpreter/interpreterobjects.py | 5 +++-- mesonbuild/interpreter/kwargs.py | 4 +++- mesonbuild/interpreter/type_checking.py | 4 +++- mesonbuild/mtest.py | 7 +++++-- .../common/278 success returncode/failing.c | 3 +++ .../common/278 success returncode/meson.build | 4 ++++ test cases/common/68 should fail/meson.build | 3 ++- 11 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 test cases/common/278 success returncode/failing.c create mode 100644 test cases/common/278 success returncode/meson.build diff --git a/docs/markdown/Unit-tests.md b/docs/markdown/Unit-tests.md index 898366095b05..fe7e6f9faee7 100644 --- a/docs/markdown/Unit-tests.md +++ b/docs/markdown/Unit-tests.md @@ -130,7 +130,12 @@ In addition, sometimes a test fails set up so that it should fail even if it is marked as an expected failure. The GNU standard approach in this case is to exit the program with error code 99. Again, Meson will detect this and report these tests as `ERROR`, ignoring the setting of -`should_fail`. This behavior was added in version 0.50.0. +`should_fail`. This behavior was added in version 0.50.0. In version +1.6.0 `should_fail` has been deprecated and renamed to `expected_fail`. + +In version 1.6.0, `success_returncode` has been introduced. This makes +it possible to positively test for non-zero return codes. An example +of this would be to test if failure injection is detected in a test. ## Testing tool diff --git a/docs/yaml/functions/benchmark.yaml b/docs/yaml/functions/benchmark.yaml index 7a555a42d6a0..32c9068dd55e 100644 --- a/docs/yaml/functions/benchmark.yaml +++ b/docs/yaml/functions/benchmark.yaml @@ -40,11 +40,29 @@ kwargs: should_fail: type: bool + deprecated: 1.6.0 default: false description: | when true the test is considered passed if the executable returns a non-zero return value (i.e. reports an error) + expected_fail: + type: bool + since: 1.6.0 + default: false + description: | + when true the test is considered passed if the + executable returns a non-zero return value (i.e. reports an error) + + success_returncode: + type: int + since: 1.6.0 + default: 0 + description: | + the test is considered passed if the + executable returns the specified returncode, + only has effect when protocol is set to exitcode + suite: type: str | list[str] description: | diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index e35660b69a51..750ce965b0c2 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -197,7 +197,8 @@ class TestSerialisation: is_parallel: bool cmd_args: T.List[str] env: mesonlib.EnvironmentVariables - should_fail: bool + expected_fail: bool + success_returncode: int timeout: T.Optional[int] workdir: T.Optional[str] extra_paths: T.List[str] @@ -1284,7 +1285,7 @@ def create_test_serialisation(self, tests: T.List['Test']) -> T.List[TestSeriali ts = TestSerialisation(t.get_name(), t.project_name, t.suite, cmd, is_cross, exe_wrapper, self.environment.need_exe_wrapper(), t.is_parallel, cmd_args, t_env, - t.should_fail, t.timeout, t.workdir, + t.expected_fail, t.success_returncode, t.timeout, t.workdir, extra_paths, t.protocol, t.priority, isinstance(exe, (build.Target, build.CustomTargetIndex)), isinstance(exe, build.Executable), diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index 9de0f8762e17..22c27a70686a 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -2242,6 +2242,14 @@ def make_test(self, node: mparser.BaseNode, if kwargs['timeout'] <= 0: FeatureNew.single_use('test() timeout <= 0', '0.57.0', self.subproject, location=node) + if kwargs['should_fail'] is not None and kwargs['expected_fail'] is not None: + raise InvalidArguments('Tried to use both \'should_fail\' and \'expected_fail\'') + elif kwargs['should_fail'] is not None: + mlog.deprecation('Use expected_fail instead of should_fail.', location=node) + kwargs['expected_fail'] = kwargs['should_fail'] + elif kwargs['expected_fail'] is None: + kwargs['expected_fail'] = False + prj = self.subproject if self.is_subproject() else self.build.project_name suite: T.List[str] = [] @@ -2258,7 +2266,8 @@ def make_test(self, node: mparser.BaseNode, kwargs.get('is_parallel', False), kwargs['args'], env, - kwargs['should_fail'], + kwargs['expected_fail'], + kwargs['success_returncode'], kwargs['timeout'], kwargs['workdir'], kwargs['protocol'], diff --git a/mesonbuild/interpreter/interpreterobjects.py b/mesonbuild/interpreter/interpreterobjects.py index 155cfd23e56e..b6fa184aa3ed 100644 --- a/mesonbuild/interpreter/interpreterobjects.py +++ b/mesonbuild/interpreter/interpreterobjects.py @@ -755,7 +755,7 @@ def __init__(self, name: str, project: str, suite: T.List[str], is_parallel: bool, cmd_args: T.List[T.Union[str, mesonlib.File, build.Target, ExternalProgram]], env: mesonlib.EnvironmentVariables, - should_fail: bool, timeout: int, workdir: T.Optional[str], protocol: str, + expected_fail: bool, success_returncode: int, timeout: int, workdir: T.Optional[str], protocol: str, priority: int, verbose: bool): super().__init__() self.name = name @@ -766,7 +766,8 @@ def __init__(self, name: str, project: str, suite: T.List[str], self.is_parallel = is_parallel self.cmd_args = cmd_args self.env = env - self.should_fail = should_fail + self.expected_fail = expected_fail + self.success_returncode = success_returncode self.timeout = timeout self.workdir = workdir self.protocol = TestProtocol.from_str(protocol) diff --git a/mesonbuild/interpreter/kwargs.py b/mesonbuild/interpreter/kwargs.py index eee53c5ff413..95793cf1e05b 100644 --- a/mesonbuild/interpreter/kwargs.py +++ b/mesonbuild/interpreter/kwargs.py @@ -39,7 +39,9 @@ class BaseTest(TypedDict): """Shared base for the Rust module.""" args: T.List[T.Union[str, File, build.Target, ExternalProgram]] - should_fail: bool + should_fail: T.Optional[bool] + expected_fail: T.Optional[bool] + success_returncode: int timeout: int workdir: T.Optional[str] depends: T.List[T.Union[build.CustomTarget, build.BuildTarget]] diff --git a/mesonbuild/interpreter/type_checking.py b/mesonbuild/interpreter/type_checking.py index ed34be950065..1beb41105414 100644 --- a/mesonbuild/interpreter/type_checking.py +++ b/mesonbuild/interpreter/type_checking.py @@ -487,7 +487,9 @@ def link_whole_validator(values: T.List[T.Union[StaticLibrary, CustomTarget, Cus TEST_KWS: T.List[KwargInfo] = [ KwargInfo('args', ContainerTypeInfo(list, (str, File, BuildTarget, CustomTarget, CustomTargetIndex, ExternalProgram)), listify=True, default=[]), - KwargInfo('should_fail', bool, default=False), + KwargInfo('should_fail', (bool, NoneType), deprecated='1.6.0', deprecated_message='Use expected_fail instead of should_fail'), + KwargInfo('expected_fail', (bool, NoneType), since='1.6.0'), + KwargInfo('success_returncode', int, default=0, since='1.6.0'), KwargInfo('timeout', int, default=30), KwargInfo('workdir', (str, NoneType), default=None, validator=lambda x: 'must be an absolute path' if not os.path.isabs(x) else None), diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index c417bc0b38b7..eae184ef5a51 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -934,7 +934,8 @@ def __init__(self, test: TestSerialisation, test_env: T.Dict[str, str], self.additional_error = '' self.cmd: T.Optional[T.List[str]] = None self.env = test_env - self.should_fail = test.should_fail + self.expected_fail = test.expected_fail + self.success_returncode = test.success_returncode self.project = test.project_name self.junit: T.Optional[et.ElementTree] = None self.is_parallel = is_parallel @@ -982,7 +983,7 @@ def _complete(self) -> None: if self.res == TestResult.RUNNING: self.res = TestResult.OK assert isinstance(self.res, TestResult) - if self.should_fail and self.res in (TestResult.OK, TestResult.FAIL): + if self.expected_fail and self.res in (TestResult.OK, TestResult.FAIL): self.res = TestResult.UNEXPECTEDPASS if self.res is TestResult.OK else TestResult.EXPECTEDFAIL if self.stdo and not self.stdo.endswith('\n'): self.stdo += '\n' @@ -1038,6 +1039,8 @@ class TestRunExitCode(TestRun): def complete(self) -> None: if self.res != TestResult.RUNNING: pass + elif self.returncode == self.success_returncode: + self.res = TestResult.OK elif self.returncode == GNU_SKIP_RETURNCODE: self.res = TestResult.SKIP elif self.returncode == GNU_ERROR_RETURNCODE: diff --git a/test cases/common/278 success returncode/failing.c b/test cases/common/278 success returncode/failing.c new file mode 100644 index 000000000000..3e70e5079304 --- /dev/null +++ b/test cases/common/278 success returncode/failing.c @@ -0,0 +1,3 @@ +int main(void) { + return 1; +} diff --git a/test cases/common/278 success returncode/meson.build b/test cases/common/278 success returncode/meson.build new file mode 100644 index 000000000000..fe9a2afb1d02 --- /dev/null +++ b/test cases/common/278 success returncode/meson.build @@ -0,0 +1,4 @@ +project('success returncode', 'c') + +exe = executable('prog', 'failing.c') +test('errorcode', exe, success_returncode: 1) diff --git a/test cases/common/68 should fail/meson.build b/test cases/common/68 should fail/meson.build index dffbbb38180f..0167956fc6ad 100644 --- a/test cases/common/68 should fail/meson.build +++ b/test cases/common/68 should fail/meson.build @@ -1,4 +1,5 @@ project('should fail', 'c') exe = executable('prog', 'failing.c') -test('failing', exe, should_fail : true) +test('should-failing', exe, should_fail : true) +test('expected-failing', exe, expected_fail : true)