tc-testing: Allow test cases to be skipped

By adding a check for an optional key/value pair to the test case
data, individual test cases may be skipped to prevent tdc from
aborting a test run due to setup or teardown failure.

If a test case is skipped, it will still appear in the results
output to allow for a consistent number of executed tests in each
run. However, the test will be marked as skipped.

This support for skipping extends to any plugins that may generate
additional results for each executed test.

Signed-off-by: Lucas Bates <lucasb@mojatatu.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/tools/testing/selftests/tc-testing/TdcPlugin.py b/tools/testing/selftests/tc-testing/TdcPlugin.py
index 1d9e279331ebd990cba41db912da239de28bce24..b980a565fa89995ece1505b963d42d7c7a84f565 100644 (file)
--- a/tools/testing/selftests/tc-testing/TdcPlugin.py
+++ b/tools/testing/selftests/tc-testing/TdcPlugin.py
@@ -18,13 +18,13 @@ class TdcPlugin:
         if self.args.verbose > 1:
             print(' -- {}.post_suite'.format(self.sub_class))
 
-    def pre_case(self, test_ordinal, testid, test_name):
+    def pre_case(self, testid, test_name, test_skip):
         '''run commands before test_runner does one test'''
         if self.args.verbose > 1:
             print(' -- {}.pre_case'.format(self.sub_class))
         self.args.testid = testid
         self.args.test_name = test_name
-        self.args.test_ordinal = test_ordinal
+        self.args.test_skip = test_skip
 
     def post_case(self):
         '''run commands after test_runner does one test'''

diff --git a/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt b/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt
index 17b267dedbd93e85bcb2a7ca0c7a169c0b1960c6..a28571aff0e1ded2fd7d373985b7f25699b71648 100644 (file)
--- a/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt
+++ b/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt
@@ -33,6 +33,11 @@ Each test case has required data:
 
 id:           A unique alphanumeric value to identify a particular test case
 name:         Descriptive name that explains the command under test
+skip:         A completely optional key, if the corresponding value is "yes"
+              then tdc will not execute the test case in question. However,
+              this test case will still appear in the results output but
+              marked as skipped. This key can be placed anywhere inside the
+              test case at the top level.
 category:     A list of single-word descriptions covering what the command
               under test is testing. Example: filter, actions, u32, gact, etc.
 setup:        The list of commands required to ensure the command under test

diff --git a/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py b/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py
index e00c798de0bbe07e18b9c7cf7dc85f81843e55e9..4bb866575ea18a63ae0f2faae49e99d0c9977f25 100644 (file)
--- a/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py
+++ b/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py
@@ -102,6 +102,14 @@ class SubPlugin(TdcPlugin):
         if not self.args.valgrind:
             return
 
+        res = TestResult('{}-mem'.format(self.args.testid),
+              '{} memory leak check'.format(self.args.test_name))
+        if self.args.test_skip:
+            res.set_result(ResultState.skip)
+            res.set_errormsg('Test case designated as skipped.')
+            self._add_results(res)
+            return
+
         self.definitely_lost_re = re.compile(
             r'definitely lost:\s+([,0-9]+)\s+bytes in\s+([,0-9]+)\sblocks', re.MULTILINE | re.DOTALL)
         self.indirectly_lost_re = re.compile(
@@ -134,8 +142,6 @@ class SubPlugin(TdcPlugin):
                 nle_num = int(nle_mo.group(1))
 
         mem_results = ''
-        res = TestResult('{}-mem'.format(self.args.testid),
-              '{} memory leak check'.format(self.args.test_name))
         if (def_num > 0) or (ind_num > 0) or (pos_num > 0) or (nle_num > 0):
             mem_results += 'not '
             res.set_result(ResultState.fail)
@@ -146,12 +152,6 @@ class SubPlugin(TdcPlugin):
 
         self._add_results(res)
 
-        mem_results += 'ok {} - {}-mem # {}\n'.format(
-            self.args.test_ordinal, self.args.testid, 'memory leak check')
-        self._add_to_tap(mem_results)
-        if mem_results.startswith('not '):
-            print('{}'.format(content))
-            self._add_to_tap(content)
 
     def _add_results(self, res):
         self._tsr.add_resultdata(res)

diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py
index e6e4ce80a726774d73828c143833b4cca0cb103e..5cee15659e5f5ae4ffd527a2ed2ee789740f75e6 100755 (executable)
--- a/tools/testing/selftests/tc-testing/tdc.py
+++ b/tools/testing/selftests/tc-testing/tdc.py
@@ -61,10 +61,10 @@ class PluginMgr:
         for pgn_inst in reversed(self.plugin_instances):
             pgn_inst.post_suite(index)
 
-    def call_pre_case(self, test_ordinal, testid, test_name):
+    def call_pre_case(self, testid, test_name, *, test_skip=False):
         for pgn_inst in self.plugin_instances:
             try:
-                pgn_inst.pre_case(test_ordinal, testid, test_name)
+                pgn_inst.pre_case(testid, test_name, test_skip)
             except Exception as ee:
                 print('exception {} in call to pre_case for {} plugin'.
                       format(ee, pgn_inst.__class__))
@@ -192,10 +192,19 @@ def run_one_test(pm, args, index, tidx):
         print("\t====================\n=====> ", end="")
     print("Test " + tidx["id"] + ": " + tidx["name"])
 
+    if 'skip' in tidx:
+        if tidx['skip'] == 'yes':
+            res = TestResult(tidx['id'], tidx['name'])
+            res.set_result(ResultState.skip)
+            res.set_errormsg('Test case designated as skipped.')
+            pm.call_pre_case(tidx['id'], tidx['name'], test_skip=True)
+            pm.call_post_execute()
+            return res
+
     # populate NAMES with TESTID for this test
     NAMES['TESTID'] = tidx['id']
 
-    pm.call_pre_case(index, tidx['id'], tidx['name'])
+    pm.call_pre_case(tidx['id'], tidx['name'])
     prepare_env(args, pm, 'setup', "-----> prepare stage", tidx["setup"])
 
     if (args.verbose > 0):