perf test: Print result for each LLVM subtest

Currently 'perf test llvm' and 'perf test BPF' have multiple sub-tests,
but the result is provided in only one line:

  # perf test LLVM
  35: Test LLVM searching and compiling                        : Ok

This patch introduces sub-tests support, allowing 'perf test' to report
result for each sub-tests:

  # perf test LLVM
  35: Test LLVM searching and compiling                        :
  35.1: Basic BPF llvm compiling test                          : Ok
  35.2: Test kbuild searching                                  : Ok
  35.3: Compile source for BPF prologue generation test        : Ok

When a failure happens:

  # cat ~/.perfconfig
  [llvm]
       clang-path = "/bin/false"
  # perf test LLVM
  35: Test LLVM searching and compiling                        :
  35.1: Basic BPF llvm compiling test                          : FAILED!
  35.2: Test kbuild searching                                  : Skip
  35.3: Compile source for BPF prologue generation test        : Skip

And:

  # rm ~/.perfconfig
  # ./perf test LLVM
  35: Test LLVM searching and compiling                        :
  35.1: Basic BPF llvm compiling test                          : Skip
  35.2: Test kbuild searching                                  : Skip
  35.3: Compile source for BPF prologue generation test        : Skip

Skip by user:

  # ./perf test -s 1,`seq -s , 3 42`
   1: vmlinux symtab matches kallsyms                          : Skip (user override)
   2: detect openat syscall event                              : Ok
  ...
  35: Test LLVM searching and compiling                        : Skip (user override)
  ...

Suggested-and-Tested-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: Wang Nan <wangnan0@huawei.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Zefan Li <lizefan@huawei.com>
Cc: pi3orama@163.com
Link: http://lkml.kernel.org/r/1447749170-175898-4-git-send-email-wangnan0@huawei.com
[ Changed so that func is not on an anonymous union ]
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>

diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
index 9cf4892c061d76340dec23ccc6abc12b646f5abb..813660976217494f92b57aae9b0a9b1b79e04e41 100644 (file)
--- a/tools/perf/tests/builtin-test.c
+++ b/tools/perf/tests/builtin-test.c
@@ -160,6 +160,11 @@ static struct test generic_tests[] = {
        {
                .desc = "Test LLVM searching and compiling",
                .func = test__llvm,
+               .subtest = {
+                       .skip_if_fail   = true,
+                       .get_nr         = test__llvm_subtest_get_nr,
+                       .get_desc       = test__llvm_subtest_get_desc,
+               },
        },
        {
                .desc = "Test topology in session",
@@ -237,6 +242,40 @@ static int run_test(struct test *test, int subtest)
        for (j = 0; j < ARRAY_SIZE(tests); j++) \
                for (t = &tests[j][0]; t->func; t++)
 
+static int test_and_print(struct test *t, bool force_skip, int subtest)
+{
+       int err;
+
+       if (!force_skip) {
+               pr_debug("\n--- start ---\n");
+               err = run_test(t, subtest);
+               pr_debug("---- end ----\n");
+       } else {
+               pr_debug("\n--- force skipped ---\n");
+               err = TEST_SKIP;
+       }
+
+       if (!t->subtest.get_nr)
+               pr_debug("%s:", t->desc);
+       else
+               pr_debug("%s subtest %d:", t->desc, subtest);
+
+       switch (err) {
+       case TEST_OK:
+               pr_info(" Ok\n");
+               break;
+       case TEST_SKIP:
+               color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
+               break;
+       case TEST_FAIL:
+       default:
+               color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
+               break;
+       }
+
+       return err;
+}
+
 static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)
 {
        struct test *t;
@@ -264,21 +303,43 @@ static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)
                        continue;
                }
 
-               pr_debug("\n--- start ---\n");
-               err = run_test(t, i);
-               pr_debug("---- end ----\n%s:", t->desc);
-
-               switch (err) {
-               case TEST_OK:
-                       pr_info(" Ok\n");
-                       break;
-               case TEST_SKIP:
-                       color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
-                       break;
-               case TEST_FAIL:
-               default:
-                       color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
-                       break;
+               if (!t->subtest.get_nr) {
+                       test_and_print(t, false, -1);
+               } else {
+                       int subn = t->subtest.get_nr();
+                       /*
+                        * minus 2 to align with normal testcases.
+                        * For subtest we print additional '.x' in number.
+                        * for example:
+                        *
+                        * 35: Test LLVM searching and compiling                        :
+                        * 35.1: Basic BPF llvm compiling test                          : Ok
+                        */
+                       int subw = width > 2 ? width - 2 : width;
+                       bool skip = false;
+                       int subi;
+
+                       if (subn <= 0) {
+                               color_fprintf(stderr, PERF_COLOR_YELLOW,
+                                             " Skip (not compiled in)\n");
+                               continue;
+                       }
+                       pr_info("\n");
+
+                       for (subi = 0; subi < subn; subi++) {
+                               int len = strlen(t->subtest.get_desc(subi));
+
+                               if (subw < len)
+                                       subw = len;
+                       }
+
+                       for (subi = 0; subi < subn; subi++) {
+                               pr_info("%2d.%1d: %-*s:", i, subi + 1, subw,
+                                       t->subtest.get_desc(subi));
+                               err = test_and_print(t, skip, subi);
+                               if (err != TEST_OK && t->subtest.skip_if_fail)
+                                       skip = true;
+                       }
                }
        }
 

diff --git a/tools/perf/tests/llvm.c b/tools/perf/tests/llvm.c
index 4350c455d06c2a1f6bfc0c96bcfd7d25119c05ae..06f45c1d42561df030e35d55078d70d910e79998 100644 (file)
--- a/tools/perf/tests/llvm.c
+++ b/tools/perf/tests/llvm.c
@@ -46,7 +46,7 @@ static struct {
        },
        [LLVM_TESTCASE_BPF_PROLOGUE] = {
                .source = test_llvm__bpf_test_prologue_prog,
-               .desc = "Test BPF prologue generation",
+               .desc = "Compile source for BPF prologue generation test",
        },
 };
 
@@ -131,44 +131,39 @@ out:
        return ret;
 }
 
-int test__llvm(int subtest __maybe_unused)
+int test__llvm(int subtest)
 {
-       enum test_llvm__testcase i;
+       int ret;
+       void *obj_buf = NULL;
+       size_t obj_buf_sz = 0;
 
-       for (i = 0; i < __LLVM_TESTCASE_MAX; i++) {
-               int ret;
-               void *obj_buf = NULL;
-               size_t obj_buf_sz = 0;
+       if ((subtest < 0) || (subtest >= __LLVM_TESTCASE_MAX))
+               return TEST_FAIL;
 
-               ret = test_llvm__fetch_bpf_obj(&obj_buf, &obj_buf_sz,
-                                              i, false);
+       ret = test_llvm__fetch_bpf_obj(&obj_buf, &obj_buf_sz,
+                                      subtest, false);
 
-               if (ret == TEST_OK) {
-                       ret = test__bpf_parsing(obj_buf, obj_buf_sz);
-                       if (ret != TEST_OK)
-                               pr_debug("Failed to parse test case '%s'\n",
-                                        bpf_source_table[i].desc);
-               }
-               free(obj_buf);
-
-               switch (ret) {
-               case TEST_SKIP:
-                       return TEST_SKIP;
-               case TEST_OK:
-                       break;
-               default:
-                       /*
-                        * Test 0 is the basic LLVM test. If test 0
-                        * fail, the basic LLVM support not functional
-                        * so the whole test should fail. If other test
-                        * case fail, it can be fixed by adjusting
-                        * config so don't report error.
-                        */
-                       if (i == 0)
-                               return TEST_FAIL;
-                       else
-                               return TEST_SKIP;
+       if (ret == TEST_OK) {
+               ret = test__bpf_parsing(obj_buf, obj_buf_sz);
+               if (ret != TEST_OK) {
+                       pr_debug("Failed to parse test case '%s'\n",
+                                bpf_source_table[subtest].desc);
                }
        }
-       return TEST_OK;
+       free(obj_buf);
+
+       return ret;
+}
+
+int test__llvm_subtest_get_nr(void)
+{
+       return __LLVM_TESTCASE_MAX;
+}
+
+const char *test__llvm_subtest_get_desc(int subtest)
+{
+       if ((subtest < 0) || (subtest >= __LLVM_TESTCASE_MAX))
+               return NULL;
+
+       return bpf_source_table[subtest].desc;
 }

diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h
index 204e4eeadea294419feafe6f9cb250721afe9095..f92af527f0801f1ce882797f0385a959bd5c6fa9 100644 (file)
--- a/tools/perf/tests/tests.h
+++ b/tools/perf/tests/tests.h
@@ -1,6 +1,8 @@
 #ifndef TESTS_H
 #define TESTS_H
 
+#include <stdbool.h>
+
 #define TEST_ASSERT_VAL(text, cond)                                     \
 do {                                                                    \
        if (!(cond)) {                                                   \
@@ -27,6 +29,11 @@ enum {
 struct test {
        const char *desc;
        int (*func)(int subtest);
+       struct {
+               bool skip_if_fail;
+               int (*get_nr)(void);
+               const char *(*get_desc)(int subtest);
+       } subtest;
 };
 
 /* Tests */
@@ -66,6 +73,8 @@ int test__fdarray__add(int subtest);
 int test__kmod_path__parse(int subtest);
 int test__thread_map(int subtest);
 int test__llvm(int subtest);
+const char *test__llvm_subtest_get_desc(int subtest);
+int test__llvm_subtest_get_nr(void);
 int test__bpf(int subtest);
 int test_session_topology(int subtest);