selftests/bpf: add few verifier scale tests

Add 3 basic tests that stress verifier scalability.

test_verif_scale1.c calls non-inlined jhash() function 90 times on
different position in the packet.
This test simulates network packet parsing.
jhash function is ~140 instructions and main program is ~1200 insns.

test_verif_scale2.c force inlines jhash() function 90 times.
This program is ~15k instructions long.

test_verif_scale3.c calls non-inlined jhash() function 90 times on
But this time jhash has to process 32-bytes from the packet
instead of 14-bytes in tests 1 and 2.
jhash function is ~230 insns and main program is ~1200 insns.

$ test_progs -s
can be used to see verifier stats.

Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
new file mode 100644 (file)
index 0000000..23b159d
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2019 Facebook
+#include <test_progs.h>
+static int libbpf_debug_print(enum libbpf_print_level level,
+                             const char *format, va_list args)
+{
+       if (level != LIBBPF_DEBUG)
+               return 0;
+
+       if (!strstr(format, "verifier log"))
+               return 0;
+       return vfprintf(stderr, "%s", args);
+}
+
+static int check_load(const char *file)
+{
+       struct bpf_prog_load_attr attr;
+       struct bpf_object *obj;
+       int err, prog_fd;
+
+       memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
+       attr.file = file;
+       attr.prog_type = BPF_PROG_TYPE_SCHED_CLS;
+       attr.log_level = 4;
+       err = bpf_prog_load_xattr(&attr, &obj, &prog_fd);
+       bpf_object__close(obj);
+       if (err)
+               error_cnt++;
+       return err;
+}
+
+void test_bpf_verif_scale(void)
+{
+       const char *file1 = "./test_verif_scale1.o";
+       const char *file2 = "./test_verif_scale2.o";
+       const char *file3 = "./test_verif_scale3.o";
+       int err;
+
+       if (verifier_stats)
+               libbpf_set_print(libbpf_debug_print);
+
+       err = check_load(file1);
+       err |= check_load(file2);
+       err |= check_load(file3);
+       if (!err)
+               printf("test_verif_scale:OK\n");
+       else
+               printf("test_verif_scale:FAIL\n");
+}

diff --git a/tools/testing/selftests/bpf/progs/test_jhash.h b/tools/testing/selftests/bpf/progs/test_jhash.h
new file mode 100644 (file)
index 0000000..3d12c11
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_jhash.h
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2019 Facebook
+
+typedef unsigned int u32;
+
+static __attribute__((always_inline)) u32 rol32(u32 word, unsigned int shift)
+{
+       return (word << shift) | (word >> ((-shift) & 31));
+}
+
+#define __jhash_mix(a, b, c)                   \
+{                                              \
+       a -= c;  a ^= rol32(c, 4);  c += b;     \
+       b -= a;  b ^= rol32(a, 6);  a += c;     \
+       c -= b;  c ^= rol32(b, 8);  b += a;     \
+       a -= c;  a ^= rol32(c, 16); c += b;     \
+       b -= a;  b ^= rol32(a, 19); a += c;     \
+       c -= b;  c ^= rol32(b, 4);  b += a;     \
+}
+
+#define __jhash_final(a, b, c)                 \
+{                                              \
+       c ^= b; c -= rol32(b, 14);              \
+       a ^= c; a -= rol32(c, 11);              \
+       b ^= a; b -= rol32(a, 25);              \
+       c ^= b; c -= rol32(b, 16);              \
+       a ^= c; a -= rol32(c, 4);               \
+       b ^= a; b -= rol32(a, 14);              \
+       c ^= b; c -= rol32(b, 24);              \
+}
+
+#define JHASH_INITVAL          0xdeadbeef
+
+static ATTR
+u32 jhash(const void *key, u32 length, u32 initval)
+{
+       u32 a, b, c;
+       const unsigned char *k = key;
+
+       a = b = c = JHASH_INITVAL + length + initval;
+
+       while (length > 12) {
+               a += *(volatile u32 *)(k);
+               b += *(volatile u32 *)(k + 4);
+               c += *(volatile u32 *)(k + 8);
+               __jhash_mix(a, b, c);
+               length -= 12;
+               k += 12;
+       }
+       switch (length) {
+       case 12: c += (u32)k[11]<<24;
+       case 11: c += (u32)k[10]<<16;
+       case 10: c += (u32)k[9]<<8;
+       case 9:  c += k[8];
+       case 8:  b += (u32)k[7]<<24;
+       case 7:  b += (u32)k[6]<<16;
+       case 6:  b += (u32)k[5]<<8;
+       case 5:  b += k[4];
+       case 4:  a += (u32)k[3]<<24;
+       case 3:  a += (u32)k[2]<<16;
+       case 2:  a += (u32)k[1]<<8;
+       case 1:  a += k[0];
+                c ^= a;
+                __jhash_final(a, b, c);
+       case 0: /* Nothing left to add */
+               break;
+       }
+
+       return c;
+}

diff --git a/tools/testing/selftests/bpf/progs/test_verif_scale1.c b/tools/testing/selftests/bpf/progs/test_verif_scale1.c
new file mode 100644 (file)
index 0000000..f3236ce
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_verif_scale1.c
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2019 Facebook
+#include <linux/bpf.h>
+#include "bpf_helpers.h"
+#define ATTR __attribute__((noinline))
+#include "test_jhash.h"
+
+SEC("scale90_noinline")
+int balancer_ingress(struct __sk_buff *ctx)
+{
+       void *data_end = (void *)(long)ctx->data_end;
+       void *data = (void *)(long)ctx->data;
+       void *ptr;
+       int ret = 0, nh_off, i = 0;
+
+       nh_off = 14;
+
+       /* pragma unroll doesn't work on large loops */
+
+#define C do { \
+       ptr = data + i; \
+       if (ptr + nh_off > data_end) \
+               break; \
+       ctx->tc_index = jhash(ptr, nh_off, ctx->cb[0] + i++); \
+       } while (0);
+#define C30 C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;
+       C30;C30;C30; /* 90 calls */
+       return 0;
+}
+char _license[] SEC("license") = "GPL";

diff --git a/tools/testing/selftests/bpf/progs/test_verif_scale2.c b/tools/testing/selftests/bpf/progs/test_verif_scale2.c
new file mode 100644 (file)
index 0000000..7783069
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_verif_scale2.c
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2019 Facebook
+#include <linux/bpf.h>
+#include "bpf_helpers.h"
+#define ATTR __attribute__((always_inline))
+#include "test_jhash.h"
+
+SEC("scale90_inline")
+int balancer_ingress(struct __sk_buff *ctx)
+{
+       void *data_end = (void *)(long)ctx->data_end;
+       void *data = (void *)(long)ctx->data;
+       void *ptr;
+       int ret = 0, nh_off, i = 0;
+
+       nh_off = 14;
+
+       /* pragma unroll doesn't work on large loops */
+
+#define C do { \
+       ptr = data + i; \
+       if (ptr + nh_off > data_end) \
+               break; \
+       ctx->tc_index = jhash(ptr, nh_off, ctx->cb[0] + i++); \
+       } while (0);
+#define C30 C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;
+       C30;C30;C30; /* 90 calls */
+       return 0;
+}
+char _license[] SEC("license") = "GPL";

diff --git a/tools/testing/selftests/bpf/progs/test_verif_scale3.c b/tools/testing/selftests/bpf/progs/test_verif_scale3.c
new file mode 100644 (file)
index 0000000..1848da0
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_verif_scale3.c
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2019 Facebook
+#include <linux/bpf.h>
+#include "bpf_helpers.h"
+#define ATTR __attribute__((noinline))
+#include "test_jhash.h"
+
+SEC("scale90_noinline32")
+int balancer_ingress(struct __sk_buff *ctx)
+{
+       void *data_end = (void *)(long)ctx->data_end;
+       void *data = (void *)(long)ctx->data;
+       void *ptr;
+       int ret = 0, nh_off, i = 0;
+
+       nh_off = 32;
+
+       /* pragma unroll doesn't work on large loops */
+
+#define C do { \
+       ptr = data + i; \
+       if (ptr + nh_off > data_end) \
+               break; \
+       ctx->tc_index = jhash(ptr, nh_off, ctx->cb[0] + i++); \
+       } while (0);
+#define C30 C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;
+       C30;C30;C30; /* 90 calls */
+       return 0;
+}
+char _license[] SEC("license") = "GPL";

diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c
index 5d10aee9e27744f75db45baf5889f56381ce287c..bf5c90998916a60961ede884b9ea2d5393813cdd 100644 (file)
--- a/tools/testing/selftests/bpf/test_progs.c
+++ b/tools/testing/selftests/bpf/test_progs.c
@@ -9,6 +9,7 @@
 
 int error_cnt, pass_cnt;
 bool jit_enabled;
+bool verifier_stats = false;
 
 struct ipv4_packet pkt_v4 = {
        .eth.h_proto = __bpf_constant_htons(ETH_P_IP),
@@ -162,12 +163,15 @@ void *spin_lock_thread(void *arg)
 #include <prog_tests/tests.h>
 #undef DECLARE
 
-int main(void)
+int main(int ac, char **av)
 {
        srand(time(NULL));
 
        jit_enabled = is_jit_enabled();
 
+       if (ac == 2 && strcmp(av[1], "-s") == 0)
+               verifier_stats = true;
+
 #define CALL
 #include <prog_tests/tests.h>
 #undef CALL

diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h
index 51a07367cd433f5ae9055083b4d1a7e6868fdae0..f095e1d4c657f4ee56a2198ce26561274d9b7115 100644 (file)
--- a/tools/testing/selftests/bpf/test_progs.h
+++ b/tools/testing/selftests/bpf/test_progs.h
@@ -40,6 +40,7 @@ typedef __u16 __sum16;
 
 extern int error_cnt, pass_cnt;
 extern bool jit_enabled;
+extern bool verifier_stats;
 
 #define MAGIC_BYTES 123