From patchwork Fri Mar 15 11:08:32 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jan Beulich X-Patchwork-Id: 10854561 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id B347715AC for ; Fri, 15 Mar 2019 11:10:14 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 922282A94D for ; Fri, 15 Mar 2019 11:10:14 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 7DBD22A94F; Fri, 15 Mar 2019 11:10:14 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) (using TLSv1.2 with cipher AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 9DDB32A94D for ; Fri, 15 Mar 2019 11:10:13 +0000 (UTC) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.89) (envelope-from ) id 1h4khX-0002La-9i; Fri, 15 Mar 2019 11:08:35 +0000 Received: from all-amaz-eas1.inumbo.com ([34.197.232.57] helo=us1-amaz-eas2.inumbo.com) by lists.xenproject.org with esmtp (Exim 4.89) (envelope-from ) id 1h4khW-0002LL-DM for xen-devel@lists.xenproject.org; Fri, 15 Mar 2019 11:08:34 +0000 X-Inumbo-ID: ab2986d8-4712-11e9-871f-c7ab690a713f Received: from prv1-mh.provo.novell.com (unknown [137.65.248.33]) by us1-amaz-eas2.inumbo.com (Halon) with ESMTPS id ab2986d8-4712-11e9-871f-c7ab690a713f; Fri, 15 Mar 2019 11:08:32 +0000 (UTC) Received: from INET-PRV1-MTA by prv1-mh.provo.novell.com with Novell_GroupWise; Fri, 15 Mar 2019 05:08:31 -0600 Message-Id: <5C8B87B0020000780021F32F@prv1-mh.provo.novell.com> X-Mailer: Novell GroupWise Internet Agent 18.1.0 Date: Fri, 15 Mar 2019 05:08:32 -0600 From: "Jan Beulich" To: "xen-devel" References: <5B6BF83602000078001DC548@prv1-mh.provo.novell.com> <5C8B7EC0020000780021F10B@prv1-mh.provo.novell.com> In-Reply-To: <5C8B7EC0020000780021F10B@prv1-mh.provo.novell.com> Mime-Version: 1.0 Content-Disposition: inline Subject: [Xen-devel] [PATCH v8 50/50] x86emul: add a PCLMUL/VPCLMUL test case to the harness X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: George Dunlap , Andrew Cooper , Wei Liu , Roger Pau Monne Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" X-Virus-Scanned: ClamAV using ClamSMTP Also use this for AVX512_VBMI2 VPSH{L,R}D{,V}{D,Q,W} testing (only the quad word right shifts get actually used; the assumption is that their "left" counterparts as well as the double word and word forms then work as well). Signed-off-by: Jan Beulich Acked-by: Andrew Cooper (subject to all --- v8: New. --- a/tools/tests/x86_emulator/Makefile +++ b/tools/tests/x86_emulator/Makefile @@ -20,9 +20,10 @@ SIMD := 3dnow sse sse2 sse4 avx avx2 xop FMA := fma4 fma SG := avx2-sg avx512f-sg avx512vl-sg AES := ssse3-aes avx-aes avx2-vaes avx512bw-vaes +CLMUL := ssse3-pclmul avx-pclmul avx2-vpclmulqdq avx512bw-vpclmulqdq avx512vbmi2-vpclmulqdq SHA := sse4-sha avx-sha avx512f-sha GF := sse2-gf avx2-gf avx512bw-gf -TESTCASES := blowfish $(SIMD) $(FMA) $(SG) $(AES) $(SHA) $(GF) +TESTCASES := blowfish $(SIMD) $(FMA) $(SG) $(AES) $(CLMUL) $(SHA) $(GF) OPMASK := avx512f avx512dq avx512bw @@ -89,6 +90,7 @@ avx512er-flts := 4 8 avx512vbmi-vecs := $(avx512bw-vecs) avx512vbmi-ints := $(avx512bw-ints) avx512vbmi-flts := $(avx512bw-flts) +avx512vbmi2-vecs := $(avx512bw-vecs) avx512f-opmask-vecs := 2 avx512dq-opmask-vecs := 1 2 @@ -149,6 +151,10 @@ define simd-aes-defs $(1)-cflags := $(foreach vec,$($(patsubst %-aes,sse,$(1))-vecs) $($(patsubst %-vaes,%,$(1))-vecs), \ "-D_$(vec) -maes $(addprefix -m,$(subst -,$(space),$(1))) $(call non-sse,$(1)) -Os -DVEC_SIZE=$(vec)") endef +define simd-clmul-defs +$(1)-cflags := $(foreach vec,$($(patsubst %-pclmul,sse,$(1))-vecs) $($(patsubst %-vpclmulqdq,%,$(1))-vecs), \ + "-D_$(vec) -mpclmul $(addprefix -m,$(subst -,$(space),$(1))) $(call non-sse,$(1)) -Os -DVEC_SIZE=$(vec)") +endef define simd-sha-defs $(1)-cflags := $(foreach vec,$(sse-vecs), \ "-D_$(vec) $(addprefix -m,$(subst -,$(space),$(1))) -Os -DVEC_SIZE=$(vec)") @@ -164,6 +170,7 @@ endef $(foreach flavor,$(SIMD) $(FMA),$(eval $(call simd-defs,$(flavor)))) $(foreach flavor,$(SG),$(eval $(call simd-sg-defs,$(flavor)))) $(foreach flavor,$(AES),$(eval $(call simd-aes-defs,$(flavor)))) +$(foreach flavor,$(CLMUL),$(eval $(call simd-clmul-defs,$(flavor)))) $(foreach flavor,$(SHA),$(eval $(call simd-sha-defs,$(flavor)))) $(foreach flavor,$(GF),$(eval $(call simd-gf-defs,$(flavor)))) $(foreach flavor,$(OPMASK),$(eval $(call opmask-defs,$(flavor)))) @@ -218,13 +225,16 @@ $(addsuffix .c,$(SG)): $(addsuffix .c,$(AES)): ln -sf simd-aes.c $@ +$(addsuffix .c,$(CLMUL)): + ln -sf simd-clmul.c $@ + $(addsuffix .c,$(SHA)): ln -sf simd-sha.c $@ $(addsuffix .c,$(GF)): ln -sf simd-gf.c $@ -$(addsuffix .h,$(SIMD) $(FMA) $(SG) $(AES) $(SHA) $(GF)): simd.h +$(addsuffix .h,$(SIMD) $(FMA) $(SG) $(AES) $(CLMUL) $(SHA) $(GF)): simd.h xop.h avx512f.h: simd-fma.c --- /dev/null +++ b/tools/tests/x86_emulator/simd-clmul.c @@ -0,0 +1,150 @@ +#define UINT_SIZE 8 + +#include "simd.h" +ENTRY(clmul_test); + +#ifdef __AVX512F__ /* AVX512BW may get enabled only below */ +# define ALL_TRUE (~0ULL >> (64 - ELEM_COUNT)) +# define eq(x, y) (B(pcmpeqq, _mask, (vdi_t)(x), (vdi_t)(y), -1) == ALL_TRUE) +# define lane_shr_unit(x) \ + ((vec_t)B(palignr, _mask, (vdi_t)(x), (vdi_t)(x), 64, (vdi_t){}, \ + 0x00ff00ff00ff00ffULL & (~0ULL >> (64 - VEC_SIZE)))) +#else +# if defined(__AVX2__) && VEC_SIZE == 32 +# define to_bool(cmp) B(ptestc, , cmp, (vdi_t){} == 0) +# else +# define to_bool(cmp) (__builtin_ia32_pmovmskb128(cmp) == 0xffff) +# endif +# define eq(x, y) to_bool((x) == (y)) +# define lane_shr_unit(x) ((vec_t)B(palignr, , (vdi_t){}, (vdi_t)(x), 64)) +#endif + +#define CLMUL(op, x, y, c) (vec_t)(__builtin_ia32_ ## op((vdi_t)(x), (vdi_t)(y), c)) + +#if VEC_SIZE == 16 +# define clmul(x, y, c) CLMUL(pclmulqdq128, x, y, c) +# define vpshrd __builtin_ia32_vpshrd_v2di +#elif VEC_SIZE == 32 +# define clmul(x, y, c) CLMUL(vpclmulqdq_v4di, x, y, c) +# define vpshrd __builtin_ia32_vpshrd_v4di +#elif VEC_SIZE == 64 +# define clmul(x, y, c) CLMUL(vpclmulqdq_v8di, x, y, c) +# define vpshrd __builtin_ia32_vpshrd_v8di +#endif + +#define clmul_ll(x, y) clmul(x, y, 0x00) +#define clmul_hl(x, y) clmul(x, y, 0x01) +#define clmul_lh(x, y) clmul(x, y, 0x10) +#define clmul_hh(x, y) clmul(x, y, 0x11) + +#if defined(__AVX512VBMI2__) +# pragma GCC target ( "avx512bw" ) +# define lane_shr_i(x, n) ({ \ + vec_t h_ = lane_shr_unit(x); \ + touch(h_); \ + (n) < 64 ? (vec_t)vpshrd((vdi_t)(x), (vdi_t)(h_), n) : h_ >> ((n) - 64); \ +}) +# define lane_shr_v(x, n) ({ \ + vec_t t_ = (x), h_ = lane_shr_unit(x); \ + typeof(t_[0]) n_ = (n); \ + if ( (n) < 64 ) \ + /* gcc does not support embedded broadcast */ \ + asm ( "vpshrdvq %2%{1to%c3%}, %1, %0" \ + : "+v" (t_) : "v" (h_), "m" (n_), "i" (ELEM_COUNT) ); \ + else \ + t_ = h_ >> ((n) - 64); \ + t_; \ +}) +#else +# define lane_shr_i lane_shr_v +# define lane_shr_v(x, n) ({ \ + vec_t t_ = (n) > 0 ? lane_shr_unit(x) : (x); \ + (n) < 64 ? ((x) >> (n)) | (t_ << (-(n) & 0x3f)) \ + : t_ >> ((n) - 64); \ +}) +#endif + +int clmul_test(void) +{ + unsigned int i; + vec_t src; + vqi_t raw = {}; + + for ( i = 1; i < VEC_SIZE; ++i ) + raw[i] = i; + src = (vec_t)raw; + + for ( i = 0; i < 256; i += VEC_SIZE ) + { + vec_t x = {}, y, z, lo, hi; + unsigned int j; + + touch(x); + y = clmul_ll(src, x); + touch(x); + + if ( !eq(y, x) ) return __LINE__; + + for ( j = 0; j < ELEM_COUNT; j += 2 ) + x[j] = 1; + + touch(src); + y = clmul_ll(x, src); + touch(src); + z = clmul_lh(x, src); + touch(src); + + for ( j = 0; j < ELEM_COUNT; j += 2 ) + y[j + 1] = z[j]; + + if ( !eq(y, src) ) return __LINE__; + + /* + * Besides the obvious property of the low and high half products + * being the same either direction, the "square" of a number has the + * property of simply being the original bit pattern with a zero bit + * inserted between any two bits. This is what the code below checks. + */ + + x = src; + touch(src); + y = clmul_lh(x, src); + touch(src); + z = clmul_hl(x, src); + + if ( !eq(y, z) ) return __LINE__; + + touch(src); + y = lo = clmul_ll(x, src); + touch(src); + z = hi = clmul_hh(x, src); + touch(src); + + for ( j = 0; j < 64; ++j ) + { + vec_t l = lane_shr_v(lo, 2 * j); + vec_t h = lane_shr_v(hi, 2 * j); + unsigned int n; + + if ( !eq(l, y) ) return __LINE__; + if ( !eq(h, z) ) return __LINE__; + + x = src >> j; + + for ( n = 0; n < ELEM_COUNT; n += 2 ) + { + if ( (x[n + 0] & 1) != (l[n] & 3) ) return __LINE__; + if ( (x[n + 1] & 1) != (h[n] & 3) ) return __LINE__; + } + + touch(y); + y = lane_shr_i(y, 2); + touch(z); + z = lane_shr_i(z, 2); + } + + src += 0x0101010101010101ULL * VEC_SIZE; + } + + return 0; +} --- a/tools/tests/x86_emulator/simd.h +++ b/tools/tests/x86_emulator/simd.h @@ -381,6 +381,7 @@ OVR(movntdq); OVR(movntdqa); OVR(movshdup); OVR(movsldup); +OVR(pclmulqdq); OVR(permd); OVR(permq); OVR(pmovsxbd); --- a/tools/tests/x86_emulator/test_x86_emulator.c +++ b/tools/tests/x86_emulator/test_x86_emulator.c @@ -13,16 +13,19 @@ asm ( ".pushsection .test, \"ax\", @prog #include "sse2.h" #include "sse2-gf.h" #include "ssse3-aes.h" +#include "ssse3-pclmul.h" #include "sse4.h" #include "sse4-sha.h" #include "avx.h" #include "avx-aes.h" +#include "avx-pclmul.h" #include "avx-sha.h" #include "fma4.h" #include "fma.h" #include "avx2.h" #include "avx2-sg.h" #include "avx2-vaes.h" +#include "avx2-vpclmulqdq.h" #include "avx2-gf.h" #include "xop.h" #include "avx512f-opmask.h" @@ -34,10 +37,12 @@ asm ( ".pushsection .test, \"ax\", @prog #include "avx512vl-sg.h" #include "avx512bw.h" #include "avx512bw-vaes.h" +#include "avx512bw-vpclmulqdq.h" #include "avx512bw-gf.h" #include "avx512dq.h" #include "avx512er.h" #include "avx512vbmi.h" +#include "avx512vbmi2-vpclmulqdq.h" #define verbose false /* Switch to true for far more logging. */ @@ -108,6 +113,16 @@ static bool simd_check_avx_aes(void) return cpu_has_aesni && cpu_has_avx; } +static bool simd_check_ssse3_pclmul(void) +{ + return cpu_has_pclmulqdq && cpu_has_ssse3; +} + +static bool simd_check_avx_pclmul(void) +{ + return cpu_has_pclmulqdq && cpu_has_avx; +} + static bool simd_check_avx512f(void) { return cpu_has_avx512f; @@ -189,6 +204,31 @@ static bool simd_check_avx512bw_vaes_vl( cpu_has_avx512bw && cpu_has_avx512vl; } +static bool simd_check_avx2_vpclmulqdq(void) +{ + return cpu_has_vpclmulqdq && cpu_has_avx2; +} + +static bool simd_check_avx512bw_vpclmulqdq(void) +{ + return cpu_has_vpclmulqdq && cpu_has_avx512bw; +} + +static bool simd_check_avx512bw_vpclmulqdq_vl(void) +{ + return cpu_has_vpclmulqdq && cpu_has_avx512bw && cpu_has_avx512vl; +} + +static bool simd_check_avx512vbmi2_vpclmulqdq(void) +{ + return cpu_has_avx512_vbmi2 && simd_check_avx512bw_vpclmulqdq(); +} + +static bool simd_check_avx512vbmi2_vpclmulqdq_vl(void) +{ + return cpu_has_avx512_vbmi2 && simd_check_avx512bw_vpclmulqdq_vl(); +} + static bool simd_check_sse2_gf(void) { return cpu_has_gfni && cpu_has_sse2; @@ -369,6 +409,8 @@ static const struct { SIMD(XOP i64x4, xop, 32i8), SIMD(AES (legacy), ssse3_aes, 16), SIMD(AES (VEX/x16), avx_aes, 16), + SIMD(PCLMUL (legacy), ssse3_pclmul, 16), + SIMD(PCLMUL (VEX/x2), avx_pclmul, 16), SIMD(OPMASK/w, avx512f_opmask, 2), SIMD(OPMASK+DQ/b, avx512dq_opmask, 1), SIMD(OPMASK+DQ/w, avx512dq_opmask, 2), @@ -475,6 +517,13 @@ static const struct { SIMD(VAES (EVEX/x64), avx512bw_vaes, 64), AVX512VL(VL+VAES (x16), avx512bw_vaes, 16), AVX512VL(VL+VAES (x32), avx512bw_vaes, 32), + SIMD(VPCLMUL (VEX/x4), avx2_vpclmulqdq, 32), + SIMD(VPCLMUL (EVEX/x8), avx512bw_vpclmulqdq, 64), + AVX512VL(VL+VPCLMUL (x4), avx512bw_vpclmulqdq, 16), + AVX512VL(VL+VPCLMUL (x8), avx512bw_vpclmulqdq, 32), + SIMD(AVX512_VBMI2+VPCLMUL (x8), avx512vbmi2_vpclmulqdq, 64), + AVX512VL(_VBMI2+VL+VPCLMUL (x2), avx512vbmi2_vpclmulqdq, 16), + AVX512VL(_VBMI2+VL+VPCLMUL (x4), avx512vbmi2_vpclmulqdq, 32), SIMD(GFNI (legacy), sse2_gf, 16), SIMD(GFNI (VEX/x16), avx2_gf, 16), SIMD(GFNI (VEX/x32), avx2_gf, 32), --- a/tools/tests/x86_emulator/x86-emulate.h +++ b/tools/tests/x86_emulator/x86-emulate.h @@ -125,6 +125,7 @@ static inline bool xcr0_mask(uint64_t ma #define cpu_has_sse cp.basic.sse #define cpu_has_sse2 cp.basic.sse2 #define cpu_has_sse3 cp.basic.sse3 +#define cpu_has_pclmulqdq cp.basic.pclmulqdq #define cpu_has_ssse3 cp.basic.ssse3 #define cpu_has_fma (cp.basic.fma && xcr0_mask(6)) #define cpu_has_sse4_1 cp.basic.sse4_1