From patchwork Wed Dec 16 01:30:49 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Luck, Tony" X-Patchwork-Id: 7864131 Return-Path: X-Original-To: patchwork-linux-nvdimm@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork2.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork2.web.kernel.org (Postfix) with ESMTP id 02B19BEEE1 for ; Wed, 16 Dec 2015 17:23:49 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id 0173E203C1 for ; Wed, 16 Dec 2015 17:23:48 +0000 (UTC) Received: from ml01.01.org (ml01.01.org [198.145.21.10]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id E8C84202B8 for ; Wed, 16 Dec 2015 17:23:45 +0000 (UTC) Received: from ml01.vlan14.01.org (localhost [IPv6:::1]) by ml01.01.org (Postfix) with ESMTP id DC5BA1A1FAB; Wed, 16 Dec 2015 09:23:45 -0800 (PST) X-Original-To: linux-nvdimm@ml01.01.org Delivered-To: linux-nvdimm@ml01.01.org Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by ml01.01.org (Postfix) with ESMTP id 570AA1A1F7A for ; Wed, 16 Dec 2015 09:23:44 -0800 (PST) Received: from orsmga001.jf.intel.com ([10.7.209.18]) by fmsmga102.fm.intel.com with ESMTP; 16 Dec 2015 09:23:44 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.20,437,1444719600"; d="scan'208";a="842580583" Received: from agluck-desk.sc.intel.com ([10.3.79.148]) by orsmga001.jf.intel.com with SMTP; 16 Dec 2015 09:23:43 -0800 Message-Id: In-Reply-To: References: From: Tony Luck Date: Tue, 15 Dec 2015 17:30:49 -0800 Subject: [PATCHV3 3/3] x86, ras: Add mcsafe_memcpy() function to recover from machine checks To: Ingo Molnar Cc: linux-nvdimm@ml01.01.org, x86@kernel.org, linux-kernel@vger.kernel.org, linux-mm@kvack.org, Borislav Petkov , Andy Lutomirski , Andrew Morton , Elliott@ml01.01.org X-BeenThere: linux-nvdimm@lists.01.org X-Mailman-Version: 2.1.17 Precedence: list List-Id: "Linux-nvdimm developer list." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: linux-nvdimm-bounces@lists.01.org Sender: "Linux-nvdimm" X-Spam-Status: No, score=-1.6 required=5.0 tests=BAYES_00, DATE_IN_PAST_12_24, RCVD_IN_DNSWL_LOW, T_RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Using __copy_user_nocache() as inspiration create a memory copy routine for use by kernel code with annotations to allow for recovery from machine checks. Notes: 1) We align the source address rather than the destination. This means we never have to deal with a memory read that spans two cache lines ... so we can provide a precise indication of where the error occurred without having to re-execute at a byte-by-byte level to find the exact spot like the original did. 2) We 'or' BIT(63) into the return because this is the first in a series of machine check safe functions. Some will copy from user addresses, so may need to indicate an invalid user address instead of a machine check. 3) This code doesn't play any cache games. Future functions can use non-temporal loads/stores to meet needs of different callers. 4) Provide helpful macros to decode the return value. Signed-off-by: Tony Luck --- arch/x86/include/asm/mcsafe_copy.h | 11 +++ arch/x86/kernel/x8664_ksyms_64.c | 5 ++ arch/x86/lib/Makefile | 1 + arch/x86/lib/mcsafe_copy.S | 142 +++++++++++++++++++++++++++++++++++++ 4 files changed, 159 insertions(+) create mode 100644 arch/x86/include/asm/mcsafe_copy.h create mode 100644 arch/x86/lib/mcsafe_copy.S diff --git a/arch/x86/include/asm/mcsafe_copy.h b/arch/x86/include/asm/mcsafe_copy.h new file mode 100644 index 000000000000..d4dbd5a667a3 --- /dev/null +++ b/arch/x86/include/asm/mcsafe_copy.h @@ -0,0 +1,11 @@ +#ifndef _ASM_X86_MCSAFE_COPY_H +#define _ASM_X86_MCSAFE_COPY_H + +u64 mcsafe_memcpy(void *dst, const void *src, unsigned size); + +#define COPY_MCHECK_ERRBIT BIT(63) +#define COPY_HAD_MCHECK(ret) ((ret) & COPY_MCHECK_ERRBIT) +#define COPY_MCHECK_REMAIN(ret) ((ret) & ~COPY_MCHECK_ERRBIT) + +#endif /* _ASM_MCSAFE_COPY_H */ + diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c index a0695be19864..afab8b25dbc0 100644 --- a/arch/x86/kernel/x8664_ksyms_64.c +++ b/arch/x86/kernel/x8664_ksyms_64.c @@ -37,6 +37,11 @@ EXPORT_SYMBOL(__copy_user_nocache); EXPORT_SYMBOL(_copy_from_user); EXPORT_SYMBOL(_copy_to_user); +#ifdef CONFIG_MCE_KERNEL_RECOVERY +#include +EXPORT_SYMBOL(mcsafe_memcpy); +#endif + EXPORT_SYMBOL(copy_page); EXPORT_SYMBOL(clear_page); diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index f2587888d987..82bb0bf46b6b 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -21,6 +21,7 @@ lib-y += usercopy_$(BITS).o usercopy.o getuser.o putuser.o lib-y += memcpy_$(BITS).o lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o +lib-$(CONFIG_MCE_KERNEL_RECOVERY) += mcsafe_copy.o obj-y += msr.o msr-reg.o msr-reg-export.o diff --git a/arch/x86/lib/mcsafe_copy.S b/arch/x86/lib/mcsafe_copy.S new file mode 100644 index 000000000000..059b3a9642eb --- /dev/null +++ b/arch/x86/lib/mcsafe_copy.S @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2015 Intel Corporation + * Author: Tony Luck + * + * This software may be redistributed and/or modified under the terms of + * the GNU General Public License ("GPL") version 2 only as published by the + * Free Software Foundation. + */ + +#include +#include + +/* + * mcsafe_memcpy - memory copy with machine check exception handling + * Note that we only catch machine checks when reading the source addresses. + * Writes to target are posted and don't generate machine checks. + */ +ENTRY(mcsafe_memcpy) + cmpl $8,%edx + jb 20f /* less then 8 bytes, go to byte copy loop */ + + /* check for bad alignment of source */ + movl %esi,%ecx + andl $7,%ecx + jz 102f /* already aligned */ + subl $8,%ecx + negl %ecx + subl %ecx,%edx +0: movb (%rsi),%al + movb %al,(%rdi) + incq %rsi + incq %rdi + decl %ecx + jnz 0b +102: + movl %edx,%ecx + andl $63,%edx + shrl $6,%ecx + jz 17f +1: movq (%rsi),%r8 +2: movq 1*8(%rsi),%r9 +3: movq 2*8(%rsi),%r10 +4: movq 3*8(%rsi),%r11 + mov %r8,(%rdi) + mov %r9,1*8(%rdi) + mov %r10,2*8(%rdi) + mov %r11,3*8(%rdi) +9: movq 4*8(%rsi),%r8 +10: movq 5*8(%rsi),%r9 +11: movq 6*8(%rsi),%r10 +12: movq 7*8(%rsi),%r11 + mov %r8,4*8(%rdi) + mov %r9,5*8(%rdi) + mov %r10,6*8(%rdi) + mov %r11,7*8(%rdi) + leaq 64(%rsi),%rsi + leaq 64(%rdi),%rdi + decl %ecx + jnz 1b +17: movl %edx,%ecx + andl $7,%edx + shrl $3,%ecx + jz 20f +18: movq (%rsi),%r8 + mov %r8,(%rdi) + leaq 8(%rsi),%rsi + leaq 8(%rdi),%rdi + decl %ecx + jnz 18b +20: andl %edx,%edx + jz 23f + movl %edx,%ecx +21: movb (%rsi),%al + movb %al,(%rdi) + incq %rsi + incq %rdi + decl %ecx + jnz 21b +23: xorl %eax,%eax + sfence + ret + + .section .fixup,"ax" +30: + addl %ecx,%edx + jmp 100f +31: + shll $6,%ecx + addl %ecx,%edx + jmp 100f +32: + shll $6,%ecx + leal -8(%ecx,%edx),%edx + jmp 100f +33: + shll $6,%ecx + leal -16(%ecx,%edx),%edx + jmp 100f +34: + shll $6,%ecx + leal -24(%ecx,%edx),%edx + jmp 100f +35: + shll $6,%ecx + leal -32(%ecx,%edx),%edx + jmp 100f +36: + shll $6,%ecx + leal -40(%ecx,%edx),%edx + jmp 100f +37: + shll $6,%ecx + leal -48(%ecx,%edx),%edx + jmp 100f +38: + shll $6,%ecx + leal -56(%ecx,%edx),%edx + jmp 100f +39: + lea (%rdx,%rcx,8),%rdx + jmp 100f +40: + movl %ecx,%edx +100: + sfence + movabsq $0x8000000000000000, %rax + orq %rdx,%rax + ret + .previous + + _ASM_MCEXTABLE(0b,30b) + _ASM_MCEXTABLE(1b,31b) + _ASM_MCEXTABLE(2b,32b) + _ASM_MCEXTABLE(3b,33b) + _ASM_MCEXTABLE(4b,34b) + _ASM_MCEXTABLE(9b,35b) + _ASM_MCEXTABLE(10b,36b) + _ASM_MCEXTABLE(11b,37b) + _ASM_MCEXTABLE(12b,38b) + _ASM_MCEXTABLE(18b,39b) + _ASM_MCEXTABLE(21b,40b) +ENDPROC(mcsafe_memcpy)