From patchwork Sat Jul 27 09:03:02 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alice Ryhl X-Patchwork-Id: 13743650 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by smtp.lore.kernel.org (Postfix) with ESMTP id F301CC3DA61 for ; Sat, 27 Jul 2024 09:03:09 +0000 (UTC) Received: by kanga.kvack.org (Postfix) id 12B836B0083; Sat, 27 Jul 2024 05:03:09 -0400 (EDT) Received: by kanga.kvack.org (Postfix, from userid 40) id 0DCCD6B0088; Sat, 27 Jul 2024 05:03:09 -0400 (EDT) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id EBE306B0089; Sat, 27 Jul 2024 05:03:08 -0400 (EDT) X-Delivered-To: linux-mm@kvack.org Received: from relay.hostedemail.com (smtprelay0010.hostedemail.com [216.40.44.10]) by kanga.kvack.org (Postfix) with ESMTP id C65206B0083 for ; Sat, 27 Jul 2024 05:03:08 -0400 (EDT) Received: from smtpin15.hostedemail.com (a10.router.float.18 [10.200.18.1]) by unirelay06.hostedemail.com (Postfix) with ESMTP id 7B41CA1F67 for ; Sat, 27 Jul 2024 09:03:08 +0000 (UTC) X-FDA: 82384943256.15.3625078 Received: from mail-yw1-f202.google.com (mail-yw1-f202.google.com [209.85.128.202]) by imf21.hostedemail.com (Postfix) with ESMTP id 920651C000B for ; Sat, 27 Jul 2024 09:03:06 +0000 (UTC) Authentication-Results: imf21.hostedemail.com; dkim=pass header.d=google.com header.s=20230601 header.b=jmJaPNFD; dmarc=pass (policy=reject) header.from=google.com; spf=pass (imf21.hostedemail.com: domain of 3ybekZgkKCAgitqkmz6ptowwotm.kwutqv25-uus3iks.wzo@flex--aliceryhl.bounces.google.com designates 209.85.128.202 as permitted sender) smtp.mailfrom=3ybekZgkKCAgitqkmz6ptowwotm.kwutqv25-uus3iks.wzo@flex--aliceryhl.bounces.google.com ARC-Seal: i=1; s=arc-20220608; d=hostedemail.com; t=1722070984; a=rsa-sha256; cv=none; b=5C5rNxL5e+hl5F9sw67Vu1yIlrmtU0mgplMJnGzZ+a5NF4kSKjaA+RBMe5lgmeFqL0lY70 ORpSvjMdsagwiyE/hg5tdtf61TpkpGYzAMYcWQ7hb7OEmtarVCf4gGlJeRRly0PBan5/oH 2xO/hExjf3bvI+2No7rldepZbWWwLIw= ARC-Authentication-Results: i=1; imf21.hostedemail.com; dkim=pass header.d=google.com header.s=20230601 header.b=jmJaPNFD; dmarc=pass (policy=reject) header.from=google.com; spf=pass (imf21.hostedemail.com: domain of 3ybekZgkKCAgitqkmz6ptowwotm.kwutqv25-uus3iks.wzo@flex--aliceryhl.bounces.google.com designates 209.85.128.202 as permitted sender) smtp.mailfrom=3ybekZgkKCAgitqkmz6ptowwotm.kwutqv25-uus3iks.wzo@flex--aliceryhl.bounces.google.com ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=hostedemail.com; s=arc-20220608; t=1722070984; h=from:from:sender:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-type:content-transfer-encoding:in-reply-to: references:dkim-signature; bh=amnmit01xBmRSFGd7PIgnlHscQiD3s0tRRVe9bRcPf8=; b=JVgT/0k7L+lpT/0U6iamJbS1jA40MKFUkuplf7O6m8QWoC191c1+5uQXGRgwles2bb/fKb Q94T0K25Y/l0y9Ne92fMQ0eRDU2DsClva0Ro/VmAxGY6MNCxAKnl/5o8NeimYgi/6AQlcT TXOfdleKmfjBkj8eAZICtVxYug3l5eg= Received: by mail-yw1-f202.google.com with SMTP id 00721157ae682-6643016423fso14561597b3.3 for ; Sat, 27 Jul 2024 02:03:06 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20230601; t=1722070985; x=1722675785; darn=kvack.org; h=cc:to:from:subject:message-id:mime-version:date:from:to:cc:subject :date:message-id:reply-to; bh=amnmit01xBmRSFGd7PIgnlHscQiD3s0tRRVe9bRcPf8=; b=jmJaPNFDg5wLVQcyrXkB3uNV1jPR4Y3r9hYEyMgrLzPuWRKK7f4owx8LJih9SYtwJr rt05/hTzJHz6bZv3wUMX1O56qdOz1oZmfIlZEIoGjTE+ARkcZ5+2ie9fMwQ8upW0EL3J QOMVNxz0YbRCXjDShPryA2swCIb0wK7Vgf1mVjREQS/iPi9y8GcAmFL3ChenkCEVVAru qPcZdKC7Z+QaNIYMo5wgfD1YY26uk/WySy1RMzZzL4iqfKzgwegoWgpRh50AkqA/p+Ti TjX4p/CuTUXkEOeu9ejTo987c3ScNo3qRTuKNBjlMEi4kIVF+CT5HFj3/xgJ4TTKzc9y Z63Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1722070985; x=1722675785; h=cc:to:from:subject:message-id:mime-version:date:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=amnmit01xBmRSFGd7PIgnlHscQiD3s0tRRVe9bRcPf8=; b=iaboj9Fde7Rb9IN1pRNmPTI8hTxFizGZKrCfRxk9yG267+OqpaFlBp7gYA3tkkXatP OuvVyZfWqkNWeB5TGyCzTbK+XcY7Upe+pqqvGtVQdO3aLWdrogzObUVcSebTxi7plcT3 BoR+0LDD5YQYEmrnQnN/X4A0/iRN0ALtC3cLA7Jon+r5o2tuUlUKzIts3y6p9xE8eQXd AvGob9Xs5N7q0XdLsiz1R/qSXyfEcge7J4ro5jxPunba5te/5rYRu3F9mjnuNDGQnCkb WNNZfw1ss72FUgqkN3jTgIJRXhEKG3sZMsksu+8hUf1PfRqr7OMgI3TgvCNa55nAjEAp cyaQ== X-Forwarded-Encrypted: i=1; AJvYcCVcjpc6M3lQGJ9/lvM6rCu167mS4ZqynniGT6ZhjJcY7cwmNtkHixFaPFCBqchF3bo51rEW8g1Q2xgHkkwz1YFEoeI= X-Gm-Message-State: AOJu0Yy1h+eth1zbQJpZV9L5sP0J+MD4qvq5pgaLygsvXlexaP/laiyc PjonLo6qDTcHdVd9oUEl1YrZBEH2k0J/bAtkdmcQj0OTSe+Hzj0aqhZ9uf5VWTb2Z0iuvlJLIrO 4voFoL649zjW7GQ== X-Google-Smtp-Source: AGHT+IGJZbkSTgSNzuICE4FkNF0bC5DXtVy19tevln3MpQ+dr32zBBtTwFoTxQ0D879MfEKPIBTGeWyjq9AZXhE= X-Received: from aliceryhl2.c.googlers.com ([fda3:e722:ac3:cc00:68:949d:c0a8:572]) (user=aliceryhl job=sendgmr) by 2002:a05:690c:2b81:b0:62c:67f4:4f5 with SMTP id 00721157ae682-67a0a7fdf7emr293227b3.9.1722070985337; Sat, 27 Jul 2024 02:03:05 -0700 (PDT) Date: Sat, 27 Jul 2024 09:03:02 +0000 Mime-Version: 1.0 X-B4-Tracking: v=1; b=H4sIAMW3pGYC/1WMyw6CMBBFf4XM2jF9SARX/odhUWFaJhHGtKbRk P67lZ3Lc3PP2SBRZEpwaTaIlDmxrBXMoYFxdmsg5KkyGGVO6mws5sWh75TWve/93bZQn89Int9 75TZUnjm9JH72aNa/9d/PGjVa46bW1VBHdA0i4UHHURYYSilffFp6JpcAAAA= X-Developer-Key: i=aliceryhl@google.com; a=openpgp; fpr=49F6C1FAA74960F43A5B86A1EE7A392FDE96209F X-Developer-Signature: v=1; a=openpgp-sha256; l=22908; i=aliceryhl@google.com; h=from:subject:message-id; bh=2jAnQ+Mj54soanof2b2IodM75IcnBD3IoxW3HnpSVMQ=; b=owEBbQKS/ZANAwAKAQRYvu5YxjlGAcsmYgBmpLfGvIh2/eiJwCrumvRcagFiw1Fi9eFyjMDvL gRBoTqru2uJAjMEAAEKAB0WIQSDkqKUTWQHCvFIvbIEWL7uWMY5RgUCZqS3xgAKCRAEWL7uWMY5 RkNIEACq6pN5nrt3pMOKJpsPn0K/paIqZS/xw0CEfPxso6dFnWgnt2LdEWnD+Ul7N/27OXwsmmL emwOXvgyUwHFzw0auz+xT5ylk5b2VMw7ciYu6c7tlJaZcX+MdfVNaxjXFzuAkjw93LV3dPa3uCo pwFBDLvvyUh6O7+WHt9iI82gPNZrD+yr6IgnpZDNhZd5sS4ztRf4WgHP7HlF7gxw+k7jQtzUER5 TrdNQKkYWzMpHHaw8mGOk4+FQgbG7qketxaCqKrbGVKLESkevIDWOBhw30Q3RqVPK/3B6B81csn s5G0aq1TuuW82KcwLKAvKa8N4VWOS0kYmsHOs1xXl6xNpUNt05n3LD3L0x0Zk+0bQl5KJ0PHYQi 6UJ2hvmrxRgf73xD17pQnhzLgDBBcR3eMAUKiCY5jdG6Xve8FX/Z8Qt1OmjBiN03XLeldAwej9+ hddaEPsl/QiUWlWmubq5VkZO09ya4sqxaAjt6iwRRn4ks4D1DwckywCSTl7uEH3bPaA8rvUylMU 2e9VreMPE8/139Z7ARzHEooiwQtLgZ05AC2pbtS48WCAOew7F0hzlGx+JH/N8eaMdMP8qVxh32h tgCGURd9gaWkKIIgRh3Z/Ee6fAnloT1wg5JwK7iKH3ERB/fPzd29/rIaBoGhZMvEXoaerGT7ckd 2NKGFuDc5/vqUvA== X-Mailer: b4 0.13-dev-26615 Message-ID: <20240727-vma-v2-1-ab3e5927dc3a@google.com> Subject: [PATCH v2] rust: mm: add abstractions for mm_struct and vm_area_struct From: Alice Ryhl To: Miguel Ojeda , Andrew Morton Cc: Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , " =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= " , Benno Lossin , Andreas Hindborg , Matthew Wilcox , linux-kernel@vger.kernel.org, linux-mm@kvack.org, rust-for-linux@vger.kernel.org, Alice Ryhl X-Rspam-User: X-Rspamd-Queue-Id: 920651C000B X-Rspamd-Server: rspam01 X-Stat-Signature: egn6ngfw1jmpss95sef7r57397nx8dym X-HE-Tag: 1722070986-267086 X-HE-Meta: U2FsdGVkX1+8zqdIxK6q0AlhrWIp1WuI38wTivIZnpAWFkH1Eq5VNPP8K481ozy6BaU/WcikzAp+obfBnwpDq0eD5uEOaPTdmZ/p+3/zgW4CPp0xdgreF5njeW9db7H1b0foCRc03e6zPTOht9Idmp01W/DxZy5s2KhUQo8KrggjPf1bMYmMaaagrGq27yADK4/3Cuqkh5WkKTThteg8JMm95/mJmJiBZN3KURVugolZ44vJqQ5nObAquNBj4AFBBU9/xVKyaCPubPhhQZxGdVLIpBX8g6KKO0plmIweADBK8Olh9/vlSVU6pEHnXxsbdaDRd5Vu5FytdWQoMN5mMwIQxs8KJ7/vMuuHeORY3lxEJXXIqBK9lEvYiH58dQLj+4VdBG8br3CN95umOGEa2mpsZyqWs71A2JQYjpIlZxNZpSQPq1L8DsA4dy+P+zPyCvqGnzQZ6YsSdUoDxH4R284Epe9apLR7tBHkHBT7tgoeG2xQXXhle14G6zK6D27Pq64ifPM+fj7+0QNAwQ2sOMBQxmSSP/sBbCjSVFKHGWQlh7nZtbbGkTjmi+g05FPTGRcUmLJbfNqkG3+b7JWZXYHp3sOiWeSWQvJwKzweTLdOFaStzYYl5ypHffVIuNiZT7CwncU+/WVC2X1WKwLpuWiQ9bRZTX+bOSAxQAUCXZpTV1ZUlasnPIKQyQqTMOGITjTyqh6bHJZG2tT2XqTjEAYSO6oktarpyenzLbC5MNkOBQtIM0LJOdA9mQoStgK1974BJYMhYjI1H6QMnxjVNPSF8Lumyz5M6BKkKJqUZ8d5K1dGgeJCtL+EXh9Lo+R+rP56DgaG86WE63jpNnQ14cYPnVx1F6oStd6j2eaYVURJL/bfHn1NJ8OxyHUVHuuvxGrzxpIhNNgCehc75f5Kls8zfKwtONUjwF/CMS+NIRzSnKz8Tp2ytkiIO7PynmtMvmZ7cveX3aIZMndZ9u9 uKj2hXx1 OBquhrIG1jwWwBPpAFsdnU5p79gFUGRuRO7ho8W9bB0gSjqIU2aDIbpDxHTEougvaJVkDCQY8p/4AXBMv6cPnrsHdysMH8qEvhZ0c1OiIQ2whMn+Hk2eoWnC3qptvJA7IDgCPAKONIDS8Wh+KJ0tZOcMa9ClQjcMg/QAj6ONnHCCDDdfvkqWc0hfFQPqGge85zWmWJ+GvnsNHC2oePKCGjddU4rtfp5TfoBtV/lcidzfO848XDahxmfcUxDQTXuNGVlrO8tocTKfQfYaAnLO0hapj+L17Lav9ILfP1WAsmxHivv0XvgqS9qBf213tfnvPaHLwbNxnNIFQ1pvuSwGBfkQHv5J/I/GuXXnkmLyznFK0qk7qCwyfeCY8uZkLt8iuob+RT1qxg2OnHZZg+IVx5K/JwHLzxu3OAHc0X4PC+mazu7pxe2Y8/SipxN9RStdlDUPWSDitE969AgGoG5MNbo1soRWPjHcaWdg1D7O2yG9lXCbkQPCNxeCdwmoDaKWbdzq+TwvIfIzLLEIFe/B1OE6DXvMo7+1zz957YjtY38yLXxDjIc7dBtCGVVLs1Da7WV4Bbe+X5lxJo0Ghd87NIdCEud5r7RvOp8unK3sNvk3lcsWlU5m0UU+NwZ4hu4+cgIT9J1AUFX6NDvZf/KZ3vQ3rCffTeDi6/mCBa48dlZFv2lKHcAtNxc5ZKUY3bomSz4U2rSv2DMINwY5tlU057zd2F8id4syzX3uQKi2DgXhOA+eAGTrGiSlzig== X-Bogosity: Ham, tests=bogofilter, spamicity=0.000000, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: List-Subscribe: List-Unsubscribe: This is a follow-up to the page abstractions [1] that were recently merged in 6.11. Rust Binder will need these abstractions to manipulate the vma in its implementation of the mmap fop on the Binder file. This patch is based on Wedson's implementation on the old rust branch, but has been changed significantly. All mistakes are Alice's. Link: https://lore.kernel.org/r/20240528-alice-mm-v7-4-78222c31b8f4@google.com [1] Co-developed-by: Wedson Almeida Filho Signed-off-by: Wedson Almeida Filho Signed-off-by: Alice Ryhl --- Changes in v2: - mm.rs is redesigned from scratch making use of AsRef - Add notes about whether destructors may sleep - Rename Area to VmArea - Link to v1: https://lore.kernel.org/r/20240723-vma-v1-1-32ad5a0118ee@google.com --- rust/helpers.c | 61 +++++++++ rust/kernel/lib.rs | 1 + rust/kernel/mm.rs | 335 +++++++++++++++++++++++++++++++++++++++++++++++++ rust/kernel/mm/virt.rs | 199 +++++++++++++++++++++++++++++ rust/kernel/types.rs | 9 ++ 5 files changed, 605 insertions(+) --- base-commit: b1263411112305acf2af728728591465becb45b0 change-id: 20240723-vma-f80119f9fb35 Best regards, diff --git a/rust/helpers.c b/rust/helpers.c index 305f0577fae9..d2205f980112 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -199,6 +199,67 @@ rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) } EXPORT_SYMBOL_GPL(rust_helper_krealloc); +void rust_helper_mmgrab(struct mm_struct *mm) +{ + mmgrab(mm); +} +EXPORT_SYMBOL_GPL(rust_helper_mmgrab); + +void rust_helper_mmdrop(struct mm_struct *mm) +{ + mmdrop(mm); +} +EXPORT_SYMBOL_GPL(rust_helper_mmdrop); + +void rust_helper_mmget(struct mm_struct *mm) +{ + mmget(mm); +} +EXPORT_SYMBOL_GPL(rust_helper_mmget); + +bool rust_helper_mmget_not_zero(struct mm_struct *mm) +{ + return mmget_not_zero(mm); +} +EXPORT_SYMBOL_GPL(rust_helper_mmget_not_zero); + +void rust_helper_mmap_read_lock(struct mm_struct *mm) +{ + mmap_read_lock(mm); +} +EXPORT_SYMBOL_GPL(rust_helper_mmap_read_lock); + +bool rust_helper_mmap_read_trylock(struct mm_struct *mm) +{ + return mmap_read_trylock(mm); +} +EXPORT_SYMBOL_GPL(rust_helper_mmap_read_trylock); + +void rust_helper_mmap_read_unlock(struct mm_struct *mm) +{ + mmap_read_unlock(mm); +} +EXPORT_SYMBOL_GPL(rust_helper_mmap_read_unlock); + +void rust_helper_mmap_write_lock(struct mm_struct *mm) +{ + mmap_write_lock(mm); +} +EXPORT_SYMBOL_GPL(rust_helper_mmap_write_lock); + +void rust_helper_mmap_write_unlock(struct mm_struct *mm) +{ + mmap_write_unlock(mm); +} +EXPORT_SYMBOL_GPL(rust_helper_mmap_write_unlock); + +struct vm_area_struct *rust_helper_vma_lookup(struct mm_struct *mm, + unsigned long addr) +{ + return vma_lookup(mm, addr); +} +EXPORT_SYMBOL_GPL(rust_helper_vma_lookup); + /* * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can * use it in contexts where Rust expects a `usize` like slice (array) indices. diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 5d310e79485f..3cbc4cf847a2 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -33,6 +33,7 @@ pub mod ioctl; #[cfg(CONFIG_KUNIT)] pub mod kunit; +pub mod mm; #[cfg(CONFIG_NET)] pub mod net; pub mod page; diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs new file mode 100644 index 000000000000..ffa778384b6b --- /dev/null +++ b/rust/kernel/mm.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Memory management. +//! +//! C header: [`include/linux/mm.h`](../../../../include/linux/mm.h) + +use crate::{ + bindings, + types::{ARef, AlwaysRefCounted, Opaque}, +}; + +use core::{ + ops::Deref, + ptr::{self, NonNull}, +}; + +pub mod virt; + +/// A wrapper for the kernel's `struct mm_struct`. +/// +/// Since `mm_users` may be zero, the associated address space may not exist anymore. You must use +/// [`mmget_not_zero`] before accessing the address space. +/// +/// The `ARef` smart pointer holds an `mmgrab` refcount. Its destructor may sleep. +/// +/// # Invariants +/// +/// Values of this type are always refcounted. +/// +/// [`mmget_not_zero`]: Mm::mmget_not_zero +pub struct Mm { + mm: Opaque, +} + +/// A wrapper for the kernel's `struct mm_struct`. +/// +/// This type is used only when `mm_users` is known to be non-zero at compile-time. It can be used +/// to access the associated address space. +/// +/// The `ARef` smart pointer holds an `mmget` refcount. Its destructor may sleep. +/// +/// # Invariants +/// +/// Values of this type are always refcounted. The value of `mm_users` is non-zero. +#[repr(transparent)] +pub struct MmWithUser { + mm: Mm, +} + +/// Equivalent to `ARef` but uses `mmput_async` in destructor. +/// +/// The destructor of this type will never sleep. +/// +/// # Invariants +/// +/// `inner` points to a valid `mm_struct` and the `ARefMmWithUserAsync` owns an `mmget` refcount. +pub struct ARefMmWithUserAsync { + inner: NonNull, +} + +// Make all `Mm` methods available on `MmWithUser`. +impl Deref for MmWithUser { + type Target = Mm; + + #[inline] + fn deref(&self) -> &Mm { + &self.mm + } +} + +// These methods are safe to call even if `mm_users` is zero. +impl Mm { + /// Call `mmgrab` on `current.mm`. + #[inline] + pub fn mmgrab_current() -> Option> { + // SAFETY: It's safe to get the `mm` field from current. + let mm = unsafe { + let current = bindings::get_current(); + (*current).mm + }; + + let mm = NonNull::new(mm)?; + + // SAFETY: We just checked that `mm` is not null. + unsafe { bindings::mmgrab(mm.as_ptr()) }; + + // SAFETY: We just created an `mmgrab` refcount. Layouts are compatible due to + // repr(transparent). + Some(unsafe { ARef::from_raw(mm.cast()) }) + } + + /// Returns a raw pointer to the inner `mm_struct`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::mm_struct { + self.mm.get() + } + + /// Obtain a reference from a raw pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` points at an `mm_struct`, and that it is not deallocated + /// during the lifetime 'a. + #[inline] + pub unsafe fn from_raw_mm<'a>(ptr: *const bindings::mm_struct) -> &'a Mm { + // SAFETY: Caller promises that the pointer is valid for 'a. Layouts are compatible due to + // repr(transparent). + unsafe { &*ptr.cast() } + } + + /// Check whether this vma is associated with this mm. + #[inline] + pub fn is_same_mm(&self, area: &virt::VmArea) -> bool { + // SAFETY: The `vm_mm` field of the area is immutable, so we can read it without + // synchronization. + let vm_mm = unsafe { (*area.as_ptr()).vm_mm }; + + ptr::eq(vm_mm, self.as_raw()) + } + + /// Calls `mmget_not_zero` and returns a handle if it succeeds. + #[inline] + pub fn mmget_not_zero(&self) -> Option> { + // SAFETY: The pointer is valid since self is a reference. + let success = unsafe { bindings::mmget_not_zero(self.as_raw()) }; + + if success { + // SAFETY: We just created an `mmget` refcount. + Some(unsafe { ARef::from_raw(NonNull::new_unchecked(self.as_raw().cast())) }) + } else { + None + } + } +} + +// These methods require `mm_users` to be non-zero. +impl MmWithUser { + /// Obtain a reference from a raw pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` points at an `mm_struct`, and that `mm_users` remains + /// non-zero for the duration of the lifetime 'a. + #[inline] + pub unsafe fn from_raw_mm<'a>(ptr: *const bindings::mm_struct) -> &'a MmWithUser { + // SAFETY: Caller promises that the pointer is valid for 'a. The layout is compatible due + // to repr(transparent). + unsafe { &*ptr.cast() } + } + + /// Lock the mmap write lock. + #[inline] + pub fn mmap_write_lock(&self) -> MmapWriteLock<'_> { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmap_write_lock(self.as_raw()) }; + + // INVARIANT: We just acquired the write lock. + MmapWriteLock { mm: self } + } + + /// Lock the mmap read lock. + #[inline] + pub fn mmap_read_lock(&self) -> MmapReadLock<'_> { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmap_read_lock(self.as_raw()) }; + + // INVARIANT: We just acquired the read lock. + MmapReadLock { mm: self } + } + + /// Try to lock the mmap read lock. + #[inline] + pub fn mmap_read_trylock(&self) -> Option> { + // SAFETY: The pointer is valid since self is a reference. + let success = unsafe { bindings::mmap_read_trylock(self.as_raw()) }; + + if success { + // INVARIANT: We just acquired the read lock. + Some(MmapReadLock { mm: self }) + } else { + None + } + } +} + +// SAFETY: By the type invariants, this type is always refcounted. +unsafe impl AlwaysRefCounted for Mm { + fn inc_ref(&self) { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmgrab(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The caller is giving up their refcount. + unsafe { bindings::mmdrop(obj.cast().as_ptr()) }; + } +} + +// SAFETY: By the type invariants, this type is always refcounted. +unsafe impl AlwaysRefCounted for MmWithUser { + fn inc_ref(&self) { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmget(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The caller is giving up their refcount. + unsafe { bindings::mmput(obj.cast().as_ptr()) }; + } +} + +// SAFETY: It is safe to call `mmdrop` on another thread than where `mmgrab` was called. +unsafe impl Send for Mm {} +// SAFETY: All methods on `Mm` can be called in parallel from several threads. +unsafe impl Sync for Mm {} + +// SAFETY: It is safe to call `mmput` on another thread than where `mmget` was called. +unsafe impl Send for MmWithUser {} +// SAFETY: All methods on `MmWithUser` can be called in parallel from several threads. +unsafe impl Sync for MmWithUser {} + +impl From> for ARefMmWithUserAsync { + fn from(mm: ARef) -> ARefMmWithUserAsync { + ARefMmWithUserAsync { + inner: ARef::into_raw(mm).cast(), + } + } +} + +impl From for ARef { + fn from(mm: ARefMmWithUserAsync) -> ARef { + let ptr = mm.inner; + // Skip the destructor. + core::mem::forget(mm); + + // SAFETY: We skipped the destructor, so we can transfer ownership of the refcount to the + // `ARef`. + unsafe { ARef::from_raw(ptr.cast()) } + } +} + +// Make MmWithUser methods available on ARefMmWithUserAsync values. +impl Deref for ARefMmWithUserAsync { + type Target = MmWithUser; + + fn deref(&self) -> &MmWithUser { + // SAFETY: The signature of this method ensures that `self` outlives the returned + // reference. Since `self` owns an `mmget` refcount, `m_users` will remain non-zero for the + // duration of the returned reference. + unsafe { MmWithUser::from_raw_mm(self.inner.as_ptr()) } + } +} + +impl Drop for ARefMmWithUserAsync { + fn drop(&mut self) { + // SAFETY: Values of this type own an `mmget` refcount by the type invariants. + unsafe { bindings::mmput_async(self.as_raw()) }; + } +} + +/// A guard for the mmap read lock. +/// +/// # Invariants +/// +/// This `MmapReadLock` guard owns the mmap read lock. +pub struct MmapReadLock<'a> { + mm: &'a MmWithUser, +} + +impl<'a> MmapReadLock<'a> { + /// Look up a vma at the given address. + #[inline] + pub fn vma_lookup(&self, vma_addr: usize) -> Option<&virt::VmArea> { + // SAFETY: We hold a reference to the mm, so the pointer must be valid. Any value is okay + // for `vma_addr`. + let vma = unsafe { bindings::vma_lookup(self.mm.as_raw(), vma_addr as _) }; + + if vma.is_null() { + None + } else { + // SAFETY: We just checked that a vma was found, so the pointer is valid. Furthermore, + // the returned area will borrow from this read lock guard, so it can only be used + // while the read lock is still held. The returned reference is immutable, so the + // reference cannot be used to modify the area. + unsafe { Some(virt::VmArea::from_raw_vma(vma)) } + } + } +} + +impl Drop for MmapReadLock<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: We hold the read lock by the type invariants. + unsafe { bindings::mmap_read_unlock(self.mm.as_raw()) }; + } +} + +/// A guard for the mmap write lock. +/// +/// # Invariants +/// +/// This `MmapReadLock` guard owns the mmap write lock. +pub struct MmapWriteLock<'a> { + mm: &'a MmWithUser, +} + +impl<'a> MmapWriteLock<'a> { + /// Look up a vma at the given address. + #[inline] + pub fn vma_lookup(&mut self, vma_addr: usize) -> Option<&mut virt::VmArea> { + // SAFETY: We hold a reference to the mm, so the pointer must be valid. Any value is okay + // for `vma_addr`. + let vma = unsafe { bindings::vma_lookup(self.mm.as_raw(), vma_addr as _) }; + + if vma.is_null() { + None + } else { + // SAFETY: We just checked that a vma was found, so the pointer is valid. Furthermore, + // the returned area will borrow from this write lock guard, so it can only be used + // while the write lock is still held. We hold the write lock, so mutable operations on + // the area are okay. + unsafe { Some(virt::VmArea::from_raw_vma_mut(vma)) } + } + } +} + +impl Drop for MmapWriteLock<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: We hold the write lock by the type invariants. + unsafe { bindings::mmap_write_unlock(self.mm.as_raw()) }; + } +} diff --git a/rust/kernel/mm/virt.rs b/rust/kernel/mm/virt.rs new file mode 100644 index 000000000000..2e97ef1eac58 --- /dev/null +++ b/rust/kernel/mm/virt.rs @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Virtual memory. + +use crate::{ + bindings, + error::{to_result, Result}, + page::Page, + types::Opaque, +}; + +/// A wrapper for the kernel's `struct vm_area_struct`. +/// +/// It represents an area of virtual memory. +#[repr(transparent)] +pub struct VmArea { + vma: Opaque, +} + +impl VmArea { + /// Access a virtual memory area given a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `vma` is valid for the duration of 'a, with shared access. The + /// caller must ensure that using the pointer for immutable operations is okay. + #[inline] + pub unsafe fn from_raw_vma<'a>(vma: *const bindings::vm_area_struct) -> &'a Self { + // SAFETY: The caller ensures that the pointer is valid. + unsafe { &*vma.cast() } + } + + /// Access a virtual memory area given a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `vma` is valid for the duration of 'a, with exclusive access. The + /// caller must ensure that using the pointer for immutable and mutable operations is okay. + #[inline] + pub unsafe fn from_raw_vma_mut<'a>(vma: *mut bindings::vm_area_struct) -> &'a mut Self { + // SAFETY: The caller ensures that the pointer is valid. + unsafe { &mut *vma.cast() } + } + + /// Returns a raw pointer to this area. + #[inline] + pub fn as_ptr(&self) -> *mut bindings::vm_area_struct { + self.vma.get() + } + + /// Returns the flags associated with the virtual memory area. + /// + /// The possible flags are a combination of the constants in [`flags`]. + #[inline] + pub fn flags(&self) -> usize { + // SAFETY: The pointer is valid since self is a reference. The field is valid for reading + // given a shared reference. + unsafe { (*self.as_ptr()).__bindgen_anon_2.vm_flags as _ } + } + + /// Sets the flags associated with the virtual memory area. + /// + /// The possible flags are a combination of the constants in [`flags`]. + #[inline] + pub fn set_flags(&mut self, flags: usize) { + // SAFETY: The pointer is valid since self is a reference. The field is valid for writing + // given an exclusive reference. + unsafe { (*self.as_ptr()).__bindgen_anon_2.vm_flags = flags as _ }; + } + + /// Returns the start address of the virtual memory area. + #[inline] + pub fn start(&self) -> usize { + // SAFETY: The pointer is valid since self is a reference. The field is valid for reading + // given a shared reference. + unsafe { (*self.as_ptr()).__bindgen_anon_1.__bindgen_anon_1.vm_start as _ } + } + + /// Returns the end address of the virtual memory area. + #[inline] + pub fn end(&self) -> usize { + // SAFETY: The pointer is valid since self is a reference. The field is valid for reading + // given a shared reference. + unsafe { (*self.as_ptr()).__bindgen_anon_1.__bindgen_anon_1.vm_end as _ } + } + + /// Make this vma anonymous. + #[inline] + pub fn set_anonymous(&mut self) { + // SAFETY: The pointer is valid since self is a reference. The field is valid for writing + // given an exclusive reference. + unsafe { (*self.as_ptr()).vm_ops = core::ptr::null() }; + } + + /// Maps a single page at the given address within the virtual memory area. + /// + /// This operation does not take ownership of the page. + #[inline] + pub fn vm_insert_page(&mut self, address: usize, page: &Page) -> Result { + // SAFETY: The page is guaranteed to be order 0. The range of `address` is already checked + // by `vm_insert_page`. `self.vma` and `page.as_ptr()` are guaranteed by their respective + // type invariants to be valid. + to_result(unsafe { bindings::vm_insert_page(self.as_ptr(), address as _, page.as_ptr()) }) + } + + /// Unmap pages in the given page range. + #[inline] + pub fn zap_page_range_single(&self, address: usize, size: usize) { + // SAFETY: The pointer is valid since self is a reference. Any value of `address` and + // `size` is allowed. + unsafe { + bindings::zap_page_range_single( + self.as_ptr(), + address as _, + size as _, + core::ptr::null_mut(), + ) + }; + } +} + +/// Container for [`VmArea`] flags. +pub mod flags { + use crate::bindings; + + /// No flags are set. + pub const NONE: usize = bindings::VM_NONE as _; + + /// Mapping allows reads. + pub const READ: usize = bindings::VM_READ as _; + + /// Mapping allows writes. + pub const WRITE: usize = bindings::VM_WRITE as _; + + /// Mapping allows execution. + pub const EXEC: usize = bindings::VM_EXEC as _; + + /// Mapping is shared. + pub const SHARED: usize = bindings::VM_SHARED as _; + + /// Mapping may be updated to allow reads. + pub const MAYREAD: usize = bindings::VM_MAYREAD as _; + + /// Mapping may be updated to allow writes. + pub const MAYWRITE: usize = bindings::VM_MAYWRITE as _; + + /// Mapping may be updated to allow execution. + pub const MAYEXEC: usize = bindings::VM_MAYEXEC as _; + + /// Mapping may be updated to be shared. + pub const MAYSHARE: usize = bindings::VM_MAYSHARE as _; + + /// Do not copy this vma on fork. + pub const DONTCOPY: usize = bindings::VM_DONTCOPY as _; + + /// Cannot expand with mremap(). + pub const DONTEXPAND: usize = bindings::VM_DONTEXPAND as _; + + /// Lock the pages covered when they are faulted in. + pub const LOCKONFAULT: usize = bindings::VM_LOCKONFAULT as _; + + /// Is a VM accounted object. + pub const ACCOUNT: usize = bindings::VM_ACCOUNT as _; + + /// should the VM suppress accounting. + pub const NORESERVE: usize = bindings::VM_NORESERVE as _; + + /// Huge TLB Page VM. + pub const HUGETLB: usize = bindings::VM_HUGETLB as _; + + /// Synchronous page faults. + pub const SYNC: usize = bindings::VM_SYNC as _; + + /// Architecture-specific flag. + pub const ARCH_1: usize = bindings::VM_ARCH_1 as _; + + /// Wipe VMA contents in child.. + pub const WIPEONFORK: usize = bindings::VM_WIPEONFORK as _; + + /// Do not include in the core dump. + pub const DONTDUMP: usize = bindings::VM_DONTDUMP as _; + + /// Not soft dirty clean area. + pub const SOFTDIRTY: usize = bindings::VM_SOFTDIRTY as _; + + /// Can contain "struct page" and pure PFN pages. + pub const MIXEDMAP: usize = bindings::VM_MIXEDMAP as _; + + /// MADV_HUGEPAGE marked this vma. + pub const HUGEPAGE: usize = bindings::VM_HUGEPAGE as _; + + /// MADV_NOHUGEPAGE marked this vma. + pub const NOHUGEPAGE: usize = bindings::VM_NOHUGEPAGE as _; + + /// KSM may merge identical pages. + pub const MERGEABLE: usize = bindings::VM_MERGEABLE as _; +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index bd189d646adb..143a2bf06941 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -366,6 +366,15 @@ pub unsafe fn from_raw(ptr: NonNull) -> Self { _p: PhantomData, } } + + /// Pass ownership of the refcount to a raw pointer. + pub fn into_raw(self) -> NonNull { + let ptr = self.ptr; + // Skip the destructor. + core::mem::forget(self); + + ptr + } } impl Clone for ARef {