From patchwork Mon Oct 14 16:00:20 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alice Ryhl X-Patchwork-Id: 13835299 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by smtp.lore.kernel.org (Postfix) with ESMTP id 96910D1812B for ; Mon, 14 Oct 2024 16:00:56 +0000 (UTC) Received: by kanga.kvack.org (Postfix) id 1FD926B007B; Mon, 14 Oct 2024 12:00:56 -0400 (EDT) Received: by kanga.kvack.org (Postfix, from userid 40) id 1ADEB6B0083; Mon, 14 Oct 2024 12:00:56 -0400 (EDT) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id 04E8D6B0085; Mon, 14 Oct 2024 12:00:56 -0400 (EDT) X-Delivered-To: linux-mm@kvack.org Received: from relay.hostedemail.com (smtprelay0017.hostedemail.com [216.40.44.17]) by kanga.kvack.org (Postfix) with ESMTP id D90796B007B for ; Mon, 14 Oct 2024 12:00:55 -0400 (EDT) Received: from smtpin20.hostedemail.com (a10.router.float.18 [10.200.18.1]) by unirelay07.hostedemail.com (Postfix) with ESMTP id EC3B416104D for ; Mon, 14 Oct 2024 16:00:46 +0000 (UTC) X-FDA: 82672670934.20.8C5505A Received: from mail-yb1-f201.google.com (mail-yb1-f201.google.com [209.85.219.201]) by imf10.hostedemail.com (Postfix) with ESMTP id BA15EC0013 for ; Mon, 14 Oct 2024 16:00:50 +0000 (UTC) Authentication-Results: imf10.hostedemail.com; dkim=pass header.d=google.com header.s=20230601 header.b=nY6wN7Za; dmarc=pass (policy=reject) header.from=google.com; spf=pass (imf10.hostedemail.com: domain of 3NEANZwkKCCkFQNHJWdMQLTTLQJ.HTRQNSZc-RRPaFHP.TWL@flex--aliceryhl.bounces.google.com designates 209.85.219.201 as permitted sender) smtp.mailfrom=3NEANZwkKCCkFQNHJWdMQLTTLQJ.HTRQNSZc-RRPaFHP.TWL@flex--aliceryhl.bounces.google.com ARC-Seal: i=1; s=arc-20220608; d=hostedemail.com; t=1728921622; a=rsa-sha256; cv=none; b=EGT6ban/A/k75r9O2PfIVwupb+ebY/hkf2kAgQM636GjpJZ9XtSBThBXOScwAwgsb/74Zk ue8SAiXDhmHnJZ+lydX3jW/A3ds8QXqfAZdebsVu3kE9BcFwjqLwA9wGqThpuZlOO+QG32 5dzOMaCED/y0CI7FAid/DDU+8N77A+8= ARC-Authentication-Results: i=1; imf10.hostedemail.com; dkim=pass header.d=google.com header.s=20230601 header.b=nY6wN7Za; dmarc=pass (policy=reject) header.from=google.com; spf=pass (imf10.hostedemail.com: domain of 3NEANZwkKCCkFQNHJWdMQLTTLQJ.HTRQNSZc-RRPaFHP.TWL@flex--aliceryhl.bounces.google.com designates 209.85.219.201 as permitted sender) smtp.mailfrom=3NEANZwkKCCkFQNHJWdMQLTTLQJ.HTRQNSZc-RRPaFHP.TWL@flex--aliceryhl.bounces.google.com ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=hostedemail.com; s=arc-20220608; t=1728921622; h=from:from:sender:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-type:content-transfer-encoding:in-reply-to: references:dkim-signature; bh=RKRYvQc5u4z80Pon/cd7Cmgo0D0IQWgEKKFqzih4+Zw=; b=qwMPKNqffgtTQkZ48TccVq7dc4YR8nSit+opfv3JXuMg+puu8y5pe8Icim6Sgiw2CMBCEG glVutxLKKZhJxBANB1yCbIpzowkyo65BQeWZr5SfrjL2vj6TuOYIY2g3yXVhN++krEaZdw RHcOh7F/+khDzFvu9kb00ds3/iO2Wbk= Received: by mail-yb1-f201.google.com with SMTP id 3f1490d57ef6-e290947f6f8so6699035276.2 for ; Mon, 14 Oct 2024 09:00:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20230601; t=1728921652; x=1729526452; darn=kvack.org; h=cc:to:from:subject:message-id:mime-version:date:from:to:cc:subject :date:message-id:reply-to; bh=RKRYvQc5u4z80Pon/cd7Cmgo0D0IQWgEKKFqzih4+Zw=; b=nY6wN7ZaLpRZGB9oKmwlnjRWnT4w9QZGTP/9a9vyO9XkH+x2Jn/jz9UOR0qQd8tmI2 5O5m1q2JxvmfLREGm7kZCY0WPVcLKGHiAlUMPFjiOnIrRvMohs3meek5mLXR2LoHTAJD HX3GZGRl1JKxfwfSfuPrA9EorbS7LxtAYAk520icJ8zHhSMfT8Si9CVkV/0DaGt1dYox VnSp12RLCkHoOmZPZfLOhAVMIBvkKcaiRCVJX0UqB01+Vq32PSizqKqhLHIH2TnHcFbR fQvQoC2D1frfFkGF+F8SyRzO9tdcWWCs6WTxXdPJcWzbF5QuVETR2zHA8EfMT3VYADrr S8nw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1728921652; x=1729526452; h=cc:to:from:subject:message-id:mime-version:date:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=RKRYvQc5u4z80Pon/cd7Cmgo0D0IQWgEKKFqzih4+Zw=; b=bHoEDHpXVE66qj0qGXb9+6HB26VSv3Wwtp27A8tflkrV0PNQ+As8ejLTqjbrtOKkxG gKm64r1m2/dQAt/kpjMZiwW4/1doJvLVFAm5qIIm28IPnTeaIVjr9ahPzgwPU5x1mm6G DLC6ntE6GMI1qdYqH5jUTyr9HLB++qda5zM+9GhBP7FQnKbCxPGLPpx9ky/qSnxyw6q7 Dx5lhHmxLzYqUk5BRHmQnoKzw40agxgcHQ4X8zauLMPSTVkjM0FAcwosHkzgy1kujgv2 ExxYsgtoF9aVGDww7IZT0BhkNnxgeKV2/TgReWRLitvhD/tTOfwkegKYJK6byNyEfD8b CTLQ== X-Forwarded-Encrypted: i=1; AJvYcCUJiFxPtoZInbB6e35PjZ0yxWv6JLKIhMbWYo6Mh6rZYsGtCx6a5reFJEknNmyMWJtJ7507rI6bfg==@kvack.org X-Gm-Message-State: AOJu0YzUjxxJOmOeWfHvhUrifjCX2zj96a8Jc7Ne0qX4kmHBm7ovyhpF x5JTxnOoDkGED1CLFB9rwK6ADHUJsVdujIgN0qoCRkcZr+Ynbjb5MMHjoglCYl6u30T5WVp/fZ2 UXPO9Ll2lfsH1Zg== X-Google-Smtp-Source: AGHT+IG5FwhCjh1GLIoAJhFPEeSSbvHOtmfOHQWD2Zr/YvqWPnbrIUZTRNTcz1wpS5hNI1L8NcC6FaZK79o5KOs= X-Received: from aliceryhl.c.googlers.com ([fda3:e722:ac3:cc00:28:9cb1:c0a8:35bd]) (user=aliceryhl job=sendgmr) by 2002:a5b:284:0:b0:e1a:9ed2:67f4 with SMTP id 3f1490d57ef6-e2919c5ac92mr8791276.2.1728921652517; Mon, 14 Oct 2024 09:00:52 -0700 (PDT) Date: Mon, 14 Oct 2024 16:00:20 +0000 Mime-Version: 1.0 X-B4-Tracking: v=1; b=H4sIABNADWcC/0XMTQrCMBCG4auUWRvJj7HRlfeQLto6SQY1kUSCU nJ3Y0FkVu/w8SyQMRFmOHYLJCyUKYYWctPB7MfgkNGlNUgud/wgBMs+UbhiYtaoXoyW82lvoM0 fCS29Vuo8tPaUnzG9V7mI7/eHyD9SBGtnpt4KqZXW6uRidDfczvEOQ631A8u/NUmhAAAA X-Developer-Key: i=aliceryhl@google.com; a=openpgp; fpr=49F6C1FAA74960F43A5B86A1EE7A392FDE96209F X-Developer-Signature: v=1; a=openpgp-sha256; l=16804; i=aliceryhl@google.com; h=from:subject:message-id; bh=kk1X9i84u1Iu7BNt/VNlbzR35kOrvNPxRSHSbE4ofbY=; b=owEBbQKS/ZANAwAKAQRYvu5YxjlGAcsmYgBnDUAVAyU7uKSyj8He8g4rJDHEvi6UaJ9gi0S82 JkB+tnT6JGJAjMEAAEKAB0WIQSDkqKUTWQHCvFIvbIEWL7uWMY5RgUCZw1AFQAKCRAEWL7uWMY5 RohcD/4vri9xp+vINrOeHveWZnfVKP92H7DZKk6AWsojrBpBaoZ+QdjWP5zq7c1hUOyhomlgm/b 85cWTJy7fmZQAM3t/4I7d1e7X/ig89tznLsZNpYv5RQKsmAbIsERnlaYM0tF6t+Gm+NbgnBEgiA Y1ybl40JF1ESckWI2D122OuG+r9CO0u4TtyHrarSsqSBTdD3/taPmGf+eUJTyTXY8sSwhpzSdsn yKZB5egd7ZFnA4krLCrmyv7E0lQBAM9xbfnPJ3TvF43qc7herNQFEjQ6QDBOpbez0SNWyNH/Dcc BDBWFaM55XBZp4YIFwupmE21+erNvGnNAtfbNjSXzmxX6NTzx6nTt5HwXBt2mRtqEkVfG2DEt+H jaWm0w6vttv3vlIF2hnse910qfx29CLMXtMw2OafjCg7I+AoR8bOJQXYntKlcGXNv4dAddgaerP pW6Ffjbnb9SGk1asazfxFnIOKTlw/i71MXuXNCxrAzllx6oIGMcyYuhNh+KDOWE90RD4OUSj0FT IkLdPOGp09HyP8LPpZ6HmwUUejgt0zBe2DIBFzu8XCa4OkUaCVcIdO88J+lAnsIGy8R4oumxQiL TRHfWd54YTzA13nTXZS4fDcC1112zQudCU4DjYM5ql21pjqHBmMMCeK5bSwwicTYnSm+HVgia26 Ya2SaMApsN7AXlA== X-Mailer: b4 0.13.0 Message-ID: <20241014-shrinker-v2-1-04719efd2342@google.com> Subject: [PATCH v2] rust: shrinker: add shrinker abstraction From: Alice Ryhl To: Miguel Ojeda , Andrew Morton , Dave Chinner , Qi Zheng , Roman Gushchin , Muchun Song Cc: Boqun Feng , Gary Guo , " =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= " , Benno Lossin , Andreas Hindborg , Trevor Gross , linux-kernel@vger.kernel.org, linux-mm@kvack.org, rust-for-linux@vger.kernel.org, Alice Ryhl X-Rspam-User: X-Stat-Signature: hoooh9imoqmwp8hzqnqj18g5eejtbo9r X-Rspamd-Queue-Id: BA15EC0013 X-Rspamd-Server: rspam02 X-HE-Tag: 1728921650-389777 X-HE-Meta: U2FsdGVkX18F2ER89sSAW/eetCKyVp193RqH1PEpJDOt3yAK7pG26pfqyYb00yPDAPEsTGvOgEIAsSbjysvGdx5pWBlHZECuqwa2eNHvhyR4r15Uud20hA/rRKvQu6PCGR891Y4lLLsE0tzlLcFJ9O80L9hlyDdxhud9tY/DXoOpOu3DTCZ4TsFwhbAg+dxAeklE2vmYtB8bSlbl70KOEuH75zEUyrtZr9EfNz9025KguOkXZ7blhP9k08uF6SZkAof3AgFGlQg7ttvInN2fIZdMsVOIbZPKztK4YJjnSuEZHFgajoYnaWy2ANrXYBR2rd2ExBvAYTmVnN7WeRjH97H8kROlGfhOqzx4ohNartJlyuehk7Ixa4tAKcaZ0LE+tuX3Uw2UfEclqhgiZ113jOMeetP7Kk5UV+dGzT91kHL+DzlLa5krRBCEpTnjCnjMmRqS4wvwi51uAc+vvf24QYtlB4XeV6FX2GhCT4tJTJeDisp47y8e8yLgBOrl5WNjyEWH3ui6GuQHf3jNssG5brQrhJIOPq3KhcnOLB77pxO8J538Q+ovtSxTCOqHtIe6JavfyDe6eZIfdrVrFKlmanz7XI8UnF6/Z3f2gLcz2lE8RPWXi+mj6mPLe5zC/WM2CjblR3peORL5WimnQW1IaN9Ekbp/Th2ArwjT9Qx0AO23Ah0ylAc5a5i92eg9Y+6vpvnVEZi3eOrxhe8OdbcYABewDG0mfTqNV0H5JX7ZL5lUxR/A2JAXv8t8R7MZ+Sz7wqAsjP2fNMkzGcBuX3S2RZWQh/6jkqtbtSUFdZxf5wccj2dh0Z5eihWZ6OQq0ffy0pKvra03+y+4Y285VNRzmhT6TY4K02WBpKZnIMB2Ksq+Z10o88AnddfRiXRIECxC53oQ5ovg5nBK0XOANV4sKS3xvMNOmf0Oo4XmO2slgDsgVXp9uYMw/14DYENw6vQnYpGGQiGcMa+QdcaQ8NL iUXWM/80 6NNfpFqWLdErhwbmyOihsGgnzERx20A9wGF+fQvoqJiXsalLITWi26K87JuUpdMpqA+lQfjY3Xm0sP1jqf5h59KC5q88x98n5TusqRz6ir+3+VhI0BVJGIaOkHPXdnzMlZu4WaX+yJpHTz2C462LOoeotT49K7BwZVLBgo2MRbbiEGPcYGCvJl0FoeZcwa9Mo7fUrVXYzXjOCsZpDO9rAwdRcVdwgZNMIEoP2T87GMRx2/S6XLNg/eftN38UgQ51Gxx3BlGbJfR2yuUmNDo2n+1fKm7AaLrtRV11MkY36NPEnx8ereRJt+Y5o1lLSyx5pViY/IH+6s7rcFZT3hnYfV1oiOkXpJJhdhAA3Re+c3nNuVllIOMWZ69FbqY5aiUtXKio16yLA9yUI4wLHYN762hRc8nbVo9T53cKMZsWFWDWGhj4iwuccXheIvCnehvGpW9KVVTdTXt8pUUtnLDF41MiBSXUYGITZ2trtp9dMD6O4jUpCycjSm9ab9bTUhDDn/50t3hPGN40XT983Vy8sLe1xfYhW9C4yOg0kSbBolVhWBcAgaBbHwaY5SP/TTFoagsM6OnD+/WkEOZMQeYW/e6gLJ7K90wG3OQ1uLq7TCN8d/ZTU2Y+NMBD5yF4Rje1MXKBD5YA3rWa6OyJJ03r1Q560RLEeszix1LNz X-Bogosity: Ham, tests=bogofilter, spamicity=0.000000, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: List-Subscribe: List-Unsubscribe: Rust Binder holds incoming transactions in a read-only mmap'd region where it manually manages the pages. These pages are only in use until the incoming transaction is consumed by userspace, but the kernel will keep the pages around for future transactions. Rust Binder registers a shrinker with the kernel so that it can give back these pages if the system comes under memory pressure. Creating a shrinker is done via the ShrinkerBuilder type. Using a builder means that some options (seeks, batch) can be optional, while other options (name, private data) can be mandatory. Unlike seeks/batch, the private data is not set using a `set_private_data` method, as this makes it mandatory to provide a private data pointer. The user specifies the callbacks in use by implementing the Shrinker trait for the type used for the private data. This requires specifying three things: implementations for count_objects and scan_objects, and the pointer type that the private data will be wrapped in. The return values of count_objects and scan_objects are provided using newtypes called CountObjects and ScanObjects respectively. These types prevent the user from e.g. returning SHRINK_STOP from count_objects or returning SHRINK_EMPTY from scan_objects. The CountObjects newtype treats CountObjects::new(0) as "the count is unknown" instead of "the count is zero" for consistency with the way that the C code works today. ShrinkControl is not using Opaque because we need mutable access. Support for numa/memcg aware shrinkers is not included here, as they cannot usefully be used without having list_lru bindings. Support for that will happen as a follow-up. Signed-off-by: Alice Ryhl --- Dave, I still had a few outstanding questions in the thread on the last version. Most prominently the question on what to do with the builder instead. I didn't change it in this version as I'm not sure which alternative you prefer. --- Changes in v2: - Rename to ShrinkerBuilder / ShrinkerRegistration. - Rename `alloc` to `new`. - Update CountOjects to match the way C does it. - Change ScanObjects::MAX to SHRINK_STOP-1. - Rename gfp_fs to reclaim_fs_allowed, and add reclaim_io_allowed. - Remove max check in set_nr_scanned. - Comment that numa/memcg aware shrinkers aren't supported yet. - Link to v1: https://lore.kernel.org/r/20240912-shrinker-v1-1-18b7f1253553@google.com --- rust/bindings/bindings_helper.h | 3 + rust/kernel/lib.rs | 1 + rust/kernel/shrinker.rs | 335 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 339 insertions(+) --- base-commit: 8cf0b93919e13d1e8d4466eb4080a4c4d9d66d7b change-id: 20240911-shrinker-f8371af00b68 Best regards, diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index ae82e9c941af..fd6d15f5dde1 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -31,4 +32,6 @@ const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; const gfp_t RUST_CONST_HELPER_GFP_NOWAIT = GFP_NOWAIT; const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO; const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; +const gfp_t RUST_CONST_HELPER___GFP_FS = ___GFP_FS; +const gfp_t RUST_CONST_HELPER___GFP_IO = ___GFP_IO; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index b5f4b3ce6b48..2e9ca7d413c4 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -45,6 +45,7 @@ pub mod prelude; pub mod print; pub mod rbtree; +pub mod shrinker; pub mod sizes; mod static_assert; #[doc(hidden)] diff --git a/rust/kernel/shrinker.rs b/rust/kernel/shrinker.rs new file mode 100644 index 000000000000..d87ad08e917c --- /dev/null +++ b/rust/kernel/shrinker.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Shrinker for handling memory pressure. +//! +//! C header: [`include/linux/shrinker.h`](srctree/include/linux/shrinker.h) + +use crate::{alloc::AllocError, bindings, c_str, str::CStr, types::ForeignOwnable}; + +use core::{ + ffi::{c_int, c_long, c_ulong, c_void}, + marker::PhantomData, + ptr::NonNull, +}; + +const SHRINK_STOP: c_ulong = bindings::SHRINK_STOP as c_ulong; +const SHRINK_EMPTY: c_ulong = bindings::SHRINK_EMPTY as c_ulong; + +/// The default value for the number of seeks needed to recreate an object. +pub const DEFAULT_SEEKS: u32 = bindings::DEFAULT_SEEKS; + +/// An unregistered shrinker. +/// +/// This type can be used to modify the settings of the shrinker before it is registered. +/// +/// # Invariants +/// +/// The `shrinker` pointer references an unregistered shrinker. +pub struct ShrinkerBuilder { + shrinker: NonNull, +} + +// SAFETY: Moving an unregistered shrinker between threads is okay. +unsafe impl Send for ShrinkerBuilder {} +// SAFETY: An unregistered shrinker is thread safe. +unsafe impl Sync for ShrinkerBuilder {} + +impl ShrinkerBuilder { + /// Create a new shrinker. + pub fn new(name: &CStr) -> Result { + // TODO: Support numa/memcg aware shrinkers once list_lru is available. + let flags = 0; + + // SAFETY: Passing `0` as flags is okay. Using `%s` as the format string is okay when we + // pass a nul-terminated string as the string for `%s` to print. + let ptr = unsafe { + bindings::shrinker_alloc(flags, c_str!("%s").as_char_ptr(), name.as_char_ptr()) + }; + + let shrinker = NonNull::new(ptr).ok_or(AllocError)?; + + // INVARIANT: The allocated shrinker is unregistered. + Ok(Self { shrinker }) + } + + /// Create a new shrinker using format arguments for the name. + pub fn new_fmt(name: core::fmt::Arguments<'_>) -> Result { + // TODO: Support numa/memcg aware shrinkers once list_lru is available. + let flags = 0; + + // SAFETY: Passing `0` as flags is okay. Using `%pA` as the format string is okay when we + // pass a `fmt::Arguments` as the value to print. + let ptr = unsafe { + bindings::shrinker_alloc( + flags, + c_str!("%pA").as_char_ptr(), + &name as *const _ as *const c_void, + ) + }; + + let shrinker = NonNull::new(ptr).ok_or(AllocError)?; + + // INVARIANT: The allocated shrinker is unregistered. + Ok(Self { shrinker }) + } + + /// Set the number of seeks needed to recreate an object. + pub fn set_seeks(&mut self, seeks: u32) { + unsafe { (*self.shrinker.as_ptr()).seeks = seeks as c_int }; + } + + /// Set the batch size for reclaiming on this shrinker. + pub fn set_batch(&mut self, batch: usize) { + unsafe { (*self.shrinker.as_ptr()).batch = batch as c_long }; + } + + /// Register the shrinker. + /// + /// The provided pointer is used as the private data, and the type `T` determines the callbacks + /// that the shrinker will use. + pub fn register(self, private_data: T::Ptr) -> ShrinkerRegistration { + let shrinker = self.shrinker; + let ptr = shrinker.as_ptr(); + + // The destructor of `self` calls `shrinker_free`, so skip the destructor. + core::mem::forget(self); + + let private_data_ptr = ::into_foreign(private_data); + + // SAFETY: We own the private data, so we can assign to it. + unsafe { (*ptr).private_data = private_data_ptr.cast_mut() }; + // SAFETY: The shrinker is not yet registered, so we can update this field. + unsafe { (*ptr).count_objects = Some(rust_count_objects::) }; + // SAFETY: The shrinker is not yet registered, so we can update this field. + unsafe { (*ptr).scan_objects = Some(rust_scan_objects::) }; + + // SAFETY: The shrinker is unregistered, so it's safe to register it. + unsafe { bindings::shrinker_register(ptr) }; + + ShrinkerRegistration { + shrinker, + _phantom: PhantomData, + } + } +} + +impl Drop for ShrinkerBuilder { + fn drop(&mut self) { + // SAFETY: The shrinker is a valid but unregistered shrinker, and we will not use it + // anymore. + unsafe { bindings::shrinker_free(self.shrinker.as_ptr()) }; + } +} + +/// A shrinker that is registered with the kernel. +/// +/// # Invariants +/// +/// The `shrinker` pointer refers to a registered shrinker using `T` as the private data. +pub struct ShrinkerRegistration { + shrinker: NonNull, + _phantom: PhantomData, +} + +// SAFETY: This allows you to deregister the shrinker from a different thread, which means that +// private data could be dropped from any thread. +unsafe impl Send for ShrinkerRegistration where T::Ptr: Send {} +// SAFETY: The only thing you can do with an immutable reference is access the private data, which +// is okay to access in parallel as the `Shrinker` trait requires the private data to be `Sync`. +unsafe impl Sync for ShrinkerRegistration {} + +impl ShrinkerRegistration { + /// Access the private data in this shrinker. + pub fn private_data(&self) -> ::Borrowed<'_> { + // SAFETY: We own the private data, so we can access it. + let private = unsafe { (*self.shrinker.as_ptr()).private_data }; + // SAFETY: By the type invariants, the private data is `T`. This access could happen in + // parallel with a shrinker callback, but that's okay as the `Shrinker` trait ensures that + // `T::Ptr` is `Sync`. + unsafe { ::borrow(private) } + } +} + +impl Drop for ShrinkerRegistration { + fn drop(&mut self) { + // SAFETY: We own the private data, so we can access it. + let private = unsafe { (*self.shrinker.as_ptr()).private_data }; + // SAFETY: We will not access the shrinker after this call. + unsafe { bindings::shrinker_free(self.shrinker.as_ptr()) }; + // SAFETY: The above call blocked until the completion of any shrinker callbacks, so there + // are no longer any users of the private data. + drop(unsafe { ::from_foreign(private) }); + } +} + +/// Callbacks for a shrinker. +pub trait Shrinker { + /// The pointer type used to store the private data of the shrinker. + /// + /// Needs to be `Sync` because the shrinker callback could access this value immutably from + /// several thread in parallel. + type Ptr: ForeignOwnable + Sync; + + /// Count the number of freeable items in the cache. + fn count_objects( + me: ::Borrowed<'_>, + sc: ShrinkControl<'_>, + ) -> CountObjects; + + /// Free some objects in this cache. + fn scan_objects( + me: ::Borrowed<'_>, + sc: ShrinkControl<'_>, + ) -> ScanObjects; +} + +/// How many objects are there in the cache? +/// +/// This is used as the return value of [`Shrinker::count_objects`]. +pub struct CountObjects { + inner: c_ulong, +} + +impl CountObjects { + /// Indicates that the number of objects is zero. + pub const EMPTY: Self = Self { + inner: SHRINK_EMPTY, + }; + + /// The maximum possible number of freeable objects. + pub const MAX: Self = Self { + // The shrinker code assumes that it can multiply this value by two without overflow. + inner: c_ulong::MAX / 2, + }; + + /// Creates a new `CountObjects` with the given value. + /// + /// This should be the number of objects that were actually freed. Objects that were scanned + /// but not freed should be counted in `nr_scanned` but not here. + /// + /// If `count` is zero, then this indicates that the real count is unknown. Use + /// `CountObjects::EMPTY` to indicate that the shrinker is empty. + pub fn new(count: usize) -> Self { + if count > Self::MAX.inner as usize { + return Self::MAX; + } + + Self { + inner: count as c_ulong, + } + } +} + +/// How many objects were freed? +/// +/// This is used as the return value of [`Shrinker::scan_objects`]. +pub struct ScanObjects { + inner: c_ulong, +} + +impl ScanObjects { + /// Indicates that the shrinker should stop trying to free objects from this cache due to + /// potential deadlocks. + pub const STOP: Self = Self { inner: SHRINK_STOP }; + + /// The maximum possible number of freeable objects. + pub const MAX: Self = Self { + inner: SHRINK_STOP - 1, + }; + + /// Creates a new `CountObjects` with the given value. + pub fn from_count(count: usize) -> Self { + if count > Self::MAX.inner as usize { + return Self::MAX; + } + + Self { + inner: count as c_ulong, + } + } +} + +/// This struct is used to pass information from page reclaim to the shrinkers. +/// +/// # Invariants +/// +/// `ptr` has exclusive access to a valid `struct shrink_control`. +pub struct ShrinkControl<'a> { + ptr: NonNull, + _phantom: PhantomData<&'a bindings::shrink_control>, +} + +impl<'a> ShrinkControl<'a> { + /// Create a `ShrinkControl` from a raw pointer. + /// + /// # Safety + /// + /// The pointer should point at a valid `shrink_control` for the duration of 'a. + pub unsafe fn from_raw(ptr: *mut bindings::shrink_control) -> Self { + Self { + // SAFETY: Caller promises that this pointer is valid. + ptr: unsafe { NonNull::new_unchecked(ptr) }, + _phantom: PhantomData, + } + } + + /// Determines whether it is safe to call into filesystem code. + pub fn reclaim_fs_allowed(&self) -> bool { + // SAFETY: Okay by type invariants. + let mask = unsafe { (*self.ptr.as_ptr()).gfp_mask }; + + (mask & bindings::__GFP_FS) != 0 + } + + /// Determines whether it is safe to call into IO code. + pub fn reclaim_io_allowed(&self) -> bool { + // SAFETY: Okay by type invariants. + let mask = unsafe { (*self.ptr.as_ptr()).gfp_mask }; + + (mask & bindings::__GFP_IO) != 0 + } + + /// Returns the number of objects that `scan_objects` should try to reclaim. + pub fn nr_to_scan(&self) -> usize { + // SAFETY: Okay by type invariants. + unsafe { (*self.ptr.as_ptr()).nr_to_scan as usize } + } + + /// The callback should set this value to the number of objects inspected by the shrinker. + pub fn set_nr_scanned(&mut self, val: usize) { + // SAFETY: Okay by type invariants. + unsafe { (*self.ptr.as_ptr()).nr_scanned = val as c_ulong }; + } +} + +unsafe extern "C" fn rust_count_objects( + shrink: *mut bindings::shrinker, + sc: *mut bindings::shrink_control, +) -> c_ulong { + // SAFETY: We own the private data, so we can access it. + let private = unsafe { (*shrink).private_data }; + // SAFETY: This function is only used with shrinkers where `T` is the type of the private data. + let private = unsafe { ::borrow(private) }; + // SAFETY: The caller passes a valid `sc` pointer. + let sc = unsafe { ShrinkControl::from_raw(sc) }; + + let ret = T::count_objects(private, sc); + ret.inner +} + +unsafe extern "C" fn rust_scan_objects( + shrink: *mut bindings::shrinker, + sc: *mut bindings::shrink_control, +) -> c_ulong { + // SAFETY: We own the private data, so we can access it. + let private = unsafe { (*shrink).private_data }; + // SAFETY: This function is only used with shrinkers where `T` is the type of the private data. + let private = unsafe { ::borrow(private) }; + // SAFETY: The caller passes a valid `sc` pointer. + let sc = unsafe { ShrinkControl::from_raw(sc) }; + + let ret = T::scan_objects(private, sc); + ret.inner +}