diff options
54 files changed, 1460 insertions, 376 deletions
diff --git a/Documentation/rust/quick-start.rst b/Documentation/rust/quick-start.rst index a6ec3fa94d33..f79aa3c7138e 100644 --- a/Documentation/rust/quick-start.rst +++ b/Documentation/rust/quick-start.rst @@ -90,7 +90,7 @@ they should generally work out of the box, e.g.:: Ubuntu ****** -Ubuntu 25.10 and 26.04 LTS provide recent Rust releases and thus they should +Ubuntu 26.04 LTS provides recent Rust releases and thus it should generally work out of the box, e.g.:: apt install rustc rust-src bindgen rustfmt rust-clippy diff --git a/MAINTAINERS b/MAINTAINERS index 0a1ba8f21b1c..1d93f008fb33 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18179,7 +18179,7 @@ F: include/linux/module*.h F: kernel/module/ F: lib/test_kmod.c F: lib/tests/module/ -F: rust/kernel/module_param.rs +F: rust/kernel/module*.rs F: rust/macros/module.rs F: scripts/module* F: tools/testing/selftests/kmod/ diff --git a/drivers/android/binder/rust_binder_main.rs b/drivers/android/binder/rust_binder_main.rs index dc1941cd2407..d6ceebbd5f94 100644 --- a/drivers/android/binder/rust_binder_main.rs +++ b/drivers/android/binder/rust_binder_main.rs @@ -17,6 +17,7 @@ use kernel::{ bindings::{self, seq_file}, fs::File, list::{ListArc, ListArcSafe, ListLinksSelfPtr, TryNewListArc}, + module::this_module, prelude::*, seq_file::SeqFile, seq_print, @@ -318,7 +319,7 @@ pub static rust_binder_fops: AssertSync<kernel::bindings::file_operations> = { let zeroed_ops = unsafe { core::mem::MaybeUninit::zeroed().assume_init() }; let ops = kernel::bindings::file_operations { - owner: THIS_MODULE.as_ptr(), + owner: this_module::<LocalModule>().as_ptr(), poll: Some(rust_binder_poll), unlocked_ioctl: Some(rust_binder_ioctl), compat_ioctl: bindings::compat_ptr_ioctl, diff --git a/drivers/block/rnull/configfs.rs b/drivers/block/rnull/configfs.rs index 7c2eb5c0b722..32c10c3f4d0f 100644 --- a/drivers/block/rnull/configfs.rs +++ b/drivers/block/rnull/configfs.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -use super::{NullBlkDevice, THIS_MODULE}; +use super::NullBlkDevice; use kernel::{ block::mq::gen_disk::{GenDisk, GenDiskBuilder}, configfs::{self, AttributeOperations}, diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 070de0731e95..3c68a66770d3 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -645,8 +645,8 @@ impl CmdqInner { // SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer // fails. unsafe { - msg_element.__init(core::ptr::from_mut(dst.header))?; - command.init().__init(core::ptr::from_mut(cmd))?; + pin_init::raw_try_init(core::ptr::from_mut(dst.header), msg_element)?; + pin_init::raw_try_init(core::ptr::from_mut(cmd), command.init())?; } // Fill the variable-length payload, which may be empty. diff --git a/rust/Makefile b/rust/Makefile index 627ed79dc6f5..fbe0accc51a3 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -350,7 +350,7 @@ rusttestlib-pin_init: $(src)/pin-init/src/lib.rs rusttestlib-macros \ rusttestlib-kernel: private rustc_target_flags = --extern ffi \ --extern build_error --extern macros --extern pin_init \ --extern bindings --extern uapi \ - --extern zerocopy --extern zerocopy_derive + --extern zerocopy=$(objtree)/$(obj)/test/libzerocopy.rlib --extern zerocopy_derive rusttestlib-kernel: $(src)/kernel/lib.rs rusttestlib-bindings rusttestlib-uapi \ rusttestlib-build_error rusttestlib-pin_init $(obj)/$(libmacros_name) \ $(obj)/bindings.o rusttestlib-zerocopy rusttestlib-zerocopy_derive FORCE diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 35d1e015848d..c63d6acdbb6f 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -372,13 +372,13 @@ where // - `ptr` is a valid pointer to uninitialized memory. // - `ptr` is not used if an error is returned. // - `ptr` won't be moved until it is dropped, i.e. it is pinned. - unsafe { init(i).__pinned_init(ptr)? }; + unsafe { pin_init::raw_try_init(ptr, init(i))? }; // SAFETY: // - `i + 1 <= len`, hence we don't exceed the capacity, due to the call to // `with_capacity()` above. // - The new value at index buffer.len() + 1 is the only element being added here, and - // it has been initialized above by `init(i).__pinned_init(ptr)`. + // it has been initialized above by `raw_try_init(ptr, i)`. unsafe { buffer.inc_len(1) }; } @@ -463,7 +463,7 @@ where let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. - unsafe { init.__init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { Box::assume_init(self) }) } @@ -473,7 +473,7 @@ where let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { Box::assume_init(self) }.into()) } diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index c42928d5a239..cc9745fbf179 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -69,7 +69,7 @@ unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { // SAFETY: `adrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::__auxiliary_driver_register(adrv.get(), module.0, name.as_char_ptr()) + bindings::__auxiliary_driver_register(adrv.get(), module.as_ptr(), name.as_char_ptr()) }) } diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs index 35ede53f2b8e..a0d089423f21 100644 --- a/rust/kernel/bitfield.rs +++ b/rust/kernel/bitfield.rs @@ -581,6 +581,7 @@ mod tests { } impl From<MemoryType> for Bounded<u64, 4> { + #[inline(always)] fn from(mt: MemoryType) -> Bounded<u64, 4> { Bounded::from_expr(mt as u64) } @@ -606,6 +607,7 @@ mod tests { } impl From<Priority> for Bounded<u16, 2> { + #[inline(always)] fn from(p: Priority) -> Bounded<u16, 2> { Bounded::from_expr(p as u16) } diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs index ed943960f851..3566f0234ca4 100644 --- a/rust/kernel/bug.rs +++ b/rust/kernel/bug.rs @@ -8,6 +8,7 @@ #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] #[cfg(CONFIG_DEBUG_BUGVERBOSE)] macro_rules! warn_flags { @@ -47,12 +48,17 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] #[cfg(not(CONFIG_DEBUG_BUGVERBOSE))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags; + if false { + _ = $file; + } + // SAFETY: // - `flags` and `size` are all compile-time constants, preventing // any invalid memory access. @@ -73,14 +79,19 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, CONFIG_UML))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { + if false { + _ = $file; + } + // SAFETY: It is always safe to call `warn_slowpath_fmt()` // with a valid null-terminated string. unsafe { $crate::bindings::warn_slowpath_fmt( - $crate::c_str!(::core::file!()).as_char_ptr(), + $crate::str::CStrExt::as_char_ptr($crate::c_str!(::core::file!())), line!() as $crate::ffi::c_int, $flags as $crate::ffi::c_uint, ::core::ptr::null(), @@ -91,9 +102,15 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, any(CONFIG_LOONGARCH, CONFIG_ARM)))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + // SAFETY: It is always safe to call `WARN_ON()`. unsafe { $crate::bindings::WARN_ON(true) } }; @@ -101,9 +118,14 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] -#[cfg(not(CONFIG_BUG))] +#[cfg(any(testlib, not(CONFIG_BUG)))] macro_rules! warn_flags { - ($file:expr, $flags:expr) => {}; + ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + }; } #[doc(hidden)] @@ -118,14 +140,14 @@ macro_rules! warn_on { let cond = $cond; #[cfg(CONFIG_DEBUG_BUGVERBOSE_DETAILED)] - const _COND_STR: &str = concat!("[", stringify!($cond), "] ", file!()); + const COND_STR: &str = concat!("[", stringify!($cond), "] ", file!()); #[cfg(not(CONFIG_DEBUG_BUGVERBOSE_DETAILED))] - const _COND_STR: &str = file!(); + const COND_STR: &str = file!(); if cond { const WARN_ON_FLAGS: u32 = $crate::bug::bugflag_taint($crate::bindings::TAINT_WARN); - $crate::warn_flags!(_COND_STR, WARN_ON_FLAGS); + $crate::warn_flags!(COND_STR, WARN_ON_FLAGS); } cond }}; diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs index 2339c6467325..cd082b83e9e7 100644 --- a/rust/kernel/configfs.rs +++ b/rust/kernel/configfs.rs @@ -875,13 +875,14 @@ impl<Container, Data> ItemType<Container, Data> { /// configfs::Subsystem<Configuration>, /// Configuration /// >::new_with_child_ctor::<N,Child>( -/// &THIS_MODULE, +/// ::kernel::module::this_module::<crate::LocalModule>(), /// &CONFIGURATION_ATTRS /// ); /// /// &CONFIGURATION_TPE /// } /// ``` +#[allow(clippy::crate_in_macro_def)] #[macro_export] macro_rules! configfs_attrs { ( @@ -1021,7 +1022,8 @@ macro_rules! configfs_attrs { static [< $data:upper _TPE >] : $crate::configfs::ItemType<$container, $data> = $crate::configfs::ItemType::<$container, $data>::new::<N>( - &THIS_MODULE, &[<$ data:upper _ATTRS >] + $crate::module::this_module::<crate::LocalModule>(), + &[<$ data:upper _ATTRS >] ); )? @@ -1030,7 +1032,8 @@ macro_rules! configfs_attrs { $crate::configfs::ItemType<$container, $data> = $crate::configfs::ItemType::<$container, $data>:: new_with_child_ctor::<N, $child>( - &THIS_MODULE, &[<$ data:upper _ATTRS >] + $crate::module::this_module::<crate::LocalModule>(), + &[<$ data:upper _ATTRS >] ); )? diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 200def84fb69..8e36a4e7f514 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -449,7 +449,7 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> { // - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on // error cannot leave the element in an invalid state. // - The DMA address has not been exposed yet, so there is no concurrent device access. - unsafe { init.__init(ptr)? }; + unsafe { pin_init::raw_try_init(ptr, init)? }; Ok(()) } @@ -791,10 +791,10 @@ impl<T: AsBytes + FromBytes> Coherent<T> { // SAFETY: // - `ptr` is valid, properly aligned, and points to exclusively owned memory. - // - If `__init` fails, `self` is dropped, which safely frees the underlying `Coherent`'s - // DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` requirements - // we are bypassing. - unsafe { init.__init(ptr)? }; + // - If `raw_try_init` fails, `self` is dropped, which safely frees the underlying + // `Coherent`'s DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` + // requirements we are bypassing. + unsafe { pin_init::raw_try_init(ptr, init)? }; Ok(dmem) } diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 477cf771fb10..81f9f7e59817 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -203,7 +203,8 @@ impl<T: drm::Driver> UnregisteredDevice<T> { fops: &Self::GEM_FOPS, }; - const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(); + const GEM_FOPS: bindings::file_operations = + drm::gem::create_fops(crate::module::this_module::<T::OwnerModule>().as_ptr()); /// Create a new `UnregisteredDevice` for a `drm::Driver`. /// @@ -244,7 +245,7 @@ impl<T: drm::Driver> UnregisteredDevice<T> { // SAFETY: // - `raw_data` is a valid pointer to uninitialized memory. // - `raw_data` will not move until it is dropped. - unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| { + unsafe { pin_init::raw_try_init(raw_data, data) }.inspect_err(|_| { // SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the // refcount must be non-zero. unsafe { bindings::drm_dev_put(drm_dev) }; diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index c8b66d816871..a7ba1453d40b 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -387,10 +387,10 @@ impl<T: DriverObject, Ctx: DeviceContext> AllocImpl for Object<T, Ctx> { }; } -pub(super) const fn create_fops() -> bindings::file_operations { +pub(super) const fn create_fops(owner: *mut bindings::module) -> bindings::file_operations { let mut fops: bindings::file_operations = pin_init::zeroed(); - fops.owner = core::ptr::null_mut(); + fops.owner = owner; fops.open = Some(bindings::drm_open); fops.release = Some(bindings::drm_release); fops.unlocked_ioctl = Some(bindings::drm_ioctl); diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs index 0b09fe44ab39..bf927b8e6fbb 100644 --- a/rust/kernel/drm/gpuvm/va.rs +++ b/rust/kernel/drm/gpuvm/va.rs @@ -116,7 +116,7 @@ impl<T: DriverGpuVm> GpuVaAlloc<T> { pub(super) fn prepare(mut self, va_data: impl PinInit<T::VaData>) -> *mut bindings::drm_gpuva { let va_ptr = MaybeUninit::as_mut_ptr(&mut self.0); // SAFETY: The `data` field is pinned. - let Ok(()) = unsafe { va_data.__pinned_init(&raw mut (*va_ptr).data) }; + unsafe { pin_init::raw_init(&raw mut (*va_ptr).data, va_data) }; KBox::into_raw(self.0).cast() } } diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs index c064ac63897b..ab12b710267e 100644 --- a/rust/kernel/drm/gpuvm/vm_bo.rs +++ b/rust/kernel/drm/gpuvm/vm_bo.rs @@ -181,7 +181,7 @@ impl<T: DriverGpuVm> GpuVmBoAlloc<T> { }; let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; // SAFETY: `ptr->data` is a valid pinned location. - let Ok(()) = unsafe { value.__pinned_init(&raw mut (*raw_ptr).data) }; + unsafe { pin_init::raw_init(&raw mut (*raw_ptr).data, value) }; // INVARIANTS: We just created the vm_bo so it's absent from lists, and the data is valid // as we just initialized it. Ok(GpuVmBoAlloc(ptr)) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index a56ba6309594..e52793f77196 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -30,6 +30,7 @@ pub mod code { }; } + // From `include/uapi/asm-generic/errno-base.h`. declare_err!(EPERM, "Operation not permitted."); declare_err!(ENOENT, "No such file or directory."); declare_err!(ESRCH, "No such process."); @@ -64,9 +65,110 @@ pub mod code { declare_err!(EPIPE, "Broken pipe."); declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); + + // From `include/uapi/asm-generic/errno.h`. + declare_err!(EDEADLK, "Resource deadlock would occur."); + declare_err!(ENAMETOOLONG, "File name too long."); + declare_err!(ENOLCK, "No record locks available."); + declare_err!(ENOSYS, "Invalid system call number."); + declare_err!(ENOTEMPTY, "Directory not empty."); + declare_err!(ELOOP, "Too many symbolic links encountered."); + declare_err!(ENOMSG, "No message of desired type."); + declare_err!(EIDRM, "Identifier removed."); + declare_err!(ECHRNG, "Channel number out of range."); + declare_err!(EL2NSYNC, "Level 2 not synchronized."); + declare_err!(EL3HLT, "Level 3 halted."); + declare_err!(EL3RST, "Level 3 reset."); + declare_err!(ELNRNG, "Link number out of range."); + declare_err!(EUNATCH, "Protocol driver not attached."); + declare_err!(ENOCSI, "No CSI structure available."); + declare_err!(EL2HLT, "Level 2 halted."); + declare_err!(EBADE, "Invalid exchange."); + declare_err!(EBADR, "Invalid request descriptor."); + declare_err!(EXFULL, "Exchange full."); + declare_err!(ENOANO, "No anode."); + declare_err!(EBADRQC, "Invalid request code."); + declare_err!(EBADSLT, "Invalid slot."); + declare_err!(EBFONT, "Bad font file format."); + declare_err!(ENOSTR, "Device not a stream."); + declare_err!(ENODATA, "No data available."); + declare_err!(ETIME, "Timer expired."); + declare_err!(ENOSR, "Out of streams resources."); + declare_err!(ENONET, "Machine is not on the network."); + declare_err!(ENOPKG, "Package not installed."); + declare_err!(EREMOTE, "Object is remote."); + declare_err!(ENOLINK, "Link has been severed."); + declare_err!(EADV, "Advertise error."); + declare_err!(ESRMNT, "Srmount error."); + declare_err!(ECOMM, "Communication error on send."); + declare_err!(EPROTO, "Protocol error."); + declare_err!(EMULTIHOP, "Multihop attempted."); + declare_err!(EDOTDOT, "RFS specific error."); + declare_err!(EBADMSG, "Not a data message."); + declare_err!(EFSBADCRC, "Bad CRC detected."); declare_err!(EOVERFLOW, "Value too large for defined data type."); + declare_err!(ENOTUNIQ, "Name not unique on network."); + declare_err!(EBADFD, "File descriptor in bad state."); + declare_err!(EREMCHG, "Remote address changed."); + declare_err!(ELIBACC, "Can not access a needed shared library."); + declare_err!(ELIBBAD, "Accessing a corrupted shared library."); + declare_err!(ELIBSCN, ".lib section in a.out corrupted."); + declare_err!(ELIBMAX, "Attempting to link in too many shared libraries."); + declare_err!(ELIBEXEC, "Cannot exec a shared library directly."); + declare_err!(EILSEQ, "Illegal byte sequence."); + declare_err!(ERESTART, "Interrupted system call should be restarted."); + declare_err!(ESTRPIPE, "Streams pipe error."); + declare_err!(EUSERS, "Too many users."); + declare_err!(ENOTSOCK, "Socket operation on non-socket."); + declare_err!(EDESTADDRREQ, "Destination address required."); declare_err!(EMSGSIZE, "Message too long."); + declare_err!(EPROTOTYPE, "Protocol wrong type for socket."); + declare_err!(ENOPROTOOPT, "Protocol not available."); + declare_err!(EPROTONOSUPPORT, "Protocol not supported."); + declare_err!(ESOCKTNOSUPPORT, "Socket type not supported."); + declare_err!(EOPNOTSUPP, "Operation not supported on transport endpoint."); + declare_err!(EPFNOSUPPORT, "Protocol family not supported."); + declare_err!(EAFNOSUPPORT, "Address family not supported by protocol."); + declare_err!(EADDRINUSE, "Address already in use."); + declare_err!(EADDRNOTAVAIL, "Cannot assign requested address."); + declare_err!(ENETDOWN, "Network is down."); + declare_err!(ENETUNREACH, "Network is unreachable."); + declare_err!(ENETRESET, "Network dropped connection because of reset."); + declare_err!(ECONNABORTED, "Software caused connection abort."); + declare_err!(ECONNRESET, "Connection reset by peer."); + declare_err!(ENOBUFS, "No buffer space available."); + declare_err!(EISCONN, "Transport endpoint is already connected."); + declare_err!(ENOTCONN, "Transport endpoint is not connected."); + declare_err!(ESHUTDOWN, "Cannot send after transport endpoint shutdown."); + declare_err!(ETOOMANYREFS, "Too many references: cannot splice."); declare_err!(ETIMEDOUT, "Connection timed out."); + declare_err!(ECONNREFUSED, "Connection refused."); + declare_err!(EHOSTDOWN, "Host is down."); + declare_err!(EHOSTUNREACH, "No route to host."); + declare_err!(EALREADY, "Operation already in progress."); + declare_err!(EINPROGRESS, "Operation now in progress."); + declare_err!(ESTALE, "Stale file handle."); + declare_err!(EUCLEAN, "Structure needs cleaning."); + declare_err!(EFSCORRUPTED, "Filesystem is corrupted."); + declare_err!(ENOTNAM, "Not a XENIX named type file."); + declare_err!(ENAVAIL, "No XENIX semaphores available."); + declare_err!(EISNAM, "Is a named type file."); + declare_err!(EREMOTEIO, "Remote I/O error."); + declare_err!(EDQUOT, "Quota exceeded."); + declare_err!(ENOMEDIUM, "No medium found."); + declare_err!(EMEDIUMTYPE, "Wrong medium type."); + declare_err!(ECANCELED, "Operation Canceled."); + declare_err!(ENOKEY, "Required key not available."); + declare_err!(EKEYEXPIRED, "Key has expired."); + declare_err!(EKEYREVOKED, "Key has been revoked."); + declare_err!(EKEYREJECTED, "Key was rejected by service."); + declare_err!(EOWNERDEAD, "Owner died."); + declare_err!(ENOTRECOVERABLE, "State not recoverable."); + declare_err!(ERFKILL, "Operation not possible due to RF-kill."); + declare_err!(EHWPOISON, "Memory page has hardware error."); + declare_err!(EFTYPE, "Wrong file type for the intended operation."); + + // From `include/linux/errno.h`. declare_err!(ERESTARTSYS, "Restart the system call."); declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); declare_err!(ERESTARTNOHAND, "Restart if no handler."); diff --git a/rust/kernel/fmt.rs b/rust/kernel/fmt.rs index 73afbc51ba33..29582b053ab1 100644 --- a/rust/kernel/fmt.rs +++ b/rust/kernel/fmt.rs @@ -4,6 +4,8 @@ //! //! This module is intended to be used in place of `core::fmt` in kernel code. +use kernel::prelude::*; + pub use core::fmt::{ Arguments, Debug, @@ -39,11 +41,115 @@ use core::fmt::{ LowerExp, LowerHex, Octal, - Pointer, UpperExp, UpperHex, // }; -impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, Pointer, LowerExp, UpperExp); +use core::ptr::NonNull; +impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, LowerExp, UpperExp); + +/// A copy of [`core::fmt::Pointer`] that allows implementing pointer formatting for foreign types. +/// +/// Together with the [`Adapter`] type and [`fmt!`] macro, it enables raw pointer formatting to be +/// intercepted and routed to [`HashedPtr`] (kernel's `%p` hashed format), preventing kernel address +/// leaks. +/// +/// [`fmt!`]: crate::prelude::fmt! +pub trait Pointer { + /// Same as [`core::fmt::Pointer::fmt`]. + fn fmt(&self, f: &mut Formatter<'_>) -> Result; +} + +/// A wrapper for pointers that formats them using kernel's `%p` format specifier. +/// +/// By default, `%p` prints a hashed representation of the pointer address to prevent kernel address +/// leaks. When the `no_hash_pointers` kernel command-line parameter is enabled, the real address is +/// printed instead (for debugging purposes). +pub struct HashedPtr<T: ?Sized>(pub *const T); + +impl<T: ?Sized> Pointer for HashedPtr<T> { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + use crate::str::CStrExt as _; + + let mut buf = [0u8; 32]; + + // Use `%#0*p` for the `0x` prefix and zero-padding; `+2` compensates for + // the prefix counting toward the field width. + let default_width = (2 * size_of::<usize>() + 2) as c_int; + let width = match (f.sign_aware_zero_pad(), f.width()) { + (true, Some(w)) if w > 0 => w.min(buf.len() - 1) as c_int, + _ => default_width, + }; + + // SAFETY: `buf` is a valid, writable 32-byte buffer, sufficient for + // all architectures (max 19 bytes for 64-bit under the default width). + // The format string is null-terminated; `width` (c_int) and pointer + // match the `%*` and `%p` specifiers. + let len = unsafe { + crate::bindings::scnprintf( + buf.as_mut_ptr().cast(), + buf.len(), + c"%#0*p".as_char_ptr(), + width, + self.0.cast::<c_void>(), + ) + }; + + // SAFETY: `%#0*p` produces only ASCII, which is valid UTF-8. + let s = unsafe { core::str::from_utf8_unchecked(&buf[..len as usize]) }; + + if f.sign_aware_zero_pad() { + // `scnprintf` already applied the width and zero-padding via `%#0*p`. + f.write_str(s) + } else { + f.pad(s) + } + } +} + +// Raw pointers are formatted via `HashedPtr` (kernel `%p`: hashed by default, plain with +// `no_hash_pointers`). +impl<T: ?Sized> Pointer for *const T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl<T: ?Sized> Pointer for *mut T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl<T: ?Sized> Pointer for &T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl<T: ?Sized> Pointer for &mut T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(core::ptr::from_ref(*self)), f) + } +} + +impl<T: ?Sized> Pointer for NonNull<T> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(self.as_ptr()), f) + } +} + +// `Adapter<&T>` bridges our `Pointer` trait to `core::fmt::Pointer` +impl<T: Pointer> core::fmt::Pointer for Adapter<&T> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(self.0, f) + } +} /// A copy of [`core::fmt::Display`] that allows us to implement it for foreign types. /// @@ -105,3 +211,88 @@ impl_display_forward!( {<T: ?Sized>} crate::sync::Arc<T> {where crate::sync::Arc<T>: core::fmt::Display}, {<T: ?Sized>} crate::sync::UniqueArc<T> {where crate::sync::UniqueArc<T>: core::fmt::Display}, ); + +#[macros::kunit_tests(rust_kernel_fmt)] +mod tests { + use crate::{ + bindings, + prelude::fmt, + str::CString, // + }; + + #[cfg(CONFIG_64BIT)] + mod expected { + pub(super) const PTR_VALUE: usize = 0xffffffffdeadbeef; + pub(super) const PTR_VAL_NO_CRNG: &str = "(____ptrval____)"; + pub(super) const HASHED_PREFIX: &str = "0x00000000"; + pub(super) const RAW_POINTER: &str = "0xffffffffdeadbeef"; + pub(super) const PADDED_RIGHT: &str = " 0xffffffffdeadbeef"; + pub(super) const ZERO_PADDED: &str = "0x000000ffffffffdeadbeef"; + pub(super) const HASHED_PADDED_RIGHT_PREFIX: &str = " "; + pub(super) const HASHED_ZERO_PADDED_PREFIX: &str = "0x00000000000000"; + pub(super) const CLAMPED: &str = "0x0000000000000ffffffffdeadbeef"; + } + + #[cfg(not(CONFIG_64BIT))] + mod expected { + pub(super) const PTR_VALUE: usize = 0xdeadbeef; + pub(super) const PTR_VAL_NO_CRNG: &str = "(ptrval)"; + pub(super) const HASHED_PREFIX: &str = "0x"; + pub(super) const RAW_POINTER: &str = "0xdeadbeef"; + pub(super) const PADDED_RIGHT: &str = " 0xdeadbeef"; + pub(super) const ZERO_PADDED: &str = "0x00000000000000deadbeef"; + pub(super) const HASHED_PADDED_RIGHT_PREFIX: &str = " "; + pub(super) const HASHED_ZERO_PADDED_PREFIX: &str = "0x00000000000000"; + pub(super) const CLAMPED: &str = "0x0000000000000000000000deadbeef"; + } + + #[test] + fn test_ptr_formatting() -> core::result::Result<(), crate::error::Error> { + let ptr: *const u8 = core::ptr::without_provenance(expected::PTR_VALUE); + + // SAFETY: `no_hash_pointers` is a global variable that is never concurrently modified — + // KUnit tests may run at boot (before `mark_readonly()`) or manually afterwards (when the + // variable is read-only). Reading is always safe. + let no_hash = unsafe { bindings::no_hash_pointers }; + + if no_hash { + let cstr = CString::try_from_fmt(fmt!("{:p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::RAW_POINTER); + + let cstr = CString::try_from_fmt(fmt!("{:>24p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::PADDED_RIGHT); + + let cstr = CString::try_from_fmt(fmt!("{:024p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::ZERO_PADDED); + + let cstr = CString::try_from_fmt(fmt!("{:0100p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::CLAMPED); + } else { + let cstr = CString::try_from_fmt(fmt!("{:p}", ptr))?; + let formatted = cstr.to_str()?; + // If the RNG is not yet ready, `%p` falls back to a placeholder. + if formatted == expected::PTR_VAL_NO_CRNG { + return Ok(()); + } + assert!(formatted.starts_with(expected::HASHED_PREFIX)); + assert_ne!(formatted, expected::RAW_POINTER); + + let cstr = CString::try_from_fmt(fmt!("{:>24p}", ptr))?; + assert!(cstr + .to_str()? + .starts_with(expected::HASHED_PADDED_RIGHT_PREFIX)); + + let cstr = CString::try_from_fmt(fmt!("{:024p}", ptr))?; + assert!(cstr + .to_str()? + .starts_with(expected::HASHED_ZERO_PADDED_PREFIX)); + + let cstr = CString::try_from_fmt(fmt!("{:0100p}", ptr))?; + let output = cstr.to_str()?; + assert!(output.starts_with("0x")); + assert!(!output[2..].chars().all(|c| c == '0')); + } + + Ok(()) + } +} diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs index 624b971ca8b0..dd9271af5eb8 100644 --- a/rust/kernel/i2c.rs +++ b/rust/kernel/i2c.rs @@ -142,7 +142,7 @@ unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { } // SAFETY: `idrv` is guaranteed to be a valid `DriverType`. - to_result(unsafe { bindings::i2c_register_driver(module.0, idrv.get()) }) + to_result(unsafe { bindings::i2c_register_driver(module.as_ptr(), idrv.get()) }) } unsafe fn unregister(idrv: &Opaque<Self::DriverType>) { diff --git a/rust/kernel/impl_flags.rs b/rust/kernel/impl_flags.rs index e2bd7639da12..fdf44d5eea9c 100644 --- a/rust/kernel/impl_flags.rs +++ b/rust/kernel/impl_flags.rs @@ -19,7 +19,10 @@ /// # Examples /// /// ``` -/// use kernel::impl_flags; +/// use kernel::{ +/// bits::bit_u32, +/// impl_flags, // +/// }; /// /// impl_flags!( /// /// Represents multiple permissions. @@ -30,13 +33,13 @@ /// #[derive(Debug, Clone, Copy, PartialEq, Eq)] /// pub enum Permission { /// /// Read permission. -/// Read = 1 << 0, +/// Read = bit_u32(0), /// /// /// Write permission. -/// Write = 1 << 1, +/// Write = bit_u32(1), /// /// /// Execute permission. -/// Execute = 1 << 2, +/// Execute = bit_u32(2), /// } /// ); /// diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 05a12e869a57..1fdc3963e3e3 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -158,7 +158,9 @@ pub trait InPlaceInit<T>: Sized { { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { - pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) + pin_init_from_closure(|slot| { + pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e)) + }) }; Self::try_pin_init(init, flags) } @@ -176,7 +178,7 @@ pub trait InPlaceInit<T>: Sized { { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { - init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) + init_from_closure(|slot| pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e))) }; Self::try_init(init, flags) } diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 9512af7156df..59144e1e3d36 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -94,6 +94,7 @@ pub mod list; pub mod maple_tree; pub mod miscdevice; pub mod mm; +pub mod module; pub mod module_param; #[cfg(CONFIG_NET)] pub mod net; @@ -140,77 +141,29 @@ pub mod xarray; #[doc(hidden)] pub use bindings; pub use macros; +pub use module::{ + InPlaceModule, + Module, + ModuleMetadata, + ThisModule, // +}; pub use uapi; /// Prefix to appear before log messages printed from within the `kernel` crate. const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; -/// The top level entrypoint to implementing a kernel module. -/// -/// For any teardown or cleanup operations, your type may implement [`Drop`]. -pub trait Module: Sized + Sync + Send { - /// Called at module initialization time. - /// - /// Use this method to perform whatever setup or registration your module - /// should do. - /// - /// Equivalent to the `module_init` macro in the C API. - fn init(module: &'static ThisModule) -> error::Result<Self>; -} - -/// A module that is pinned and initialised in-place. -pub trait InPlaceModule: Sync + Send { - /// Creates an initialiser for the module. - /// - /// It is called when the module is loaded. - fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>; -} - -impl<T: Module> InPlaceModule for T { - fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error> { - let initer = move |slot: *mut Self| { - let m = <Self as Module>::init(module)?; +/// Dummy module type for `#[vtable]` `impl` blocks within the `kernel` crate (e.g. KUnit tests). +// The `allow` is needed since it may be unused (e.g. KUnit tests may be disabled). +#[allow(dead_code)] +struct LocalModule; - // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. - unsafe { slot.write(m) }; - Ok(()) - }; +impl ModuleMetadata for LocalModule { + const NAME: &'static str::CStr = c"rust_kernel"; - // SAFETY: On success, `initer` always fully initialises an instance of `Self`. - unsafe { pin_init::pin_init_from_closure(initer) } - } -} - -/// Metadata attached to a [`Module`] or [`InPlaceModule`]. -pub trait ModuleMetadata { - /// The name of the module as specified in the `module!` macro. - const NAME: &'static crate::str::CStr; -} - -/// Equivalent to `THIS_MODULE` in the C API. -/// -/// C header: [`include/linux/init.h`](srctree/include/linux/init.h) -pub struct ThisModule(*mut bindings::module); - -// SAFETY: `THIS_MODULE` may be used from all threads within a module. -unsafe impl Sync for ThisModule {} - -impl ThisModule { - /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer. - /// - /// # Safety - /// - /// The pointer must be equal to the right `THIS_MODULE`. - pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule { - ThisModule(ptr) - } - - /// Access the raw pointer for this module. - /// - /// It is up to the user to use it correctly. - pub const fn as_ptr(&self) -> *mut bindings::module { - self.0 - } + const THIS_MODULE: ThisModule = { + // SAFETY: `try_module_get`/`module_put` handle null module pointers gracefully. + unsafe { ThisModule::from_ptr(core::ptr::null_mut()) } + }; } #[cfg(not(testlib))] diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs index 83ce50def5ac..2a4329f98614 100644 --- a/rust/kernel/miscdevice.rs +++ b/rust/kernel/miscdevice.rs @@ -24,12 +24,13 @@ use crate::{ IovIterSource, // }, mm::virt::VmaNew, + module::this_module, prelude::*, seq_file::SeqFile, types::{ ForeignOwnable, Opaque, // - }, + }, // }; use core::marker::PhantomData; @@ -430,6 +431,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { } else { None }, + owner: this_module::<T::OwnerModule>().as_ptr(), ..pin_init::zeroed() }; diff --git a/rust/kernel/module.rs b/rust/kernel/module.rs new file mode 100644 index 000000000000..d71370598447 --- /dev/null +++ b/rust/kernel/module.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Module-related types and helpers. + +/// The entrypoint to implementing a kernel module. +/// +/// For any teardown or cleanup operations, your type may implement [`Drop`]. +pub trait Module: Sized + Sync + Send { + /// Called at module initialization time. + /// + /// Use this method to perform whatever setup or registration your module + /// should do. + /// + /// Equivalent to the `module_init` macro in the C API. + fn init(module: &'static ThisModule) -> crate::error::Result<Self>; +} + +/// A module that is pinned and initialised in-place. +pub trait InPlaceModule: Sync + Send { + /// Creates an initialiser for the module. + /// + /// It is called when the module is loaded. + fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, crate::error::Error>; +} + +impl<T: Module> InPlaceModule for T { + fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, crate::error::Error> { + let initer = move |slot: *mut Self| { + let m = <Self as Module>::init(module)?; + + // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. + unsafe { slot.write(m) }; + Ok(()) + }; + + // SAFETY: On success, `initer` always fully initialises an instance of `Self`. + unsafe { pin_init::pin_init_from_closure(initer) } + } +} + +/// Metadata attached to a [`Module`] or [`InPlaceModule`]. +pub trait ModuleMetadata { + /// The name of the module as specified in the `module!` macro. + const NAME: &'static crate::str::CStr; + + /// The module's `THIS_MODULE` pointer. + const THIS_MODULE: ThisModule; +} + +/// Returns a reference to the `THIS_MODULE` of the given module type. +#[inline] +pub const fn this_module<M: ModuleMetadata>() -> &'static ThisModule { + &M::THIS_MODULE +} + +/// Equivalent to `THIS_MODULE` in the C API. +/// +/// C header: [`include/linux/init.h`](srctree/include/linux/init.h) +pub struct ThisModule(*mut crate::bindings::module); + +// SAFETY: `THIS_MODULE` may be used from all threads within a module. +unsafe impl Sync for ThisModule {} + +impl ThisModule { + /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer. + /// + /// # Safety + /// + /// The pointer must be equal to the right `THIS_MODULE`. + pub const unsafe fn from_ptr(ptr: *mut crate::bindings::module) -> ThisModule { + ThisModule(ptr) + } + + /// Access the raw pointer for this module. + /// + /// It is up to the user to use it correctly. + pub const fn as_ptr(&self) -> *mut crate::bindings::module { + self.0 + } +} diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 3ca99db5cccf..8b7036b8fe48 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -659,7 +659,11 @@ impl Registration { // the `drivers` slice are initialized properly. `drivers` will not be moved. // So it's just an FFI call. to_result(unsafe { - bindings::phy_drivers_register(drivers[0].0.get(), drivers.len().try_into()?, module.0) + bindings::phy_drivers_register( + drivers[0].0.get(), + drivers.len().try_into()?, + module.as_ptr(), + ) })?; // INVARIANT: The `drivers` slice is successfully registered to the kernel via `phy_drivers_register`. Ok(Registration { drivers }) diff --git a/rust/kernel/num.rs b/rust/kernel/num.rs index 8532b511384c..dbe848e30efe 100644 --- a/rust/kernel/num.rs +++ b/rust/kernel/num.rs @@ -5,6 +5,8 @@ use core::ops; pub mod bounded; +pub mod casts; + pub use bounded::*; /// Designates unsigned primitive types. diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index dafe77782d79..d192610a687d 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -485,13 +485,45 @@ where /// assert_eq!(v_shifted.get(), 0xff); /// ``` pub fn shr<const SHIFT: u32, const RES: u32>(self) -> Bounded<T, RES> { - const { assert!(RES + SHIFT >= N) } + const_assert!(SHIFT < T::BITS); + const_assert!(RES + SHIFT >= N); // SAFETY: We shift the value right by `SHIFT`, reducing the number of bits needed to // represent the shifted value by as much, and just asserted that `RES >= N - SHIFT`. unsafe { Bounded::__new(self.0 >> SHIFT) } } + /// Right-shifts `self` by `SHIFT` if that loses no set bits, and returns the result as a + /// `Bounded<_, RES>`, where `RES >= N - SHIFT`. + /// + /// Returns [`None`] if any of the `SHIFT` least significant bits of `self` is set. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::<u32, 16>::new::<0xff00>(); + /// let v_shifted: Option<Bounded<u32, 8>> = v.shr_exact::<8, _>(); + /// + /// assert_eq!(v_shifted.map(|v| v.get()), Some(0xff)); + /// + /// // A set bit would be shifted out. + /// let v = Bounded::<u32, 16>::new::<0xff01>(); + /// let v_shifted: Option<Bounded<u32, 8>> = v.shr_exact::<8, _>(); + /// + /// assert!(v_shifted.is_none()); + /// ``` + #[inline] + pub fn shr_exact<const SHIFT: u32, const RES: u32>(self) -> Option<Bounded<T, RES>> { + let shifted = self.shr::<SHIFT, RES>(); + if shifted.get() << SHIFT == self.0 { + Some(shifted) + } else { + None + } + } + /// Left-shifts `self` by `SHIFT` and returns the result as a `Bounded<_, RES>`, where `RES >= /// N + SHIFT`. /// @@ -506,7 +538,7 @@ where /// assert_eq!(v_shifted.get(), 0xff00); /// ``` pub fn shl<const SHIFT: u32, const RES: u32>(self) -> Bounded<T, RES> { - const { assert!(RES >= N + SHIFT) } + const_assert!(RES >= N + SHIFT); // SAFETY: We shift the value left by `SHIFT`, augmenting the number of bits needed to // represent the shifted value by as much, and just asserted that `RES >= N + SHIFT`. diff --git a/rust/kernel/num/casts.rs b/rust/kernel/num/casts.rs new file mode 100644 index 000000000000..7e6c7dec747d --- /dev/null +++ b/rust/kernel/num/casts.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Helpers for performing lossless integer casts. +//! +//! The `as` keyword can be used to perform casts between integer types, but it unfortunately makes +//! no distinction between casts that are lossless, and casts from a larger type into a smaller one +//! that might silently strip data away. Thus, its use in the kernel is discouraged in favor of +//! [`From`] implementations. +//! +//! Conversely, there are casts that are lossless depending on the build architecture (such as +//! casting [`usize`] to [`u64`] on 32 or 64 bit archs), but not supported by [`From`] +//! implementations in the standard library because they are not portable. It does however make +//! sense for the kernel to support these, if only for code that is architecture-specific. +//! +//! This module provides ways to perform such conversions safely: +//! +//! - A series of const functions (e.g. [`usize_as_u64`]) supporting safe conversions in const +//! context. Conversions supported by [`From`] implementations in the standard library are also +//! covered as the [`From`] trait cannot be used in const context. +//! - Two extension traits, [`FromSafeCast`] and [`IntoSafeCast`], providing conversion methods +//! similar to [`From`] and [`Into`] for conversions that are safe to perform in the kernel, but +//! not supported by the standard library. +//! - Another series of const functions (e.g. [`u64_into_u8`]) supporting the conversion of a const +//! value from a larger type into a smaller one, provided the value fits into the destination +//! type. This is useful if a constant is defined as a larger type, but needs to be used as a +//! smaller one. +//! - An [`arch`] sub-module, defining more conversion functions that are only guaranteed to be +//! lossless for a given pointer size. These can only be used in code that is specific to a +//! given pointer size. +//! +//! # Examples +//! +//! ``` +//! use kernel::num::casts::{self, FromSafeCast, IntoSafeCast}; +//! +//! // Conversion from const context. +//! const USIZED_CONST: usize = casts::u8_as_usize(255u8); +//! +//! // Non-const conversions. +//! let a = u64::from_safe_cast(4096usize); +//! let b: u64 = 4096usize.into_safe_cast(); +//! ``` + +use crate::prelude::*; + +/// Implements safe `as` conversion functions from a given type into a series of target types. +/// +/// These functions can be used in place of `as`, with the guarantee that they will be lossless. +macro_rules! impl_safe_as { + ($from:ty as { $($into:ty),* }) => { + $( + $crate::macros::paste! { + #[doc = ::core::concat!( + "Losslessly converts a [`", + ::core::stringify!($from), + "`] into a [`", + ::core::stringify!($into), + "`].")] + /// + /// This conversion is allowed as it is always lossless. Prefer this over the `as` + /// keyword to ensure no lossy casts are performed. + /// + /// This is for use from a `const` context. For non `const` use, prefer the + /// [`FromSafeCast`] and [`IntoSafeCast`] traits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::casts; + /// + #[doc = ::core::concat!( + "assert_eq!(casts::", + ::core::stringify!($from), + "_as_", + ::core::stringify!($into), + "(1", + ::core::stringify!($from), + "), 1", + ::core::stringify!($into), + ");")] + /// ``` + #[inline] + pub const fn [<$from _as_ $into>](value: $from) -> $into { + $crate::static_assert!(size_of::<$into>() >= size_of::<$from>()); + + value as $into + } + } + )* + }; +} + +// Valid `Into` transformations. +impl_safe_as!(u8 as { u16, u32, u64, usize }); +impl_safe_as!(u16 as { u32, u64, usize }); +impl_safe_as!(u32 as { u64 }); +// A `usize` fits into a `u64` on all supported platforms. +impl_safe_as!(usize as { u64 }); +// A `u32` fits into a `usize` on all supported platforms. +impl_safe_as!(u32 as { usize }); + +/// Extension trait providing guaranteed lossless cast to [`Self`] from `T`. +/// +/// The standard library's [`From`] implementations do not cover conversions that are not portable +/// or future-proof. For instance, even though it is safe today, [`From<usize>`] is not implemented +/// for [`u64`] because of the possibility of needing to support larger-than-64bit architectures in +/// the future. +/// +/// The workaround is to either deal with the error handling of [`TryFrom`] for an operation that +/// technically cannot fail, or to use the `as` keyword, which can silently strip data if the +/// destination type is smaller than the source. +/// +/// Both options are hardly acceptable for the kernel. It is also a much more architecture +/// dependent environment, supporting only 32 and 64 bit architectures, with some modules +/// explicitly depending on a specific bus width that could greatly benefit from infallible +/// conversion operations. +/// +/// Thus this extension trait that provides, for all architectures supported by the kernel, +/// conversion methods between types for which such a cast is lossless. +/// +/// In other words, this trait is implemented if, for all supported targets and with `t: T`, the +/// `t as Self` operation is completely lossless. +/// +/// Prefer this over the `as` keyword to guarantee that no lossy casts are performed. +/// +/// If you need to perform a conversion in `const` context, use [`u32_as_usize`], [`usize_as_u64`], +/// etc. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::casts::FromSafeCast; +/// +/// assert_eq!(usize::from_safe_cast(0xf00u32), 0xf00usize); +/// ``` +pub trait FromSafeCast<T> { + /// Create a [`Self`] from `value`. This operation is guaranteed to be lossless. + fn from_safe_cast(value: T) -> Self; +} + +// A `usize` fits into a `u64` on all supported platforms. +impl FromSafeCast<usize> for u64 { + #[inline] + fn from_safe_cast(value: usize) -> Self { + usize_as_u64(value) + } +} + +// A `u32` fits into a `usize` on all supported platforms. +impl FromSafeCast<u32> for usize { + #[inline] + fn from_safe_cast(value: u32) -> Self { + u32_as_usize(value) + } +} + +/// Counterpart to the [`FromSafeCast`] trait, i.e. this trait is to [`FromSafeCast`] what [`Into`] +/// is to [`From`]. +/// +/// See the documentation of [`FromSafeCast`] for the motivation. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::casts::IntoSafeCast; +/// +/// assert_eq!(0xf00usize, 0xf00u32.into_safe_cast()); +/// ``` +pub trait IntoSafeCast<T> { + /// Convert `self` into a `T`. This operation is guaranteed to be lossless. + fn into_safe_cast(self) -> T; +} + +/// Reverse operation for types implementing [`FromSafeCast`]. +impl<S, T> IntoSafeCast<T> for S +where + T: FromSafeCast<S>, +{ + #[inline] + fn into_safe_cast(self) -> T { + T::from_safe_cast(self) + } +} + +/// Implements lossless conversion of a constant from a larger type into a smaller one. +macro_rules! impl_const_into { + ($from:ty => { $($into:ty),* }) => { + $( + $crate::macros::paste! { + #[doc = ::core::concat!( + "Performs a build-time safe conversion of a [`", + ::core::stringify!($from), + "`] constant value into a [`", + ::core::stringify!($into), + "`].")] + /// + /// This checks at compile-time that the conversion is lossless, and triggers a build + /// error if it isn't. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::casts; + /// + /// // Succeeds because the value of the source fits into the destination's type. + #[doc = ::core::concat!( + "assert_eq!(casts::", + ::core::stringify!($from), + "_into_", + ::core::stringify!($into), + "::<1", + ::core::stringify!($from), + ">(), 1", + ::core::stringify!($into), + ");")] + /// ``` + #[inline] + pub const fn [<$from _into_ $into>]<const N: $from>() -> $into { + // Make sure that the target type is smaller than the source one. + $crate::static_assert!($from::BITS >= $into::BITS); + // CAST: we statically enforced above that `$from` is larger than `$into`, so the + // `as` conversion will be lossless. + $crate::const_assert!(N >= $into::MIN as $from && N <= $into::MAX as $from); + + N as $into + } + } + )* + }; +} + +impl_const_into!(usize => { u8, u16, u32 }); +impl_const_into!(u64 => { u8, u16, u32 }); +impl_const_into!(u32 => { u8, u16 }); +impl_const_into!(u16 => { u8 }); + +/// Conversions that are only lossless for the current architecture. +/// +/// # Portability +/// +/// Callers of this module become dependent on the setting of `CONFIG_64BIT`. Use with caution, and +/// never in code that is portable across pointer sizes. +pub mod arch { + /// Trait identical to [`FromSafeCast`](super::FromSafeCast), but for conversions that are not + /// available on all architectures. + pub trait FromSafeCastArch<T> { + /// Create a [`Self`] from `value`. This operation is guaranteed to be lossless. + fn from_safe_cast_arch(value: T) -> Self; + } + + /// Trait identical to [`IntoSafeCast`](super::IntoSafeCast), but for conversions that are not + /// available on all architectures. + pub trait IntoSafeCastArch<T> { + /// Convert `self` into a `T`. This operation is guaranteed to be lossless. + fn into_safe_cast_arch(self) -> T; + } + + /// Reverse operation for types implementing [`FromSafeCastArch`]. + impl<S, T> IntoSafeCastArch<T> for S + where + T: FromSafeCastArch<S>, + { + #[inline] + fn into_safe_cast_arch(self) -> T { + T::from_safe_cast_arch(self) + } + } + + /// A [`u64`] fits into a [`usize`] on 64-bit platforms. + #[cfg(CONFIG_64BIT)] + #[inline] + pub const fn u64_as_usize(value: u64) -> usize { + value as usize + } + + #[cfg(CONFIG_64BIT)] + impl FromSafeCastArch<u64> for usize { + #[inline] + fn from_safe_cast_arch(value: u64) -> Self { + u64_as_usize(value) + } + } + + /// A [`usize`] fits into a [`u32`] on 32-bit platforms. + #[cfg(not(CONFIG_64BIT))] + #[inline] + pub const fn usize_as_u32(value: usize) -> u32 { + value as u32 + } + + #[cfg(not(CONFIG_64BIT))] + impl FromSafeCastArch<usize> for u32 { + #[inline] + fn from_safe_cast_arch(value: usize) -> Self { + usize_as_u32(value) + } + } +} diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 5071cae6543f..4def9ca1824c 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -86,7 +86,7 @@ unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::__pci_register_driver(pdrv.get(), module.0, name.as_char_ptr()) + bindings::__pci_register_driver(pdrv.get(), module.as_ptr(), name.as_char_ptr()) }) } diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index d41555a4b31d..5a5f4156d79b 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -83,7 +83,7 @@ unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::__platform_driver_register(pdrv.get(), module.0, name.as_char_ptr()) + bindings::__platform_driver_register(pdrv.get(), module.as_ptr(), name.as_char_ptr()) }) } diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs index 6fd84389a858..0d62beeedca5 100644 --- a/rust/kernel/print.rs +++ b/rust/kernel/print.rs @@ -99,7 +99,7 @@ pub mod format_strings { /// The format string must be one of the ones in [`format_strings`], and /// the module name must be null-terminated. /// -/// [`_printk`]: srctree/include/linux/_printk.h +/// [`_printk`]: srctree/include/linux/printk.h #[doc(hidden)] #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] pub unsafe fn call_printk( diff --git a/rust/kernel/pwm.rs b/rust/kernel/pwm.rs index 6c9d667009ef..8b3a580b4f0f 100644 --- a/rust/kernel/pwm.rs +++ b/rust/kernel/pwm.rs @@ -600,7 +600,7 @@ impl<T: PwmOps> Chip<T> { let drvdata_ptr = unsafe { bindings::pwmchip_get_drvdata(c_chip_ptr) }; // SAFETY: We construct the `T` object in-place in the allocated private memory. - unsafe { data.__pinned_init(drvdata_ptr.cast()) }.inspect_err(|_| { + unsafe { pin_init::raw_try_init(drvdata_ptr.cast(), data) }.inspect_err(|_| { // SAFETY: It is safe to call `pwmchip_put()` with a valid pointer obtained // from `pwmchip_alloc()`. We will not use pointer after this. unsafe { bindings::pwmchip_put(c_chip_ptr) } diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 5ac4961b7cd2..8ae0fe6f19ec 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -154,7 +154,7 @@ impl<T: ?Sized> ArcInner<T> { /// /// # Safety /// - /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the [`Arc`] must /// not yet have been destroyed. unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> { let refcount_layout = Layout::new::<Refcount>(); @@ -253,7 +253,7 @@ impl<T: ?Sized> Arc<T> { /// Convert the [`Arc`] into a raw pointer. /// - /// The raw pointer has ownership of the refcount that this Arc object owned. + /// The raw pointer has ownership of the refcount that this [`Arc`] object owned. pub fn into_raw(self) -> *const T { let ptr = self.ptr.as_ptr(); core::mem::forget(self); @@ -261,7 +261,7 @@ impl<T: ?Sized> Arc<T> { unsafe { core::ptr::addr_of!((*ptr).data) } } - /// Return a raw pointer to the data in this arc. + /// Return a raw pointer to the data in this [`Arc`]. pub fn as_ptr(this: &Self) -> *const T { let ptr = this.ptr.as_ptr(); @@ -305,7 +305,7 @@ impl<T: ?Sized> Arc<T> { /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. /// - /// When this destroys the `Arc`, it does so while properly avoiding races. This means that + /// When this destroys the [`Arc`], it does so while properly avoiding races. This means that /// this method will never call the destructor of the value. /// /// # Examples @@ -345,11 +345,11 @@ impl<T: ?Sized> Arc<T> { // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will // return without further touching the `Arc`. If the refcount reaches zero, then there are - // no other arcs, and we can create a `UniqueArc`. + // no other `Arc`s, and we can create a `UniqueArc`. if refcount.dec_and_test() { refcount.set(1); - // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We + // INVARIANT: We own the only refcount to this `Arc`, so we may create a `UniqueArc`. We // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin // their values. Some(Pin::from(UniqueArc { @@ -717,7 +717,7 @@ impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. - unsafe { init.__init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { self.assume_init() }) } @@ -727,7 +727,7 @@ impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { self.assume_init() }.into()) } @@ -795,7 +795,7 @@ impl<T> UniqueArc<MaybeUninit<T>> { #[inline] pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> { // SAFETY: The supplied pointer is valid for initialization. - match unsafe { init.__init(self.as_mut_ptr()) } { + match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } { // SAFETY: Initialization completed successfully. Ok(()) => Ok(unsafe { self.assume_init() }), Err(err) => Err(err), @@ -810,7 +810,7 @@ impl<T> UniqueArc<MaybeUninit<T>> { ) -> core::result::Result<Pin<UniqueArc<T>>, E> { // SAFETY: The supplied pointer is valid for initialization and we will later pin the value // to ensure it does not move. - match unsafe { init.__pinned_init(self.as_mut_ptr()) } { + match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } { // SAFETY: Initialization completed successfully. Ok(()) => Ok(unsafe { self.assume_init() }.into()), Err(err) => Err(err), diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs index b721b2e00b98..9983ee085248 100644 --- a/rust/kernel/sync/aref.rs +++ b/rust/kernel/sync/aref.rs @@ -24,6 +24,11 @@ use core::{ ptr::NonNull, // }; +use crate::{ + prelude::*, + types::ForeignOwnable, // +}; + /// Types that are _always_ reference counted. /// /// It allows such types to define their own custom ref increment and decrement functions. @@ -188,6 +193,51 @@ where } impl<T: AlwaysRefCounted + Eq> Eq for ARef<T> {} +// SAFETY: `into_foreign` returns a pointer from `NonNull::as_ptr`, so it's non-null. The +// `ARef` invariant guarantees that `ptr` points to a valid `T`, so it's aligned to `T`. +unsafe impl<T: AlwaysRefCounted> ForeignOwnable for ARef<T> { + const FOREIGN_ALIGN: usize = core::mem::align_of::<T>(); + + type Borrowed<'a> + = &'a T + where + Self: 'a; + type BorrowedMut<'a> + = &'a T + where + Self: 'a; + + #[inline] + fn into_foreign(self) -> *mut c_void { + ARef::into_raw(self).as_ptr().cast() + } + + #[inline] + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let ptr = unsafe { NonNull::new_unchecked(ptr.cast()) }; + + // SAFETY: `ptr` came from `into_foreign`, which consumed an `ARef` without decrementing + // the refcount, so we can transfer the ownership to the new `ARef`. + unsafe { ARef::from_raw(ptr) } + } + + #[inline] + unsafe fn borrow<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements of this method ensure that the object remains alive and + // immutable for the duration of 'a. + unsafe { &*ptr.cast() } + } + + #[inline] + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety + // requirements for `borrow`. + unsafe { <Self as ForeignOwnable>::borrow(ptr) } + } +} + impl<T, U> PartialEq<&'_ U> for ARef<T> where T: AlwaysRefCounted + PartialEq<U>, diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs index a32bef6e490b..42d1b26a2143 100644 --- a/rust/kernel/sync/rcu.rs +++ b/rust/kernel/sync/rcu.rs @@ -50,3 +50,23 @@ impl Drop for Guard { pub fn read_lock() -> Guard { Guard::new() } + +/// Wait until all in-flight `call_rcu()` callbacks complete. +/// +/// Note that this primitive does not necessarily wait for an RCU grace period +/// to complete. For example, if there are no RCU callbacks queued anywhere +/// in the system, then [`rcu_barrier()`] is within its rights to return +/// immediately, without waiting for anything, much less an RCU grace period. +/// In fact, [`rcu_barrier()`] will normally not result in any RCU grace periods +/// beyond those that were already destined to be executed. +/// +/// In kernels built with `CONFIG_RCU_LAZY=y`, this function also hurries all +/// pending lazy RCU callbacks. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn rcu_barrier() { + // SAFETY: `rcu_barrier()` is always safe to be called. It just might wait for a grace period. + unsafe { bindings::rcu_barrier() }; +} diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index b8463823aed9..6c0a5e8090d0 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -246,7 +246,7 @@ impl<C: ClockSource> ops::Sub for Instant<C> { #[inline] fn sub(self, other: Instant<C>) -> Delta { Delta { - nanos: self.inner - other.inner, + value: self.inner - other.inner, } } } @@ -258,7 +258,7 @@ impl<T: ClockSource> ops::Add<Delta> for Instant<T> { fn add(self, rhs: Delta) -> Self::Output { // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow // (e.g. go above `KTIME_MAX`) - let res = self.inner + rhs.nanos; + let res = self.inner + rhs.value; // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] @@ -278,7 +278,7 @@ impl<T: ClockSource> ops::Sub<Delta> for Instant<T> { fn sub(self, rhs: Delta) -> Self::Output { // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow // (e.g. go above `KTIME_MAX`) - let res = self.inner - rhs.nanos; + let res = self.inner - rhs.value; // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] @@ -291,14 +291,64 @@ impl<T: ClockSource> ops::Sub<Delta> for Instant<T> { } } +mod private { + pub trait Sealed {} + + impl Sealed for super::Nsec {} + impl Sealed for super::Jiffy {} +} + +/// A trait for time units. +pub trait TimeUnit: private::Sealed { + /// The underlying representation of the time unit. + type Repr: Copy + Clone + PartialEq + PartialOrd + Eq + Ord + core::fmt::Debug; +} + +/// A time unit of nanoseconds. +/// +/// A [`Delta<Nsec>`] stores its value as [`i64`] nanoseconds and can represent +/// any [`i64`] value, including negative, zero, and positive numbers. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum Nsec {} + +impl TimeUnit for Nsec { + type Repr = i64; +} + +/// A time unit of jiffies. +/// +/// A [`Delta<Jiffy>`] stores its value as [`isize`] jiffies and can represent +/// any [`isize`] value, including negative, zero, and positive numbers. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum Jiffy {} + +impl TimeUnit for Jiffy { + type Repr = isize; +} + /// A span of time. /// -/// This struct represents a span of time, with its value stored as nanoseconds. -/// The value can represent any valid i64 value, including negative, zero, and -/// positive numbers. +/// The span is stored in the unit given by the type parameter `U` (see +/// [`TimeUnit`]); its value has type `U::Repr`. `U` defaults to [`Nsec`], so a +/// plain [`Delta`] is a span in nanoseconds. The value can be negative, zero, or +/// positive. #[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] -pub struct Delta { - nanos: i64, +pub struct Delta<U: TimeUnit = Nsec> { + value: U::Repr, +} + +impl Delta<Jiffy> { + /// Create a new [`Delta`] from a number of jiffies. + #[inline] + pub const fn from_jiffies(jiffies: isize) -> Self { + Self { value: jiffies } + } + + /// Return the number of jiffies in the [`Delta`]. + #[inline] + pub const fn as_jiffies(self) -> isize { + self.value + } } impl ops::Add for Delta { @@ -307,7 +357,7 @@ impl ops::Add for Delta { #[inline] fn add(self, rhs: Self) -> Self { Self { - nanos: self.nanos + rhs.nanos, + value: self.value + rhs.value, } } } @@ -315,7 +365,7 @@ impl ops::Add for Delta { impl ops::AddAssign for Delta { #[inline] fn add_assign(&mut self, rhs: Self) { - self.nanos += rhs.nanos; + self.value += rhs.value; } } @@ -325,7 +375,7 @@ impl ops::Sub for Delta { #[inline] fn sub(self, rhs: Self) -> Self::Output { Self { - nanos: self.nanos - rhs.nanos, + value: self.value - rhs.value, } } } @@ -333,7 +383,7 @@ impl ops::Sub for Delta { impl ops::SubAssign for Delta { #[inline] fn sub_assign(&mut self, rhs: Self) { - self.nanos -= rhs.nanos; + self.value -= rhs.value; } } @@ -343,7 +393,7 @@ impl ops::Mul<i64> for Delta { #[inline] fn mul(self, rhs: i64) -> Self::Output { Self { - nanos: self.nanos * rhs, + value: self.value * rhs, } } } @@ -351,7 +401,7 @@ impl ops::Mul<i64> for Delta { impl ops::MulAssign<i64> for Delta { #[inline] fn mul_assign(&mut self, rhs: i64) { - self.nanos *= rhs; + self.value *= rhs; } } @@ -362,25 +412,25 @@ impl ops::Div for Delta { fn div(self, rhs: Self) -> Self::Output { #[cfg(CONFIG_64BIT)] { - self.nanos / rhs.nanos + self.value / rhs.value } #[cfg(not(CONFIG_64BIT))] { // SAFETY: This function is always safe to call regardless of the input values - unsafe { bindings::div64_s64(self.nanos, rhs.nanos) } + unsafe { bindings::div64_s64(self.value, rhs.value) } } } } impl Delta { /// A span of time equal to zero. - pub const ZERO: Self = Self { nanos: 0 }; + pub const ZERO: Self = Self { value: 0 }; /// Create a new [`Delta`] from a number of nanoseconds. #[inline] pub const fn from_nanos(nanos: i64) -> Self { - Self { nanos } + Self { value: nanos } } /// Create a new [`Delta`] from a number of microseconds. @@ -391,7 +441,7 @@ impl Delta { #[inline] pub const fn from_micros(micros: i64) -> Self { Self { - nanos: micros.saturating_mul(NSEC_PER_USEC), + value: micros.saturating_mul(NSEC_PER_USEC), } } @@ -403,7 +453,7 @@ impl Delta { #[inline] pub const fn from_millis(millis: i64) -> Self { Self { - nanos: millis.saturating_mul(NSEC_PER_MSEC), + value: millis.saturating_mul(NSEC_PER_MSEC), } } @@ -415,7 +465,7 @@ impl Delta { #[inline] pub const fn from_secs(secs: i64) -> Self { Self { - nanos: secs.saturating_mul(NSEC_PER_SEC), + value: secs.saturating_mul(NSEC_PER_SEC), } } @@ -434,29 +484,32 @@ impl Delta { /// Return the number of nanoseconds in the [`Delta`]. #[inline] pub const fn as_nanos(self) -> i64 { - self.nanos + self.value } /// Return the smallest number of microseconds greater than or equal /// to the value in the [`Delta`]. #[inline] pub fn as_micros_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. let n = self.as_nanos(); - let n = if n >= 0 { - n.saturating_add(NSEC_PER_USEC - 1) - } else { - n - }; + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; #[cfg(CONFIG_64BIT)] { - n / NSEC_PER_USEC + n / NSEC_PER_USEC + add } #[cfg(not(CONFIG_64BIT))] // SAFETY: It is always safe to call `ktime_to_us()` with any value. unsafe { - bindings::ktime_to_us(n) + bindings::ktime_to_us(n) + add } } @@ -475,6 +528,32 @@ impl Delta { } } + /// Return the smallest number of milliseconds greater than or equal + /// to the value in the [`Delta`]. + #[inline] + pub fn as_millis_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. + let n = self.as_nanos(); + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; + + #[cfg(CONFIG_64BIT)] + { + n / NSEC_PER_MSEC + add + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_ms()` with any value. + unsafe { + bindings::ktime_to_ms(n) + add + } + } + /// Return `self % dividend` where `dividend` is in nanoseconds. /// /// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is @@ -484,7 +563,7 @@ impl Delta { #[cfg(CONFIG_64BIT)] { Self { - nanos: self.as_nanos() % i64::from(dividend), + value: self.as_nanos() % i64::from(dividend), } } @@ -496,7 +575,7 @@ impl Delta { unsafe { bindings::div_s64_rem(self.as_nanos(), dividend, &mut rem) }; Self { - nanos: i64::from(rem), + value: i64::from(rem), } } } diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index ac316fd7b538..67b3874cb3d2 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -417,13 +417,13 @@ impl<T> Opaque<T> { impl<T> Wrapper<T> for Opaque<T> { /// Create an opaque pin-initializer from the given pin-initializer. - fn pin_init<E>(slot: impl PinInit<T, E>) -> impl PinInit<Self, E> { - Self::try_ffi_init(|ptr: *mut T| { + fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> { + Self::try_ffi_init(|slot: *mut T| { // SAFETY: - // - `ptr` is a valid pointer to uninitialized memory, + // - `slot` is a valid pointer to uninitialized memory, // - `slot` is not accessed on error, // - `slot` is pinned in memory. - unsafe { PinInit::<T, E>::__pinned_init(slot, ptr) } + unsafe { pin_init::raw_try_init(slot, init) } }) } } diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs index 7aff0c82d0af..870423806e4f 100644 --- a/rust/kernel/usb.rs +++ b/rust/kernel/usb.rs @@ -63,7 +63,7 @@ unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { // SAFETY: `udrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::usb_register_driver(udrv.get(), module.0, name.as_char_ptr()) + bindings::usb_register_driver(udrv.get(), module.as_ptr(), name.as_char_ptr()) }) } diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 4a48fabbc268..408a90567f7e 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -177,12 +177,29 @@ pub fn module(input: TokenStream) -> TokenStream { /// /// This macro should not be used when all functions are required. /// +/// Additionally, this macro automatically handles the `OwnerModule` +/// associated type: on the trait side, `type OwnerModule: ModuleMetadata;` +/// is added as a required associated type if not already defined; on the +/// impl side, `type OwnerModule = LocalModule;` is automatically inserted +/// if not explicitly defined. +/// /// # Examples /// /// ``` /// use kernel::error::VTABLE_DEFAULT_ERROR; /// use kernel::prelude::*; /// +/// # struct LocalModule; +/// # impl kernel::ModuleMetadata for LocalModule { +/// # const NAME: &'static kernel::str::CStr = c"vtable_doctest"; +/// # +/// # // SAFETY: This doctest runs on the host: there is no `THIS_MODULE`. +/// # const THIS_MODULE: kernel::ThisModule = unsafe { +/// # kernel::ThisModule::from_ptr(core::ptr::null_mut()) +/// # }; +/// # } +/// # +/// # fn main() { /// // Declares a `#[vtable]` trait /// #[vtable] /// pub trait Operations: Send + Sync + Sized { @@ -208,6 +225,7 @@ pub fn module(input: TokenStream) -> TokenStream { /// /// assert_eq!(<Foo as Operations>::HAS_FOO, true); /// assert_eq!(<Foo as Operations>::HAS_BAR, false); +/// # } /// ``` /// /// [`kernel::error::VTABLE_DEFAULT_ERROR`]: ../kernel/error/constant.VTABLE_DEFAULT_ERROR.html diff --git a/rust/macros/module.rs b/rust/macros/module.rs index 06c18e207508..bd69d8dc4bbb 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -497,28 +497,28 @@ pub(crate) fn module(info: ModuleInfo) -> Result<TokenStream> { /// Used by the printing macros, e.g. [`info!`]. const __LOG_PREFIX: &[u8] = #name_cstr.to_bytes_with_nul(); - // SAFETY: `__this_module` is constructed by the kernel at load time and will not be - // freed until the module is unloaded. - #[cfg(MODULE)] - static THIS_MODULE: ::kernel::ThisModule = unsafe { - extern "C" { - static __this_module: ::kernel::types::Opaque<::kernel::bindings::module>; - }; - - ::kernel::ThisModule::from_ptr(__this_module.get()) - }; - - #[cfg(not(MODULE))] - static THIS_MODULE: ::kernel::ThisModule = unsafe { - ::kernel::ThisModule::from_ptr(::core::ptr::null_mut()) - }; - /// The `LocalModule` type is the type of the module created by `module!`, /// `module_pci_driver!`, `module_platform_driver!`, etc. type LocalModule = #type_; impl ::kernel::ModuleMetadata for #type_ { const NAME: &'static ::kernel::str::CStr = #name_cstr; + + #[cfg(MODULE)] + const THIS_MODULE: ::kernel::ThisModule = { + extern "C" { + static __this_module: ::kernel::types::Opaque<::kernel::bindings::module>; + } + + // SAFETY: `__this_module` is constructed by the kernel at load time + // and lives until the module is unloaded. + unsafe { ::kernel::ThisModule::from_ptr(__this_module.get()) } + }; + + #[cfg(not(MODULE))] + const THIS_MODULE: ::kernel::ThisModule = unsafe { + ::kernel::ThisModule::from_ptr(::core::ptr::null_mut()) + }; } // Double nested modules, since then nobody can access the public items inside. @@ -616,12 +616,12 @@ pub(crate) fn module(info: ModuleInfo) -> Result<TokenStream> { /// This function must only be called once. unsafe fn __init() -> ::kernel::ffi::c_int { let initer = <super::super::LocalModule as ::kernel::InPlaceModule>::init( - &super::super::THIS_MODULE + ::kernel::module::this_module::<super::super::LocalModule>() ); // SAFETY: No data race, since `__MOD` can only be accessed by this module // and there only `__init` and `__exit` access it. These functions are only // called once and `__exit` cannot be called before or during `__init`. - match unsafe { initer.__pinned_init(__MOD.as_mut_ptr()) } { + match unsafe { ::pin_init::raw_try_init(__MOD.as_mut_ptr(), initer) } { Ok(m) => 0, Err(e) => e.to_errno(), } diff --git a/rust/macros/vtable.rs b/rust/macros/vtable.rs index c6510b0c4ea1..be9a5ed8abe5 100644 --- a/rust/macros/vtable.rs +++ b/rust/macros/vtable.rs @@ -30,6 +30,22 @@ fn handle_trait(mut item: ItemTrait) -> Result<ItemTrait> { const USE_VTABLE_ATTR: (); }); + // Add `type OwnerModule: ModuleMetadata` as a required associated type if + // the trait does not already define it. + if !item + .items + .iter() + .any(|i| matches!(i, TraitItem::Type(t) if t.ident == "OwnerModule")) + { + gen_items.push(parse_quote! { + /// The module implementing this vtable trait. + /// + /// Automatically set to `crate::LocalModule` by the `#[vtable]` + /// impl macro. + type OwnerModule: ::kernel::ModuleMetadata; + }); + } + for item in &item.items { if let TraitItem::Fn(fn_item) = item { let name = &fn_item.sig.ident; @@ -57,12 +73,18 @@ fn handle_trait(mut item: ItemTrait) -> Result<ItemTrait> { fn handle_impl(mut item: ItemImpl) -> Result<ItemImpl> { let mut gen_items = Vec::new(); - let mut defined_consts = HashSet::new(); + let mut defined_items = HashSet::new(); - // Iterate over all user-defined constants to gather any possible explicit overrides. + // Iterate over all user-defined items to gather any possible explicit overrides. for item in &item.items { - if let ImplItem::Const(const_item) = item { - defined_consts.insert(const_item.ident.clone()); + match item { + ImplItem::Const(const_item) => { + defined_items.insert(const_item.ident.clone()); + } + ImplItem::Type(type_item) => { + defined_items.insert(type_item.ident.clone()); + } + _ => {} } } @@ -70,6 +92,15 @@ fn handle_impl(mut item: ItemImpl) -> Result<ItemImpl> { const USE_VTABLE_ATTR: () = (); }); + // Auto-insert `type OwnerModule = crate::LocalModule` if not explicitly defined. + // `crate::LocalModule` resolves to the real module type (via `module!`) or a + // dummy fallback in non-module contexts (e.g., doctests). + if !defined_items.contains(&parse_quote!(OwnerModule)) { + gen_items.push(parse_quote! { + type OwnerModule = crate::LocalModule; + }); + } + for item in &item.items { if let ImplItem::Fn(fn_item) = item { let name = &fn_item.sig.ident; @@ -78,7 +109,7 @@ fn handle_impl(mut item: ItemImpl) -> Result<ItemImpl> { name.span(), ); // Skip if it's declared already -- this allows user override. - if defined_consts.contains(&gen_const_name) { + if defined_items.contains(&gen_const_name) { continue; } let cfg_attrs = crate::helpers::gather_cfg_attrs(&fn_item.attrs); diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs index 35ecb5f68dc3..e8d4dbb664fe 100644 --- a/rust/pin-init/examples/mutex.rs +++ b/rust/pin-init/examples/mutex.rs @@ -79,11 +79,7 @@ impl<T> CMutex<T> { wait_list <- ListHead::new(), spin_lock: SpinLock::new(), locked: Cell::new(false), - data <- unsafe { - pin_init_from_closure(|slot: *mut UnsafeCell<T>| { - val.__pinned_init(slot.cast::<T>()) - }) - }, + data <- UnsafeCell::pin_init(val), }) } @@ -91,7 +87,7 @@ impl<T> CMutex<T> { pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> { let mut sguard = self.spin_lock.acquire(); if self.locked.get() { - stack_pin_init!(let wait_entry = WaitEntry::insert_new(&self.wait_list)); + stack_pin_init!(let _wait_entry = WaitEntry::insert_new(&self.wait_list)); // println!("wait list length: {}", self.wait_list.size()); while self.locked.get() { drop(sguard); @@ -99,9 +95,6 @@ impl<T> CMutex<T> { thread::park(); sguard = self.spin_lock.acquire(); } - // This does have an effect, as the ListHead inside wait_entry implements Drop! - #[expect(clippy::drop_non_drop)] - drop(wait_entry); } self.locked.set(true); unsafe { diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs index 58cd4241b78c..8dd52313c1b8 100644 --- a/rust/pin-init/examples/static_init.rs +++ b/rust/pin-init/examples/static_init.rs @@ -59,7 +59,7 @@ impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> { println!("doing init"); let ptr = self.cell.get().cast::<T>(); match self.init.take() { - Some(f) => unsafe { f.__pinned_init(ptr).unwrap() }, + Some(f) => unsafe { pin_init::raw_init(ptr, f) }, None => unsafe { core::hint::unreachable_unchecked() }, } self.present.set(true); @@ -71,13 +71,11 @@ impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> { pub struct CountInit; unsafe impl PinInit<CMutex<usize>> for CountInit { - unsafe fn __pinned_init( - self, - slot: *mut CMutex<usize>, - ) -> Result<(), core::convert::Infallible> { + unsafe fn __init(self, slot: *mut CMutex<usize>) -> Result<(), core::convert::Infallible> { let init = CMutex::new(0); std::thread::sleep(std::time::Duration::from_millis(1000)); - unsafe { init.__pinned_init(slot) } + unsafe { pin_init::raw_init(slot, init) }; + Ok(()) } } diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 28d30805d06b..fd0b5ea4a0a3 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -233,10 +233,12 @@ fn init_fields( InitializerKind::Value { ident, .. } => ident, InitializerKind::Init { ident, .. } => ident, InitializerKind::Code { block, .. } => { + let stmt = &block.stmts; res.extend(quote! { #(#attrs)* - #[allow(unused_braces)] - #block + { + #(#stmt)* + } }); continue; } @@ -334,7 +336,7 @@ fn make_field_check( }), }; quote! { - #[allow(unreachable_code, clippy::diverging_sub_expression)] + #[allow(unreachable_code)] // We use unreachable code to perform field checks. They're still checked by the compiler. // SAFETY: this code is never executed. let _ = || unsafe { diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 9fbbd25bcaac..ff194d27565e 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -1,13 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::TokenStream; -use quote::{format_ident, quote}; +use quote::{format_ident, quote, ToTokens}; use syn::{ parse::{End, Nothing, Parse}, parse_quote, parse_quote_spanned, spanned::Spanned, visit_mut::VisitMut, - Attribute, Field, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, + Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, }; use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; @@ -35,10 +35,18 @@ impl Parse for Args { } } +impl ToTokens for Args { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Nothing(_) => (), + Self::PinnedDrop(kw) => kw.to_tokens(tokens), + } + } +} + struct FieldInfo<'a> { field: &'a Field, pinned: bool, - cfg_attrs: Vec<&'a Attribute>, } pub(crate) fn pin_data( @@ -68,6 +76,55 @@ pub(crate) fn pin_data( } }; + // Handling cfg can gets very complicated, especially for tuple structs. Therefore, resolve all + // field cfgs first before continuing. + // + // We need to perform this after parsing so we can reliably detect field cfgs. + for (field_idx, field) in struct_.fields.iter_mut().enumerate() { + let cfg: Vec<_> = field + .attrs + .iter() + .filter(|a| a.path().is_ident("cfg")) + .map(|a| { + a.parse_args::<TokenStream>() + .expect("parse as token stream cannot fail") + }) + .collect(); + + if cfg.is_empty() { + continue; + } + + field.attrs.retain(|a| !a.path().is_ident("cfg")); + let cfg_true_struct = quote!(#struct_); + + let punctuated = match &mut struct_.fields { + Fields::Named(fields) => &mut fields.named, + Fields::Unnamed(fields) => &mut fields.unnamed, + Fields::Unit => unreachable!(), + }; + *punctuated = std::mem::take(punctuated) + .into_pairs() + .enumerate() + .filter(|&(i, _)| i != field_idx) + .map(|(_, p)| p) + .collect(); + let cfg_false_struct = quote!(#struct_); + + // Resolve one field at a time until we've got no more field cfgs. + // + // This is linear time because macro invocations with false cfg will not be expanded. + return Ok(quote!( + #[cfg(all(#(#cfg,)*))] + #[::pin_init::pin_data(#args)] + #cfg_true_struct + + #[cfg(not(all(#(#cfg,)*)))] + #[::pin_init::pin_data(#args)] + #cfg_false_struct + )); + } + // The generics might contain the `Self` type. Since this macro will define a new type with the // same generics and bounds, this poses a problem: `Self` will refer to the new type as opposed // to this struct definition. Therefore we have to replace `Self` with the concrete name. @@ -85,18 +142,19 @@ pub(crate) fn pin_data( .map(|field| { let len = field.attrs.len(); field.attrs.retain(|a| !a.path().is_ident("pin")); - let pinned = len != field.attrs.len(); + let pinned_count = len - field.attrs.len(); + if pinned_count > 1 { + dcx.error(&field, "#[pin] attribute specified more than once"); + } - let cfg_attrs = field - .attrs - .iter() - .filter(|a| a.path().is_ident("cfg")) - .collect(); + assert!( + !field.attrs.iter().any(|a| a.path().is_ident("cfg")), + "cfgs should be all resolved at this point" + ); FieldInfo { field: &*field, - pinned, - cfg_attrs, + pinned: pinned_count != 0, } }) .collect(); @@ -182,9 +240,7 @@ fn generate_unpin_impl( let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| { let ident = f.field.ident.as_ref().unwrap(); let ty = &f.field.ty; - let cfg_attrs = &f.cfg_attrs; quote!( - #(#cfg_attrs)* #ident: #ty ) }); @@ -242,7 +298,6 @@ fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenSt // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, // if it also implements `Drop` trait MustNotImplDrop {} - #[expect(drop_bounds)] impl<T: ::core::ops::Drop + ?::core::marker::Sized> MustNotImplDrop for T {} impl #impl_generics MustNotImplDrop for #ident #ty_generics #whr @@ -250,7 +305,6 @@ fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenSt // We also take care to prevent users from writing a useless `PinnedDrop` implementation. // They might implement `PinnedDrop` correctly for the struct, but forget to give // `PinnedDrop` as the parameter to `#[pin_data]`. - #[expect(non_camel_case_types)] trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} impl<T: ::pin_init::PinnedDrop + ?::core::marker::Sized> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} @@ -279,7 +333,6 @@ fn generate_projections( .iter() .map(|field| { let Field { vis, ident, ty, .. } = &field.field; - let cfg_attrs = &field.cfg_attrs; let ident = ident .as_ref() @@ -287,11 +340,9 @@ fn generate_projections( if field.pinned { ( quote!( - #(#cfg_attrs)* #vis #ident: ::core::pin::Pin<&'__pin mut #ty>, ), quote!( - #(#cfg_attrs)* // SAFETY: this field is structurally pinned. #ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) }, ), @@ -299,11 +350,9 @@ fn generate_projections( } else { ( quote!( - #(#cfg_attrs)* #vis #ident: &'__pin mut #ty, ), quote!( - #(#cfg_attrs)* #ident: &mut #this.#ident, ), ) @@ -373,7 +422,6 @@ fn generate_the_pin_data( .iter() .map(|f| { let Field { vis, ident, ty, .. } = f.field; - let cfg_attrs = &f.cfg_attrs; let field_name = ident .as_ref() @@ -390,7 +438,6 @@ fn generate_the_pin_data( /// - `(*slot).#field_name` is properly aligned. /// - `(*slot).#field_name` points to uninitialized and exclusively accessed /// memory. - #(#cfg_attrs)* // Allow `non_snake_case` since the same warning will be emitted on // the struct definition. #[allow(non_snake_case)] @@ -421,6 +468,7 @@ fn generate_the_pin_data( impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics #whr { + #[inline] fn clone(&self) -> Self { *self } } @@ -429,7 +477,6 @@ fn generate_the_pin_data( {} #[allow(dead_code)] // Some functions might never be used and private. - #[expect(clippy::missing_safety_doc)] impl #impl_generics __ThePinData #ty_generics #whr { @@ -453,6 +500,7 @@ fn generate_the_pin_data( { type PinData = __ThePinData #ty_generics; + #[inline] unsafe fn __pin_data() -> Self::PinData { __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() } } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 56dc655e323e..8e9fd18b993f 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -105,6 +105,7 @@ pub unsafe trait HasInitData { pub struct AllData<T: ?Sized>(PhantomInvariant<T>); impl<T: ?Sized> Clone for AllData<T> { + #[inline] fn clone(&self) -> Self { *self } @@ -127,6 +128,7 @@ impl<T: ?Sized> AllData<T> { unsafe impl<T: ?Sized> HasInitData for T { type InitData = AllData<T>; + #[inline] unsafe fn __init_data() -> Self::InitData { AllData(PhantomInvariant::new()) } @@ -181,7 +183,7 @@ impl<T> StackInit<T> { unsafe { this.value.assume_init_drop() }; } // SAFETY: The memory slot is valid and this type ensures that it will stay pinned. - unsafe { init.__pinned_init(this.value.as_mut_ptr())? }; + unsafe { init.__init(this.value.as_mut_ptr())? }; // INVARIANT: `this.value` is initialized above. this.is_init = true; // SAFETY: The slot is now pinned, since we will never give access to `&mut T`. @@ -289,7 +291,7 @@ impl<T: ?Sized> Slot<Pinned, T> { // - when `Err` is returned, we also propagate the error without touching `ptr`; // also `self` is consumed so it cannot be touched further. // - the drop guard will not hand out `&mut` (only `Pin<&mut T>`). - unsafe { init.__pinned_init(self.ptr)? }; + unsafe { init.__init(self.ptr)? }; // SAFETY: // - `self.ptr` is valid, properly aligned and pinned per type invariant. @@ -385,20 +387,23 @@ pub struct AlwaysFail<T: ?Sized> { impl<T: ?Sized> AlwaysFail<T> { /// Creates a new initializer that always fails. + #[inline] pub fn new() -> Self { Self { _t: PhantomData } } } impl<T: ?Sized> Default for AlwaysFail<T> { + #[inline] fn default() -> Self { Self::new() } } -// SAFETY: `__pinned_init` always fails, which is always okay. +// SAFETY: `__init` always fails, which is always okay. unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> { - unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> { + #[inline] + unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> { Err(()) } } diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs index 5017f57442d8..471652e8663a 100644 --- a/rust/pin-init/src/alloc.rs +++ b/rust/pin-init/src/alloc.rs @@ -35,10 +35,11 @@ pub trait InPlaceInit<T>: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] fn pin_init(init: impl PinInit<T>) -> Result<Pin<Self>, AllocError> { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { - pin_init_from_closure(|slot| match init.__pinned_init(slot) { + pin_init_from_closure(|slot| match init.__init(slot) { Ok(()) => Ok(()), Err(i) => match i {}, }) @@ -52,6 +53,7 @@ pub trait InPlaceInit<T>: Sized { E: From<AllocError>; /// Use the given initializer to in-place initialize a `T`. + #[inline] fn init(init: impl Init<T>) -> Result<Self, AllocError> { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { @@ -109,7 +111,7 @@ impl<T> InPlaceInit<T> for Arc<T> { let slot = slot.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: All fields have been initialized and this is the only `Arc` to that data. Ok(unsafe { Pin::new_unchecked(this.assume_init()) }) } @@ -136,6 +138,7 @@ impl<T> InPlaceInit<T> for Arc<T> { impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { type Initialized = Box<T>; + #[inline] fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, @@ -145,11 +148,12 @@ impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { Ok(unsafe { self.assume_init() }) } + #[inline] fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: All fields have been initialized. Ok(unsafe { self.assume_init() }.into()) } diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index fd40c8f244a1..7600cdbbbf98 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -70,7 +70,6 @@ //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. //! //! ```rust -//! # #![expect(clippy::disallowed_names)] //! # #![feature(allocator_api)] //! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; //! # use core::pin::Pin; @@ -94,7 +93,6 @@ //! (or just the stack) to actually initialize a `Foo`: //! //! ```rust -//! # #![expect(clippy::disallowed_names)] //! # #![feature(allocator_api)] //! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; //! # use core::{alloc::AllocError, pin::Pin}; @@ -456,7 +454,6 @@ pub use ::pin_init_internal::MaybeZeroable; /// # Examples /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; /// # use pin_init::*; @@ -508,7 +505,6 @@ macro_rules! stack_pin_init { /// # Examples /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/error.rs"] mod error; use error::Error; /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; @@ -535,7 +531,6 @@ macro_rules! stack_pin_init { /// ``` /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/error.rs"] mod error; use error::Error; /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; @@ -658,7 +653,6 @@ macro_rules! stack_try_pin_init { /// Users of `Foo` can now create it like this: /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # use pin_init::*; /// # use core::pin::Pin; /// # #[pin_data] @@ -895,7 +889,7 @@ macro_rules! assert_pinned { /// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. /// -/// The [`PinInit::__pinned_init`] function: +/// The [`PinInit::__init`] function: /// - returns `Ok(())` if it initialized every field of `slot`, /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: /// - `slot` can be deallocated without UB occurring, @@ -915,15 +909,33 @@ macro_rules! assert_pinned { #[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { + /// Alias of [`PinInit::__init`]. + /// + /// New code should use `__init` instead. + /// + /// # Safety + /// + /// Same as `__init`. + #[inline(always)] + #[cfg(not(kernel))] + #[deprecated = "use `raw_try_init` instead"] + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { self.__init(slot) } + } + /// Initializes `slot`. /// + /// It is not recommended to call this directly. Use [`raw_init`] or [`raw_try_init`]. + /// /// # Safety /// /// - `slot` is a valid pointer to uninitialized memory. /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to /// deallocate. /// - `slot` will not move until it is dropped, i.e. it will be pinned. - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; + /// If `Self: Init<T, E>`, this requirement is cancelled and it may be moved. + unsafe fn __init(self, slot: *mut T) -> Result<(), E>; /// First initializes the value using `self` then calls the function `f` with the initialized /// value. @@ -943,6 +955,7 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// Ok(()) /// }); /// ``` + #[inline] fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> where F: FnOnce(Pin<&mut T>) -> Result<(), E>, @@ -951,10 +964,38 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { } } +/// Initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_init<T>(slot: *mut T, init: impl PinInit<T>) { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot).unwrap_or_else(|e| match e {}) } +} + +/// Fallibly initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to +/// deallocate. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_try_init<T, E>(slot: *mut T, init: impl PinInit<T, E>) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot) } +} + /// An initializer returned by [`PinInit::pin_chain`]. pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>); -// SAFETY: The `__pinned_init` function is implemented such that it +// SAFETY: The `__init` function is implemented such that it // - returns `Ok(())` on successful initialization, // - returns `Err(err)` on error and in this case `slot` will be dropped. // - considers `slot` pinned. @@ -963,15 +1004,14 @@ where I: PinInit<T, E>, F: FnOnce(Pin<&mut T>) -> Result<(), E>, { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: All requirements fulfilled since this function is `__pinned_init`. - unsafe { self.0.__pinned_init(slot)? }; - // SAFETY: The above call initialized `slot` and we still have unique access. - let val = unsafe { &mut *slot }; - // SAFETY: `slot` is considered pinned. - let val = unsafe { Pin::new_unchecked(val) }; - // SAFETY: `slot` was initialized above. - (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) }) + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) } } @@ -988,19 +1028,8 @@ where /// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. /// -/// The [`Init::__init`] function: -/// - returns `Ok(())` if it initialized every field of `slot`, -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: -/// - `slot` can be deallocated without UB occurring, -/// - `slot` does not need to be dropped, -/// - `slot` is not partially initialized. -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. -/// -/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same -/// code as `__init`. -/// -/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to -/// move the pointee after initialization. +/// The [`PinInit::__init`] function must work without the pinning requirement; the caller is +/// allowed to move the pointee after initialization. /// #[cfg_attr( kernel, @@ -1014,15 +1043,6 @@ where #[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { - /// Initializes `slot`. - /// - /// # Safety - /// - /// - `slot` is a valid pointer to uninitialized memory. - /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to - /// deallocate. - unsafe fn __init(self, slot: *mut T) -> Result<(), E>; - /// First initializes the value using `self` then calls the function `f` with the initialized /// value. /// @@ -1031,7 +1051,6 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// # Examples /// /// ```rust - /// # #![expect(clippy::disallowed_names)] /// use pin_init::{init, init_zeroed, Init}; /// /// struct Foo { @@ -1051,6 +1070,7 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// Ok(()) /// }); /// ``` + #[inline] fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> where F: FnOnce(&mut T) -> Result<(), E>, @@ -1062,62 +1082,55 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// An initializer returned by [`Init::chain`]. pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>); -// SAFETY: The `__init` function is implemented such that it -// - returns `Ok(())` on successful initialization, -// - returns `Err(err)` on error and in this case `slot` will be dropped. +// SAFETY: The `__init` function does not rely on the pinning requirement. unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> where I: Init<T, E>, F: FnOnce(&mut T) -> Result<(), E>, { - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: All requirements fulfilled since this function is `__init`. - unsafe { self.0.__pinned_init(slot)? }; - // SAFETY: The above call initialized `slot` and we still have unique access. - (self.1)(unsafe { &mut *slot }).inspect_err(|_| - // SAFETY: `slot` was initialized above. - unsafe { core::ptr::drop_in_place(slot) }) - } } -// SAFETY: `__pinned_init` behaves exactly the same as `__init`. +// SAFETY: The `__init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> where I: Init<T, E>, F: FnOnce(&mut T) -> Result<(), E>, { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. - unsafe { self.__init(slot) } + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) } } /// Implement `PinInit` and `Init` for closures. /// /// It is unsafe to create this type, since the closure needs to fulfill the same safety -/// requirement as the `__pinned_init`/`__init` functions. +/// requirement as the `__init` functions. struct InitClosure<F, T: ?Sized>(F, __internal::PhantomInvariant<T>); -// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the -// `__init` invariants. -unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T> -where - F: FnOnce(*mut T) -> Result<(), E>, +// SAFETY: When constructing via `init_from_closure`, the `__init` function does not rely on the +// pinning requirement. When constructing via `pin_init_from_closure`, the opaque type prevents this +// implementation from being visible. +unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T> where + F: FnOnce(*mut T) -> Result<(), E> { - #[inline] - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - (self.0)(slot) - } } // SAFETY: While constructing the `InitClosure`, the user promised that it upholds the -// `__pinned_init` invariants. +// `__init` invariants. unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T> where F: FnOnce(*mut T) -> Result<(), E>, { #[inline] - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { (self.0)(slot) } } @@ -1166,10 +1179,11 @@ pub const unsafe fn init_from_closure<T: ?Sized, E>( /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. +#[inline] pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl PinInit<U, E> { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. - unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::<T>())) } + unsafe { pin_init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) } } /// Changes the to be initialized type. @@ -1178,6 +1192,7 @@ pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl Pin /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. +#[inline] pub const unsafe fn cast_init<T, U, E>(init: impl Init<T, E>) -> impl Init<U, E> { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. @@ -1193,6 +1208,77 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { unsafe { init_from_closure(|_| Ok(())) } } +/// Array initializer from element initializer. +struct ArrayInit<T: ?Sized, F>(F, __internal::PhantomInvariant<T>); + +// SAFETY: On success, all `N` elements of the array have been initialized. On error or panic, the +// elements that have been initialized so far are dropped, thus leaving the array uninitialized and +// ready to deallocate. +unsafe impl<T, F, I, E, const N: usize> PinInit<[T; N], E> for ArrayInit<T, F> +where + F: FnMut(usize) -> I, + I: PinInit<T, E>, +{ + unsafe fn __init(mut self, slot: *mut [T; N]) -> Result<(), E> { + /// # Invariants + /// + /// - `ptr[..num_init]` contains initialized elements of type `T` + /// - `ptr[num_init..N]` (where N is the size of the array) contains uninitialized memory + struct ArrayInitGuard<T> { + /// A pointer to the first element of the array. + ptr: *mut T, + /// The number of initialized elements in the array. + num_init: usize, + } + + impl<T> Drop for ArrayInitGuard<T> { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariant, `self.ptr[..self.num_init]` are initialized. + unsafe { + core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.ptr, + self.num_init, + )) + }; + } + } + + // INVARIANT: nothing is initialized yet. + let mut guard = ArrayInitGuard { + ptr: slot.cast::<T>(), + num_init: 0, + }; + + for i in 0..N { + // INVARIANT: Elements `self.ptr[..self.num_init]` have been initialized + // thus far. This holds true for every `self.num_init = i`. + guard.num_init = i; + + let init = (self.0)(i); + // SAFETY: + // - The subslot is derived from `slot` with a valid offset. + // - If `Err` is touched, the subslot is not touched further, the guard will drop + // previously initialized elements only. + // - `slot` is pinned so is the subslot. + unsafe { init.__init(&raw mut (*slot)[i]) }?; + } + + // Dismiss the drop guard now that all elements are initialized. + core::mem::forget(guard); + Ok(()) + } +} + +// SAFETY: `I: Init` cancels out the pinning requirement on subslots, which is the only place in the +// `__init` function that relies on `slot` being pinned. +unsafe impl<T, F, I, E, const N: usize> Init<[T; N], E> for ArrayInit<T, F> +where + F: FnMut(usize) -> I, + I: Init<T, E>, +{ +} + /// Initializes an array by initializing each element via the provided initializer. /// /// # Examples @@ -1203,32 +1289,14 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { /// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` +#[inline] pub fn init_array_from_fn<I, const N: usize, T, E>( - mut make_init: impl FnMut(usize) -> I, + make_init: impl FnMut(usize) -> I, ) -> impl Init<[T; N], E> where I: Init<T, E>, { - let init = move |slot: *mut [T; N]| { - let slot = slot.cast::<T>(); - for i in 0..N { - let init = make_init(i); - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. - let ptr = unsafe { slot.add(i) }; - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` - // requirements. - if let Err(e) = unsafe { init.__init(ptr) } { - // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return - // `Err` below, `slot` will be considered uninitialized memory. - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; - return Err(e); - } - } - Ok(()) - }; - // SAFETY: The initializer above initializes every element of the array. On failure it drops - // any initialized elements and returns `Err`. - unsafe { init_from_closure(init) } + ArrayInit(make_init, __internal::PhantomInvariant::new()) } /// Initializes an array by initializing each element via the provided initializer. @@ -1246,32 +1314,14 @@ where /// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` +#[inline] pub fn pin_init_array_from_fn<I, const N: usize, T, E>( - mut make_init: impl FnMut(usize) -> I, + make_init: impl FnMut(usize) -> I, ) -> impl PinInit<[T; N], E> where I: PinInit<T, E>, { - let init = move |slot: *mut [T; N]| { - let slot = slot.cast::<T>(); - for i in 0..N { - let init = make_init(i); - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. - let ptr = unsafe { slot.add(i) }; - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` - // requirements. - if let Err(e) = unsafe { init.__pinned_init(ptr) } { - // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return - // `Err` below, `slot` will be considered uninitialized memory. - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; - return Err(e); - } - } - Ok(()) - }; - // SAFETY: The initializer above initializes every element of the array. On failure it drops - // any initialized elements and returns `Err`. - unsafe { pin_init_from_closure(init) } + ArrayInit(make_init, __internal::PhantomInvariant::new()) } /// Construct an initializer in a closure and run it. @@ -1300,6 +1350,7 @@ where /// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the /// initializer itself will fail with that error. If it returned `Ok`, then it will run the /// initializer returned by the [`pin_init!`] invocation. +#[inline] pub fn pin_init_scope<T, E, F, I>(make_init: F) -> impl PinInit<T, E> where F: FnOnce() -> Result<I, E>, @@ -1307,13 +1358,13 @@ where { // SAFETY: // - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized, - // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__pinned_init`. - // - The safety requirements of `init.__pinned_init` are fulfilled, since it's being called - // from an initializer. + // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`. + // - The safety requirements of `init.__init` are fulfilled, since it's being called from an + // initializer. unsafe { pin_init_from_closure(move |slot: *mut T| -> Result<(), E> { let init = make_init()?; - init.__pinned_init(slot) + init.__init(slot) }) } } @@ -1343,6 +1394,7 @@ where /// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the /// initializer itself will fail with that error. If it returned `Ok`, then it will run the /// initializer returned by the [`init!`] invocation. +#[inline] pub fn init_scope<T, E, F, I>(make_init: F) -> impl Init<T, E> where F: FnOnce() -> Result<I, E>, @@ -1361,41 +1413,29 @@ where } } -// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of `slot`. -unsafe impl<T> Init<T> for T { - unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { - // SAFETY: `slot` is valid for writes by the safety requirements of this function. - unsafe { slot.write(self) }; - Ok(()) - } -} +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl<T> Init<T> for T {} -// SAFETY: the `__pinned_init` function always returns `Ok(())` and initializes every field of +// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of // `slot`. Additionally, all pinning invariants of `T` are upheld. unsafe impl<T> PinInit<T> for T { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), Infallible> { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self) }; Ok(()) } } -// SAFETY: when the `__init` function returns with -// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. -// - `Err(err)`, slot was not written to. -unsafe impl<T, E> Init<T, E> for Result<T, E> { - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `slot` is valid for writes by the safety requirements of this function. - unsafe { slot.write(self?) }; - Ok(()) - } -} +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl<T, E> Init<T, E> for Result<T, E> {} -// SAFETY: when the `__pinned_init` function returns with +// SAFETY: when the `__init` function returns with // - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. // - `Err(err)`, slot was not written to. unsafe impl<T, E> PinInit<T, E> for Result<T, E> { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self?) }; Ok(()) @@ -1421,6 +1461,7 @@ pub trait InPlaceWrite<T> { impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { type Initialized = &'static mut T; + #[inline] fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); @@ -1431,6 +1472,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { unsafe { Ok(self.assume_init_mut()) } } + #[inline] fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); @@ -1438,7 +1480,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { // // The `'static` borrow guarantees the data will not be // moved/invalidated until it gets dropped (which is never). - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: The above call initialized the memory. Ok(Pin::static_mut(unsafe { self.assume_init_mut() })) @@ -1510,10 +1552,13 @@ pub unsafe trait Zeroable { /// Whenever a type implements [`Zeroable`], this function should be preferred over /// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`. /// + /// As const traits are not yet stable, [`pin_init::zeroed()`] can be used instead + /// when initialization is required in a `const` context. + /// /// # Examples /// /// ``` - /// use pin_init::{Zeroable, zeroed}; + /// use pin_init::Zeroable; /// /// #[derive(Zeroable)] /// struct Point { @@ -1521,10 +1566,11 @@ pub unsafe trait Zeroable { /// y: u32, /// } /// - /// let point: Point = zeroed(); + /// let point: Point = Zeroable::zeroed(); /// assert_eq!(point.x, 0); /// assert_eq!(point.y, 0); /// ``` + #[inline] fn zeroed() -> Self where Self: Sized, @@ -1553,6 +1599,9 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> { /// Whenever a type implements [`Zeroable`], this function should be preferred over /// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`. /// +/// While const traits remain unstable, this function serves as the `const` version of +/// [`Zeroable::zeroed()`]. +/// /// # Examples /// /// ``` @@ -1568,6 +1617,7 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> { /// assert_eq!(point.x, 0); /// assert_eq!(point.y, 0); /// ``` +#[inline] pub const fn zeroed<T: Zeroable>() -> T { // SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`. unsafe { core::mem::zeroed() } @@ -1728,6 +1778,7 @@ pub trait Wrapper<T> { } impl<T> Wrapper<T> for UnsafeCell<T> { + #[inline] fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `UnsafeCell<T>` has a compatible layout to `T`. unsafe { cast_pin_init(value_init) } @@ -1735,6 +1786,7 @@ impl<T> Wrapper<T> for UnsafeCell<T> { } impl<T> Wrapper<T> for MaybeUninit<T> { + #[inline] fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `MaybeUninit<T>` has a compatible layout to `T`. unsafe { cast_pin_init(value_init) } @@ -1743,6 +1795,7 @@ impl<T> Wrapper<T> for MaybeUninit<T> { #[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))] impl<T> Wrapper<T> for core::pin::UnsafePinned<T> { + #[inline] fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `UnsafePinned<T>` has a compatible layout to `T`. unsafe { cast_pin_init(init) } diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 06d7d1a2e8da..1c4aa4292dce 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -6,11 +6,11 @@ * Sorted alphabetically. */ -#include <uapi/asm-generic/ioctl.h> #include <uapi/drm/drm.h> #include <uapi/drm/nova_drm.h> #include <uapi/drm/panthor_drm.h> #include <uapi/linux/android/binder.h> +#include <uapi/linux/ioctl.h> #include <uapi/linux/mdio.h> #include <uapi/linux/mii.h> #include <uapi/linux/ethtool.h> diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index 551f1ebd0dcb..c30983562a2f 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -208,6 +208,20 @@ if [ "$bindgen_libclang_cversion" -lt "$bindgen_libclang_min_cversion" ]; then exit 1 fi +if [ "$bindgen_libclang_cversion" -ge 2200000 ] && + [ "$rust_bindings_generator_cversion" -lt 7201 ]; then + # Distributions may have patched the issue. + if ! "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang_22.h | grep -q 'pub foo'; then + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' < 0.72.1 together with libclang >= 22" + echo >&2 "*** may not work due to a bug (https://github.com/rust-lang/rust-bindgen/pull/3278)." + echo >&2 "*** Your bindgen version: $rust_bindings_generator_version" + echo >&2 "*** Your libclang version: $bindgen_libclang_version" + echo >&2 "***" + warning=1 + fi +fi + # If the C compiler is Clang, then we can also check whether its version # matches the `libclang` version used by the Rust bindings generator. # diff --git a/scripts/rust_is_available_bindgen_libclang_22.h b/scripts/rust_is_available_bindgen_libclang_22.h new file mode 100644 index 000000000000..6b33544c14a8 --- /dev/null +++ b/scripts/rust_is_available_bindgen_libclang_22.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +struct S; +struct S { + int foo; +}; diff --git a/scripts/rust_is_available_test.py b/scripts/rust_is_available_test.py index d6d54b7ea42a..22bdff980c35 100755 --- a/scripts/rust_is_available_test.py +++ b/scripts/rust_is_available_test.py @@ -54,16 +54,23 @@ else: """) @classmethod - def generate_bindgen(cls, version_stdout, libclang_stderr): + def generate_bindgen(cls, version_stdout, libclang_stderr, libclang_22_patched=False): if libclang_stderr is None: libclang_case = f"raise SystemExit({cls.bindgen_default_bindgen_libclang_failure_exit_code})" else: libclang_case = f"print({repr(libclang_stderr)}, file=sys.stderr)" + if libclang_22_patched: + libclang_22_case = "print('pub foo: ::std::os::raw::c_int,')" + else: + libclang_22_case = "pass" + return cls.generate_executable(f"""#!/usr/bin/env python3 import sys if "rust_is_available_bindgen_libclang.h" in " ".join(sys.argv): {libclang_case} +elif "rust_is_available_bindgen_libclang_22.h" in " ".join(sys.argv): + {libclang_22_case} else: print({repr(version_stdout)}) """) @@ -177,7 +184,13 @@ else: def test_rustc_nonexecutable(self): result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.nonexecutable }) - self.assertIn(f"Running '{self.nonexecutable}' to check the Rust compiler version failed with", result.stderr) + self.assertTrue( + # `dash`. + f"Running '{self.nonexecutable}' to check the Rust compiler version failed with" in result.stderr or + # `bash`. + f"Rust compiler '{self.nonexecutable}' could not be found." in result.stderr, + f"Unexpected `stderr`:\n{result.stderr}" + ) def test_rustc_unexpected_binary(self): result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.unexpected_binary }) @@ -205,7 +218,13 @@ else: def test_bindgen_nonexecutable(self): result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.nonexecutable }) - self.assertIn(f"Running '{self.nonexecutable}' to check the Rust bindings generator version failed with", result.stderr) + self.assertTrue( + # `dash`. + f"Running '{self.nonexecutable}' to check the Rust bindings generator version failed with" in result.stderr or + # `bash`. + f"Rust bindings generator '{self.nonexecutable}' could not be found." in result.stderr, + f"Unexpected `stderr`:\n{result.stderr}" + ) def test_bindgen_unexpected_binary(self): result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.unexpected_binary }) @@ -248,6 +267,27 @@ else: result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) self.assertIn(f"libclang (used by the Rust bindings generator '{bindgen}') is too old.", result.stderr) + def test_bindgen_bad_libclang_22(self): + for (bindgen_version, libclang_version, expected_not_patched) in ( + ("0.71.1", "21.1.0", self.Expected.SUCCESS), + ("0.71.1", "22.0.0", self.Expected.SUCCESS_WITH_WARNINGS), + ("0.71.1", "22.1.0", self.Expected.SUCCESS_WITH_WARNINGS), + + ("0.72.0", "22.0.0", self.Expected.SUCCESS_WITH_WARNINGS), + + ("0.72.1", "22.0.0", self.Expected.SUCCESS), + ): + with self.subTest(bindgen_version=bindgen_version, libclang_version=libclang_version): + cc = self.generate_clang(f"clang version {libclang_version}") + libclang_stderr = f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {libclang_version} [-W#pragma-messages], err: false" + bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr) + result = self.run_script(expected_not_patched, { "BINDGEN": bindgen, "CC": cc }) + if expected_not_patched == self.Expected.SUCCESS_WITH_WARNINGS: + self.assertIn(f"Rust bindings generator '{bindgen}' < 0.72.1 together with libclang >= 22", result.stderr) + + bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr, libclang_22_patched=True) + result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen, "CC": cc }) + def test_clang_matches_bindgen_libclang_different_bindgen(self): bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version 999.0.0 [-W#pragma-messages], err: false") result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) diff --git a/scripts/rustdoc_test_gen.rs b/scripts/rustdoc_test_gen.rs index d61a77219a8c..d087c0d9fcb3 100644 --- a/scripts/rustdoc_test_gen.rs +++ b/scripts/rustdoc_test_gen.rs @@ -31,8 +31,15 @@ use std::{ fs, fs::File, - io::{BufWriter, Read, Write}, - path::{Path, PathBuf}, + io::{ + BufWriter, + Read, + Write, // + }, + path::{ + Path, + PathBuf, // + }, // }; /// Find the real path to the original file based on the `file` portion of the test name. @@ -232,6 +239,24 @@ pub extern "C" fn {kunit_name}(__kunit_test: *mut ::kernel::bindings::kunit) {{ const __LOG_PREFIX: &[u8] = b"rust_doctests_kernel\0"; +/// Dummy module type for doctest context. +struct LocalModule; + +use kernel::{{ + str::CStr, + ModuleMetadata, + ThisModule, // +}}; +use core::ptr::null_mut; + +impl ModuleMetadata for LocalModule {{ + const NAME: &'static CStr = c"rust_doctests_kernel"; + const THIS_MODULE: ThisModule = {{ + // SAFETY: `try_module_get`/`module_put` handle null module pointers gracefully. + unsafe {{ ThisModule::from_ptr(null_mut()) }} + }}; +}} + {rust_tests} "# ) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index f03dd59e7fca..87db9f4ed9e2 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -194,6 +194,7 @@ static bool is_rust_noreturn(const struct symbol *func) */ return str_ends_with(func->name, "_4core3num20from_str_radix_panic") || str_ends_with(func->name, "_4core3num22from_ascii_radix_panic") || + str_ends_with(func->name, "_4core3num28from_ascii_bytes_radix_panic") || str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || str_ends_with(func->name, "_4core6option13expect_failed") || str_ends_with(func->name, "_4core6option13unwrap_failed") || |