Turbopack: bincode: Add traits for types that require TurboBincodeEncoder or TurboBincodeDecoder (#86633)

**Example usage:** See `CachedTaskType`​ in `turbopack/crates/turbo-tasks/src/backend.rs`​ on #86631 for an example of how this is used. That needs to call the encoder/decoder function pointer stored on `native_fn.arg_meta.bincode` with a concrete `TurboBincodeEncoder`/`TurboBincodeDecoder`, so it uses this trait.

---

For `SharedReference` (value cell contents) and `TaskInput`s, we need to generate function pointers to functions that can encode/decode the type.

Those generated functions cannot contain generics, so they require `TurboBincodeEncoder` or `TurboBincodeDecoder<()>` as arguments. That's okay because we don't want to use multiple encoders/decoders anyways, because that would be really bad for binary size and compilation time.

However, `SharedReference` and `TaskInput`s are wrapped in other container types that need to be encoded/decoded (e.g. `TypedSharedReference`. In these cases, the wrapper types need to be able to implement `Encode`/`Decode`, but can't because of the fixed `Encoder`/`Decoder` implementations. So this provides a way to do that, by panicking at runtime if the type doesn't match.

This uses the `unty` crate which is part of the bincode project: https://github.com/bincode-org/unty

Author: bgw

Cargo.lock

@@ -470,6 +470,7 @@ tracing = "0.1.37"
 tracing-subscriber = "0.3.16"
 triomphe = { git = "https://github.com/sokra/triomphe", branch = "sokra/unstable" }
 unsize = "1.1.0"
+unty = "0.0.4"
 url = "2.2.2"
 urlencoding = "2.1.2"
 uuid = "1.18.1"

Cargo.toml

@@ -19,3 +19,4 @@ ringmap = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 smallvec = { workspace = true }
+unty = { workspace = true }

turbopack/crates/turbo-bincode/Cargo.toml

@@ -1,3 +1,6 @@
+#[doc(hidden)]
+pub mod macro_helpers;
+
 use std::ptr::copy_nonoverlapping;
 
 use ::smallvec::SmallVec;
@@ -115,6 +118,57 @@ impl Reader for TurboBincodeReader<'_> {
     }
 }
 
+/// Represents a type that can only be encoded with a [`TurboBincodeEncoder`].
+///
+/// All traits implementing this must also implement the more generic [`Encode`] trait, but they
+/// should panic if any other encoder is used.
+///
+/// Use [`impl_encode_for_turbo_bincode_encode`] to automatically implement the [`Encode`] trait
+/// from this one.
+pub trait TurboBincodeEncode: Encode {
+    fn encode(&self, encoder: &mut TurboBincodeEncoder) -> Result<(), EncodeError>;
+}
+
+/// Represents a type that can only be decoded with a [`TurboBincodeDecoder`] and an empty `()`
+/// context.
+///
+/// All traits implementing this must also implement the more generic [`Decode`] trait, but they
+/// should panic if any other encoder is used.
+///
+/// Use [`impl_decode_for_turbo_bincode_decode`] to automatically implement the [`Decode`] trait
+/// from this one.
+pub trait TurboBincodeDecode<Context>: Decode<Context> {
+    fn decode(decoder: &mut TurboBincodeDecoder) -> Result<Self, DecodeError>;
+}
+
+#[macro_export]
+macro_rules! impl_encode_for_turbo_bincode_encode {
+    ($ty:ty) => {
+        impl $crate::macro_helpers::bincode::Encode for $ty {
+            fn encode<'a, E: $crate::macro_helpers::bincode::enc::Encoder>(
+                &self,
+                encoder: &'a mut E,
+            ) -> ::std::result::Result<(), $crate::macro_helpers::bincode::error::EncodeError> {
+                $crate::macro_helpers::encode_for_turbo_bincode_encode_impl(self, encoder)
+            }
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! impl_decode_for_turbo_bincode_decode {
+    ($ty:ty) => {
+        impl<Context> $crate::macro_helpers::bincode::Decode<Context> for $ty {
+            fn decode<D: $crate::macro_helpers::bincode::de::Decoder<Context = Context>>(
+                decoder: &mut D,
+            ) -> ::std::result::Result<Self, $crate::macro_helpers::bincode::error::DecodeError>
+            {
+                $crate::macro_helpers::decode_for_turbo_bincode_decode_impl(decoder)
+            }
+        }
+    };
+}
+
 pub mod indexmap {
     use std::hash::{BuildHasher, Hash};
 

turbopack/crates/turbo-bincode/src/lib.rs

@@ -0,0 +1,62 @@
+use std::{any::type_name, mem::transmute};
+
+pub use bincode;
+use bincode::{
+    de::Decoder,
+    enc::Encoder,
+    error::{DecodeError, EncodeError},
+};
+
+use crate::{TurboBincodeDecode, TurboBincodeDecoder, TurboBincodeEncode, TurboBincodeEncoder};
+
+#[track_caller]
+pub fn encode_for_turbo_bincode_encode_impl<'a, T: TurboBincodeEncode, E: Encoder>(
+    value: &T,
+    encoder: &'a mut E,
+) -> Result<(), EncodeError> {
+    let encoder = if unty::type_equal::<E, TurboBincodeEncoder>() {
+        // SAFETY: Transmute is safe because `&mut E` is `&mut TurboBincodeEncoder`:
+        // - `unty::type_equal::<E, TurboBincodeEncoder>()` does not check lifetimes, but does check
+        //   the type and layout, so we know those are correct.
+        // - The transmuted encoder cannot escape this function, and we know that the lifetime of
+        //   `'a` is at least as long as the function.
+        // - Lifetimes don't change layout. This is not strictly guaranteed, but if this assumption
+        //   is broken, we'd have a different type id (type ids are derived from layout
+        //   information), `type_equal` would return `false`, and we'd panic instead of violating
+        //   memory safety.
+        // - Two mutable references have the same layout and alignment when they reference exactly
+        //   the same type.
+        // - The explicit lifetime ('a) avoids creating an implitly unbounded lifetime.
+        unsafe { transmute::<&'a mut E, &'a mut TurboBincodeEncoder>(encoder) }
+    } else {
+        unreachable!(
+            "{} implements TurboBincodeEncode, but was called with a {} encoder implementation",
+            type_name::<T>(),
+            type_name::<E>(),
+        )
+    };
+    TurboBincodeEncode::encode(value, encoder)
+}
+
+#[track_caller]
+pub fn decode_for_turbo_bincode_decode_impl<
+    'a,
+    Context,
+    T: TurboBincodeDecode<Context>,
+    D: Decoder<Context = Context>,
+>(
+    decoder: &'a mut D,
+) -> Result<T, DecodeError> {
+    let decoder = if unty::type_equal::<D, TurboBincodeDecoder>() {
+        // SAFETY: See notes on the `Encode::encode` implementation on
+        // `encode_for_turbo_bincode_encode_impl`.
+        unsafe { transmute::<&'a mut D, &'a mut TurboBincodeDecoder<'a>>(decoder) }
+    } else {
+        unreachable!(
+            "{} implements TurboBincodeDecode, but was called with a {} decoder implementation",
+            type_name::<T>(),
+            type_name::<D>(),
+        )
+    };
+    TurboBincodeDecode::decode(decoder)
+}