From d50c41f9b136e8c82a5b33325ceb00f7ba72a406 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lomig=20Me=CC=81gard?= <dev@lomig.ch>
Date: Sun, 8 Nov 2026 09:07:05 +0100
Subject: [PATCH] fix(openssl): sign RSA-PSS with a salt as long as the digest

The PSS salt length was left to the linked OpenSSL. 3.0 signs with the
longest salt the key allows and later versions with the digest length, so the
same step wrote different signatures depending on the build, and a verifier
that expects the usual 32 bytes refused the 3.0 ones. Signing now sets the
length to the digest's. Verification still reads it from the signature.
---
 crates/rite-openssl/src/backend.rs | 44 ++++++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/crates/rite-openssl/src/backend.rs b/crates/rite-openssl/src/backend.rs
index ea4bef3..8a1a434 100644
--- a/crates/rite-openssl/src/backend.rs
+++ b/crates/rite-openssl/src/backend.rs
@@ -18,7 +18,7 @@ use openssl::nid::Nid;
 use openssl::pkey::{HasPublic, Id, PKey, PKeyRef, Private};
 use openssl::pkey_ctx::PkeyCtx;
 use openssl::rsa::{Padding, Rsa};
-use openssl::sign::{Signer, Verifier};
+use openssl::sign::{RsaPssSaltlen, Signer, Verifier};
 use openssl::symm::Cipher;
 use openssl::x509::{X509Builder, X509Extension, X509NameBuilder};
 use rite_sdk::cms;
@@ -466,6 +466,8 @@ fn generate_ml_dsa(algorithm: KeyAlgorithm) -> Result<PKey<Private>, BackendErro
 trait RsaPadding {
     fn padding(&mut self, padding: Padding) -> Result<(), openssl::error::ErrorStack>;
     fn mgf1_md(&mut self, md: MessageDigest) -> Result<(), openssl::error::ErrorStack>;
+    /// The PSS salt length, for the side that chooses one.
+    fn pss_salt(&mut self) -> Result<(), openssl::error::ErrorStack>;
 }
 
 impl RsaPadding for Signer<'_> {
@@ -475,6 +477,12 @@ impl RsaPadding for Signer<'_> {
     fn mgf1_md(&mut self, md: MessageDigest) -> Result<(), openssl::error::ErrorStack> {
         self.set_rsa_mgf1_md(md)
     }
+    /// As long as the digest, the length RFC 8017 recommends and the one
+    /// PKCS#11 tokens and X.509 tooling expect. Left unset, the length is the
+    /// linked OpenSSL's default, which 3.0 sets to the maximum the key allows.
+    fn pss_salt(&mut self) -> Result<(), openssl::error::ErrorStack> {
+        self.set_rsa_pss_saltlen(RsaPssSaltlen::DIGEST_LENGTH)
+    }
 }
 
 impl RsaPadding for Verifier<'_> {
@@ -484,6 +492,11 @@ impl RsaPadding for Verifier<'_> {
     fn mgf1_md(&mut self, md: MessageDigest) -> Result<(), openssl::error::ErrorStack> {
         self.set_rsa_mgf1_md(md)
     }
+    /// Left to OpenSSL, which reads the length from the signature, so a
+    /// signature another signer made with a longer salt still verifies.
+    fn pss_salt(&mut self) -> Result<(), openssl::error::ErrorStack> {
+        Ok(())
+    }
 }
 
 /// Apply the padding scheme an RSA algorithm names. A no-op for everything else.
@@ -505,7 +518,10 @@ fn apply_rsa_padding<T: RsaPadding>(
                 .map_err(|e| ossl_err("Set PSS padding", &e))?;
             operation
                 .mgf1_md(MessageDigest::sha256())
-                .map_err(|e| ossl_err("Set MGF1 MD", &e))
+                .map_err(|e| ossl_err("Set MGF1 MD", &e))?;
+            operation
+                .pss_salt()
+                .map_err(|e| ossl_err("Set PSS salt length", &e))
         }
         _ => Ok(()),
     }
@@ -3192,6 +3208,30 @@ mod tests {
         );
     }
 
+    /// The salt is as long as the digest whatever OpenSSL is linked: a
+    /// verifier told to expect 32 bytes refuses any other length.
+    #[test]
+    fn a_pss_signature_carries_a_salt_as_long_as_the_digest() {
+        let mut backend = OpenSslBackend::try_new("test").unwrap();
+        let meta = backend
+            .generate_key(spec(KeyAlgorithm::Rsa2048, "signer"))
+            .unwrap();
+        let message = b"salt length";
+        let signature = backend
+            .sign(&meta.key_id, message, SignAlgorithm::RsaPssSha256)
+            .unwrap();
+
+        let public =
+            PKey::public_key_from_der(meta.public_key.as_ref().unwrap().as_bytes()).unwrap();
+        let mut verifier = Verifier::new(MessageDigest::sha256(), &public).unwrap();
+        verifier.set_rsa_padding(Padding::PKCS1_PSS).unwrap();
+        verifier.set_rsa_mgf1_md(MessageDigest::sha256()).unwrap();
+        verifier
+            .set_rsa_pss_saltlen(RsaPssSaltlen::DIGEST_LENGTH)
+            .unwrap();
+        assert!(verifier.verify_oneshot(&signature, message).unwrap());
+    }
+
     /// PKCS#8 material for a key this build can make.
     fn pkcs8_of(key: &PKey<Private>) -> Vec<u8> {
         key.private_key_to_pkcs8().unwrap()