[release-branch.go1.21] crypto/tls: add GODEBUG to control max RSA key size

Add a new GODEBUG setting, tlsmaxrsasize, which allows controlling the
maximum RSA key size we will accept during TLS handshakes.

Fixes #61967

Change-Id: I52f060be132014d219f4cd438f59990011a35c96
Reviewed-on: https://go-review.googlesource.com/c/go/+/517495
Auto-Submit: Roland Shoemaker <roland@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>
Run-TryBot: Roland Shoemaker <roland@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-on: https://go-review.googlesource.com/c/go/+/518535
Auto-Submit: Dmitri Shuralyov <dmitshur@google.com>

doc/godebug.md

@@ -126,6 +126,14 @@ for example,
 see the [runtime documentation](/pkg/runtime#hdr-Environment_Variables)
 and the [go command documentation](/cmd/go#hdr-Build_and_test_caching).
 
+### Go 1.22
+
+Go 1.22 adds a configurable limit to control the maximum acceptable RSA key size
+that can be used in TLS handshakes, controlled by the [`tlsmaxrsasize`setting](/pkg/crypto/tls#Conn.Handshake).
+The default is tlsmaxrsasize=8192, limiting RSA to 8192-bit keys. To avoid
+denial of service attacks, this setting and default was backported to Go
+1.19.13, Go 1.20.8, and Go 1.21.1.
+
 ### Go 1.21
 
 Go 1.21 made it a run-time error to call `panic` with a nil interface value,

src/crypto/tls/conn.go

@@ -1467,6 +1467,11 @@ func (c *Conn) closeNotify() error {
 //
 // For control over canceling or setting a timeout on a handshake, use
 // HandshakeContext or the Dialer's DialContext method instead.
+//
+// In order to avoid denial of service attacks, the maximum RSA key size allowed
+// in certificates sent by either the TLS server or client is limited to 8192
+// bits. This limit can be overridden by setting tlsmaxrsasize in the GODEBUG
+// environment variable (e.g. GODEBUG=tlsmaxrsasize=4096).
 func (c *Conn) Handshake() error {
 	return c.HandshakeContext(context.Background())
 }

src/crypto/tls/handshake_client.go

@@ -17,8 +17,10 @@ import (
 	"errors"
 	"fmt"
 	"hash"
+	"internal/godebug"
 	"io"
 	"net"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -936,9 +938,23 @@ func (hs *clientHandshakeState) sendFinished(out []byte) error {
 	return nil
 }
 
-// maxRSAKeySize is the maximum RSA key size in bits that we are willing
+// defaultMaxRSAKeySize is the maximum RSA key size in bits that we are willing
 // to verify the signatures of during a TLS handshake.
-const maxRSAKeySize = 8192
+const defaultMaxRSAKeySize = 8192
+
+var tlsmaxrsasize = godebug.New("tlsmaxrsasize")
+
+func checkKeySize(n int) (max int, ok bool) {
+	if v := tlsmaxrsasize.Value(); v != "" {
+		if max, err := strconv.Atoi(v); err == nil {
+			if (n <= max) != (n <= defaultMaxRSAKeySize) {
+				tlsmaxrsasize.IncNonDefault()
+			}
+			return max, n <= max
+		}
+	}
+	return defaultMaxRSAKeySize, n <= defaultMaxRSAKeySize
+}
 
 // verifyServerCertificate parses and verifies the provided chain, setting
 // c.verifiedChains and c.peerCertificates or sending the appropriate alert.
@@ -951,9 +967,12 @@ func (c *Conn) verifyServerCertificate(certificates [][]byte) error {
 			c.sendAlert(alertBadCertificate)
 			return errors.New("tls: failed to parse certificate from server: " + err.Error())
 		}
-		if cert.cert.PublicKeyAlgorithm == x509.RSA && cert.cert.PublicKey.(*rsa.PublicKey).N.BitLen() > maxRSAKeySize {
-			c.sendAlert(alertBadCertificate)
-			return fmt.Errorf("tls: server sent certificate containing RSA key larger than %d bits", maxRSAKeySize)
+		if cert.cert.PublicKeyAlgorithm == x509.RSA {
+			n := cert.cert.PublicKey.(*rsa.PublicKey).N.BitLen()
+			if max, ok := checkKeySize(n); !ok {
+				c.sendAlert(alertBadCertificate)
+				return fmt.Errorf("tls: server sent certificate containing RSA key larger than %d bits", max)
+			}
 		}
 		activeHandles[i] = cert
 		certs[i] = cert.cert

src/crypto/tls/handshake_client_test.go

@@ -2783,19 +2783,44 @@ u58=
 -----END CERTIFICATE-----`
 
 func TestHandshakeRSATooBig(t *testing.T) {
-	testCert, _ := pem.Decode([]byte(largeRSAKeyCertPEM))
+	for _, tc := range []struct {
+		name              string
+		godebug           string
+		expectedServerErr string
+		expectedClientErr string
+	}{
+		{
+			name:              "key too large",
+			expectedServerErr: "tls: server sent certificate containing RSA key larger than 8192 bits",
+			expectedClientErr: "tls: client sent certificate containing RSA key larger than 8192 bits",
+		},
+		{
+			name:    "acceptable key (GODEBUG=tlsmaxrsasize=8193)",
+			godebug: "tlsmaxrsasize=8193",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.godebug != "" {
+				t.Setenv("GODEBUG", tc.godebug)
+			}
 
-	c := &Conn{conn: &discardConn{}, config: testConfig.Clone()}
+			testCert, _ := pem.Decode([]byte(largeRSAKeyCertPEM))
 
-	expectedErr := "tls: server sent certificate containing RSA key larger than 8192 bits"
-	err := c.verifyServerCertificate([][]byte{testCert.Bytes})
-	if err == nil || err.Error() != expectedErr {
-		t.Errorf("Conn.verifyServerCertificate unexpected error: want %q, got %q", expectedErr, err)
-	}
+			c := &Conn{conn: &discardConn{}, config: testConfig.Clone()}
 
-	expectedErr = "tls: client sent certificate containing RSA key larger than 8192 bits"
-	err = c.processCertsFromClient(Certificate{Certificate: [][]byte{testCert.Bytes}})
-	if err == nil || err.Error() != expectedErr {
-		t.Errorf("Conn.processCertsFromClient unexpected error: want %q, got %q", expectedErr, err)
+			err := c.verifyServerCertificate([][]byte{testCert.Bytes})
+			if tc.expectedServerErr == "" && err != nil {
+				t.Errorf("Conn.verifyServerCertificate unexpected error: %s", err)
+			} else if tc.expectedServerErr != "" && (err == nil || err.Error() != tc.expectedServerErr) {
+				t.Errorf("Conn.verifyServerCertificate unexpected error: want %q, got %q", tc.expectedServerErr, err)
+			}
+
+			err = c.processCertsFromClient(Certificate{Certificate: [][]byte{testCert.Bytes}})
+			if tc.expectedClientErr == "" && err != nil {
+				t.Errorf("Conn.processCertsFromClient unexpected error: %s", err)
+			} else if tc.expectedClientErr != "" && (err == nil || err.Error() != tc.expectedClientErr) {
+				t.Errorf("Conn.processCertsFromClient unexpected error: want %q, got %q", tc.expectedClientErr, err)
+			}
+		})
 	}
 }

src/crypto/tls/handshake_server.go

@@ -864,9 +864,12 @@ func (c *Conn) processCertsFromClient(certificate Certificate) error {
 			c.sendAlert(alertBadCertificate)
 			return errors.New("tls: failed to parse client certificate: " + err.Error())
 		}
-		if certs[i].PublicKeyAlgorithm == x509.RSA && certs[i].PublicKey.(*rsa.PublicKey).N.BitLen() > maxRSAKeySize {
-			c.sendAlert(alertBadCertificate)
-			return fmt.Errorf("tls: client sent certificate containing RSA key larger than %d bits", maxRSAKeySize)
+		if certs[i].PublicKeyAlgorithm == x509.RSA {
+			n := certs[i].PublicKey.(*rsa.PublicKey).N.BitLen()
+			if max, ok := checkKeySize(n); !ok {
+				c.sendAlert(alertBadCertificate)
+				return fmt.Errorf("tls: client sent certificate containing RSA key larger than %d bits", max)
+			}
 		}
 	}
 

src/internal/godebugs/table.go

@@ -42,6 +42,7 @@ var All = []Info{
 	{Name: "panicnil", Package: "runtime", Changed: 21, Old: "1"},
 	{Name: "randautoseed", Package: "math/rand"},
 	{Name: "tarinsecurepath", Package: "archive/tar"},
+	{Name: "tlsmaxrsasize", Package: "crypto/tls"},
 	{Name: "x509sha1", Package: "crypto/x509"},
 	{Name: "x509usefallbackroots", Package: "crypto/x509"},
 	{Name: "zipinsecurepath", Package: "archive/zip"},

src/runtime/metrics/doc.go

@@ -290,6 +290,10 @@ Below is the full list of supported metrics, ordered lexicographically.
 		package due to a non-default GODEBUG=tarinsecurepath=...
 		setting.
 
+	/godebug/non-default-behavior/tlsmaxrsasize:events
+		The number of non-default behaviors executed by the crypto/tls
+		package due to a non-default GODEBUG=tlsmaxrsasize=... setting.
+
 	/godebug/non-default-behavior/x509sha1:events
 		The number of non-default behaviors executed by the crypto/x509
 		package due to a non-default GODEBUG=x509sha1=... setting.