Enlarge TCP recv_buf to improve throughput (#6690)

Image uploads performed via the [web console are 3-4x slower than
uploads performed via the Oxide
CLI](oxidecomputer/console#2096). We found
that the CLI creates 8 separate TCP connections to upload the image
chunks, while the console uses HTTP/2 to multiplex a single TCP
connection six ways. The default TCP `recv_buf` size on Helios is 128
KB, which limits window size and therefore the number of packets that
can be sent in parallel. By increasing this value to 1 MB, we can
increase single-connection
throughput by ~3x, bringing console performance to rough parity with the
CLI.
      
This does increase the amount of memory a potential DoS attack could
consume, but 1 MB is still quite small relative to the total resources
available on a compute sled.
      
While we're at it, also update the TCP congestion control algorithm to
`cubic` from its default value of `sunreno`, which may also help improve
throughput.
      
Closes #6601

Author: wfchandler

illumos-utils/src/ipadm.rs

@@ -141,7 +141,7 @@ impl Ipadm {
         Ok(Self::addrobj_addr(addrobj)?.is_some())
     }
 
-    // Set MTU to 9000 on both IPv4 and IPv6
+    /// Set MTU to 9000 on both IPv4 and IPv6
     pub fn set_interface_mtu(datalink: &str) -> Result<(), ExecutionError> {
         let mut cmd = std::process::Command::new(PFEXEC);
         let cmd = cmd.args(&[
@@ -197,4 +197,44 @@ impl Ipadm {
         Self::ensure_ip_addrobj_exists(&addrobj, AddrObjType::DHCP)?;
         Ok(())
     }
+
+    /// Set TCP recv_buf to 1 MB.
+    pub fn set_tcp_recv_buf() -> Result<(), ExecutionError> {
+        let mut cmd = std::process::Command::new(PFEXEC);
+
+        // This is to improve single-connection throughput on large uploads
+        // from clients, e.g., images. Modern browsers will almost always use
+        // HTTP/2, which will multiplex concurrent writes to the same host over
+        // a single TCP connection. The small default receive window size is a
+        // major bottleneck, see
+        // https://github.com/oxidecomputer/console/issues/2096 for further
+        // details.
+        let cmd = cmd.args(&[
+            IPADM,
+            "set-prop",
+            "-t",
+            "-p",
+            "recv_buf=1000000",
+            "tcp",
+        ]);
+        execute(cmd)?;
+
+        Ok(())
+    }
+
+    /// Set TCP congestion control algorithm to `cubic`.
+    pub fn set_tcp_congestion_control() -> Result<(), ExecutionError> {
+        let mut cmd = std::process::Command::new(PFEXEC);
+        let cmd = cmd.args(&[
+            IPADM,
+            "set-prop",
+            "-t",
+            "-p",
+            "congestion_control=cubic",
+            "tcp",
+        ]);
+        execute(cmd)?;
+
+        Ok(())
+    }
 }

zone-setup/src/bin/zone-setup.rs

@@ -563,6 +563,17 @@ async fn common_nw_set_up(
     Ipadm::set_interface_mtu(&datalink)
         .with_context(|| format!("failed to set MTU on datalink {datalink}"))?;
 
+    info!(
+        log, "Setting TCP recv_buf size to 1 MB";
+    );
+    Ipadm::set_tcp_recv_buf().context("failed to set TCP recv_buf")?;
+
+    info!(
+        log, "Setting TCP congestion control algorithm to cubic";
+    );
+    Ipadm::set_tcp_congestion_control()
+        .context("failed to set TCP congestion_control")?;
+
     if static_addrs.is_empty() {
         info!(
             log,