Skip to content

Commit a9575c0

Browse files
Limit zstd window size to web-safe value
zstd's memory buffer is limited to 8MB by Chromium and Firefox (at least), but the encoder can exceed that at level 20 and above. Ensure it's limited to 8MB.
1 parent bb0d1b2 commit a9575c0

File tree

1 file changed

+23
-1
lines changed

1 file changed

+23
-1
lines changed

tower-http/src/compression/body.rs

+23-1
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,29 @@ where
340340
type Output = ZstdEncoder<Self::Input>;
341341

342342
fn apply(input: Self::Input, quality: CompressionLevel) -> Self::Output {
343-
ZstdEncoder::with_quality(input, quality.into_async_compression())
343+
// See https://issues.chromium.org/issues/41493659:
344+
// "For memory usage reasons, Chromium limits the window size to 8MB"
345+
// See https://datatracker.ietf.org/doc/html/rfc8878#name-window-descriptor
346+
// "For improved interoperability, it's recommended for decoders to support values
347+
// of Window_Size up to 8 MB and for encoders not to generate frames requiring a
348+
// Window_Size larger than 8 MB."
349+
// Level 17 in zstd (as of v1.5.6) is the first level with a window size of 8 MB (2^23):
350+
// https://github.com/facebook/zstd/blob/v1.5.6/lib/compress/clevels.h#L25-L51
351+
// Set the parameter for all levels >= 17. This will either have no effect (but reduce
352+
// the risk of future changes in zstd) or limit the window log to 8MB.
353+
let needs_window_limit = match quality {
354+
CompressionLevel::Best => true, // level 20
355+
CompressionLevel::Precise(level) => level >= 17,
356+
_ => false,
357+
};
358+
// The parameter is not set for levels below 17 as it will increase the window size
359+
// for those levels.
360+
if needs_window_limit {
361+
let params = [async_compression::zstd::CParameter::window_log(23)];
362+
ZstdEncoder::with_quality_and_params(input, quality.into_async_compression(), &params)
363+
} else {
364+
ZstdEncoder::with_quality(input, quality.into_async_compression())
365+
}
344366
}
345367

346368
fn get_pin_mut(pinned: Pin<&mut Self::Output>) -> Pin<&mut Self::Input> {

0 commit comments

Comments
 (0)