24 bytes past the end: how FFmpeg's sws_scale corrupted our heap

The symptom

• AddressSanitizer found a handful of genuine bugs (thread-unsafe localtime, a shared static HMAC buffer, a lambda capturing a loop variable by reference), but never this one.

• Our test videos were 1920 and 960 pixels wide. As you'll see, those widths never trigger it.

The code that looked fine

// once per video
sws_ctx = sws_getContext(w, h, AV_PIX_FMT_YUV420P,
                         w, h, AV_PIX_FMT_BGR24,
                         SWS_BILINEAR, NULL, NULL, NULL);
av_image_alloc(buffer, line_size, w, h, AV_PIX_FMT_BGR24, 1);  // tightly packed

// per frame
sws_scale(sws_ctx, frame->data, frame->linesize, 0, h, buffer, line_size);
cv::Mat image(h, w, CV_8UC3, buffer[0], line_size[0]);

/* buffer is going to be written to rawvideo file, no alignment */
if ((ret = av_image_alloc(dst_data, dst_linesize,
                          dst_w, dst_h, dst_pix_fmt, 1)) < 0) {

What swscale actually does

#define YUV2RGB_LOOP(depth)                                   \
    h_size = (c->opts.dst_w + 7) & ~7;      /* round up to 8 */ \
    if (h_size * depth > FFABS(dstStride[0]))                 \
        h_size -= 8;                        /* ...or down    */ \
    ...
        x86_reg index = -h_size / 2;

    movu [imageq], m0          ; 3 × 16-byte stores = 16 BGR pixels
    movu [imageq + 16], m1
    movu [imageq + 32], m2
    ...
add imageq, 8 * depth * time_num   ; time_num = 2 for XMM: 16 pixels
add indexq, 4 * time_num
js .loop0                          ; loop until index >= 0

From 24 stray bytes to a crash in free()

         your BGR frame (w × h × 3 bytes)          next chunk
... ─────────────────────────────────┬────────────┬────────────┬────────────────
     ... last row ...                │ prev_size  │   size     │ neighbour's data
                                     └────────────┴────────────┴────────────────
                                     ▲ end of buffer
                                     ├──────────── 24 bytes ────────────┤
                                      overwritten with BGR pixels by sws_scale

Reproduce it yourself

sudo apt install build-essential pkg-config libswscale-dev libavutil-dev

// sws_overrun.c - sws_scale (x86 SSSE3, YUV420P -> BGR24) writes past the end
// of a tightly packed destination buffer for some frame widths.
//
// Build:  gcc -O2 sws_overrun.c -o sws_overrun $(pkg-config --cflags --libs libswscale libavutil)
// Run:    ./sws_overrun           measure the overrun for a set of widths
//         ./sws_overrun crash     corrupt the glibc heap the way a real program would
//         ./sws_overrun padded    the same, with one spare row of padding (no crash)
//         ./sws_overrun accurate  measure again with SWS_ACCURATE_RND (no SIMD path)
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define CANARY 256

static int sws_flags = SWS_BILINEAR;

// Convert one mid-grey YUV420P frame of w x h into dst (tightly packed BGR24).
static void convert(int w, int h, uint8_t *dst)
{
    uint8_t *src[4];
    int src_stride[4];
    av_image_alloc(src, src_stride, w, h, AV_PIX_FMT_YUV420P, 32);
    memset(src[0], 0x80, (size_t)src_stride[0] * h);
    memset(src[1], 0x80, (size_t)src_stride[1] * h / 2);
    memset(src[2], 0x80, (size_t)src_stride[2] * h / 2);

    uint8_t *dst_data[4] = {dst, NULL, NULL, NULL};
    int dst_stride[4] = {w * 3, 0, 0, 0};

    struct SwsContext *ctx = sws_getContext(w, h, AV_PIX_FMT_YUV420P, w, h, AV_PIX_FMT_BGR24,
                                            sws_flags, NULL, NULL, NULL);
    sws_scale(ctx, (const uint8_t *const *)src, src_stride, 0, h, dst_data, dst_stride);
    sws_freeContext(ctx);
    av_freep(&src[0]);
}

static int measure(void)
{
    const int widths[] = {1920, 1280, 960, 1352, 1080, 856, 540, 1366};
    const int h = 720;
    int failures = 0;

    printf("swscale %d.%d.%d\n", LIBSWSCALE_VERSION_MAJOR, LIBSWSCALE_VERSION_MINOR,
           LIBSWSCALE_VERSION_MICRO);
    for (size_t i = 0; i < sizeof(widths) / sizeof(widths[0]); i++) {
        int w = widths[i];
        size_t size = (size_t)w * 3 * h;
        uint8_t *buf = malloc(size + CANARY);
        memset(buf + size, 0xAB, CANARY); // canary right after the image

        convert(w, h, buf);

        int overrun = 0;
        for (int j = 0; j < CANARY; j++)
            if (buf[size + j] != 0xAB)
                overrun = j + 1;
        printf("%4dx%d  width %% 16 = %2d  ", w, h, w % 16);
        if (overrun) {
            printf("OVERRUN: %d bytes written past the end\n", overrun);
            failures++;
        } else {
            printf("ok\n");
        }
        free(buf);
    }
    return failures ? 2 : 0;
}

static int crash(int padded)
{
    const int w = 1352, h = 760;
    size_t size = (size_t)w * 3 * h; // exactly what av_image_alloc(..., align=1) asks for
    size_t alloc = padded ? size + w * 3 + 64 : size;

    // A 3 MB request is normally served by mmap(), whose page rounding hides a
    // small overrun. But once a program frees such a block, glibc raises its
    // mmap threshold and later ones come from the heap, packed against their
    // neighbours. Long-running video programs reach this state almost at once.
    void *volatile warm_up = malloc(alloc); // volatile: stop gcc eliding the pair
    free(warm_up);

    uint8_t *bgr = malloc(alloc);
    char *neighbour = malloc(64); // whatever happens to live next to the frame
    strcpy(neighbour, "innocent bystander");
    printf("frame  %p .. %p\nneighbour %p\n", (void *)bgr, (void *)(bgr + size),
           (void *)neighbour);

    convert(w, h, bgr);

    printf("conversion done, freeing the neighbour...\n");
    fflush(stdout);
    free(neighbour); // glibc now reads a chunk header full of grey pixels
    free(bgr);
    printf("no crash\n");
    return 0;
}

int main(int argc, char **argv)
{
    if (argc > 1 && strcmp(argv[1], "crash") == 0)
        return crash(0);
    if (argc > 1 && strcmp(argv[1], "padded") == 0)
        return crash(1);
    if (argc > 1 && strcmp(argv[1], "accurate") == 0)
        sws_flags |= SWS_ACCURATE_RND;
    return measure();
}

$ gcc -O2 sws_overrun.c -o sws_overrun $(pkg-config --cflags --libs libswscale libavutil)
$ ./sws_overrun
swscale 7.5.100
1920x720  width % 16 =  0  ok
1280x720  width % 16 =  0  ok
 960x720  width % 16 =  0  ok
1352x720  width % 16 =  8  OVERRUN: 24 bytes written past the end
1080x720  width % 16 =  8  OVERRUN: 24 bytes written past the end
 856x720  width % 16 =  8  OVERRUN: 24 bytes written past the end
 540x720  width % 16 = 12  OVERRUN: 12 bytes written past the end
1366x720  width % 16 =  6  ok

$ ./sws_overrun crash
frame  0x55a02c01a340 .. 0x55a02c30ac80
neighbour 0x55a02c30ac90
conversion done, freeing the neighbour...
munmap_chunk(): invalid pointer
Aborted (core dumped)

Is it really FFmpeg's bug?

1. The API contract: sws_scale's documentation sets no padding requirement on dst, and av_image_alloc documents align as a free parameter.

2. FFmpeg's own example allocates its sws_scale destination with align = 1.

3. FFmpeg's intent: the h_size guard exists specifically to keep writes inside the caller's stride. It just computes the wrong bound for the 16-pixel loop. My guess is that it was written for the older 8-pixel MMX loop and never updated when the SSSE3 version arrived.

• #9254 (2021): sws_scale() writes out of buffer on ssse3 YUV→RGB conversion

• #9791 (2022): yuv420_bgr32_ssse3 writes out of buffer boundary during sws_scale() call

• #10852 (2024): sws_scale overflows buffer for some resolutions using ssse3 instructions. yuv420 → bgr24, 8 bytes, "only for some image resolution". Exactly this bug.

The fix

int size = av_image_get_buffer_size(AV_PIX_FMT_BGR24, w, h, 1);
uint8_t *data = (uint8_t *)av_malloc(size + w * 3 + 64);   // tail padding
av_image_fill_arrays(buffer, line_size, data, AV_PIX_FMT_BGR24, w, h, 1);

Takeaways

• Pad buffers that SIMD code writes into. Hand-written vector code tends to process whole vectors and trust that a little overrun is harmless. Many libraries quietly assume padded buffers even when their docs don't say so.

• Test odd sizes. Our test set of 1920 and 960 was a set of lucky numbers. If you process images, include widths that aren't multiples of 16 (or 32, or 64).

• ASan can't see into assembly. For heap corruption that survives ASan, try MALLOC_CHECK_=3 / libc_malloc_debug.so, MALLOC_PERTURB_, and canary-based reproducers like the one above.

• Don't run your service as PID 1 in Docker without --init. We lost several crash reports because the kernel ignores signals a PID-1 process hasn't set a handler for, including the SIGABRT that glibc raises when it detects corruption. abort() then falls back to executing hlt, which faults in user mode, and you get a confusing "general protection fault" in libc with no message and no core dump.

References

• FFmpeg ticket #9254: sws_scale() writes out of buffer on ssse3 YUV→RGB conversion

• FFmpeg ticket #9791: yuv420_bgr32_ssse3 writes out of buffer boundary during sws_scale() call

• FFmpeg ticket #10852: sws_scale overflows buffer for some resolutions using ssse3 instructions

• libswscale/x86/yuv2rgb.c: the h_size guard (FFmpeg n9.0.2)

• libswscale/x86/yuv_2_rgb.asm: the SSSE3 conversion loop (FFmpeg n9.0.2)

• libswscale/swscale.h: the sws_scale() API documentation

• doc/examples/scale_video.c: FFmpeg's official scaling example

• tests/checkasm/sw_yuv2rgb.c: FFmpeg's own test for this converter

• mallopt(3): M_MMAP_THRESHOLD and glibc's dynamic mmap threshold