Backport of:

From fe9d5d37234aca04fef7248184177168905a7a69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= <sebastian@centricular.com>
Date: Fri, 27 Sep 2024 00:12:57 +0300
Subject: [PATCH] qtdemux: Fix length checks and offsets in stsd entry parsing

Thanks to Antonio Morales for finding and reporting the issue.

Fixes GHSL-2024-242
Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3845

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/8060>
---
 .../gst-plugins-good/gst/isomp4/qtdemux.c     | 218 +++++++-----------
 1 file changed, 79 insertions(+), 139 deletions(-)

--- a/gst/isomp4/qtdemux.c
+++ b/gst/isomp4/qtdemux.c
@@ -11018,43 +11018,35 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
           case FOURCC_avc1:
           case FOURCC_avc3:
           {
-            guint len = QT_UINT32 (stsd_entry_data);
+            guint32 len = QT_UINT32 (stsd_entry_data);
             len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *avc_data = stsd_entry_data + 0x56;
 
             /* find avcC */
-            while (len >= 0x8) {
-              guint size;
+            while (len >= 8) {
+              guint32 size = QT_UINT32 (avc_data);
 
-              if (QT_UINT32 (avc_data) <= 0x8)
-                size = 0;
-              else if (QT_UINT32 (avc_data) <= len)
-                size = QT_UINT32 (avc_data) - 0x8;
-              else
-                size = len - 0x8;
-
-              /* No real data, so skip */
-              if (size < 1) {
-                len -= 8;
-                avc_data += 8;
-                continue;
-              }
+              if (size < 8 || size > len)
+                break;
 
-              switch (QT_FOURCC (avc_data + 0x4)) {
+              switch (QT_FOURCC (avc_data + 4)) {
                 case FOURCC_avcC:
                 {
                   /* parse, if found */
                   GstBuffer *buf;
 
+                  if (size < 8 + 1)
+                    break;
+
                   GST_DEBUG_OBJECT (qtdemux, "found avcC codec_data in stsd");
 
                   /* First 4 bytes are the length of the atom, the next 4 bytes
                    * are the fourcc, the next 1 byte is the version, and the
                    * subsequent bytes are profile_tier_level structure like data. */
                   gst_codec_utils_h264_caps_set_level_and_profile (entry->caps,
-                      avc_data + 8 + 1, size - 1);
-                  buf = gst_buffer_new_and_alloc (size);
-                  gst_buffer_fill (buf, 0, avc_data + 0x8, size);
+                      avc_data + 8 + 1, size - 8 - 1);
+                  buf = gst_buffer_new_and_alloc (size - 8);
+                  gst_buffer_fill (buf, 0, avc_data + 8, size - 8);
                   gst_caps_set_simple (entry->caps,
                       "codec_data", GST_TYPE_BUFFER, buf, NULL);
                   gst_buffer_unref (buf);
@@ -11065,6 +11057,9 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
                 {
                   GstBuffer *buf;
 
+                  if (size < 8 + 40 + 1)
+                    break;
+
                   GST_DEBUG_OBJECT (qtdemux, "found strf codec_data in stsd");
 
                   /* First 4 bytes are the length of the atom, the next 4 bytes
@@ -11072,17 +11067,14 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
                    * next 1 byte is the version, and the
                    * subsequent bytes are sequence parameter set like data. */
 
-                  size -= 40;   /* we'll be skipping BITMAPINFOHEADER */
-                  if (size > 1) {
-                    gst_codec_utils_h264_caps_set_level_and_profile
-                        (entry->caps, avc_data + 8 + 40 + 1, size - 1);
+                  gst_codec_utils_h264_caps_set_level_and_profile
+                      (entry->caps, avc_data + 8 + 40 + 1, size - 8 - 40 - 1);
 
-                    buf = gst_buffer_new_and_alloc (size);
-                    gst_buffer_fill (buf, 0, avc_data + 8 + 40, size);
-                    gst_caps_set_simple (entry->caps,
-                        "codec_data", GST_TYPE_BUFFER, buf, NULL);
-                    gst_buffer_unref (buf);
-                  }
+                  buf = gst_buffer_new_and_alloc (size - 8 - 40);
+                  gst_buffer_fill (buf, 0, avc_data + 8 + 40, size - 8 - 40);
+                  gst_caps_set_simple (entry->caps,
+                      "codec_data", GST_TYPE_BUFFER, buf, NULL);
+                  gst_buffer_unref (buf);
                   break;
                 }
                 case FOURCC_btrt:
@@ -11090,11 +11082,11 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
                   guint avg_bitrate, max_bitrate;
 
                   /* bufferSizeDB, maxBitrate and avgBitrate - 4 bytes each */
-                  if (size < 12)
+                  if (size < 8 + 12)
                     break;
 
-                  max_bitrate = QT_UINT32 (avc_data + 0xc);
-                  avg_bitrate = QT_UINT32 (avc_data + 0x10);
+                  max_bitrate = QT_UINT32 (avc_data + 8 + 4);
+                  avg_bitrate = QT_UINT32 (avc_data + 8 + 8);
 
                   if (!max_bitrate && !avg_bitrate)
                     break;
@@ -11126,8 +11118,8 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
                   break;
               }
 
-              len -= size + 8;
-              avc_data += size + 8;
+              len -= size;
+              avc_data += size;
             }
 
             break;
@@ -11136,44 +11128,36 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
           case FOURCC_hvc1:
           case FOURCC_hev1:
           {
-            guint len = QT_UINT32 (stsd_entry_data);
+            guint32 len = QT_UINT32 (stsd_entry_data);
             len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *hevc_data = stsd_entry_data + 0x56;
 
             /* find hevc */
-            while (len >= 0x8) {
-              guint size;
+            while (len >= 8) {
+              guint32 size = QT_UINT32 (hevc_data);
 
-              if (QT_UINT32 (hevc_data) <= 0x8)
-                size = 0;
-              else if (QT_UINT32 (hevc_data) <= len)
-                size = QT_UINT32 (hevc_data) - 0x8;
-              else
-                size = len - 0x8;
-
-              /* No real data, so skip */
-              if (size < 1) {
-                len -= 8;
-                hevc_data += 8;
-                continue;
-              }
+              if (size < 8 || size > len)
+                break;
 
-              switch (QT_FOURCC (hevc_data + 0x4)) {
+              switch (QT_FOURCC (hevc_data + 4)) {
                 case FOURCC_hvcC:
                 {
                   /* parse, if found */
                   GstBuffer *buf;
 
+                  if (size < 8 + 1)
+                    break;
+
                   GST_DEBUG_OBJECT (qtdemux, "found hvcC codec_data in stsd");
 
                   /* First 4 bytes are the length of the atom, the next 4 bytes
                    * are the fourcc, the next 1 byte is the version, and the
                    * subsequent bytes are sequence parameter set like data. */
                   gst_codec_utils_h265_caps_set_level_tier_and_profile
-                      (entry->caps, hevc_data + 8 + 1, size - 1);
+                      (entry->caps, hevc_data + 8 + 1, size - 8 - 1);
 
-                  buf = gst_buffer_new_and_alloc (size);
-                  gst_buffer_fill (buf, 0, hevc_data + 0x8, size);
+                  buf = gst_buffer_new_and_alloc (size - 8);
+                  gst_buffer_fill (buf, 0, hevc_data + 8, size - 8);
                   gst_caps_set_simple (entry->caps,
                       "codec_data", GST_TYPE_BUFFER, buf, NULL);
                   gst_buffer_unref (buf);
@@ -11182,8 +11166,8 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
                 default:
                   break;
               }
-              len -= size + 8;
-              hevc_data += size + 8;
+              len -= size;
+              hevc_data += size;
             }
             break;
           }
@@ -11563,36 +11547,25 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
           }
           case FOURCC_vc_1:
           {
-            guint len = QT_UINT32 (stsd_entry_data);
+            guint32 len = QT_UINT32 (stsd_entry_data);
             len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *vc1_data = stsd_entry_data + 0x56;
 
             /* find dvc1 */
             while (len >= 8) {
-              guint size;
+              guint32 size = QT_UINT32 (vc1_data);
 
-              if (QT_UINT32 (vc1_data) <= 8)
-                size = 0;
-              else if (QT_UINT32 (vc1_data) <= len)
-                size = QT_UINT32 (vc1_data) - 8;
-              else
-                size = len - 8;
-
-              /* No real data, so skip */
-              if (size < 1) {
-                len -= 8;
-                vc1_data += 8;
-                continue;
-              }
+              if (size < 8 || size > len)
+                break;
 
-              switch (QT_FOURCC (vc1_data + 0x4)) {
+              switch (QT_FOURCC (vc1_data + 4)) {
                 case GST_MAKE_FOURCC ('d', 'v', 'c', '1'):
                 {
                   GstBuffer *buf;
 
                   GST_DEBUG_OBJECT (qtdemux, "found dvc1 codec_data in stsd");
-                  buf = gst_buffer_new_and_alloc (size);
-                  gst_buffer_fill (buf, 0, vc1_data + 8, size);
+                  buf = gst_buffer_new_and_alloc (size - 8);
+                  gst_buffer_fill (buf, 0, vc1_data + 8, size - 8);
                   gst_caps_set_simple (entry->caps,
                       "codec_data", GST_TYPE_BUFFER, buf, NULL);
                   gst_buffer_unref (buf);
@@ -11601,36 +11574,25 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
                 default:
                   break;
               }
-              len -= size + 8;
-              vc1_data += size + 8;
+              len -= size;
+              vc1_data += size;
             }
             break;
           }
           case FOURCC_av01:
           {
-            guint len = QT_UINT32 (stsd_entry_data);
+            guint32 len = QT_UINT32 (stsd_entry_data);
             len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *av1_data = stsd_entry_data + 0x56;
 
             /* find av1C */
-            while (len >= 0x8) {
-              guint size;
+            while (len >= 8) {
+              guint32 size = QT_UINT32 (av1_data);
 
-              if (QT_UINT32 (av1_data) <= 0x8)
-                size = 0;
-              else if (QT_UINT32 (av1_data) <= len)
-                size = QT_UINT32 (av1_data) - 0x8;
-              else
-                size = len - 0x8;
-
-              /* No real data, so skip */
-              if (size < 1) {
-                len -= 8;
-                av1_data += 8;
-                continue;
-              }
+              if (size < 8 || size > len)
+                break;
 
-              switch (QT_FOURCC (av1_data + 0x4)) {
+              switch (QT_FOURCC (av1_data + 4)) {
                 case FOURCC_av1C:
                 {
                   /* parse, if found */
@@ -11641,7 +11603,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
                       "found av1C codec_data in stsd of size %d", size);
 
                   /* not enough data, just ignore and hope for the best */
-                  if (size < 5)
+                  if (size < 8 + 5)
                     break;
 
                   /* Content is:
@@ -11667,10 +11629,10 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
                         "presentation-delay", G_TYPE_INT,
                         (gint) (pres_delay_field & 0x0F) + 1, NULL);
                   }
-                  if (size > 5) {
-                    buf = gst_buffer_new_and_alloc (size - 5);
+                  if (size > 8 + 5) {
+                    buf = gst_buffer_new_and_alloc (size - 8 - 5);
                     GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_HEADER);
-                    gst_buffer_fill (buf, 0, av1_data + 13, size - 5);
+                    gst_buffer_fill (buf, 0, av1_data + 13, size - 8 - 5);
                     gst_caps_set_simple (entry->caps,
                         "codec_data", GST_TYPE_BUFFER, buf, NULL);
                     gst_buffer_unref (buf);
@@ -11681,8 +11643,8 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
                   break;
               }
 
-              len -= size + 8;
-              av1_data += size + 8;
+              len -= size;
+              av1_data += size;
             }
 
             break;
@@ -11945,7 +11907,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
         }
         case FOURCC_wma_:
         {
-          guint len = QT_UINT32 (stsd_entry_data);
+          guint32 len = QT_UINT32 (stsd_entry_data);
           len = len <= offset ? 0 : len - offset;
           const guint8 *wfex_data = stsd_entry_data + offset;
           const gchar *codec_name = NULL;
@@ -11970,21 +11932,10 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
 
           /* find wfex */
           while (len >= 8) {
-            guint size;
+            guint32 size = QT_UINT32 (wfex_data);
 
-            if (QT_UINT32 (wfex_data) <= 0x8)
-              size = 0;
-            else if (QT_UINT32 (wfex_data) <= len)
-              size = QT_UINT32 (wfex_data) - 8;
-            else
-              size = len - 8;
-
-            /* No real data, so skip */
-            if (size < 1) {
-              len -= 8;
-              wfex_data += 8;
-              continue;
-            }
+            if (size < 8 || size > len)
+              break;
 
             switch (QT_FOURCC (wfex_data + 4)) {
               case GST_MAKE_FOURCC ('w', 'f', 'e', 'x'):
@@ -12029,12 +11980,12 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
                     "width", G_TYPE_INT, wfex.wBitsPerSample,
                     "depth", G_TYPE_INT, wfex.wBitsPerSample, NULL);
 
-                if (size > wfex.cbSize) {
+                if (size > 8 + wfex.cbSize) {
                   GstBuffer *buf;
 
-                  buf = gst_buffer_new_and_alloc (size - wfex.cbSize);
+                  buf = gst_buffer_new_and_alloc (size - 8 - wfex.cbSize);
                   gst_buffer_fill (buf, 0, wfex_data + 8 + wfex.cbSize,
-                      size - wfex.cbSize);
+                      size - 8 - wfex.cbSize);
                   gst_caps_set_simple (entry->caps,
                       "codec_data", GST_TYPE_BUFFER, buf, NULL);
                   gst_buffer_unref (buf);
@@ -12051,8 +12002,8 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
               default:
                 break;
             }
-            len -= size + 8;
-            wfex_data += size + 8;
+            len -= size;
+            wfex_data += size;
           }
           break;
         }
