Backport of:

From fd96fc23c53dcd95becfcca06d471e92923265ab Mon Sep 17 00:00:00 2001
From: Justin Chadwell <me@jedevc.com>
Date: Wed, 2 Sep 2020 10:49:40 +0100
Subject: [PATCH] qtdemux: use unsigned int types to store result of QT_UINT32

In a few cases throughout qtdemux, the results of QT_UINT32 were being
stored in a signed integer, which could cause subtle bugs in the case of
an integer overflow, even allowing the the result to equal a negative
number!

This patch prevents this by simply storing the results of this function
call properly in an unsigned integer type. Additionally, we fix up the
length checking with stsd parsing to prevent cases of child atoms
exceeding their parent atom sizes.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3344>
---
 .../gst-plugins-good/gst/isomp4/qtdemux.c     | 76 ++++++++++++-------
 1 file changed, 47 insertions(+), 29 deletions(-)

--- a/gst/isomp4/qtdemux.c
+++ b/gst/isomp4/qtdemux.c
@@ -9772,8 +9772,8 @@ qtdemux_parse_segments (GstQTDemux * qtd
   stream->segments = NULL;
   if ((edts = qtdemux_tree_get_child_by_type (trak, FOURCC_edts))) {
     GNode *elst;
-    gint n_segments;
-    gint segment_number, entry_size;
+    guint n_segments;
+    guint segment_number, entry_size;
     guint64 time;
     GstClockTime stime;
     const guint8 *buffer;
@@ -10449,6 +10449,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
   guint8 tkhd_version = 0;
   guint32 w = 0, h = 0;
   guint value_size, stsd_len, len;
+  guint32 lenb;
   guint32 track_id;
   guint32 dummy;
 
@@ -10789,7 +10790,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
             break;
         }
       } else {
-        gint i, j, start, end;
+        guint i, j, start, end;
 
         if (len < 94)
           goto corrupt_file;
@@ -10905,7 +10906,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
 
       if (pasp) {
         const guint8 *pasp_data = (const guint8 *) pasp->data;
-        gint len = QT_UINT32 (pasp_data);
+        guint len = QT_UINT32 (pasp_data);
 
         if (len == 16) {
           CUR_STREAM (stream)->par_w = QT_UINT32 (pasp_data + 8);
@@ -10921,7 +10922,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
 
       if (fiel) {
         const guint8 *fiel_data = (const guint8 *) fiel->data;
-        gint len = QT_UINT32 (fiel_data);
+        guint len = QT_UINT32 (fiel_data);
 
         if (len == 10) {
           CUR_STREAM (stream)->interlace_mode = GST_READ_UINT8 (fiel_data + 8);
@@ -10931,7 +10932,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
 
       if (colr) {
         const guint8 *colr_data = (const guint8 *) colr->data;
-        gint len = QT_UINT32 (colr_data);
+        guint len = QT_UINT32 (colr_data);
 
         if (len == 19 || len == 18) {
           guint32 color_type = GST_READ_UINT32_LE (colr_data + 8);
@@ -11017,14 +11018,17 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
           case FOURCC_avc1:
           case FOURCC_avc3:
           {
-            gint len = QT_UINT32 (stsd_entry_data) - 0x56;
+            guint len = QT_UINT32 (stsd_entry_data);
+            len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *avc_data = stsd_entry_data + 0x56;
 
             /* find avcC */
             while (len >= 0x8) {
-              gint size;
+              guint size;
 
-              if (QT_UINT32 (avc_data) <= len)
+              if (QT_UINT32 (avc_data) <= 0x8)
+                size = 0;
+              else if (QT_UINT32 (avc_data) <= len)
                 size = QT_UINT32 (avc_data) - 0x8;
               else
                 size = len - 0x8;
@@ -11129,14 +11133,17 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
           case FOURCC_hvc1:
           case FOURCC_hev1:
           {
-            gint len = QT_UINT32 (stsd_entry_data) - 0x56;
+            guint len = QT_UINT32 (stsd_entry_data);
+            len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *hevc_data = stsd_entry_data + 0x56;
 
             /* find hevc */
             while (len >= 0x8) {
-              gint size;
+              guint size;
 
-              if (QT_UINT32 (hevc_data) <= len)
+              if (QT_UINT32 (hevc_data) <= 0x8)
+                size = 0;
+              else if (QT_UINT32 (hevc_data) <= len)
                 size = QT_UINT32 (hevc_data) - 0x8;
               else
                 size = len - 0x8;
@@ -11192,7 +11199,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
             if (glbl) {
               guint8 *data;
               GstBuffer *buf;
-              gint len;
+              guint len;
 
               GST_DEBUG_OBJECT (qtdemux, "found glbl data in stsd");
               data = glbl->data;
@@ -11376,7 +11383,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
             /* add codec_data if provided */
             if (prefix) {
               GstBuffer *buf;
-              gint len;
+              guint len;
 
               GST_DEBUG_OBJECT (qtdemux, "found prefix data in stsd");
               data = prefix->data;
@@ -11398,7 +11405,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
             GstBuffer *buf;
             GstBuffer *seqh = NULL;
             const guint8 *gamma_data = NULL;
-            gint len = QT_UINT32 (stsd_data);   /* FIXME review - why put the whole stsd in codec data? */
+            guint len = QT_UINT32 (stsd_data);  /* FIXME review - why put the whole stsd in codec data? */
 
             qtdemux_parse_svq3_stsd_data (qtdemux, stsd_entry_data, &gamma_data,
                 &seqh);
@@ -11550,14 +11557,17 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
           }
           case FOURCC_vc_1:
           {
-            gint len = QT_UINT32 (stsd_entry_data) - 0x56;
+            guint len = QT_UINT32 (stsd_entry_data);
+            len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *vc1_data = stsd_entry_data + 0x56;
 
             /* find dvc1 */
             while (len >= 8) {
-              gint size;
+              guint size;
 
-              if (QT_UINT32 (vc1_data) <= len)
+              if (QT_UINT32 (vc1_data) <= 8)
+                size = 0;
+              else if (QT_UINT32 (vc1_data) <= len)
                 size = QT_UINT32 (vc1_data) - 8;
               else
                 size = len - 8;
@@ -11589,14 +11599,17 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
           }
           case FOURCC_av01:
           {
-            gint len = QT_UINT32 (stsd_entry_data) - 0x56;
+            guint len = QT_UINT32 (stsd_entry_data);
+            len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *av1_data = stsd_entry_data + 0x56;
 
             /* find av1C */
             while (len >= 0x8) {
-              gint size;
+              guint size;
 
-              if (QT_UINT32 (av1_data) <= len)
+              if (QT_UINT32 (av1_data) <= 0x8)
+                size = 0;
+              else if (QT_UINT32 (av1_data) <= len)
                 size = QT_UINT32 (av1_data) - 0x8;
               else
                 size = len - 0x8;
@@ -11673,7 +11686,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
 
     } else if (stream->subtype == FOURCC_soun) {
       GNode *wave;
-      int version, samplesize;
+      guint version, samplesize;
       guint16 compression_id;
       gboolean amrwb = FALSE;
 
@@ -11920,7 +11933,8 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
         }
         case FOURCC_wma_:
         {
-          gint len = QT_UINT32 (stsd_entry_data) - offset;
+          guint len = QT_UINT32 (stsd_entry_data);
+          len = len <= offset ? 0 : len - offset;
           const guint8 *wfex_data = stsd_entry_data + offset;
           const gchar *codec_name = NULL;
           gint version = 1;
@@ -11944,9 +11958,11 @@ qtdemux_parse_trak (GstQTDemux * qtdemux
 
           /* find wfex */
           while (len >= 8) {
-            gint size;
+            guint size;
 
-            if (QT_UINT32 (wfex_data) <= len)
+            if (QT_UINT32 (wfex_data) <= 0x8)
+              size = 0;
+            else if (QT_UINT32 (wfex_data) <= len)
               size = QT_UINT32 (wfex_data) - 8;
             else
               size = len - 8;
