[base, sfnt] Better checks for invalid cmaps (1/2).

* src/base/ftobjs.c (FT_Get_Char_Index): Don't return out-of-bounds
glyph indices.
(FT_Get_First_Char): Updated.

* src/sfnt/ttcmap.c (tt_cmap6_char_next): Don't return character
codes greater than 0xFFFF.

(tt_cmap8_char_index): Avoid integer overflow in computation of
glyph index.
(tt_cmap8_char_next): Avoid integer overflows in computation of
both next character code and glyph index.

(tt_cmap10_char_index): Fix unsigned integer logic.
(tt_cmap10_char_next): Avoid integer overflow in computation of
next character code.

(tt_cmap12_next): Avoid integer overflows in computation of both
next character code and glyph index.
(tt_cmap12_char_map_binary): Ditto.
(tt_cmap12_char_next): Simplify.

(tt_cmap13_char_map_binary): Avoid integer overflow in computation
of next character code.
(tt_cmap13_char_next): Simplify.

ChangeLog

@@ -1,3 +1,32 @@
+2015-09-23  Werner Lemberg  <wl@gnu.org>
+
+	[base, sfnt] Better checks for invalid cmaps (1/2).
+
+	* src/base/ftobjs.c (FT_Get_Char_Index): Don't return out-of-bounds
+	glyph indices.
+	(FT_Get_First_Char): Updated.
+
+	* src/sfnt/ttcmap.c (tt_cmap6_char_next): Don't return character
+	codes greater than 0xFFFF.
+
+	(tt_cmap8_char_index): Avoid integer overflow in computation of
+	glyph index.
+	(tt_cmap8_char_next): Avoid integer overflows in computation of
+	both next character code and glyph index.
+
+	(tt_cmap10_char_index): Fix unsigned integer logic.
+	(tt_cmap10_char_next): Avoid integer overflow in computation of
+	next character code.
+
+	(tt_cmap12_next): Avoid integer overflows in computation of both
+	next character code and glyph index.
+	(tt_cmap12_char_map_binary): Ditto.
+	(tt_cmap12_char_next): Simplify.
+
+	(tt_cmap13_char_map_binary): Avoid integer overflow in computation
+	of next character code.
+	(tt_cmap13_char_next): Simplify.
+
 2015-09-21  suzuki toshiya  <mpsuzuki@hiroshima-u.ac.jp>
 
 	[base] Check too long POST and sfnt resource (#45919). 

src/base/ftobjs.c

@@ -3382,8 +3382,12 @@
         FT_TRACE1(( "FT_Get_Char_Index: too large charcode" ));
         FT_TRACE1(( " 0x%x is truncated\n", charcode ));
       }
+
       result = cmap->clazz->char_index( cmap, (FT_UInt32)charcode );
+      if ( result >= (FT_UInt)face->num_glyphs )
+        result = 0;
     }
+
     return result;
   }
 
@@ -3402,7 +3406,7 @@
     if ( face && face->charmap && face->num_glyphs )
     {
       gindex = FT_Get_Char_Index( face, 0 );
-      if ( gindex == 0 || gindex >= (FT_UInt)face->num_glyphs )
+      if ( gindex == 0 )
         result = FT_Get_Next_Char( face, 0, &gindex );
     }
 

src/sfnt/ttcmap.c

@@ -51,6 +51,13 @@
 #define TT_NEXT_ULONG   FT_NEXT_ULONG
 
 
+  /* Too large glyph index return values are caught in `FT_Get_Char_Index' */
+  /* and `FT_Get_Next_Char' (the latter calls the internal `next' function */
+  /* again in this case).  To mark character code return values as invalid */
+  /* it is sufficient to set the corresponding glyph index return value to */
+  /* zero.                                                                 */
+
+
   FT_CALLBACK_DEF( FT_Error )
   tt_cmap_init( TT_CMap   cmap,
                 FT_Byte*  table )
@@ -1533,7 +1540,7 @@
 
 
     if ( char_code >= 0x10000UL )
-      goto Exit;
+      return 0;
 
     if ( char_code < start )
       char_code = start;
@@ -1549,10 +1556,13 @@
         result = char_code;
         break;
       }
+
+      if ( char_code >= 0xFFFFU )
+        return 0;
+
       char_code++;
     }
 
-  Exit:
     *pchar_code = result;
     return gindex;
   }
@@ -1772,7 +1782,10 @@
 
       if ( char_code <= end )
       {
-        result = (FT_UInt)( start_id + char_code - start );
+        if ( start_id > 0xFFFFFFFFUL - ( char_code - start ) )
+          return 0;
+
+        result = (FT_UInt)( start_id + ( char_code - start ) );
         break;
       }
     }
@@ -1785,7 +1798,7 @@
                       FT_UInt32  *pchar_code )
   {
     FT_UInt32  result     = 0;
-    FT_UInt32  char_code  = *pchar_code + 1;
+    FT_UInt32  char_code;
     FT_UInt    gindex     = 0;
     FT_Byte*   table      = cmap->data;
     FT_Byte*   p          = table + 8204;
@@ -1793,6 +1806,11 @@
     FT_UInt32  start, end, start_id;
 
 
+    if ( *pchar_code >= 0xFFFFFFFFUL )
+      return 0;
+
+    char_code = *pchar_code + 1;
+
     p = table + 8208;
 
     for ( ; num_groups > 0; num_groups-- )
@@ -1804,18 +1822,30 @@
       if ( char_code < start )
         char_code = start;
 
+    Again:
       if ( char_code <= end )
       {
-        gindex = (FT_UInt)( char_code - start + start_id );
+        /* ignore invalid group */
-        if ( gindex != 0 )
+        if ( start_id > 0xFFFFFFFFUL - ( char_code - start ) )
+          continue;
+
+        gindex = (FT_UInt)( start_id + ( char_code - start ) );
+
+        /* does first element of group point to `.notdef' glyph? */
+        if ( gindex == 0 )
         {
-          result = char_code;
+          if ( char_code >= 0xFFFFFFFFUL )
-          goto Exit;
+            break;
+
+          char_code++;
+          goto Again;
         }
+
+        result = char_code;
+        break;
       }
     }
 
-  Exit:
     *pchar_code = result;
     return gindex;
   }
@@ -1932,14 +1962,20 @@
     FT_Byte*   p      = table + 12;
     FT_UInt32  start  = TT_NEXT_ULONG( p );
     FT_UInt32  count  = TT_NEXT_ULONG( p );
-    FT_UInt32  idx    = (FT_ULong)( char_code - start );
+    FT_UInt32  idx;
 
 
+    if ( char_code < start )
+      return 0;
+
+    idx = char_code - start;
+
     if ( idx < count )
     {
       p     += 2 * idx;
       result = TT_PEEK_USHORT( p );
     }
+
     return result;
   }
 
@@ -1949,7 +1985,7 @@
                        FT_UInt32  *pchar_code )
   {
     FT_Byte*   table     = cmap->data;
-    FT_UInt32  char_code = *pchar_code + 1;
+    FT_UInt32  char_code;
     FT_UInt    gindex    = 0;
     FT_Byte*   p         = table + 12;
     FT_UInt32  start     = TT_NEXT_ULONG( p );
@@ -1957,10 +1993,15 @@
     FT_UInt32  idx;
 
 
+    if ( *pchar_code >= 0xFFFFFFFFUL )
+      return 0;
+
+    char_code = *pchar_code + 1;
+
     if ( char_code < start )
       char_code = start;
 
-    idx = (FT_UInt32)( char_code - start );
+    idx = char_code - start;
     p  += 2 * idx;
 
     for ( ; idx < count; idx++ )
@@ -1968,6 +2009,10 @@
       gindex = TT_NEXT_USHORT( p );
       if ( gindex != 0 )
         break;
+
+      if ( char_code >= 0xFFFFFFFFUL )
+        return 0;
+
       char_code++;
     }
 
@@ -2157,20 +2202,32 @@
       if ( char_code < start )
         char_code = start;
 
-      for ( ; char_code <= end; char_code++ )
+    Again:
+      if ( char_code <= end )
       {
-        gindex = (FT_UInt)( start_id + char_code - start );
+        /* ignore invalid group */
+        if ( start_id > 0xFFFFFFFFUL - ( char_code - start ) )
+          continue;
 
-        if ( gindex )
+        gindex = (FT_UInt)( start_id + ( char_code - start ) );
+
+        /* does first element of group point to `.notdef' glyph? */
+        if ( gindex == 0 )
         {
-          cmap->cur_charcode = char_code;;
+          if ( char_code >= 0xFFFFFFFFUL )
+            goto Fail;
+
+          char_code++;
+          goto Again;
+        }
+
+        cmap->cur_charcode = char_code;
         cmap->cur_gindex   = gindex;
         cmap->cur_group    = n;
 
         return;
       }
     }
-    }
 
   Fail:
     cmap->valid = 0;
@@ -2198,7 +2255,12 @@
     end = 0xFFFFFFFFUL;
 
     if ( next )
+    {
+      if ( char_code >= 0xFFFFFFFFUL )
+        return 0;
+
       char_code++;
+    }
 
     min = 0;
     max = num_groups;
@@ -2219,8 +2281,12 @@
       else
       {
         start_id = TT_PEEK_ULONG( p );
-        gindex = (FT_UInt)( start_id + char_code - start );
 
+        /* reject invalid glyph index */
+        if ( start_id > 0xFFFFFFFFUL - ( char_code - start ) )
+          gindex = 0;
+        else
+          gindex = (FT_UInt)( start_id + ( char_code - start ) );
         break;
       }
     }
@@ -2254,7 +2320,6 @@
       else
         cmap12->cur_gindex = gindex;
 
-      if ( gindex )
       *pchar_code = cmap12->cur_charcode;
     }
 
@@ -2275,12 +2340,9 @@
                        FT_UInt32  *pchar_code )
   {
     TT_CMap12  cmap12 = (TT_CMap12)cmap;
-    FT_ULong   gindex;
+    FT_UInt    gindex;
 
 
-    if ( cmap12->cur_charcode >= 0xFFFFFFFFUL )
-      return 0;
-
     /* no need to search */
     if ( cmap12->valid && cmap12->cur_charcode == *pchar_code )
     {
@@ -2288,9 +2350,6 @@
       if ( cmap12->valid )
       {
         gindex      = cmap12->cur_gindex;
-
-        /* XXX: check cur_charcode overflow is expected */
-        if ( gindex )
         *pchar_code = (FT_UInt32)cmap12->cur_charcode;
       }
       else
@@ -2299,8 +2358,7 @@
     else
       gindex = tt_cmap12_char_map_binary( cmap, pchar_code, 1 );
 
-    /* XXX: check gindex overflow is expected */
+    return gindex;
-    return (FT_UInt32)gindex;
   }
 
 
@@ -2521,7 +2579,12 @@
     end = 0xFFFFFFFFUL;
 
     if ( next )
+    {
+      if ( char_code >= 0xFFFFFFFFUL )
+        return 0;
+
       char_code++;
+    }
 
     min = 0;
     max = num_groups;
@@ -2576,7 +2639,6 @@
       else
         cmap13->cur_gindex = gindex;
 
-      if ( gindex )
       *pchar_code = cmap13->cur_charcode;
     }
 
@@ -2600,9 +2662,6 @@
     FT_UInt    gindex;
 
 
-    if ( cmap13->cur_charcode >= 0xFFFFFFFFUL )
-      return 0;
-
     /* no need to search */
     if ( cmap13->valid && cmap13->cur_charcode == *pchar_code )
     {
@@ -2610,7 +2669,6 @@
       if ( cmap13->valid )
       {
         gindex      = cmap13->cur_gindex;
-        if ( gindex )
         *pchar_code = cmap13->cur_charcode;
       }
       else