[sfnt] Better checks for invalid cmaps (2/2) (#46019).

While the current code in `FT_Get_Next_Char' correctly rejects
out-of-bounds glyph indices, it can be extremely slow for malformed
cmaps that use 32bit values.  This commit tries to improve that.

* src/sfnt/ttcmap.c (tt_cmap8_char_next, tt_cmap12_next,
tt_cmap12_char_map_binary, tt_cmap13_next,
tt_cmap13_char_map_binary): Reject glyph indices larger than or
equal to the number of glyphs.

ChangeLog

@@ -1,3 +1,16 @@
+2015-09-23  Werner Lemberg  <wl@gnu.org>
+
+	[sfnt] Better checks for invalid cmaps (2/2) (#46019).
+
+	While the current code in `FT_Get_Next_Char' correctly rejects
+	out-of-bounds glyph indices, it can be extremely slow for malformed
+	cmaps that use 32bit values.  This commit tries to improve that.
+
+	* src/sfnt/ttcmap.c (tt_cmap8_char_next, tt_cmap12_next,
+	tt_cmap12_char_map_binary, tt_cmap13_next,
+	tt_cmap13_char_map_binary): Reject glyph indices larger than or
+	equal to the number of glyphs.
+
 2015-09-23  Werner Lemberg  <wl@gnu.org>
 
 	[base, sfnt] Better checks for invalid cmaps (1/2).

include/freetype/freetype.h

@@ -3372,6 +3372,13 @@ FT_BEGIN_HEADER
   /*      }                                                                */
   /*    }                                                                  */
   /*                                                                       */
+  /*    Be aware that character codes can have values up to 0xFFFFFFFF;    */
+  /*    this might happen for non-Unicode or malformed cmaps.  However,    */
+  /*    even with regular Unicode encoding, so-called `last resort fonts'  */
+  /*    (using SFNT cmap format 13, see function @FT_Get_CMap_Format)      */
+  /*    normally have entries for all Unicode characters up to 0x1FFFFF,   */
+  /*    which can cause *a lot* of iterations.                             */
+  /*                                                                       */
   /*    Note that `*agindex' is set to~0 if the charmap is empty.  The     */
   /*    result itself can be~0 in two cases: if the charmap is empty or    */
   /*    if the value~0 is the first valid character code.                  */

include/freetype/internal/services/svttcmap.h

@@ -48,11 +48,12 @@ FT_BEGIN_HEADER
   /*      `ttnameid.h'.                                                    */
   /*                                                                       */
   /*    format ::                                                          */
-  /*      The cmap format.  OpenType 1.5 defines the formats 0 (byte       */
+  /*      The cmap format.  OpenType 1.6 defines the formats 0 (byte       */
   /*      encoding table), 2~(high-byte mapping through table), 4~(segment */
   /*      mapping to delta values), 6~(trimmed table mapping), 8~(mixed    */
   /*      16-bit and 32-bit coverage), 10~(trimmed array), 12~(segmented   */
-  /*      coverage), and 14 (Unicode Variation Sequences).                 */
+  /*      coverage), 13~(last resort font), and 14 (Unicode Variation      */
+  /*      Sequences).                                                      */
   /*                                                                       */
   typedef struct  TT_CMapInfo_
   {

src/sfnt/ttcmap.c

@@ -1797,6 +1797,7 @@
   tt_cmap8_char_next( TT_CMap     cmap,
                       FT_UInt32  *pchar_code )
   {
+    FT_Face    face       = cmap->cmap.charmap.face;
     FT_UInt32  result     = 0;
     FT_UInt32  char_code;
     FT_UInt    gindex     = 0;
@@ -1841,6 +1842,11 @@
           goto Again;
         }
 
+        /* if `gindex' is invalid, the remaining values */
+        /* in this group are invalid, too               */
+        if ( gindex >= (FT_UInt)face->num_glyphs )
+          continue;
+
         result = char_code;
         break;
       }
@@ -2181,6 +2187,7 @@
   static void
   tt_cmap12_next( TT_CMap12  cmap )
   {
+    FT_Face   face = cmap->cmap.cmap.charmap.face;
     FT_Byte*  p;
     FT_ULong  start, end, start_id, char_code;
     FT_ULong  n;
@@ -2221,6 +2228,11 @@
           goto Again;
         }
 
+        /* if `gindex' is invalid, the remaining values */
+        /* in this group are invalid, too               */
+        if ( gindex >= (FT_UInt)face->num_glyphs )
+          continue;
+
         cmap->cur_charcode = char_code;
         cmap->cur_gindex   = gindex;
         cmap->cur_group    = n;
@@ -2293,6 +2305,7 @@
 
     if ( next )
     {
+      FT_Face    face   = cmap->cmap.charmap.face;
       TT_CMap12  cmap12 = (TT_CMap12)cmap;
 
 
@@ -2310,6 +2323,9 @@
       cmap12->cur_charcode = char_code;
       cmap12->cur_group    = mid;
 
+      if ( gindex >= (FT_UInt)face->num_glyphs )
+        gindex = 0;
+
       if ( !gindex )
       {
         tt_cmap12_next( cmap12 );
@@ -2517,6 +2533,7 @@
   static void
   tt_cmap13_next( TT_CMap13  cmap )
   {
+    FT_Face   face = cmap->cmap.cmap.charmap.face;
     FT_Byte*  p;
     FT_ULong  start, end, glyph_id, char_code;
     FT_ULong  n;
@@ -2542,7 +2559,7 @@
       {
         gindex = (FT_UInt)glyph_id;
 
-        if ( gindex )
+        if ( gindex && gindex < (FT_UInt)face->num_glyphs )
         {
           cmap->cur_charcode = char_code;
           cmap->cur_gindex   = gindex;
@@ -2612,6 +2629,7 @@
 
     if ( next )
     {
+      FT_Face    face   = cmap->cmap.charmap.face;
       TT_CMap13  cmap13 = (TT_CMap13)cmap;
 
 
@@ -2629,6 +2647,9 @@
       cmap13->cur_charcode = char_code;
       cmap13->cur_group    = mid;
 
+      if ( gindex >= (FT_UInt)face->num_glyphs )
+        gindex = 0;
+
       if ( !gindex )
       {
         tt_cmap13_next( cmap13 );