Fix ToUnicode with reverse CMap lookup

When dvipdfmx is provided a regular CMap in the following format:

<unicode codepoint> cid

We can try to utilize the data and do a reverse lookup when all we
know is the cids used.

Initially, when parsing the cmap specified, we build a reverseMap
which maps CID to the first Unicode codepoint it corresponds to.
(Which is pretty much the reverse of CMap_decode().)

Since this cmap is already parsed when we create the fontmap cache,
we can just pass the cmap_id all the way down to
otf_create_ToUnicode_stream() and get CMap* back from this cmap_id.

Second, we refactor create_ToUnicode_cmap{4, 12}() functions into
one create_ToUnicode_cmap() and combine the common code.

In the third step, in create_ToUnicode_cmap() we try to see if the
font is indeed a CFF font (with CID-keyed), and we do have a cmap,
we can use the newly added CMap_reverse_decode() to convert all
used cids back to Unicode codepoints. With that mapping information
we can successfully create the ToUnicode PDF stream we needed.

git-svn-id: svn://tug.org/texlive/trunk@34831 c570f23f-e606-0410-a88d-b1316a301751

7 files changed, 146 insertions, 83 deletions

diff --git a/Build/source/texk/dvipdfm-x/ChangeLog b/Build/source/texk/dvipdfm-x/ChangeLog
index 77fb6a8fdf5..e0e5efbea02 100644
--- a/Build/source/texk/dvipdfm-x/ChangeLog
+++ b/Build/source/texk/dvipdfm-x/ChangeLog
@@ -7,6 +7,11 @@
 	handles the dvi file when Unicode to CID mapping file is specified,
 	where pdf_dev_set_string() got string of cids instead of glyphs.
 
+	* cmap.c: Add reverse lookup from CID to Unicode codepoints.
+
+	* tt_cmap.c, type0.c: Refactor create_ToUnicode_cmap{4, 12}(), use
+	  reverse CMap lookup to construct ToUnicode streams.
+
 2014-08-03  Akira Kakuto  <kakuto@fuk.kindai.ac.jp>
 
 	* pdfobj.c: Fix a typo in check_for_pdf().
diff --git a/Build/source/texk/dvipdfm-x/cmap.c b/Build/source/texk/dvipdfm-x/cmap.c
index 0629ac1cc49..d5c909cda01 100644
--- a/Build/source/texk/dvipdfm-x/cmap.c
+++ b/Build/source/texk/dvipdfm-x/cmap.c
@@ -109,6 +109,9 @@ CMap_new (void)
   cmap->mapData->pos  = 0;
   cmap->mapData->data = NEW(MEM_ALLOC_SIZE, unsigned char);
 
+  cmap->reverseMap = NEW(65536, int);
+  memset(cmap->reverseMap, 0, 65536 * sizeof(int));
+
   return cmap;
 }
 
@@ -140,6 +143,9 @@ CMap_release (CMap *cmap)
     }
   }
 
+  if (cmap->reverseMap)
+    RELEASE(cmap->reverseMap);
+
   RELEASE(cmap);
 }
 
@@ -359,6 +365,14 @@ CMap_decode (CMap *cmap,
   return count;
 }
 
+int
+CMap_reverse_decode(CMap *cmap, CID cid) {
+  int ch = cmap->reverseMap ? cmap->reverseMap[cid] : -1;
+  if (ch == 0 && cmap->useCMap)
+    return CMap_reverse_decode(cmap->useCMap, cid);
+  return ch;
+}
+
 char *
 CMap_get_name (CMap *cmap)
 {
@@ -664,7 +678,7 @@ int
 CMap_add_cidrange (CMap *cmap,
 		   const unsigned char *srclo, const unsigned char *srchi, int srcdim, CID base)
 {
-  int     c;
+  int    i, c, v;
   mapDef *cur;
 
   ASSERT(cmap);
@@ -679,6 +693,11 @@ CMap_add_cidrange (CMap *cmap,
   if (locate_tbl(&cur, srclo, srcdim) < 0)
     return -1;
 
+  for (v = 0, i = 0; i < srcdim - 1; i++)
+    v = (v << 8) + srclo[i];
+
+  cmap->reverseMap[base] = v;
+
   for (c = srclo[srcdim-1]; c <= srchi[srcdim-1]; c++) {
     if (cur[c].flag != 0) {
       if (!__silent)
@@ -689,6 +708,8 @@ CMap_add_cidrange (CMap *cmap,
       cur[c].code = get_mem(cmap, 2);
       cur[c].code[0] = base >> 8;
       cur[c].code[1] = base & 0xff;
+
+      cmap->reverseMap[base] = (v << 8) + c;
     }
     if (base >= CID_MAX)
       WARN("CID number too large.");
diff --git a/Build/source/texk/dvipdfm-x/cmap.h b/Build/source/texk/dvipdfm-x/cmap.h
index 9b0f6faa345..75c39bd4b47 100644
--- a/Build/source/texk/dvipdfm-x/cmap.h
+++ b/Build/source/texk/dvipdfm-x/cmap.h
@@ -107,6 +107,8 @@ extern long CMap_decode (CMap *cmap,
 			 const unsigned char **inbuf,  long *inbytesleft,
 			 unsigned char **outbuf, long *outbytesleft);
 
+extern int  CMap_reverse_decode(CMap *cmap, CID cid);
+
 extern void  CMap_cache_init  (void);
 extern CMap *CMap_cache_get   (int id);
 extern int   CMap_cache_find  (const char *cmap_name);
diff --git a/Build/source/texk/dvipdfm-x/cmap_p.h b/Build/source/texk/dvipdfm-x/cmap_p.h
index 63639d2e3f7..7b9828bfbd3 100644
--- a/Build/source/texk/dvipdfm-x/cmap_p.h
+++ b/Build/source/texk/dvipdfm-x/cmap_p.h
@@ -97,6 +97,7 @@ struct CMap {
     int maxBytesOut;
   } profile;
 
+  int *reverseMap;
 };
 
 #endif /* _CMAP_P_H_ */
diff --git a/Build/source/texk/dvipdfm-x/tt_cmap.c b/Build/source/texk/dvipdfm-x/tt_cmap.c
index 4638b8628be..d1dc49c797c 100644
--- a/Build/source/texk/dvipdfm-x/tt_cmap.c
+++ b/Build/source/texk/dvipdfm-x/tt_cmap.c
@@ -984,27 +984,14 @@ prepare_CIDFont_from_sfnt(sfnt* sfont)
   return cffont;
 }
 
-static pdf_obj *
-create_ToUnicode_cmap4 (struct cmap4 *map,
-			const char *cmap_name, CMap *cmap_add,
-			const char *used_glyphs,
-			sfnt *sfont)
+static USHORT
+create_ToUnicode_cmap4 (CMap *cmap,
+                        struct cmap4 *map,
+                        char *used_glyphs,
+                        cff_font *cffont)
 {
-  pdf_obj  *stream = NULL;
-  CMap     *cmap;
   USHORT    c0, c1, gid, count, ch;
   USHORT    i, j, d, segCount;
-  char      used_glyphs_copy[8192];
-  cff_font *cffont = prepare_CIDFont_from_sfnt(sfont);
-
-  cmap = CMap_new();
-  CMap_set_name (cmap, cmap_name);
-  CMap_set_wmode(cmap, 0);
-  CMap_set_type (cmap, CMAP_TYPE_TO_UNICODE);
-  CMap_set_CIDSysInfo(cmap, &CSI_UNICODE);
-  CMap_add_codespacerange(cmap, srange_min, srange_max, 2);
-
-  memcpy(used_glyphs_copy, used_glyphs, 8192);
 
   segCount = map->segCountX2 / 2;
   for (count = 0, i = 0; i < segCount; i++) {
@@ -1022,7 +1009,7 @@ create_ToUnicode_cmap4 (struct cmap4 *map,
 	gid = (map->glyphIndexArray[j+d] +
 	       map->idDelta[i]) & 0xffff;
       }
-      if (is_used_char2(used_glyphs_copy, gid)) {
+      if (is_used_char2(used_glyphs, gid)) {
         unsigned int cid = cffont ? cff_charsets_lookup_inverse(cffont, gid) : gid;
 	count++;
 
@@ -1039,55 +1026,30 @@ create_ToUnicode_cmap4 (struct cmap4 *map,
          * mapping of ligatures encoded in PUA in fonts like Linux Libertine
          * and old Adobe fonts.
          */
-	if (!is_PUA_or_presentation(ch))
-	  /* Avoid duplicate entry
-	   * There are problem when two Unicode code is mapped to
-	   * single glyph...
-	   */
-	  used_glyphs_copy[gid/8] &= ~(1 << (7 - (gid % 8)));
-	count++;
+        if (!is_PUA_or_presentation(ch)) {
+          /* Avoid duplicate entry
+           * There are problem when two Unicode code is mapped to
+           * single glyph...
+           */
+          used_glyphs[gid/8] &= ~(1 << (7 - (gid % 8)));
+        }
+        count++;
       }
     }
   }
 
-  count += handle_subst_glyphs(cmap, cmap_add, used_glyphs_copy, sfont, cffont);
-
-  if (count < 1)
-    stream = NULL;
-  else {
-    stream = CMap_create_stream(cmap, 0);
-  }
-  CMap_release(cmap);
-
-  if (cffont)
-    cff_close(cffont);
-
-  return stream;
+  return count;
 }
 
-
-static pdf_obj *
-create_ToUnicode_cmap12 (struct cmap12 *map,
-			 const char *cmap_name, CMap *cmap_add,
-			 const char *used_glyphs,
-			 sfnt *sfont)
+static USHORT
+create_ToUnicode_cmap12 (CMap *cmap,
+                         struct cmap12 *map,
+                         char *used_glyphs,
+                         cff_font *cffont)
 {
-  pdf_obj  *stream = NULL;
-  CMap     *cmap;
-  ULONG     i, ch;
-  USHORT    gid, count;
-  char      used_glyphs_copy[8192];
-  cff_font *cffont = prepare_CIDFont_from_sfnt(sfont);
+  USHORT i, gid, ch, count = 0;
 
-  cmap = CMap_new();
-  CMap_set_name (cmap, cmap_name);
-  CMap_set_wmode(cmap, 0);
-  CMap_set_type (cmap, CMAP_TYPE_TO_UNICODE);
-  CMap_set_CIDSysInfo(cmap, &CSI_UNICODE);
-  CMap_add_codespacerange(cmap, srange_min, srange_max, 2);
-
-  memcpy(used_glyphs_copy, used_glyphs, 8192);
-  for (count = 0, i = 0; i < map->nGroups; i++) {
+  for (i = 0; i < map->nGroups; i++) {
     for (ch  = map->groups[i].startCharCode;
          ch <= map->groups[i].endCharCode; ch++) {
       unsigned char *p;
@@ -1097,7 +1059,7 @@ create_ToUnicode_cmap12 (struct cmap12 *map,
       p   = wbuf + 2;
       d   = ch - map->groups[i].startCharCode;
       gid = (USHORT) ((map->groups[i].startGlyphID + d) & 0xffff);
-      if (is_used_char2(used_glyphs_copy, gid)) {
+      if (is_used_char2(used_glyphs, gid)) {
         unsigned int cid = cffont ? cff_charsets_lookup_inverse(cffont, gid) : gid;
         count++;
         wbuf[0] = (cid >> 8) & 0xff;
@@ -1111,18 +1073,86 @@ create_ToUnicode_cmap12 (struct cmap12 *map,
          * mapping of ligatures encoded in PUA in fonts like Linux Libertine
          * and old Adobe fonts.
          */
-	if (!is_PUA_or_presentation(ch))
-	  /* Avoid duplicate entry
-	   * There are problem when two Unicode code is mapped to
-	   * single glyph...
-	   */
-	  used_glyphs_copy[gid/8] &= ~(1 << (7 - (gid % 8)));
+        if (!is_PUA_or_presentation(ch)) {
+          /* Avoid duplicate entry
+           * There are problem when two Unicode code is mapped to
+           * single glyph...
+           */
+          used_glyphs[gid/8] &= ~(1 << (7 - (gid % 8)));
+        }
         count++;
       }
     }
   }
 
-  count += handle_subst_glyphs(cmap, cmap_add, used_glyphs_copy, sfont, cffont);
+  return count;
+}
+
+static pdf_obj *
+create_ToUnicode_cmap (tt_cmap *ttcmap,
+                       const char *cmap_name,
+                       CMap *cmap_add,
+                       const char *used_glyphs,
+                       sfnt *sfont,
+                       CMap *cmap_loaded)
+{
+  pdf_obj  *stream = NULL;
+  CMap     *cmap;
+  USHORT    i, gid, ch, count = 0;
+  char      used_glyphs_copy[8192];
+  cff_font *cffont = prepare_CIDFont_from_sfnt(sfont);
+
+  cmap = CMap_new();
+  CMap_set_name (cmap, cmap_name);
+  CMap_set_wmode(cmap, 0);
+  CMap_set_type (cmap, CMAP_TYPE_TO_UNICODE);
+  CMap_set_CIDSysInfo(cmap, &CSI_UNICODE);
+  CMap_add_codespacerange(cmap, srange_min, srange_max, 2);
+
+  if (cmap_loaded && cffont) {
+    for (i = 0; i < 8192; i++) {
+      int   j;
+      long  len, inbytesleft, outbytesleft;
+      const unsigned char *inbuf;
+      unsigned char *outbuf;
+
+      if (used_glyphs[i] == 0)
+        continue;
+
+      for (j = 0; j < 8; j++) {
+        unsigned int cid;
+        int ch;
+        gid = 8 * i + j;
+
+        if (!is_used_char2(used_glyphs, gid))
+          continue;
+
+        cid = cff_charsets_lookup_inverse(cffont, gid);
+        ch = CMap_reverse_decode(cmap_loaded, cid);
+        if (ch >= 0) {
+          unsigned char *p = wbuf + 2;
+          wbuf[0] = (cid >> 8) & 0xff;
+          wbuf[1] =  cid & 0xff;
+          len = UC_sput_UTF16BE((long)ch, &p, wbuf + WBUF_SIZE);
+          CMap_add_bfchar(cmap, wbuf, 2, wbuf + 2, len);
+          count++;
+        }
+      }
+    }
+  } else {
+    memcpy(used_glyphs_copy, used_glyphs, 8192);
+
+    switch (ttcmap->format) {
+      case 4:
+        count = create_ToUnicode_cmap4(cmap, ttcmap->map, used_glyphs_copy, cffont);
+        break;
+      case 12:
+        count = create_ToUnicode_cmap12(cmap, ttcmap->map, used_glyphs_copy, cffont);
+        break;
+    }
+
+    count += handle_subst_glyphs(cmap, cmap_add, used_glyphs_copy, sfont, cffont);
+  }
 
   if (count < 1)
     stream = NULL;
@@ -1152,13 +1182,14 @@ static cmap_plat_enc_rec cmap_plat_encs[] = {
 
 pdf_obj *
 otf_create_ToUnicode_stream (const char *font_name,
-			     int ttc_index, /* 0 for non-TTC */
-			     const char *used_glyphs)
+                             int ttc_index, /* 0 for non-TTC */
+                             const char *used_glyphs,
+                             int cmap_id)
 {
   pdf_obj    *cmap_ref = NULL;
   long        res_id;
   pdf_obj    *cmap_obj = NULL;
-  CMap       *cmap_add;
+  CMap       *cmap_add, *cmap_loaded;
   int         cmap_add_id;
   tt_cmap    *ttcmap;
   char       *normalized_font_name;
@@ -1226,6 +1257,8 @@ otf_create_ToUnicode_stream (const char *font_name,
     ERROR("Could not read OpenType/TrueType table directory.");
   }
 
+  cmap_loaded = CMap_cache_get(cmap_id);
+
   cmap_add_id = CMap_cache_find(cmap_name);
   if (cmap_add_id < 0) {
     cmap_add = NULL;
@@ -1238,14 +1271,10 @@ otf_create_ToUnicode_stream (const char *font_name,
     ttcmap = tt_cmap_read(sfont, cmap_plat_encs[i].platform, cmap_plat_encs[i].encoding);
     if (!ttcmap)
       continue;
-    if (ttcmap->format == 4) {
-      cmap_obj = create_ToUnicode_cmap4(ttcmap->map,
-					cmap_name, cmap_add, used_glyphs, sfont);
-      break;
-    }
-    if (ttcmap->format == 12) {
-      cmap_obj = create_ToUnicode_cmap12(ttcmap->map,
-				       cmap_name, cmap_add, used_glyphs, sfont);
+
+    if (ttcmap->format == 4 || ttcmap->format == 12) {
+      cmap_obj = create_ToUnicode_cmap(ttcmap, cmap_name, cmap_add, used_glyphs,
+                                       sfont, cmap_loaded);
       break;
     }
   }
diff --git a/Build/source/texk/dvipdfm-x/tt_cmap.h b/Build/source/texk/dvipdfm-x/tt_cmap.h
index 7aefdbf0ffc..afc492cb127 100644
--- a/Build/source/texk/dvipdfm-x/tt_cmap.h
+++ b/Build/source/texk/dvipdfm-x/tt_cmap.h
@@ -67,8 +67,9 @@ extern void     tt_cmap_release (tt_cmap *cmap);
 
 /* Indirect reference */
 extern pdf_obj *otf_create_ToUnicode_stream (const char *map_name,
-					     int ttc_index,
-					     const char *used_glyphs);
+                                             int ttc_index,
+                                             const char *used_glyphs,
+                                             int cmap_id);
 /* CMap ID */
 extern int      otf_load_Unicode_CMap       (const char *map_name,
 					     int ttc_index,
diff --git a/Build/source/texk/dvipdfm-x/type0.c b/Build/source/texk/dvipdfm-x/type0.c
index 0a768982cac..9015941f20c 100644
--- a/Build/source/texk/dvipdfm-x/type0.c
+++ b/Build/source/texk/dvipdfm-x/type0.c
@@ -93,6 +93,7 @@ struct Type0Font {
   CIDFont *descendant; /* Only single descendant is allowed. */
   int      flags;
   int      wmode;
+  int      cmap_id;
 
   /*
    * PDF Font Resource
@@ -116,6 +117,7 @@ Type0Font_init_font_struct (Type0Font *font)
   font->used_glyphs = NULL;
   font->descendant = NULL;
   font->wmode      = -1;
+  font->cmap_id    = -1;
   font->flags      = FLAG_NONE;
 
   return;
@@ -161,7 +163,8 @@ Type0Font_create_ToUnicode_stream(Type0Font *font) {
 
   return otf_create_ToUnicode_stream(CIDFont_get_ident(cidfont),
                                      CIDFont_get_opt_index(cidfont),
-                                     used);
+                                     used,
+                                     font->cmap_id);
 }
 
 /* Try to load ToUnicode CMap from file system first, if not found fallback to
@@ -466,6 +469,7 @@ Type0Font_cache_find (const char *map_name, int cmap_id, fontmap_opt *fmap_opt)
     strcpy(font->encoding, "Identity-H");
   }
   font->wmode = wmode;
+  font->cmap_id = cmap_id;
 
   /*
    * Now we start font dictionary.