Fix #34 handle 66 Unicode non-characters, also improve performance and surrogate handling

test/charwidth.c

@@ -10,7 +10,7 @@ int my_isprint(int c) {
 
 int main(int argc, char **argv)
 {
-     int prevc, c, error = 0;
+     int c, error = 0;
 
      (void) argc; /* unused */
      (void) argv; /* unused */

test/graphemetest.c

@@ -6,6 +6,7 @@ int main(int argc, char **argv)
     size_t bufsize = 0;
     FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
     utf8proc_uint8_t src[1024];
+    int len;
     
     check(f != NULL, "error opening GraphemeBreakTest.txt");
     while (getline(&buf, &bufsize, f) > 0) {
@@ -29,9 +30,10 @@ int main(int argc, char **argv)
             else if (buf[bi] == '#') { /* start of comments */
                 break;
             }
-            else { /* hex-encoded codepoint */
+	    else { /* hex-encoded codepoint */
-                bi += encode((char*) (src + si), buf + bi) - 1;
+                len = encode((char*) (src + si), buf + bi) - 1;
                 while (src[si]) ++si; /* advance to NUL termination */
+                bi += len;
             }
         }
         if (si && src[si-1] == '/')
@@ -59,7 +61,7 @@ int main(int argc, char **argv)
                        utf8proc_errmsg(glen));
                  for (i = 0; i <= glen; ++i)
                       if (g[i] == 0xff)
-                           g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
+                          g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
                  check(!strcmp((char*)g, (char*)src),
                        "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
             }

test/tests.h

@@ -33,10 +33,11 @@ size_t skipspaces(const char *buf, size_t i)
    separated by whitespace, and terminated by any character not in
    [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string
    in dest, returning the number of bytes read from buf */
+utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst);
 size_t encode(char *dest, const char *buf)
 {
      size_t i = 0, j, d = 0;
-     do {
+     for (;;) {
           int c;
           i = skipspaces(buf, i);
           for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j)
@@ -45,9 +46,9 @@ size_t encode(char *dest, const char *buf)
                dest[d] = 0; /* NUL-terminate destination string */
                return i + 1;
           }
-          check(sscanf(buf + i, "%x", &c) == 1, "invalid hex input %s", buf+i);
+          check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
           i = j; /* skip to char after hex input */
-          d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d));
+          d += unsafe_encode_char(c, (utf8proc_uint8_t *) (dest + d));
-     } while (1);
+     }
 }
 

utf8proc.c

@@ -108,51 +108,57 @@ UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
   }
 }
 
+#define utf_cont(ch)  (((ch) & 0xc0) == 0x80)
 UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(
   const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
 ) {
-  int length;
+  utf8proc_uint32_t uc;
-  int i;
+  const utf8proc_uint8_t *end;
-  utf8proc_int32_t uc = -1;
+
   *dst = -1;
   if (!strlen) return 0;
-  length = utf8proc_utf8class[str[0]];
+  end = str + ((strlen < 0) ? 4 : strlen);
-  if (!length) return UTF8PROC_ERROR_INVALIDUTF8;
+  uc = *str++;
-  if (strlen >= 0 && length > strlen) return UTF8PROC_ERROR_INVALIDUTF8;
+  if (uc < 0x80) {
-  for (i=1; i<length; i++) {
+    *dst = uc;
-    if ((str[i] & 0xC0) != 0x80) return UTF8PROC_ERROR_INVALIDUTF8;
+    return 1;
   }
-  switch (length) {
+  // Must be between 0xc2 and 0xf4 inclusive to be valid
-    case 1:
+  if ((uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
-    uc = str[0];
+  if (uc < 0xe0) {         // 2-byte sequence
-    break;
+     // Must have valid continuation character
-    case 2:
+     if (!utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
-    uc = ((str[0] & 0x1F) <<  6) + (str[1] & 0x3F);
+     *dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
-    if (uc < 0x80) uc = -1;
+     return 2;
-    break;
-    case 3:
-    uc = ((str[0] & 0x0F) << 12) + ((str[1] & 0x3F) <<  6)
-      + (str[2] & 0x3F);
-    if (uc < 0x800 || (uc >= 0xD800 && uc < 0xE000) ||
-      (uc >= 0xFDD0 && uc < 0xFDF0)) uc = -1;
-    break;
-    case 4:
-    uc = ((str[0] & 0x07) << 18) + ((str[1] & 0x3F) << 12)
-      + ((str[2] & 0x3F) <<  6) + (str[3] & 0x3F);
-    if (uc < 0x10000 || uc >= 0x110000) uc = -1;
-    break;
   }
-  if (uc < 0 || ((uc & 0xFFFF) >= 0xFFFE))
+  if (uc < 0xf0) {        // 3-byte sequence
-    return UTF8PROC_ERROR_INVALIDUTF8;
+     if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
-  *dst = uc;
+        return UTF8PROC_ERROR_INVALIDUTF8;
-  return length;
+     // Check for surrogate chars
+     if (uc == 0xed && *str > 0x9f)
+         return UTF8PROC_ERROR_INVALIDUTF8;
+     uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
+     if (uc < 0x800)
+         return UTF8PROC_ERROR_INVALIDUTF8;
+     *dst = uc;
+     return 3;
+  }
+  // 4-byte sequence
+  // Must have 3 valid continuation characters
+  if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
+     return UTF8PROC_ERROR_INVALIDUTF8;
+  // Make sure in correct range (0x10000 - 0x10ffff)
+  if (uc == 0xf0) {
+    if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
+  } else if (uc == 0xf4) {
+    if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
+  }
+  *dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
+  return 4;
 }
 
 UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
-  if (uc < 0 || uc >= 0x110000 ||
+    return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
-    ((uc & 0xFFFF) >= 0xFFFE) || (uc >= 0xD800 && uc < 0xE000) ||
-    (uc >= 0xFDD0 && uc < 0xFDF0)) return false;
-  else return true;
 }
 
 UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
@@ -165,12 +171,8 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, ut
     dst[0] = 0xC0 + (uc >> 6);
     dst[1] = 0x80 + (uc & 0x3F);
     return 2;
-  } else if (uc == 0xFFFF) {
+  // Note: we allow encoding 0xd800-0xdfff here, so as not to change
-    dst[0] = 0xFF;
+  // the API, however, these are actually invalid in UTF-8
-    return 1;
-  } else if (uc == 0xFFFE) {
-    dst[0] = 0xFE;
-    return 1;
   } else if (uc < 0x10000) {
     dst[0] = 0xE0 + (uc >> 12);
     dst[1] = 0x80 + ((uc >> 6) & 0x3F);
@@ -186,7 +188,38 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, ut
 }
 
 /* internal "unsafe" version that does not check whether uc is in range */
-static const utf8proc_property_t *get_property(utf8proc_int32_t uc) {
+utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
+   if (uc < 0x00) {
+      return 0;
+   } else if (uc < 0x80) {
+      dst[0] = uc;
+      return 1;
+   } else if (uc < 0x800) {
+      dst[0] = 0xC0 + (uc >> 6);
+      dst[1] = 0x80 + (uc & 0x3F);
+      return 2;
+   } else if (uc == 0xFFFF) {
+       dst[0] = 0xFF;
+       return 1;
+   } else if (uc == 0xFFFE) {
+       dst[0] = 0xFE;
+       return 1;
+   } else if (uc < 0x10000) {
+      dst[0] = 0xE0 + (uc >> 12);
+      dst[1] = 0x80 + ((uc >> 6) & 0x3F);
+      dst[2] = 0x80 + (uc & 0x3F);
+      return 3;
+   } else if (uc < 0x110000) {
+      dst[0] = 0xF0 + (uc >> 18);
+      dst[1] = 0x80 + ((uc >> 12) & 0x3F);
+      dst[2] = 0x80 + ((uc >> 6) & 0x3F);
+      dst[3] = 0x80 + (uc & 0x3F);
+      return 4;
+   } else return 0;
+}
+
+/* internal "unsafe" version that does not check whether uc is in range */
+static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
   /* ASSERT: uc >= 0 && uc < 0x110000 */
   return utf8proc_properties + (
     utf8proc_stage2table[
@@ -196,7 +229,7 @@ static const utf8proc_property_t *get_property(utf8proc_int32_t uc) {
 }
 
 UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
-  return uc < 0 || uc >= 0x110000 ? utf8proc_properties : get_property(uc);
+  return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
 }
 
 /* return whether there is a grapheme break between boundclasses lbc and tbc */
@@ -255,7 +288,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc,
   utf8proc_propval_t category;
   utf8proc_int32_t hangul_sindex;
   if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
-  property = get_property(uc);
+  property = unsafe_get_property(uc);
   category = property->category;
   hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
   if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
@@ -401,8 +434,8 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
       const utf8proc_property_t *property1, *property2;
       uc1 = buffer[pos];
       uc2 = buffer[pos+1];
-      property1 = get_property(uc1);
+      property1 = unsafe_get_property(uc1);
-      property2 = get_property(uc2);
+      property2 = unsafe_get_property(uc2);
       if (property1->combining_class > property2->combining_class &&
           property2->combining_class > 0) {
         buffer[pos] = uc2;
@@ -460,7 +493,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer,
     utf8proc_int32_t composition;
     for (rpos = 0; rpos < length; rpos++) {
       current_char = buffer[rpos];
-      current_property = get_property(current_char);
+      current_property = unsafe_get_property(current_char);
       if (starter && current_property->combining_class > max_combining_class) {
         /* combination perhaps possible */
         utf8proc_int32_t hangul_lindex;
@@ -489,7 +522,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer,
           }
         }
         if (!starter_property) {
-          starter_property = get_property(*starter);
+          starter_property = unsafe_get_property(*starter);
         }
         if (starter_property->comb1st_index >= 0 &&
             current_property->comb2nd_index >= 0) {
@@ -498,7 +531,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer,
             current_property->comb2nd_index
           ];
           if (composition >= 0 && (!(options & UTF8PROC_STABLE) ||
-              !(get_property(composition)->comp_exclusion))) {
+              !(unsafe_get_property(composition)->comp_exclusion))) {
             *starter = composition;
             starter_property = NULL;
             continue;
@@ -522,9 +555,16 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer,
   {
     utf8proc_ssize_t rpos, wpos = 0;
     utf8proc_int32_t uc;
-    for (rpos = 0; rpos < length; rpos++) {
+    if (options & UTF8PROC_CHARBOUND) {
-      uc = buffer[rpos];
+        for (rpos = 0; rpos < length; rpos++) {
-      wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+            uc = buffer[rpos];
+            wpos += unsafe_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+        }
+    } else {
+        for (rpos = 0; rpos < length; rpos++) {
+            uc = buffer[rpos];
+            wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+        }
     }
     ((utf8proc_uint8_t *)buffer)[wpos] = 0;
     return wpos;

utf8proc.h

@@ -82,10 +82,13 @@ typedef unsigned char utf8proc_uint8_t;
 typedef short utf8proc_int16_t;
 typedef unsigned short utf8proc_uint16_t;
 typedef int utf8proc_int32_t;
+typedef unsigned int utf8proc_uint32_t;
 #  ifdef _WIN64
 typedef __int64 utf8proc_ssize_t;
+typedef unsigned __int64 utf8proc_size_t;
 #  else
 typedef int utf8proc_ssize_t;
+typedef unsigned int utf8proc_size_t;
 #  endif
 #  ifndef __cplusplus
 typedef unsigned char utf8proc_bool;
@@ -101,6 +104,8 @@ typedef uint8_t utf8proc_uint8_t;
 typedef int16_t utf8proc_int16_t;
 typedef uint16_t utf8proc_uint16_t;
 typedef int32_t utf8proc_int32_t;
+typedef uint32_t utf8proc_uint32_t;
+typedef size_t utf8proc_size_t;
 typedef ssize_t utf8proc_ssize_t;
 typedef bool utf8proc_bool;
 #endif