give up on Unifont for charwidth data (#150)

* fix CHARBOUND option for non-characters

* give up on unifont for charwidth computation

data/Makefile

@@ -1,7 +1,7 @@
 # Unicode data generation rules.  Except for the test data files, most
 # users will not use these Makefile rules, which are primarily to re-generate
 # unicode_data.c when we get a new Unicode version or charwidth data; they
-# require ruby, fontforge, and julia to be installed.
+# require ruby and julia to be installed.
 
 # programs
 CURL=curl
@@ -9,7 +9,6 @@ RUBY=ruby
 PERL=perl
 MAKE=make
 JULIA=julia
-FONTFORGE=fontforge
 CURLFLAGS = --retry 5 --location
 
 .PHONY: clean
@@ -19,19 +18,7 @@ CURLFLAGS = --retry 5 --location
 utf8proc_data.c.new: data_generator.rb UnicodeData.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt CharWidths.txt emoji-data.txt
 	$(RUBY) data_generator.rb < UnicodeData.txt > $@
 
-# GNU Unifont version for font metric calculations:
+CharWidths.txt: charwidths.jl EastAsianWidth.txt
-UNIFONT_VERSION=12.0.01
-
-unifont.ttf:
-	$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont-$(UNIFONT_VERSION).ttf
-
-unifont_upper.ttf:
-	$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont_upper-$(UNIFONT_VERSION).ttf
-
-%.sfd: %.ttf
-	$(FONTFORGE) -lang=ff -c "Open(\"$<\");Save(\"$@\");Quit(0);"
-
-CharWidths.txt: charwidths.jl unifont.sfd unifont_upper.sfd EastAsianWidth.txt
 	$(JULIA) charwidths.jl > $@
 
 # Unicode data version
@@ -65,5 +52,5 @@ emoji-data.txt:
 	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://unicode.org/Public/emoji/`echo $(UNICODE_VERSION) | cut -d. -f1-2`/emoji-data.txt
 
 clean:
-	rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt unifont*.ttf unifont*.sfd emoji-data.txt
+	rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt emoji-data.txt
 	rm -f utf8proc_data.c.new

data/charwidths.jl

@@ -1,9 +1,12 @@
 # Following work by @jiahao, we compute character widths using a combination of
-#   * advance widths from GNU Unifont (advance width 512 = 1 en)
+#   * character category
 #   * UAX 11: East Asian Width
 #   * a few exceptions as needed
 # Adapted from http://nbviewer.ipython.org/gist/jiahao/07e8b08bf6d8671e9734
 #
+# We used to also use data from GNU Unifont, but that has proven unreliable
+# and unlikely to match widths assumed by terminals.
+#
 # Requires Julia (obviously) and FontForge.
 
 #############################################################################
@@ -49,7 +52,7 @@ const UTF8PROC_CATEGORY_CO = 29
 #############################################################################
 # Use a default width of 1 for all character categories that are
 # letter/symbol/number-like, as well as for unassigned/private-use chars.
-# This can be overriden by Unifont or UAX 11
+# This can be overriden by UAX 11
 # below, but provides a useful nonzero fallback for new codepoints when
 # a new Unicode version has been released but Unifont hasn't been updated yet.
 
@@ -58,7 +61,6 @@ push!(zerowidth, UTF8PROC_CATEGORY_MN)
 push!(zerowidth, UTF8PROC_CATEGORY_MC)
 push!(zerowidth, UTF8PROC_CATEGORY_ME)
 push!(zerowidth, UTF8PROC_CATEGORY_SK)
-push!(zerowidth, UTF8PROC_CATEGORY_ZS)
 push!(zerowidth, UTF8PROC_CATEGORY_ZL)
 push!(zerowidth, UTF8PROC_CATEGORY_ZP)
 push!(zerowidth, UTF8PROC_CATEGORY_CC)
@@ -70,45 +72,9 @@ for c in 0x0000:0x110000
     end
 end
 
-#############################################################################
-# Widths from GNU Unifont
-
-#Read sfdfile for character widths
-function parsesfd(filename::AbstractString, CharWidths::Dict{Int,Int}=Dict{Int,Int}())
-    state=:seekchar
-    lineno = 0
-    codepoint = width = nothing
-    for line in readlines(open(filename))
-        lineno += 1
-        if state==:seekchar         #StartChar: nonmarkingreturn
-            if occursin("StartChar: ", line)
-                codepoint = nothing
-                width = nothing
-                state = :readdata
-            end
-        elseif state==:readdata #Encoding: 65538 -1 2, Width: 1024
-            occursin("Encoding:", line) && (codepoint = parse(Int, split(line)[3]))
-            occursin("Width:", line) && (width = parse(Int, split(line)[2]))
-            if codepoint!=nothing && width!=nothing && codepoint >= 0
-                w=div(width, 512) # 512 units to the en
-                if w > 0
-                    # only add nonzero widths, since (1) the default is zero
-                    # and (2) this circumvents some apparent bugs in Unifont
-                    # (https://savannah.gnu.org/bugs/index.php?45395)
-                    CharWidths[codepoint] = w
-                end
-                state = :seekchar
-            end
-        end
-    end
-    CharWidths
-end
-CharWidths=parsesfd("unifont.sfd", CharWidths)
-CharWidths=parsesfd("unifont_upper.sfd", CharWidths)
-
 #############################################################################
 # Widths from UAX #11: East Asian Width
-#   .. these take precedence over the Unifont width for all codepoints
+#   .. these take precedence for all codepoints
 #      listed explicitly as wide/full/narrow/half-width
 
 for line in readlines(open("EastAsianWidth.txt"))
@@ -180,16 +146,6 @@ CharWidths[0x00ad]=1
 CharWidths[0x2028]=0
 CharWidths[0x2029]=0
 
-#By definition, should be narrow = width of 1 en space
-#0x00202f ' ' category: Zs name: NARROW NO-BREAK SPACE/
-CharWidths[0x202f]=1
-
-#By definition, should be wide = width of 1 em space
-#0x002001 ' ' category: Zs name: EM QUAD/
-#0x002003 ' ' category: Zs name: EM SPACE/
-CharWidths[0x2001]=2
-CharWidths[0x2003]=2
-
 #############################################################################
 # Output (to a file or pipe) for processing by data_generator.rb,
 # encoded as a sequence of intervals.