Update tools/tools/locale

[hrs]

Summary

Update tools/tools/locale to add make targets to automatically generate locale source files. With this patch, just typing "make obj && make" should build them.

Fix issues in ja_JP ctypedef and range specification support in utf8-rollup.pl.

Details are as follows:

Makefile:

• Support .OBJDIR.
• Support pararell build.
• Add a check of the installed tools.
• Update to use CLDR v35.

tools/cldr2def.pl:

• Use eucJP for ja_JP ctypedef because eucJP is not compatible with UTF-8.

tools/convert_map.pl:

• Add a verbose error message.

tools/utf8-rollup.pl:

• Normalize entries to use Unicode, not UTF-8.

[bapt]

It looks good to me. Don't forget to mention in the commit log that it updates also Unicode to Unicode 13

[yuripv]

As I was going through MFCing locale changes, I noticed one little (BIG) problem with this review/commit -- it forgot to actually install the updated sources to share/ subdirectories, so we currently ship what was previous version in share/.

I'll take care of this in upcoming changes, no action required.

tools/tools/locale/patch/patch-UnicodeData.txt

1 ↗

(On Diff #73860)

Was this submitted upstream as it's pretty hard to maintain such patches?

head/tools/tools/locale/Makefile

 # $FreeBSD$
 # See https://wiki.freebsd.org/LocaleNewApproach
 # Taken from FreeBSD svn [base]/user/edwin/locale/cldr
 #
 # needs:
 # devel/p5-Tie-IxHash
 #
 # Modified by John Marino to suit DragonFly needs
 #
+.if ${.CURDIR} == ${.OBJDIR}
+.error Do make obj first.
+.endif
 
-.OBJDIR:        .
+LOCALESRCDIR?=        ${DESTDIR}/usr/src/share
+TMPDIR?=        /tmp
 
-.if !defined(UNIDIR)
-.error UNIDIR is not set
-.endif
-PASSON=                UNIDIR="${UNIDIR}"
+BASEDIR=        ${.CURDIR}
+ETCDIR=                ${BASEDIR}/etc
+TOOLSDIR=        ${BASEDIR}/tools
+PATCHDIR=        ${BASEDIR}/patch
+UNIDIR=         ${.OBJDIR:tA}/unicode
 
-ETCDIR=                ${.CURDIR}/etc
+PKGS=        openjdk8 \
+        apache-ant \
+        p5-XML-Parser \
+        p5-Tie-IxHash \
+        p5-Text-Iconv
+tools-test:
+        pkg info -e ${PKGS}
+        @echo tools ok.
 
 KNOWN=                monetdef numericdef msgdef colldef ctypedef # timedef
 TYPES?=         ${KNOWN}
-LOCALE_DESTDIR?=        /tmp/generated-locales/
 
 COLLATION_SPECIAL?= \
         cs_CZ ISO8859-2 \
         da_DK ISO8859-1 \
         da_DK ISO8859-15 \
         hr_HR ISO8859-2 \
         hu_HU ISO8859-2 \
         nb_NO ISO8859-1 \
         nb_NO ISO8859-15 \
         sk_SK ISO8859-2 \
         sr_Latn_RS ISO8859-2 \
         sr_Cyrl_RS ISO8859-5 \
         zh_Hans_CN GB2312 \
         zh_Hans_CN eucCN \
         zh_Hant_TW Big5 \
         zh_Hans_CN GB18030 \
         zh_Hans_CN GBK \
         ja_JP eucJP \
         nn_NO ISO8859-15 \
         nn_NO ISO8859-1
 
 .for area enc in ${COLLATION_SPECIAL}
 COLLATIONS_SPECIAL_ENV+=        ${area}.${enc}
 .endfor
-PASSON+=        COLLATIONS_SPECIAL="${COLLATIONS_SPECIAL_ENV}"
+SETENV=        env -i \
+        PATH="${PATH}" \
+        TMPDIR="${TMPDIR}" \
+        COLLATIONS_SPECIAL="${COLLATIONS_SPECIAL_ENV}" \
+        UNIDIR="${UNIDIR}" \
+        BASEDIR="${BASEDIR}" \
+        TOOLSDIR="${TOOLSDIR}" \
+        ETCDIR="${ETCDIR}"
 
-all:
+all: posix build afterbuild
+.ORDER: posix build afterbuild
+
+afterbuild: build
+        @echo ""
+        @find . -name *failed
+
 .for t in ${TYPES}
 .  if ${KNOWN:M${t}}
-        test -d ${t} || mkdir ${t}
-        make build-${t}
+build: build-${t}
+.ORDER: build-${t} afterbuild
 .  endif
 .endfor
-        @echo ""
-        @find . -name *failed
 
+diff:
 .for t in ${TYPES}
+.  if ${KNOWN:M${t}}
+diff: diff-${t}
+diff-${t}:
+        -/usr/bin/diff -ruN -x Makefile -x Makefile.depend \
+            ${LOCALESRCDIR}/${t} ${t}
+.  endif
+.endfor
+
+install:
+.for t in ${TYPES}
+.  if ${KNOWN:M${t}}
 install: install-${t}
 install-${t}:
-.  if ${KNOWN:M${t}}
-        rm -rf ${.CURDIR}/${t}.draft
-        rm -f ${.CURDIR}/../../../share/${t}/Makefile
-        rm -f ${.CURDIR}/../../../share/${t}/*.src
-        mv ${.CURDIR}/${t}/* ${.CURDIR}/../../../share/${t}/
+        cd ${LOCALESRCDIR}/${t} && \
+            rm -f Makefile *.src && \
+            install -c ${t}/* ${LOCALESRCDIR}/${t}
 .  endif
 .endfor
 
 post-install:
 .for t in ${TYPES}
 .  if ${KNOWN:M${t}}
-        (cd ${.CURDIR}/../../../share/${t} && \
-                make && make install && make clean)
+        cd ${LOCALSRCDIR}/${t} && \
+            make && make install && make clean
 .  endif
 .endfor
 
 .for t in ${TYPES}
-gen-${t}:
-        mkdir -p ${t} ${t}.draft
-        perl -I tools tools/cldr2def.pl \
-                --unidir=$$(realpath ${UNIDIR}) \
-                --etc=$$(realpath ${ETCDIR}) \
+CLEANDIRS+=        ${t} ${t}.draft
+${t}:
+        mkdir -p ${t} ${t}.draft && \
+        perl -I ${TOOLSDIR} ${TOOLSDIR}/cldr2def.pl \
+                --unidir=${UNIDIR:tA} \
+                --etc=${ETCDIR:tA} \
                 --type=${t}
 
-build-${t}: gen-${t}
-        env ${PASSON} tools/finalize ${t}
+build-${t}: ${t}
+        ${SETENV} OUTBASEDIR="${.OBJDIR}/${t}" ${TOOLSDIR}/finalize ${t}
 .endfor
 
-gen-ctypedef:        ctype-rollup
-static-colldef: gen-colldef
+static-colldef: colldef
 build-colldef:  static-colldef
 
 static-colldef:
 .for area enc in ${COLLATION_SPECIAL}
-        awk -f tools/extract-colldef.awk ${UNIDIR}/posix/${area}.${enc}.src > \
-            colldef.draft/${area}.${enc}.src
+colldef.draft/${area}.${enc}.src: posix/${area}.${enc}.src
+        awk -f ${TOOLSDIR}/extract-colldef.awk \
+            ${.ALLSRC} > ${.TARGET} || (rm -f ${.TARGET} && false)
 .endfor
 
-ctype-rollup:
-        perl -I tools tools/utf8-rollup.pl --unidir=$$(realpath ${UNIDIR})
-
-clean:
-.for t in ${TYPES}
-        rm -rf ${t} ${t}.draft
-.endfor
-
 BASE_LOCALES_OF_INTEREST?= \
         af_ZA am_ET ar_AE ar_EG ar_JO ar_MA ar_QA ar_SA \
         be_BY bg_BG ca_AD ca_ES ca_FR ca_IT \
         cs_CZ da_DK de_AT de_CH de_DE el_GR en_AU en_CA \
         en_GB en_HK en_IE en_NZ en_PH en_SG en_US en_ZA \
         es_AR es_CR es_ES es_MX et_EE eu_ES fi_FI fr_BE \
         fr_CA fr_CH fr_FR ga_IE he_IL hi_IN hr_HR hu_HU hy_AM \
         is_IS it_CH it_IT ja_JP ko_KR lt_LT lv_LV \
ENCODINGS=      Big5 \
                 ISO8859-7 \
                 ISO8859-9 \
                 KOI8-R \
                 KOI8-U \
                 SJIS \
                 US-ASCII \
                 UTF-8
 
-POSIX:        posixsrc posixcol posixcm
+# CLDR files
+CLDRFILES_CORE= https://unicode.org/Public/cldr/35/core.zip
+CLDRFILES_KEY=  https://unicode.org/Public/cldr/35/keyboards.zip
+CLDRFILES_TOOLS=https://unicode.org/Public/cldr/35/tools.zip
+CLDRFILES_UCD=  http://www.unicode.org/Public/zipped/latest/UCD.zip
+
+# fetch and extract targets
+${UNIDIR}:
+        mkdir -p ${UNIDIR}
+.for N in CORE KEY TOOLS UCD
+${CLDRFILES_${N}:T}:
+        fetch ${CLDRFILES_${N}}
+fetch: ${CLDRFILES_${N}:T}
+extract-${CLDRFILES_${N}:T}:: ${CLDRFILES_${N}:T} ${UNIDIR}
+        cd ${UNIDIR} && unzip -o ../${CLDRFILES_${N}:T}
+extract: extract-${CLDRFILES_${N}:T}
+.endfor
+patch::
+.if exists(${PATCHDIR})
+        cd ${UNIDIR} && cat ${PATCHDIR}/patch-* | patch
+.endif
+
 .if !exists(${UNIDIR}/tools/java/cldr.jar)
-.error check README about building cldr.jar
+.ORDER: extract patch
+build-tools: extract patch tools-test ${UNIDIR}
+        cd ${UNIDIR}/tools/java && ${SETENV} ant all jar
+.else
+build-tools:
+        @echo cldr.jar is ready.
 .endif
+
+JAVA_CLDR= java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar
+
+posix: posixcm post-posixcm posixsrc posixcol
+.ORDER: posixcm post-posixcm posixsrc posixcol
+${UNIDIR}/posix:
+        ln -s -f ../posix ${.TARGET}
+clean-posix:
+        rm -rf posix ${UNIDIR}/posix
+post-posixcm: ${UNIDIR}/posix
+        perl -I ${TOOLSDIR} ${TOOLSDIR}/utf8-rollup.pl \
+            --unidir=${UNIDIR}
+.for enc in ${ENCODINGS}
+posixcm: build-tools posix/${enc}.cm
+.ORDER: build-tools posix/${enc}.cm
+posix/${enc}.cm:
+        mkdir -p posix && \
+            ${JAVA_CLDR} org.unicode.cldr.posix.GenerateCharmap \
+                -d posix -c ${enc}
+.endfor
 .for area in ${BASE_LOCALES_OF_INTEREST}
-posixsrc: ${UNIDIR}/posix/${area}.UTF-8.src
-${UNIDIR}/posix/${area}.UTF-8.src:
-        java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \
-                org.unicode.cldr.posix.GeneratePOSIX \
-                -d ${UNIDIR}/posix -m ${area} -c UTF-8
+posixsrc: build-tools posix/${area}.UTF-8.src
+.ORDER: build-tools posix/${area}.UTF-8.src
+posix/${area}.UTF-8.src: 
+        mkdir -p posix && \
+            ${JAVA_CLDR} org.unicode.cldr.posix.GeneratePOSIX \
+                -d posix -m ${area} -c UTF-8
 .endfor
 .for area encoding in ${COLLATION_SPECIAL}
-posixcol: ${UNIDIR}/posix/${area}.${encoding}.src
-${UNIDIR}/posix/${area}.${encoding}.src:
-        java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \
-                org.unicode.cldr.posix.GeneratePOSIX \
-                -d ${UNIDIR}/posix -m ${area} -c ${encoding}
+posixcol: build-tools posix/${area}.${encoding}.src
+.ORDER: build-tools posix/${area}.${encoding}.src
+posix/${area}.${encoding}.src:
+        mkdir -p posix && \
+            ${JAVA_CLDR} org.unicode.cldr.posix.GeneratePOSIX \
+                -d posix -m ${area} -c ${encoding}
 .endfor
-.for enc in ${ENCODINGS}
-posixcm: ${UNIDIR}/posix/${enc}.cm
-${UNIDIR}/posix/${enc}.cm:
-        java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \
-                org.unicode.cldr.posix.GenerateCharmap \
-                -d ${UNIDIR}/posix -c ${enc}
-.endfor
 
-clean-POSIX:
-        rm -f ${UNIDIR}/posix/*
+.include <bsd.obj.mk>

head/tools/tools/locale/README

 # $FreeBSD$
 
-To generate the locales:
+Files in this directory are used to generate locale source files
+from files in CLDR (Unicode Common Locale Data Repository). 
 
-Tools needed:
-        java (openjdk >= 8)
-        perl
-        converters/p5-Text-Iconv
-        devel/apache-ant
-        devel/p5-Tie-IxHash
-        textproc/p5-XML-Parser
+To generate the files, do the following:
 
-1. Fetch CLDR data from: http://unicode.org/Public/cldr/.  You need all of the
-core.zip, keyboards.zip, and tools.zip.
-2. Fetch unidata (UCD.zip) from http://www.unicode.org/Public/zipped/latest.
-3. Extract:
-        mkdir -p ~/unicode
-        cd ~/unicode
-        unzip ~/core.zip
-        unzip ~/keyboards.zip
-        unzip ~/tools.zip
-        unzip ~/UCD.zip
-4. Export variable:
-        UNIDIR=~/unicode; export UNIDIR
-5. Build the CLDR tools:
-        cd $UNIDIR/tools/java
-        ant jar
-6. Build POSIX data files from CLDR data:
-        make POSIX
-7. Build and install new locale data:
-        make
+        cd /usr/src/tools/tools/locale
+        make obj        (mandatory)
+        make -j16       (-jN recommended)
+        make diff       (check if the changes are reasonable)
         make install
+
+"make" downloads the necessary files, build them, and install the
+results into /usr/src/share/* as source files for locales.
+
+More details are as follows:
+
+Variables:
+        LOCALESRCDIR
+                Destination path for the generated locale files.
+                Default: $DESTDIR/usr/src/share.
+        TMPDIR
+                Temporary directory.
+                Default: /tmp
+
+Targets:
+        make obj
+                Create a temporary directory for building.
+
+        make clean
+                Clean up the obj directories.
+
+        make cleandir
+                Remove the obj directories completely.
+
+        make tools-test
+                Check if necessary tools are installed or not.
+                If something is missing, install them.
+
+        make fetch
+                Download necessary files from CLDR.
+
+        make build-tools
+                Build a tool to generate locale source files.
+
+        make posix
+                Build POSIX locale source files.
+
+        make build
+                Build locale files.
+
+        make diff
+                Run diff(1) the build results against $LOCALESRCDIR.
+
+        make install
+                Install the build results into $LOCALESRCDIR.
+[EOF]

head/tools/tools/locale/patch/patch-UnicodeData.txt

+--- UnicodeData.txt.orig        2020-06-29 14:05:49.483379000 +0900
++++ UnicodeData.txt     2020-06-29 14:12:09.808622000 +0900
+@@ -12138,7 +12138,7 @@
+ 33FE;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTY-ONE;So;0;L;<compat> 0033 0031 65E5;;;;N;;;;;
+ 33FF;SQUARE GAL;So;0;ON;<square> 0067 0061 006C;;;;N;;;;;
+ 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
+-4DBF;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
++4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
+ 4DC0;HEXAGRAM FOR THE CREATIVE HEAVEN;So;0;ON;;;;;N;;;;;
+ 4DC1;HEXAGRAM FOR THE RECEPTIVE EARTH;So;0;ON;;;;;N;;;;;
+ 4DC2;HEXAGRAM FOR DIFFICULTY AT THE BEGINNING;So;0;ON;;;;;N;;;;;
+@@ -12204,7 +12204,7 @@
+ 4DFE;HEXAGRAM FOR AFTER COMPLETION;So;0;ON;;;;;N;;;;;
+ 4DFF;HEXAGRAM FOR BEFORE COMPLETION;So;0;ON;;;;;N;;;;;
+ 4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;;
+-9FFC;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
++9FEF;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
+ A000;YI SYLLABLE IT;Lo;0;L;;;;;N;;;;;
+ A001;YI SYLLABLE IX;Lo;0;L;;;;;N;;;;;
+ A002;YI SYLLABLE I;Lo;0;L;;;;;N;;;;;
+@@ -32901,7 +32901,7 @@
+ 1FBF8;SEGMENTED DIGIT EIGHT;Nd;0;EN;<font> 0038;8;8;8;N;;;;;
+ 1FBF9;SEGMENTED DIGIT NINE;Nd;0;EN;<font> 0039;9;9;9;N;;;;;
+ 20000;<CJK Ideograph Extension B, First>;Lo;0;L;;;;;N;;;;;
+-2A6DD;<CJK Ideograph Extension B, Last>;Lo;0;L;;;;;N;;;;;
++2A6D6;<CJK Ideograph Extension B, Last>;Lo;0;L;;;;;N;;;;;
+ 2A700;<CJK Ideograph Extension C, First>;Lo;0;L;;;;;N;;;;;
+ 2B734;<CJK Ideograph Extension C, Last>;Lo;0;L;;;;;N;;;;;
+ 2B740;<CJK Ideograph Extension D, First>;Lo;0;L;;;;;N;;;;;