Index: head/tools/tools/locale/Makefile =================================================================== --- head/tools/tools/locale/Makefile (revision 298115) +++ head/tools/tools/locale/Makefile (revision 298116) @@ -1,123 +1,180 @@ # $FreeBSD$ # See https://wiki.freebsd.org/LocaleNewApproach # Taken from FreeBSD svn [base]/user/edwin/locale/cldr # # needs: # devel/p5-Tie-IxHash # # Modified by John Marino to suit DragonFly needs # .if !defined(CLDRDIR) CLDRDIR!= grep ^cldr etc/unicode.conf | cut -f 2 -d " " .endif .if !defined(UNIDATADIR) UNIDATADIR!= grep ^unidata etc/unicode.conf | cut -f 2 -d " " .endif PASSON= CLDRDIR="${CLDRDIR}" UNIDATADIR="${UNIDATADIR}" ETCDIR= ${.CURDIR}/etc KNOWN= monetdef numericdef msgdef timedef colldef ctypedef TYPES?= ${KNOWN} LOCALE_DESTDIR?= /tmp/generated-locales/ +COLLATION_SPECIAL?= \ + cs_CZ ISO8859-2 \ + da_DK ISO8859-1 \ + da_DK ISO8859-15 \ + hr_HR ISO8859-2 \ + hu_HU ISO8859-2 \ + nb_NO ISO8859-1 \ + nb_NO ISO8859-15 \ + sk_SK ISO8859-2 \ + zh_Hans_CN GB2312 \ + zh_Hans_CN eucCN \ + +.for area enc in ${COLLATION_SPECIAL} +COLLATIONS_SPECIAL_ENV+= ${area}.${enc} +.endfor +PASSON+= COLLATIONS_SPECIAL="${COLLATIONS_SPECIAL_ENV}" + .if defined(LC) LC:= --lc=${LC} .endif all: cp ${ETCDIR}/common.UTF-8.src ${CLDRDIR}/posix/xx_Comm_US.UTF-8.src .for t in ${TYPES} . if ${KNOWN:M${t}} test -d ${t} || mkdir ${t} make build-${t} . endif .endfor @echo "" @find . -name *failed install: .for t in ${TYPES} . if ${KNOWN:M${t}} rm -rf ${.CURDIR}/${t}.draft rm -rf ${.CURDIR}/../../../share/${t} mv ${.CURDIR}/${t} ${.CURDIR}/../../../share/${t} . endif .endfor post-install: .for t in ${TYPES} . if ${KNOWN:M${t}} (cd ${.CURDIR}/../../../share/${t} && \ make && make install && make clean) . endif .endfor .for t in ${TYPES} -build-${t}: +gen-${t}: mkdir -p ${t} ${t}.draft perl -I tools tools/cldr2def.pl \ --cldr=$$(realpath ${CLDRDIR}) \ --unidata=$$(realpath ${UNIDATADIR}) \ --etc=$$(realpath ${ETCDIR}) \ --type=${t} ${LC} + +build-${t}: gen-${t} env ${PASSON} tools/finalize ${t} .endfor -build-ctypedef: transfer-rollup +gen-ctypedef: transfer-rollup +static-colldef: gen-colldef +build-colldef: static-colldef +static-colldef: +.for area enc in ${COLLATION_SPECIAL} + awk -f tools/extract-colldef.awk ${CLDRDIR}/posix/${area}.${enc}.src > colldef/${area}.${enc}.src +.endfor + transfer-rollup: cp ${ETCDIR}/common.UTF-8.src ${CLDRDIR}/posix/xx_Comm_US.UTF-8.src rollup: perl -I tools tools/utf8-rollup.pl \ --cldr=$$(realpath ${CLDRDIR}) \ --etc=$$(realpath ${ETCDIR}) clean: .for t in ${TYPES} rm -rf ${t} ${t}.draft .endfor BASE_LOCALES_OF_INTEREST?= \ af_ZA am_ET ar_AE ar_EG ar_JO ar_MA ar_QA ar_SA \ be_BY bg_BG ca_AD ca_ES ca_FR ca_IT \ cs_CZ da_DK de_AT de_CH de_DE el_GR en_AU en_CA \ en_GB en_HK en_IE en_NZ en_PH en_SG en_US en_ZA \ es_AR es_CR es_ES es_MX et_EE eu_ES fi_FI fr_BE \ fr_CA fr_CH fr_FR he_IL hi_IN hr_HR hu_HU hy_AM \ is_IS it_CH it_IT ja_JP ko_KR lt_LT lv_LV \ nb_NO nl_BE nl_NL nn_NO pl_PL pt_BR pt_PT ro_RO \ ru_RU se_FI se_NO sk_SK sl_SI sv_FI sv_SE tr_TR \ uk_UA \ kk_Cyrl_KZ mn_Cyrl_MN sr_Cyrl_RS sr_Latn_RS \ zh_Hans_CN zh_Hant_HK zh_Hant_TW \ - \ - \ bn_IN gu_IN or_IN ta_IN te_IN kn_IN ml_IN si_LK \ th_TH lo_LA bo_IN my_MM pa_Guru_IN ka_GE chr_US \ km_KH shi_Tfng_MA ii_CN vai_Vaii_LR vi_VN +ENCODINGS= Big5 \ + CP1251 \ + CP866 \ + CP949 \ + eucCN \ + eucJP \ + eucKR \ + GB18030 \ + GB2312 \ + GBK \ + ISO8859-1 \ + ISO8859-13 \ + ISO8859-15 \ + ISO8859-2 \ + ISO8859-5 \ + ISO8859-7 \ + ISO8859-9 \ + KOI8-R \ + KOI8-U \ + SJIS \ + US-ASCII \ + UTF-8 \ + + POSIX: .if exists (${CLDRDIR}/tools/java/cldr.jar) mkdir -p ${CLDRDIR}/posix . for area in ${BASE_LOCALES_OF_INTEREST} . if !exists(${CLDRDIR}/posix/${area}.UTF-8.src) java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \ org.unicode.cldr.posix.GeneratePOSIX \ -d ${CLDRDIR}/posix -m ${area} -c UTF-8 . endif . endfor -. if !exists(${CLDRDIR}/posix/UTF-8.cm) +. for area encoding in ${COLLATION_SPECIAL} +. if !exists(${CLDRDIR}/posix/${area}.${encoding}.src) java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \ + org.unicode.cldr.posix.GeneratePOSIX \ + -d ${CLDRDIR}/posix -m ${area} -c ${encoding} +. endif +. endfor +. for enc in ${ENCODINGS} +. if !exists(${CLDRDIR}/posix/${enc}.cm) + java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \ org.unicode.cldr.posix.GenerateCharmap \ - -d ${CLDRDIR}/posix + -d ${CLDRDIR}/posix -c ${enc} . endif +. endfor .else @echo "Please install CLDR toolset for the desired release" @echo "It should go at ${CLDRDIR}/tools" .endif clean-POSIX: rm -f {CLDRDIR}/posix/* Index: head/tools/tools/locale/etc/charmaps/charmaps.txt =================================================================== --- head/tools/tools/locale/etc/charmaps/charmaps.txt (revision 298115) +++ head/tools/tools/locale/etc/charmaps/charmaps.txt (revision 298116) @@ -1,35 +1,34 @@ $FreeBSD$ Sources: unicode.org: http://unicode.org/Public/MAPPINGS IANA: http://www.iana.org/assignments/charset-reg/ haible.de: http://haible.de/bruno/charsets/conversion-tables/ ARMSCII-8 haible.de: Armenian.html Big5 unicodeorg: OBSOLETE/EASTASIA/OTHER - Big5HKSCS haible.de: BIG5-HKSCS.html / CP1131 haible.de: CP1131.html / aix-4.3.2/IBM-1131.TXT CP1251 unicode.org: VENDORS/MICSFT/WINDOWS CP866 unicode.org: VENDORS/MICSFT/PC CP949 unicode.org: VENDORS/MICSFT/WINDOWS GB18030 haible.de: GB18030.html GB2312 haible.de: GB2312.html / unicode.org-mappings/EASTASIA/GB/GB2312.TXT GBK ISCII-DEV unicode.org: VENDORS/APPLE/DEVANAGA i/ haible.de: Indic.html ISO8859-1 unicode.org: ISO8859/ ISO8859-2 unicode.org: ISO8859/ ISO8859-4 unicode.org: ISO8859/ ISO8859-5 unicode.org: ISO8859/ ISO8859-7 unicode.org: ISO8859/ ISO8859-9 unicode.org: ISO8859/ ISO8859-13 unicode.org: ISO8859/ ISO8859-15 unicode.org: ISO8859/ KOI8-R unicode.org: VENDORS/MISC KOI8-U unicode.org: VENDORS/MISC PT154 IANA: PTCP154 SJIS haible.de: Shift_JIS.html US-ASCII From ISO8859-1.TXT eucCN haible.de: GB2312.html / unicode.org-mappings/EASTASIA/GB/GB2312.TXT eucJP haible.de: EUC-JP.html / libiconv 1.13 eucKR haible.de: EUC-KR.html Index: head/tools/tools/locale/etc/charmaps.xml =================================================================== --- head/tools/tools/locale/etc/charmaps.xml (revision 298115) +++ head/tools/tools/locale/etc/charmaps.xml (revision 298116) @@ -1,744 +1,740 @@ - - - - - - - - - - - - - - - - - - - - - - - - "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> "; ""; ""; ""; ""; ""; ""; "

"; ""; ""; "

"; "

" ]]> "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> "; ""; ""; ""; ""; ""; "

"; "

"; ""; "

"; "

"; "" ]]> "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> "; ""; ""; ""; ""; ""; "

"; "

"; ""; "

"; "

"; "" ]]> "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> Index: head/tools/tools/locale/tools/cldr2def.pl =================================================================== --- head/tools/tools/locale/tools/cldr2def.pl (revision 298115) +++ head/tools/tools/locale/tools/cldr2def.pl (revision 298116) @@ -1,973 +1,985 @@ #!/usr/local/bin/perl -wC # $FreeBSD$ use strict; use File::Copy; use XML::Parser; use Tie::IxHash; use Data::Dumper; use Getopt::Long; use Digest::SHA qw(sha1_hex); require "charmaps.pm"; if ($#ARGV < 2) { print "Usage: $0 --cldr= --unidata= --etc= --type= [--lc=]\n"; exit(1); } my $DEFENCODING = "UTF-8"; my @filter = (); my $CLDRDIR = undef; my $UNIDATADIR = undef; my $ETCDIR = undef; my $TYPE = undef; my $doonly = undef; my $result = GetOptions ( "cldr=s" => \$CLDRDIR, "unidata=s" => \$UNIDATADIR, "etc=s" => \$ETCDIR, "type=s" => \$TYPE, "lc=s" => \$doonly ); my %convertors = (); my %ucd = (); my %values = (); my %hashtable = (); my %languages = (); my %translations = (); my %encodings = (); my %alternativemonths = (); get_languages(); my %utf8map = (); my %utf8aliases = (); get_unidata($UNIDATADIR); get_utf8map("$CLDRDIR/posix/$DEFENCODING.cm"); get_encodings("$ETCDIR/charmaps"); my %keys = (); tie(%keys, "Tie::IxHash"); tie(%hashtable, "Tie::IxHash"); my %FILESNAMES = ( "monetdef" => "LC_MONETARY", "timedef" => "LC_TIME", "msgdef" => "LC_MESSAGES", "numericdef" => "LC_NUMERIC", "colldef" => "LC_COLLATE", "ctypedef" => "LC_CTYPE" ); my %callback = ( mdorder => \&callback_mdorder, altmon => \&callback_altmon, cformat => \&callback_cformat, dtformat => \&callback_dtformat, cbabmon => \&callback_abmon, data => undef, ); my %DESC = ( # numericdef "decimal_point" => "decimal_point", "thousands_sep" => "thousands_sep", "grouping" => "grouping", # monetdef "int_curr_symbol" => "int_curr_symbol (last character always " . "SPACE)", "currency_symbol" => "currency_symbol", "mon_decimal_point" => "mon_decimal_point", "mon_thousands_sep" => "mon_thousands_sep", "mon_grouping" => "mon_grouping", "positive_sign" => "positive_sign", "negative_sign" => "negative_sign", "int_frac_digits" => "int_frac_digits", "frac_digits" => "frac_digits", "p_cs_precedes" => "p_cs_precedes", "p_sep_by_space" => "p_sep_by_space", "n_cs_precedes" => "n_cs_precedes", "n_sep_by_space" => "n_sep_by_space", "p_sign_posn" => "p_sign_posn", "n_sign_posn" => "n_sign_posn", # msgdef "yesexpr" => "yesexpr", "noexpr" => "noexpr", "yesstr" => "yesstr", "nostr" => "nostr", # timedef "abmon" => "Short month names", "mon" => "Long month names (as in a date)", "abday" => "Short weekday names", "day" => "Long weekday names", "t_fmt" => "X_fmt", "d_fmt" => "x_fmt", "c_fmt" => "c_fmt", "am_pm" => "AM/PM", "d_t_fmt" => "date_fmt", "altmon" => "Long month names (without case ending)", "md_order" => "md_order", "t_fmt_ampm" => "ampm_fmt", ); if ($TYPE eq "colldef") { transform_collation(); make_makefile(); } if ($TYPE eq "ctypedef") { transform_ctypes(); make_makefile(); } if ($TYPE eq "numericdef") { %keys = ( "decimal_point" => "s", "thousands_sep" => "s", "grouping" => "ai", ); get_fields(); print_fields(); make_makefile(); } if ($TYPE eq "monetdef") { %keys = ( "int_curr_symbol" => "s", "currency_symbol" => "s", "mon_decimal_point" => "s", "mon_thousands_sep" => "s", "mon_grouping" => "ai", "positive_sign" => "s", "negative_sign" => "s", "int_frac_digits" => "i", "frac_digits" => "i", "p_cs_precedes" => "i", "p_sep_by_space" => "i", "n_cs_precedes" => "i", "n_sep_by_space" => "i", "p_sign_posn" => "i", "n_sign_posn" => "i" ); get_fields(); print_fields(); make_makefile(); } if ($TYPE eq "msgdef") { %keys = ( "yesexpr" => "s", "noexpr" => "s", "yesstr" => "s", "nostr" => "s" ); get_fields(); print_fields(); make_makefile(); } if ($TYPE eq "timedef") { %keys = ( "abmon" => " "as", "abday" => "as", "day" => "as", "t_fmt" => "s", "d_fmt" => "s", "c_fmt" => " "as", "d_fmt" => "s", "d_t_fmt" => " " " "s", ); get_fields(); print_fields(); make_makefile(); } sub callback_cformat { my $s = shift; $s =~ s/ %Z//; $s =~ s/ %z//; return $s; }; sub callback_dtformat { my $s = shift; my $nl = $callback{data}{l} . "_" . $callback{data}{c}; if ($nl eq 'ja_JP') { $s =~ s/(> )(%H)/$1%A $2/; } return $s; }; sub callback_mdorder { my $s = shift; return undef if (!defined $s); $s =~ s/[^dm]//g; return $s; }; sub callback_altmon { # if the language/country is known in %alternative months then # return that, otherwise repeat mon my $s = shift; if (defined $alternativemonths{$callback{data}{l}}{$callback{data}{c}}) { my @altnames = split(";",$alternativemonths{$callback{data}{l}}{$callback{data}{c}}); my @cleaned; foreach (@altnames) { $_ =~ s/^\s+//; $_ =~ s/\s+$//; push @cleaned, $_; } return join(";",@cleaned); } return $s; } sub callback_abmon { # for specified CJK locales, pad result with a space to enable # columns to line up (style established in FreeBSD in 2001) my $s = shift; my $nl = $callback{data}{l} . "_" . $callback{data}{c}; if ($nl eq 'ja_JP' || $nl eq 'ko_KR' || $nl eq 'zh_CN' || $nl eq 'zh_HK' || $nl eq 'zh_TW') { my @monthnames = split(";", $s); my @cleaned; foreach (@monthnames) { if ($_ =~ /^"<(two|three|four|five|six|seven|eight|nine)>/ || ($_ =~ /^"/ && $_ !~ /^"(||)/)) { $_ =~ s/^"/"/; } push @cleaned, $_; } return join(";",@cleaned); } return $s; } ############################ sub get_unidata { my $directory = shift; open(FIN, "$directory/UnicodeData.txt") or die("Cannot open $directory/UnicodeData.txt");; my @lines = ; chomp(@lines); close(FIN); foreach my $l (@lines) { my @a = split(/;/, $l); $ucd{code2name}{"$a[0]"} = $a[1]; # Unicode name $ucd{name2code}{"$a[1]"} = $a[0]; # Unicode code } } sub get_utf8map { my $file = shift; open(FIN, $file); my @lines = ; close(FIN); chomp(@lines); my $prev_k = undef; my $prev_v = ""; my $incharmap = 0; foreach my $l (@lines) { $l =~ s/\r//; next if ($l =~ /^\#/); next if ($l eq ""); if ($l eq "CHARMAP") { $incharmap = 1; next; } next if (!$incharmap); last if ($l eq "END CHARMAP"); $l =~ /^<([^\s]+)>\s+(.*)/; my $k = $1; my $v = $2; $k =~ s/_/ /g; # unicode char string $v =~ s/\\x//g; # UTF-8 char code $utf8map{$k} = $v; $utf8aliases{$k} = $prev_k if ($prev_v eq $v); $prev_v = $v; $prev_k = $k; } } sub get_encodings { my $dir = shift; foreach my $e (sort(keys(%encodings))) { if (!open(FIN, "$dir/$e.TXT")) { print "Cannot open charmap for $e\n"; next; } $encodings{$e} = 1; my @lines = ; close(FIN); chomp(@lines); foreach my $l (@lines) { $l =~ s/\r//; next if ($l =~ /^\#/); next if ($l eq ""); my @a = split(" ", $l); next if ($#a < 1); $a[0] =~ s/^0[xX]//; # local char code $a[1] =~ s/^0[xX]//; # unicode char code $convertors{$e}{uc($a[1])} = uc($a[0]); } } } sub get_languages { my %data = get_xmldata($ETCDIR); %languages = %{$data{L}}; %translations = %{$data{T}}; %alternativemonths = %{$data{AM}}; %encodings = %{$data{E}}; return if (!defined $doonly); my @a = split(/_/, $doonly); if ($#a == 1) { $filter[0] = $a[0]; $filter[1] = "x"; $filter[2] = $a[1]; } elsif ($#a == 2) { $filter[0] = $a[0]; $filter[1] = $a[1]; $filter[2] = $a[2]; } print Dumper(@filter); return; } sub transform_ctypes { foreach my $l (sort keys(%languages)) { foreach my $f (sort keys(%{$languages{$l}})) { foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { next if ($#filter == 2 && ($filter[0] ne $l || $filter[1] ne $f || $filter[2] ne $c)); next if (defined $languages{$l}{$f}{definitions} && $languages{$l}{$f}{definitions} !~ /$TYPE/); $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread my $file; $file = $l . "_"; $file .= $f . "_" if ($f ne "x"); $file .= $c; my $actfile = $file; my $filename = "$CLDRDIR/posix/xx_Comm_US.UTF-8.src"; if (! -f $filename) { print STDERR "Cannot open $filename\n"; next; } open(FIN, "$filename"); print "Reading from $filename for ${l}_${f}_${c}\n"; $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 1; # read my @lines; my $shex; my $uhex; while () { push @lines, $_; } close(FIN); $shex = sha1_hex(join("\n", @lines)); $languages{$l}{$f}{data}{$c}{$DEFENCODING} = $shex; $hashtable{$shex}{"${l}_${f}_${c}.$DEFENCODING"} = 1; open(FOUT, ">$TYPE.draft/$actfile.$DEFENCODING.src"); print FOUT @lines; close(FOUT); foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) { next if ($enc eq $DEFENCODING); $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; if (! -f $filename) { print STDERR "Cannot open $filename\n"; next; } @lines = (); open(FIN, "$filename"); while () { if ((/^comment_char\s/) || (/^escape_char\s/)){ push @lines, $_; } if (/^LC_CTYPE/../^END LC_CTYPE/) { push @lines, $_; } } close(FIN); $uhex = sha1_hex(join("\n", @lines) . $enc); $languages{$l}{$f}{data}{$c}{$enc} = $uhex; $hashtable{$uhex}{"${l}_${f}_${c}.$enc"} = 1; open(FOUT, ">$TYPE.draft/$actfile.$enc.src"); print FOUT <) { if ((/^comment_char\s/) || (/^escape_char\s/)){ push @lines, $_; } if (/^LC_COLLATE/../^END LC_COLLATE/) { $_ =~ s/[ ]+/ /g; push @lines, $_; } } close(FIN); $shex = sha1_hex(join("\n", @lines)); $languages{$l}{$f}{data}{$c}{$DEFENCODING} = $shex; $hashtable{$shex}{"${l}_${f}_${c}.$DEFENCODING"} = 1; open(FOUT, ">$TYPE.draft/$actfile.$DEFENCODING.src"); print FOUT <; chomp(@lines); close(FIN); my $continue = 0; foreach my $k (keys(%keys)) { foreach my $line (@lines) { $line =~ s/\r//; next if (!$continue && $line !~ /^$k\s/); if ($continue) { $line =~ s/^\s+//; } else { $line =~ s/^$k\s+//; } $values{$l}{$c}{$k} = "" if (!defined $values{$l}{$c}{$k}); $continue = ($line =~ /\/$/); $line =~ s/\/$// if ($continue); while ($line =~ /_/) { $line =~ s/\<([^>_]+)_([^>]+)\>/<$1 $2>/; } die "_ in data - $line" if ($line =~ /_/); $values{$l}{$c}{$k} .= $line; last if (!$continue); } } } } } } sub decodecldr { my $e = shift; my $s = shift; my $v = undef; if ($e eq "UTF-8") { # # Conversion to UTF-8 can be done from the Unicode name to # the UTF-8 character code. # $v = $utf8map{$s}; die "Cannot convert $s in $e (charmap)" if (!defined $v); } else { # # Conversion to these encodings can be done from the Unicode # name to Unicode code to the encodings code. # my $ucc = undef; $ucc = $ucd{name2code}{$s} if (defined $ucd{name2code}{$s}); $ucc = $ucd{name2code}{$utf8aliases{$s}} if (!defined $ucc && $utf8aliases{$s} && defined $ucd{name2code}{$utf8aliases{$s}}); if (!defined $ucc) { if (defined $translations{$e}{$s}{hex}) { $v = $translations{$e}{$s}{hex}; $ucc = 0; } elsif (defined $translations{$e}{$s}{ucc}) { $ucc = $translations{$e}{$s}{ucc}; } } die "Cannot convert $s in $e (ucd string)" if (!defined $ucc); $v = $convertors{$e}{$ucc} if (!defined $v); $v = $translations{$e}{$s}{hex} if (!defined $v && defined $translations{$e}{$s}{hex}); if (!defined $v && defined $translations{$e}{$s}{unicode}) { my $ucn = $translations{$e}{$s}{unicode}; $ucc = $ucd{name2code}{$ucn} if (defined $ucd{name2code}{$ucn}); $ucc = $ucd{name2code}{$utf8aliases{$ucn}} if (!defined $ucc && defined $ucd{name2code}{$utf8aliases{$ucn}}); $v = $convertors{$e}{$ucc}; } die "Cannot convert $s in $e (charmap)" if (!defined $v); } return pack("C", hex($v)) if (length($v) == 2); return pack("CC", hex(substr($v, 0, 2)), hex(substr($v, 2, 2))) if (length($v) == 4); return pack("CCC", hex(substr($v, 0, 2)), hex(substr($v, 2, 2)), hex(substr($v, 4, 2))) if (length($v) == 6); print STDERR "Cannot convert $e $s\n"; return "length = " . length($v); } sub translate { my $enc = shift; my $v = shift; return $translations{$enc}{$v} if (defined $translations{$enc}{$v}); return undef; } sub print_fields { foreach my $l (sort keys(%languages)) { foreach my $f (sort keys(%{$languages{$l}})) { foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { next if ($#filter == 2 && ($filter[0] ne $l || $filter[1] ne $f || $filter[2] ne $c)); next if (defined $languages{$l}{$f}{definitions} && $languages{$l}{$f}{definitions} !~ /$TYPE/); foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) { if ($languages{$l}{$f}{data}{$c}{$DEFENCODING} eq "0") { print "Skipping ${l}_" . ($f eq "x" ? "" : "${f}_") . "${c} - not read\n"; next; } my $file = $l; $file .= "_" . $f if ($f ne "x"); $file .= "_" . $c; print "Writing to $file in $enc\n"; if ($enc ne $DEFENCODING && !defined $convertors{$enc}) { print "Failed! Cannot convert to $enc.\n"; next; }; open(FOUT, ">$TYPE.draft/$file.$enc.new"); my $okay = 1; my $output = ""; print FOUT </) { $k = substr($f, 1); $f = $keys{$k}; } # Callback function if ($f =~ /^\(.*)/) { my $p1 = $1; $cm = $2; my $p3 = $3; my $rv = decodecldr($enc, $cm); # $rv = translate($enc, $cm) # if (!defined $rv); if (!defined $rv) { print STDERR "Could not convert $k ($cm) from $DEFENCODING to $enc\n"; $okay = 0; next; } $v = $p1 . $rv . $p3; } $output .= "$v\n"; next; } if ($f eq "as") { foreach my $v (split(/;/, $v)) { $v =~ s/^"//; $v =~ s/"$//; my $cm = ""; while ($v =~ /^(.*?)<(.*?)>(.*)/) { my $p1 = $1; $cm = $2; my $p3 = $3; my $rv = decodecldr($enc, $cm); # $rv = translate($enc, # $cm) # if (!defined $rv); if (!defined $rv) { print STDERR "Could not convert $k ($cm) from $DEFENCODING to $enc\n"; $okay = 0; next; } $v = $1 . $rv . $3; } $output .= "$v\n"; } next; } die("$k is '$f'"); } $languages{$l}{$f}{data}{$c}{$enc} = sha1_hex($output); $hashtable{sha1_hex($output)}{"${l}_${f}_${c}.$enc"} = 1; print FOUT "$output# EOF\n"; close(FOUT); if ($okay) { rename("$TYPE.draft/$file.$enc.new", "$TYPE.draft/$file.$enc.src"); } else { rename("$TYPE.draft/$file.$enc.new", "$TYPE.draft/$file.$enc.failed"); } } } } } } sub make_makefile { return if ($#filter > -1); print "Creating Makefile for $TYPE\n"; my $SRCOUT; my $SRCOUT2; my $SRCOUT3 = ""; + my $SRCOUT4 = ""; my $MAPLOC; if ($TYPE eq "colldef") { $SRCOUT = "localedef -D -U -i \${.IMPSRC} \\\n" . - "\t-f \${MAPLOC}/map.UTF-8 " . + "\t-f \${MAPLOC}/map.\${.TARGET:T:R:E} " . "\${.OBJDIR}/\${.IMPSRC:T:R}"; $MAPLOC = "MAPLOC=\t\t\${.CURDIR}/../../tools/tools/" . "locale/etc/final-maps\n"; $SRCOUT2 = "LC_COLLATE"; + $SRCOUT3 = "" . + ".for f t in \${LOCALES_MAPPED}\n" . + "FILES+=\t\$t.LC_COLLATE\n" . + "\$t.LC_COLLATE: \${.CURDIR}/\$f.src\n" . + "\tlocaledef -D -U -i \${.ALLSRC} \\\n" . + "\t\t-f \${MAPLOC}/map.\${.TARGET:T:R:E} \\\n" . + "\t\t\${.OBJDIR}/\${.TARGET:T:R}\n" . + ".endfor\n\n"; + $SRCOUT4 = "## LOCALES_MAPPED\n"; } elsif ($TYPE eq "ctypedef") { $SRCOUT = "localedef -D -U -c -w \${MAPLOC}/widths.txt \\\n" . "\t-f \${MAPLOC}/map.\${.IMPSRC:T:R:E} " . "\\\n\t-i \${.IMPSRC} \${.OBJDIR}/\${.IMPSRC:T:R} " . " || true"; $SRCOUT2 = "LC_CTYPE"; $MAPLOC = "MAPLOC=\t\t\${.CURDIR}/../../tools/tools/" . "locale/etc/final-maps\n"; $SRCOUT3 = "## SYMPAIRS\n\n" . ".for s t in \${SYMPAIRS}\n" . "\${t:S/src\$/LC_CTYPE/}: " . "\$s\n" . "\tlocaledef -D -U -c -w \${MAPLOC}/widths.txt \\\n" . "\t-f \${MAPLOC}/map.\${.TARGET:T:R:C/^.*\\.//} " . "\\\n\t-i \${.ALLSRC} \${.OBJDIR}/\${.TARGET:T:R} " . " || true\n" . ".endfor\n\n"; } else { $SRCOUT = "grep -v -E '^(\#\$\$|\#[ ])' < \${.IMPSRC} > \${.TARGET}"; $SRCOUT2 = "out"; $MAPLOC = ""; } open(FOUT, ">$TYPE.draft/Makefile"); print FOUT < 0) { my $link = shift(@files); $link =~ s/_x_/_/; # strip family if none there foreach my $file (@files) { my @a = split(/_/, $file); my @b = split(/\./, $a[-1]); $file =~ s/_x_/_/; print FOUT "SAME+=\t\t$link $file\n"; undef($languages{$a[0]}{$a[1]}{data}{$b[0]}{$b[1]}); } } } foreach my $l (sort keys(%languages)) { foreach my $f (sort keys(%{$languages{$l}})) { foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { next if ($#filter == 2 && ($filter[0] ne $l || $filter[1] ne $f || $filter[2] ne $c)); next if (defined $languages{$l}{$f}{definitions} && $languages{$l}{$f}{definitions} !~ /$TYPE/); if (defined $languages{$l}{$f}{data}{$c}{$DEFENCODING} && $languages{$l}{$f}{data}{$c}{$DEFENCODING} eq "0") { print "Skipping ${l}_" . ($f eq "x" ? "" : "${f}_") . "${c} - not read\n"; next; } foreach my $e (sort keys(%{$languages{$l}{$f}{data}{$c}})) { my $file = $l . "_"; $file .= $f . "_" if ($f ne "x"); $file .= $c; next if (!defined $languages{$l}{$f}{data}{$c}{$e}); print FOUT "LOCALES+=\t$file.$e\n"; } if (defined $languages{$l}{$f}{nc_link}) { foreach my $e (sort keys(%{$languages{$l}{$f}{data}{$c}})) { my $file = $l . "_"; $file .= $f . "_" if ($f ne "x"); $file .= $c; print FOUT "SAME+=\t\t$file.$e $languages{$l}{$f}{nc_link}.$e\t# legacy (lang/country change)\n"; } } if (defined $languages{$l}{$f}{e_link}) { foreach my $el (split(" ", $languages{$l}{$f}{e_link})) { my @a = split(/:/, $el); my $file = $l . "_"; $file .= $f . "_" if ($f ne "x"); $file .= $c; print FOUT "SAME+=\t\t$file.$a[0] $file.$a[1]\t# legacy (same charset)\n"; } } } } } print FOUT < EOF close(FOUT); } Index: head/tools/tools/locale/tools/convert_map.pl =================================================================== --- head/tools/tools/locale/tools/convert_map.pl (revision 298115) +++ head/tools/tools/locale/tools/convert_map.pl (revision 298116) @@ -1,192 +1,193 @@ #! /usr/local/bin/perl # +# $FreeBSD$ +# # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy is of the CDDL is also available via the Internet # at http://www.illumos.org/license/CDDL. # # # Copyright 2010 Nexenta Systems, Inc. All rights reserved. # Copyright 2015 John Marino # # This converts MAPPING files to localedef character maps # suitable for use with the UTF-8 derived localedef data. sub ucs_to_utf8 { my $ucs = shift; my $utf8; if ($ucs <= 0x7f) { $utf8 = sprintf("\\x%02X", $ucs).$utf8; } elsif ($ucs <= 0x7ff) { $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", $ucs | 0xc0).$utf8; } elsif ($ucs <= 0xffff) { $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", $ucs | 0xe0).$utf8; } elsif ($ucs <= 0x1fffff) { $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", $ucs | 0xf0).$utf8; } elsif ($ucs <= 0x03ffffff) { $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", $ucs | 0xf8).$utf8; } else { $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", ($ucs & 0x3f) | 0x80).$utf8; $ucs >>= 6; $utf8 = sprintf("\\x%02X", $ucs | 0xf8).$utf8; } return ($utf8); } my %unames; my %uvalues; # # This is not a general purpose Character Map parser, but its good enough # for the stock one supplied with CLDR. # sub load_utf8_cm { my $file = shift; open(UTF8, "$file") || die "open"; while () { next if (/^#/); next if (/^\s*$/); next if (/^\s*CHARMAP\s*$/); next if (/^\s*END\s*CHARMAP\s*$/); chomp; @words = split /\s+/; $name = $words[0]; $utf8val = $words[1]; if (defined($unames{$utf8val})) { $unames{$utf8val} .= "\n" .$name; } else { $unames{$utf8val} = $name; } $uvalues{$name} = $utf8val; } close(UTF8); } my %map; sub load_map { my $file = shift; open(MAP, "$file") || die "open"; while () { next if (/^#/); next if (/^\s*$/); next if (/^0x..\+0x../); next if (/^0x[0-9A-F]{4}\t0x[0-9A-F]{4} 0x[0-9A-F]{4}/); next if (/^0x[0-9A-F]{2}\s+#/); next if (/# ... NO MAPPING .../); chomp; @words = split /\s+/; $utf8 = $words[1]; $utf8 =~ s/^\\x[0]*//; $utf8 = ucs_to_utf8(hex($utf8)); $val = $words[0]; if (defined ($map{$val})) { $map{$val} .= " ".$utf8; } else { $map{$val} = $utf8; } } } sub mb_str { my $val = shift; my $str = ""; $val = hex($val); if ($val == 0) { return ("\\x00"); } while ($val) { $str = sprintf("\\x%02x", $val & 0xff).$str; $val >>= 8; } return ($str); } $mf = shift(@ARGV); $codeset = shift(@ARGV); my $max_mb; load_utf8_cm("etc/final-maps/map.UTF-8"); load_map($mf); if ($codeset eq "SJIS") { $max_mb = 2 } elsif ($codeset eq "eucCN") { $max_mb = 2 } elsif ($codeset eq "eucJP") { $max_mb = 3 } elsif ($codeset eq "eucKR") { $max_mb = 2 } elsif ($codeset eq "GBK") { $max_mb = 2 } elsif ($codeset eq "GB2312") { $max_mb = 2 } elsif ($codeset eq "Big5") { $max_mb = 2 } -elsif ($codeset eq "Big5HKSCS") { $max_mb = 2 } else { $max_mb = 1 }; print(" \"$codeset\"\n"); print(" 1\n"); print(" $max_mb\n"); print("CHARMAP\n"); foreach $val (sort (keys (%map))) { #$utf8 = $map{$val}; foreach $utf8 (split / /, $map{$val}) { $ref = $unames{$utf8}; foreach $name (sort (split /\n/, $ref)) { print "$name"; my $nt = int((64 - length($name) + 7) / 8); while ($nt) { print "\t"; $nt--; } print mb_str($val)."\n"; } } } print "END CHARMAP\n"; Index: head/tools/tools/locale/tools/extract-colldef.awk =================================================================== --- head/tools/tools/locale/tools/extract-colldef.awk (nonexistent) +++ head/tools/tools/locale/tools/extract-colldef.awk (revision 298116) @@ -0,0 +1,18 @@ +# $FreeBSD$ + +BEGIN { + print "# Warning: Do not edit. This is automatically extracted" + print "# from CLDR project data, obtained from http://cldr.unicode.org/" + print "# -----------------------------------------------------------------------------" +} +$1 == "comment_char" { print $0 } +$1 == "escape_char" { print $0 } +$1 == "LC_COLLATE" { + print $0 + while (getline line) { + print line + if (line == "END LC_COLLATE") { + break + } + } +} Property changes on: head/tools/tools/locale/tools/extract-colldef.awk ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/tools/tools/locale/tools/finalize =================================================================== --- head/tools/tools/locale/tools/finalize (revision 298115) +++ head/tools/tools/locale/tools/finalize (revision 298116) @@ -1,88 +1,118 @@ #!/bin/sh # # $FreeBSD$ # # This is a helper script for the Makefile in the parent directory. # When the localization definitions are generated in the draft area, # this script will copy base ones that others symlink to, and rearrange # the generate makefile to pull the LOCALES first. # usage () { echo "finalize ' to package standard localization" echo "type must be one of { monetdef, msgdef, numericdef, timedef, colldef, ctypedef }" exit 1 } [ $# -ne 1 ] && usage [ $1 = "monetdef" -o $1 = "msgdef" -o $1 = "colldef" -o \ $1 = "numericdef" -o $1 = "timedef" -o $1 = "ctypedef" ] || usage self=$(realpath $0) base=$(dirname ${self}) old=${base}/../${1}.draft new=${base}/../${1} TEMP=/tmp/${1}.locales TEMP2=/tmp/${1}.hashes TEMP3=/tmp/${1}.symlinks +TEMP4=/tmp/${1}.mapped FULLMAP=/tmp/utf8-map FULLEXTRACT=/tmp/extracted-names AWKCMD="/## PLACEHOLDER/ { \ while ( getline line < \"${TEMP}\" ) {print line} } \ /## SYMPAIRS/ { \ while ( getline line < \"${TEMP3}\" ) {print line} } \ + /## LOCALES_MAPPED/ { \ + while ( getline line < \"${TEMP4}\" ) {print line} } \ !/## / { print \$0 }" grep '^LOCALES+' ${old}/Makefile > ${TEMP} if [ $1 = "ctypedef" ] then keep=$(cat ${TEMP} | awk '{ print $2 ".src" }') (cd ${old} && md5 -r ${keep} | sort) > ${TEMP2} keep=$(awk '{ if ($1 != last1) print $2; last1 = $1; }' ${TEMP2}) for original in ${keep} do cp ${old}/${original} ${new}/ done awk '{ if ($1 == last1) { print "SYMPAIRS+=\t" last2 " " $2 } \ else {last1 = $1; last2 = $2}}' ${TEMP2} > ${TEMP3} rm -f ${TEMP2} /usr/bin/sed -E -e 's/[ ]+/ /g' \ ${CLDRDIR}/posix/UTF-8.cm \ > ${base}/../etc/final-maps/map.UTF-8 - CHARMAPS="ARMSCII-8 Big5 Big5HKSCS CP1131 CP1251 \ + /usr/bin/sed -E -e 's/[ ]+/ /g' \ + ${CLDRDIR}/posix/eucCN.cm \ + > ${base}/../etc/final-maps/map.eucCN + /usr/bin/sed -E -e 's/[ ]+/ /g' \ + ${CLDRDIR}/posix/eucCN.cm \ + > ${base}/../etc/final-maps/map.GB2312 + CHARMAPS="ARMSCII-8 Big5 CP1131 CP1251 \ CP866 GB2312 GBK ISCII-DEV ISO8859-1 \ ISO8859-13 ISO8859-15 ISO8859-2 ISO8859-4 \ ISO8859-5 ISO8859-7 ISO8859-9 KOI8-R KOI8-U \ - PT154 SJIS US-ASCII eucCN eucJP eucKR" + PT154 SJIS US-ASCII eucJP eucKR" # GB18030 blows up, use pre-generate Illumos version for map in ${CHARMAPS} do encoding=${map} - if [ ${map} = "Big5HKSCS" ] - then - encoding="Big5" - fi /usr/local/bin/perl ${base}/convert_map.pl \ ${base}/../etc/charmaps/${map}.TXT ${encoding} \ | /usr/bin/sed -E -e 's/ +/ /g' \ > ${base}/../etc/final-maps/map.${map} echo map ${map} converted. done +elif [ $1 = "colldef" ] +then + awk -v tmp4=${TEMP4} '$1 == "SAME+=" && $0 !~ /legacy/ { + orig=$2 + dest=$3 + gsub(/.*\./, "", orig) + gsub(/.*\./, "", dest) + if (orig != dest ) + print "LOCALES_MAPPED+=\t"$2 " "$3 > tmp4 + }' ${old}/Makefile + + for line in $(awk '{ print $3 }' ${TEMP4}); do + sed -i '' "/^SAME.*$line$/d" ${old}/Makefile + done + echo "" >> ${TEMP4} + for enc in ${COLLATIONS_SPECIAL}; do + sed -i '' "/^.*${enc}$/d" ${TEMP4} + echo "LOCALES+= ${enc}" >> ${TEMP4} + done + + keep=$(cat ${TEMP} | awk '{ print $2 }') + for original in ${keep} + do + cp ${old}/${original}.src ${new}/ + done else # below is everything but ctypedef keep=$(cat ${TEMP} | awk '{ print $2 }') for original in ${keep} do cp ${old}/${original}.src ${new}/ done fi grep -v '^LOCALES+' ${old}/Makefile | awk "${AWKCMD}" > ${new}/Makefile -rm -f ${TEMP} ${TEMP3} +rm -f ${TEMP} ${TEMP3} ${TEMP4}