Changeset View
Changeset View
Standalone View
Standalone View
tools/tools/locale/tools/cldr2def.pl
| #!/usr/local/bin/perl -wC | #!/usr/local/bin/perl -wC | ||||
| # $FreeBSD$ | # $FreeBSD$ | ||||
| use strict; | use strict; | ||||
| use File::Copy; | use File::Copy; | ||||
| use XML::Parser; | use XML::Parser; | ||||
| use Tie::IxHash; | use Tie::IxHash; | ||||
| use Text::Iconv; | use Text::Iconv; | ||||
| use Data::Dumper; | #use Data::Dumper; | ||||
| use Getopt::Long; | use Getopt::Long; | ||||
| use Digest::SHA qw(sha1_hex); | use Digest::SHA qw(sha1_hex); | ||||
| require "charmaps.pm"; | require "charmaps.pm"; | ||||
| if ($#ARGV < 2) { | if ($#ARGV < 2) { | ||||
| print "Usage: $0 --cldr=<cldrdir> --unidata=<unidatadir> --etc=<etcdir> --type=<type> [--lc=<la_CC>]\n"; | print "Usage: $0 --unidir=<unidir> --etc=<etcdir> --type=<type>\n"; | ||||
| exit(1); | exit(1); | ||||
| } | } | ||||
| my $DEFENCODING = "UTF-8"; | my $DEFENCODING = "UTF-8"; | ||||
| my @filter = (); | |||||
| my $CLDRDIR = undef; | my $UNIDIR = undef; | ||||
| my $UNIDATADIR = undef; | |||||
| my $ETCDIR = undef; | my $ETCDIR = undef; | ||||
| my $TYPE = undef; | my $TYPE = undef; | ||||
| my $doonly = undef; | |||||
| my $result = GetOptions ( | my $result = GetOptions ( | ||||
| "cldr=s" => \$CLDRDIR, | "unidir=s" => \$UNIDIR, | ||||
| "unidata=s" => \$UNIDATADIR, | |||||
| "etc=s" => \$ETCDIR, | "etc=s" => \$ETCDIR, | ||||
| "type=s" => \$TYPE, | "type=s" => \$TYPE, | ||||
| "lc=s" => \$doonly | |||||
| ); | ); | ||||
| my %convertors = (); | my %convertors = (); | ||||
| my %ucd = (); | my %ucd = (); | ||||
| my %values = (); | my %values = (); | ||||
| my %hashtable = (); | my %hashtable = (); | ||||
| my %languages = (); | my %languages = (); | ||||
| my %translations = (); | my %translations = (); | ||||
| my %encodings = (); | my %encodings = (); | ||||
| my %alternativemonths = (); | my %alternativemonths = (); | ||||
| get_languages(); | get_languages(); | ||||
| my %utf8map = (); | my %utf8map = (); | ||||
| my %utf8aliases = (); | my %utf8aliases = (); | ||||
| get_unidata($UNIDATADIR); | get_unidata($UNIDIR); | ||||
| get_utf8map("$CLDRDIR/posix/$DEFENCODING.cm"); | get_utf8map("$UNIDIR/posix/$DEFENCODING.cm"); | ||||
| get_encodings("$ETCDIR/charmaps"); | get_encodings("$ETCDIR/charmaps"); | ||||
| my %keys = (); | my %keys = (); | ||||
| tie(%keys, "Tie::IxHash"); | tie(%keys, "Tie::IxHash"); | ||||
| tie(%hashtable, "Tie::IxHash"); | tie(%hashtable, "Tie::IxHash"); | ||||
| my %FILESNAMES = ( | my %FILESNAMES = ( | ||||
| "monetdef" => "LC_MONETARY", | "monetdef" => "LC_MONETARY", | ||||
| ▲ Show 20 Lines • Show All 332 Lines • ▼ Show 20 Lines | |||||
| } | } | ||||
| sub get_languages { | sub get_languages { | ||||
| my %data = get_xmldata($ETCDIR); | my %data = get_xmldata($ETCDIR); | ||||
| %languages = %{$data{L}}; | %languages = %{$data{L}}; | ||||
| %translations = %{$data{T}}; | %translations = %{$data{T}}; | ||||
| %alternativemonths = %{$data{AM}}; | %alternativemonths = %{$data{AM}}; | ||||
| %encodings = %{$data{E}}; | %encodings = %{$data{E}}; | ||||
| return if (!defined $doonly); | |||||
| my @a = split(/_/, $doonly); | |||||
| if ($#a == 1) { | |||||
| $filter[0] = $a[0]; | |||||
| $filter[1] = "x"; | |||||
| $filter[2] = $a[1]; | |||||
| } elsif ($#a == 2) { | |||||
| $filter[0] = $a[0]; | |||||
| $filter[1] = $a[1]; | |||||
| $filter[2] = $a[2]; | |||||
| } | } | ||||
| print Dumper(@filter); | |||||
| return; | |||||
| } | |||||
| sub transform_ctypes { | sub transform_ctypes { | ||||
| # Add the C.UTF-8 | # Add the C.UTF-8 | ||||
| $languages{"C"}{"x"}{data}{"x"}{$DEFENCODING} = undef; | $languages{"C"}{"x"}{data}{"x"}{$DEFENCODING} = undef; | ||||
| foreach my $l (sort keys(%languages)) { | foreach my $l (sort keys(%languages)) { | ||||
| foreach my $f (sort keys(%{$languages{$l}})) { | foreach my $f (sort keys(%{$languages{$l}})) { | ||||
| foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { | foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { | ||||
| next if ($#filter == 2 && ($filter[0] ne $l | |||||
| || $filter[1] ne $f || $filter[2] ne $c)); | |||||
| next if (defined $languages{$l}{$f}{definitions} | next if (defined $languages{$l}{$f}{definitions} | ||||
| && $languages{$l}{$f}{definitions} !~ /$TYPE/); | && $languages{$l}{$f}{definitions} !~ /$TYPE/); | ||||
| $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread | $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread | ||||
| my $file = $l; | my $file = $l; | ||||
| $file .= "_" . $f if ($f ne "x"); | $file .= "_" . $f if ($f ne "x"); | ||||
| $file .= "_" . $c if ($c ne "x"); | $file .= "_" . $c if ($c ne "x"); | ||||
| my $actfile = $file; | my $actfile = $file; | ||||
| my $filename = "$CLDRDIR/posix/xx_Comm_C.UTF-8.src"; | my $filename = "$UNIDIR/posix/xx_Comm_C.UTF-8.src"; | ||||
| if (! -f $filename) { | if (! -f $filename) { | ||||
| print STDERR "Cannot open $filename\n"; | print STDERR "Cannot open $filename\n"; | ||||
| next; | next; | ||||
| } | } | ||||
| open(FIN, "$filename"); | open(FIN, "$filename"); | ||||
| print "Reading from $filename for ${l}_${f}_${c}\n"; | print "Reading from $filename for ${l}_${f}_${c}\n"; | ||||
| $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 1; # read | $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 1; # read | ||||
| my @lines; | my @lines; | ||||
| my $shex; | my $shex; | ||||
| my $uhex; | my $uhex; | ||||
| while (<FIN>) { | while (<FIN>) { | ||||
| push @lines, $_; | push @lines, $_; | ||||
| } | } | ||||
| close(FIN); | close(FIN); | ||||
| $shex = sha1_hex(join("\n", @lines)); | $shex = sha1_hex(join("\n", @lines)); | ||||
| $languages{$l}{$f}{data}{$c}{$DEFENCODING} = $shex; | $languages{$l}{$f}{data}{$c}{$DEFENCODING} = $shex; | ||||
| $hashtable{$shex}{"${l}_${f}_${c}.$DEFENCODING"} = 1; | $hashtable{$shex}{"${l}_${f}_${c}.$DEFENCODING"} = 1; | ||||
| open(FOUT, ">$TYPE.draft/$actfile.$DEFENCODING.src"); | open(FOUT, ">$TYPE.draft/$actfile.$DEFENCODING.src"); | ||||
| print FOUT @lines; | print FOUT @lines; | ||||
| close(FOUT); | close(FOUT); | ||||
| foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) { | foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) { | ||||
| next if ($enc eq $DEFENCODING); | next if ($enc eq $DEFENCODING); | ||||
| $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; | $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"; | ||||
| if (! -f $filename) { | if (! -f $filename) { | ||||
| print STDERR "Cannot open $filename\n"; | print STDERR "Cannot open $filename\n"; | ||||
| next; | next; | ||||
| } | } | ||||
| @lines = (); | @lines = (); | ||||
| open(FIN, "$filename"); | open(FIN, "$filename"); | ||||
| while (<FIN>) { | while (<FIN>) { | ||||
| if ((/^comment_char\s/) || (/^escape_char\s/)){ | if ((/^comment_char\s/) || (/^escape_char\s/)){ | ||||
| Show All 22 Lines | EOF | ||||
| } | } | ||||
| } | } | ||||
| sub transform_collation { | sub transform_collation { | ||||
| foreach my $l (sort keys(%languages)) { | foreach my $l (sort keys(%languages)) { | ||||
| foreach my $f (sort keys(%{$languages{$l}})) { | foreach my $f (sort keys(%{$languages{$l}})) { | ||||
| foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { | foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { | ||||
| next if ($#filter == 2 && ($filter[0] ne $l | |||||
| || $filter[1] ne $f || $filter[2] ne $c)); | |||||
| next if (defined $languages{$l}{$f}{definitions} | next if (defined $languages{$l}{$f}{definitions} | ||||
| && $languages{$l}{$f}{definitions} !~ /$TYPE/); | && $languages{$l}{$f}{definitions} !~ /$TYPE/); | ||||
| $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread | $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread | ||||
| my $file; | my $file; | ||||
| $file = $l . "_"; | $file = $l . "_"; | ||||
| $file .= $f . "_" if ($f ne "x"); | $file .= $f . "_" if ($f ne "x"); | ||||
| $file .= $c; | $file .= $c; | ||||
| my $actfile = $file; | my $actfile = $file; | ||||
| my $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; | my $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"; | ||||
| $filename = "$ETCDIR/$file.$DEFENCODING.src" | $filename = "$ETCDIR/$file.$DEFENCODING.src" | ||||
| if (! -f $filename); | if (! -f $filename); | ||||
| if (! -f $filename | if (! -f $filename | ||||
| && defined $languages{$l}{$f}{fallback}) { | && defined $languages{$l}{$f}{fallback}) { | ||||
| $file = $languages{$l}{$f}{fallback}; | $file = $languages{$l}{$f}{fallback}; | ||||
| $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; | $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"; | ||||
| } | } | ||||
| $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src" | $filename = "$UNIDIR/posix/$file.$DEFENCODING.src" | ||||
| if (! -f $filename); | if (! -f $filename); | ||||
| if (! -f $filename) { | if (! -f $filename) { | ||||
| print STDERR | print STDERR | ||||
| "Cannot open $file.$DEFENCODING.src or fallback\n"; | "Cannot open $file.$DEFENCODING.src or fallback\n"; | ||||
| next; | next; | ||||
| } | } | ||||
| open(FIN, "$filename"); | open(FIN, "$filename"); | ||||
| print "Reading from $filename for ${l}_${f}_${c}\n"; | print "Reading from $filename for ${l}_${f}_${c}\n"; | ||||
| Show All 34 Lines | EOF | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| sub get_fields { | sub get_fields { | ||||
| foreach my $l (sort keys(%languages)) { | foreach my $l (sort keys(%languages)) { | ||||
| foreach my $f (sort keys(%{$languages{$l}})) { | foreach my $f (sort keys(%{$languages{$l}})) { | ||||
| foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { | foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { | ||||
| next if ($#filter == 2 && ($filter[0] ne $l | |||||
| || $filter[1] ne $f || $filter[2] ne $c)); | |||||
| next if (defined $languages{$l}{$f}{definitions} | next if (defined $languages{$l}{$f}{definitions} | ||||
| && $languages{$l}{$f}{definitions} !~ /$TYPE/); | && $languages{$l}{$f}{definitions} !~ /$TYPE/); | ||||
| $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread | $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread | ||||
| my $file; | my $file; | ||||
| $file = $l . "_"; | $file = $l . "_"; | ||||
| $file .= $f . "_" if ($f ne "x"); | $file .= $f . "_" if ($f ne "x"); | ||||
| $file .= $c; | $file .= $c; | ||||
| my $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; | my $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"; | ||||
| $filename = "$ETCDIR/$file.$DEFENCODING.src" | $filename = "$ETCDIR/$file.$DEFENCODING.src" | ||||
| if (! -f $filename); | if (! -f $filename); | ||||
| if (! -f $filename | if (! -f $filename | ||||
| && defined $languages{$l}{$f}{fallback}) { | && defined $languages{$l}{$f}{fallback}) { | ||||
| $file = $languages{$l}{$f}{fallback}; | $file = $languages{$l}{$f}{fallback}; | ||||
| $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; | $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"; | ||||
| } | } | ||||
| $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src" | $filename = "$UNIDIR/posix/$file.$DEFENCODING.src" | ||||
| if (! -f $filename); | if (! -f $filename); | ||||
| if (! -f $filename) { | if (! -f $filename) { | ||||
| print STDERR | print STDERR | ||||
| "Cannot open $file.$DEFENCODING.src or fallback\n"; | "Cannot open $file.$DEFENCODING.src or fallback\n"; | ||||
| next; | next; | ||||
| } | } | ||||
| open(FIN, "$filename"); | open(FIN, "$filename"); | ||||
| print "Reading from $filename for ${l}_${f}_${c}\n"; | print "Reading from $filename for ${l}_${f}_${c}\n"; | ||||
| ▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines | sub translate { | ||||
| return $translations{$enc}{$v} if (defined $translations{$enc}{$v}); | return $translations{$enc}{$v} if (defined $translations{$enc}{$v}); | ||||
| return undef; | return undef; | ||||
| } | } | ||||
| sub print_fields { | sub print_fields { | ||||
| foreach my $l (sort keys(%languages)) { | foreach my $l (sort keys(%languages)) { | ||||
| foreach my $f (sort keys(%{$languages{$l}})) { | foreach my $f (sort keys(%{$languages{$l}})) { | ||||
| foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { | foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { | ||||
| next if ($#filter == 2 && ($filter[0] ne $l | |||||
| || $filter[1] ne $f || $filter[2] ne $c)); | |||||
| next if (defined $languages{$l}{$f}{definitions} | next if (defined $languages{$l}{$f}{definitions} | ||||
| && $languages{$l}{$f}{definitions} !~ /$TYPE/); | && $languages{$l}{$f}{definitions} !~ /$TYPE/); | ||||
| foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) { | foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) { | ||||
| if ($languages{$l}{$f}{data}{$c}{$DEFENCODING} eq "0") { | if ($languages{$l}{$f}{data}{$c}{$DEFENCODING} eq "0") { | ||||
| print "Skipping ${l}_" . | print "Skipping ${l}_" . | ||||
| ($f eq "x" ? "" : "${f}_") . | ($f eq "x" ? "" : "${f}_") . | ||||
| "${c} - not read\n"; | "${c} - not read\n"; | ||||
| next; | next; | ||||
| ▲ Show 20 Lines • Show All 130 Lines • ▼ Show 20 Lines | "Could not convert $k ($cm) from $DEFENCODING to $enc\n"; | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| sub make_makefile { | sub make_makefile { | ||||
| return if ($#filter > -1); | |||||
| print "Creating Makefile for $TYPE\n"; | print "Creating Makefile for $TYPE\n"; | ||||
| my $SRCOUT; | my $SRCOUT; | ||||
| my $SRCOUT2; | my $SRCOUT2; | ||||
| my $SRCOUT3 = ""; | my $SRCOUT3 = ""; | ||||
| my $SRCOUT4 = ""; | my $SRCOUT4 = ""; | ||||
| my $MAPLOC; | my $MAPLOC; | ||||
| if ($TYPE eq "colldef") { | if ($TYPE eq "colldef") { | ||||
| $SRCOUT = "localedef \${LOCALEDEF_ENDIAN} -D -U " . | $SRCOUT = "localedef \${LOCALEDEF_ENDIAN} -D -U " . | ||||
| ▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
| # \$FreeBSD\$ | # \$FreeBSD\$ | ||||
| # Warning: Do not edit. This file is automatically generated from the | # Warning: Do not edit. This file is automatically generated from the | ||||
| # tools in /usr/src/tools/tools/locale. | # tools in /usr/src/tools/tools/locale. | ||||
| LOCALEDIR= \${SHAREDIR}/locale | LOCALEDIR= \${SHAREDIR}/locale | ||||
| FILESNAME= $FILESNAMES{$TYPE} | FILESNAME= $FILESNAMES{$TYPE} | ||||
| .SUFFIXES: .src .${SRCOUT2} | .SUFFIXES: .src .${SRCOUT2} | ||||
| ${MAPLOC} | ${MAPLOC} | ||||
| EOF | |||||
| if ($TYPE eq "colldef" || $TYPE eq "ctypedef") { | |||||
| print FOUT <<EOF; | |||||
| .include <bsd.endian.mk> | .include <bsd.endian.mk> | ||||
| EOF | |||||
| } | |||||
| print FOUT <<EOF; | |||||
| .src.${SRCOUT2}: | .src.${SRCOUT2}: | ||||
| $SRCOUT | $SRCOUT | ||||
| ## PLACEHOLDER | ## PLACEHOLDER | ||||
| ${SRCOUT4} | ${SRCOUT4} | ||||
| EOF | EOF | ||||
| ▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | if ($#files > 0) { | ||||
| undef($languages{$a[0]}{$a[1]}{data}{$b[0]}{$b[1]}); | undef($languages{$a[0]}{$a[1]}{data}{$b[0]}{$b[1]}); | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| foreach my $l (sort keys(%languages)) { | foreach my $l (sort keys(%languages)) { | ||||
| foreach my $f (sort keys(%{$languages{$l}})) { | foreach my $f (sort keys(%{$languages{$l}})) { | ||||
| foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { | foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { | ||||
| next if ($#filter == 2 && ($filter[0] ne $l | |||||
| || $filter[1] ne $f || $filter[2] ne $c)); | |||||
| next if (defined $languages{$l}{$f}{definitions} | next if (defined $languages{$l}{$f}{definitions} | ||||
| && $languages{$l}{$f}{definitions} !~ /$TYPE/); | && $languages{$l}{$f}{definitions} !~ /$TYPE/); | ||||
| if (defined $languages{$l}{$f}{data}{$c}{$DEFENCODING} | if (defined $languages{$l}{$f}{data}{$c}{$DEFENCODING} | ||||
| && $languages{$l}{$f}{data}{$c}{$DEFENCODING} eq "0") { | && $languages{$l}{$f}{data}{$c}{$DEFENCODING} eq "0") { | ||||
| print "Skipping ${l}_" . ($f eq "x" ? "" : "${f}_") . | print "Skipping ${l}_" . ($f eq "x" ? "" : "${f}_") . | ||||
| "${c} - not read\n"; | "${c} - not read\n"; | ||||
| next; | next; | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 50 Lines • Show Last 20 Lines | |||||