unicode/cpmap.pl - wine - Git at Google

 #!/usr/bin/perl
 #
 # Generate code page .c files from ftp.unicode.org descriptions
 #
 # Copyright 2000 Alexandre Julliard
 #

 # base directory for ftp.unicode.org files
 $BASEDIR = "ftp.unicode.org/Public/";
 $MAPPREFIX = $BASEDIR . "MAPPINGS/";

 # UnicodeData file
 $UNICODEDATA = $BASEDIR . "UNIDATA/UnicodeData.txt";

 # Defaults mapping
 $DEFAULTS = "./defaults";

 # Default char for undefined mappings
 $DEF_CHAR = ord '?';

 @allfiles =
 (
     [ 37,    "VENDORS/MICSFT/EBCDIC/CP037.TXT",   "IBM EBCDIC US Canada" ],
     [ 424,   "VENDORS/MISC/CP424.TXT",            "IBM EBCDIC Hebrew" ],
     [ 437,   "VENDORS/MICSFT/PC/CP437.TXT",       "OEM United States" ],
     [ 500,   "VENDORS/MICSFT/EBCDIC/CP500.TXT",   "IBM EBCDIC International" ],
     [ 737,   "VENDORS/MICSFT/PC/CP737.TXT",       "OEM Greek 437G" ],
     [ 775,   "VENDORS/MICSFT/PC/CP775.TXT",       "OEM Baltic" ],
     [ 850,   "VENDORS/MICSFT/PC/CP850.TXT",       "OEM Multilingual Latin 1" ],
     [ 852,   "VENDORS/MICSFT/PC/CP852.TXT",       "OEM Slovak Latin 2" ],
     [ 855,   "VENDORS/MICSFT/PC/CP855.TXT",       "OEM Cyrillic" ],
     [ 856,   "VENDORS/MISC/CP856.TXT",            "Hebrew PC" ],
     [ 857,   "VENDORS/MICSFT/PC/CP857.TXT",       "OEM Turkish" ],
     [ 860,   "VENDORS/MICSFT/PC/CP860.TXT",       "OEM Portuguese" ],
     [ 861,   "VENDORS/MICSFT/PC/CP861.TXT",       "OEM Icelandic" ],
     [ 862,   "VENDORS/MICSFT/PC/CP862.TXT",       "OEM Hebrew" ],
     [ 863,   "VENDORS/MICSFT/PC/CP863.TXT",       "OEM Canadian French" ],
     [ 864,   "VENDORS/MICSFT/PC/CP864.TXT",       "OEM Arabic" ],
     [ 865,   "VENDORS/MICSFT/PC/CP865.TXT",       "OEM Nordic" ],
     [ 866,   "VENDORS/MICSFT/PC/CP866.TXT",       "OEM Russian" ],
     [ 869,   "VENDORS/MICSFT/PC/CP869.TXT",       "OEM Greek" ],
     [ 874,   "VENDORS/MICSFT/PC/CP874.TXT",       "ANSI/OEM Thai" ],
     [ 875,   "VENDORS/MICSFT/EBCDIC/CP875.TXT",   "IBM EBCDIC Greek" ],
     [ 878,   "VENDORS/MISC/KOI8-R.TXT",           "Russian KOI8" ],
     [ 932,   "VENDORS/MICSFT/WINDOWS/CP932.TXT",  "ANSI/OEM Japanese Shift-JIS" ],
     [ 936,   "VENDORS/MICSFT/WINDOWS/CP936.TXT",  "ANSI/OEM Simplified Chinese GBK" ],
     [ 949,   "VENDORS/MICSFT/WINDOWS/CP949.TXT",  "ANSI/OEM Korean Unified Hangul" ],
     [ 950,   "VENDORS/MICSFT/WINDOWS/CP950.TXT",  "ANSI/OEM Traditional Chinese Big5" ],
     [ 1006,  "VENDORS/MISC/CP1006.TXT",           "IBM Arabic" ],
     [ 1026,  "VENDORS/MICSFT/EBCDIC/CP1026.TXT",  "IBM EBCDIC Latin 5 Turkish" ],
     [ 1250,  "VENDORS/MICSFT/WINDOWS/CP1250.TXT", "ANSI Eastern Europe" ],
     [ 1251,  "VENDORS/MICSFT/WINDOWS/CP1251.TXT", "ANSI Cyrillic" ],
     [ 1252,  "VENDORS/MICSFT/WINDOWS/CP1252.TXT", "ANSI Latin 1" ],
     [ 1253,  "VENDORS/MICSFT/WINDOWS/CP1253.TXT", "ANSI Greek" ],
     [ 1254,  "VENDORS/MICSFT/WINDOWS/CP1254.TXT", "ANSI Turkish" ],
     [ 1255,  "VENDORS/MICSFT/WINDOWS/CP1255.TXT", "ANSI Hebrew" ],
     [ 1256,  "VENDORS/MICSFT/WINDOWS/CP1256.TXT", "ANSI Arabic" ],
     [ 1257,  "VENDORS/MICSFT/WINDOWS/CP1257.TXT", "ANSI Baltic" ],
     [ 1258,  "VENDORS/MICSFT/WINDOWS/CP1258.TXT", "ANSI/OEM Viet Nam" ],
     [ 10000, "VENDORS/MICSFT/MAC/ROMAN.TXT",      "Mac Roman" ],
     [ 10006, "VENDORS/MICSFT/MAC/GREEK.TXT",      "Mac Greek" ],
     [ 10007, "VENDORS/MICSFT/MAC/CYRILLIC.TXT",   "Mac Cyrillic" ],
     [ 10029, "VENDORS/MICSFT/MAC/LATIN2.TXT",     "Mac Latin 2" ],
     [ 10079, "VENDORS/MICSFT/MAC/ICELAND.TXT",    "Mac Icelandic" ],
     [ 10081, "VENDORS/MICSFT/MAC/TURKISH.TXT",    "Mac Turkish" ],
     [ 20866, "VENDORS/MISC/KOI8-R.TXT",           "Russian KOI8" ],
     [ 28591, "ISO8859/8859-1.TXT",                "ISO 8859-1 Latin 1" ],
     [ 28592, "ISO8859/8859-2.TXT",                "ISO 8859-2 Eastern Europe" ],
     [ 28593, "ISO8859/8859-3.TXT",                "ISO 8859-3 Turkish" ],
     [ 28594, "ISO8859/8859-4.TXT",                "ISO 8859-4 Baltic" ],
     [ 28595, "ISO8859/8859-5.TXT",                "ISO 8859-5 Cyrillic" ],
     [ 28596, "ISO8859/8859-6.TXT",                "ISO 8859-6 Arabic" ],
     [ 28597, "ISO8859/8859-7.TXT",                "ISO 8859-7 Greek" ],
     [ 28598, "ISO8859/8859-8.TXT",                "ISO 8859-8 Hebrew" ],
     [ 28599, "ISO8859/8859-9.TXT",                "ISO 8859-9 Latin 5" ]
 );

 ################################################################
 # main routine

 READ_DEFAULTS();

 foreach $file (@allfiles) { HANDLE_FILE( @$file ); }

 OUTPUT_CPTABLE();

 exit(0);


 ################################################################
 # read in the defaults file
 sub READ_DEFAULTS
 {
     @unicode_defaults = ();
     @unicode_aliases = ();

     # first setup a few default mappings

     open DEFAULTS or die "Cannot open $DEFAULTS";
     print "Loading $DEFAULTS\n";
     while (<DEFAULTS>)
     {
         next if /^\#/;  # skip comments
         next if /^$/;  # skip empty lines
         if (/^(([0-9a-fA-F]+)(,[0-9a-fA-F]+)*)\s+([0-9a-fA-F]+|'.'|none)\s+(\#.*)?/)
         {
             my @src = map hex, split /,/,$1;
             my $dst = $4;
             my $comment = $5;
             if ($#src > 0) { push @unicode_aliases, \@src; }
             next if ($dst eq "none");
             $dst = ($dst =~ /\'.\'/) ? ord substr($dst,1,1) : hex $dst;
             foreach $src (@src)
             {
                 die "Duplicate value" if defined($unicode_defaults[$src]);
                 $unicode_defaults[$src] = $dst;
             }
             next;
         }
         die "Unrecognized line $_\n";
     }

     # now build mappings from the decomposition field of the Unicode database

     open UNICODEDATA or die "Cannot open $UNICODEDATA";
     while (<UNICODEDATA>)
     {
 	# Decode the fields ...
 	($code, $name, $cat, $comb, $bidi,
 	 $decomp, $dec, $dig, $num, $mirror,
 	 $oldname, $comment, $upper, $lower, $title) = split /;/;

         next if $decomp eq "";  # no decomposition, skip it

         $src = hex $code;

         if ($decomp =~ /^<([a-zA-Z]+)>\s+([0-9a-fA-F]+)$/)
         {
             # decomposition of the form "<foo> 1234" -> use char if type is known
             next unless ($1 eq "font" ||
                          $1 eq "noBreak" ||
                          $1 eq "circle" ||
                          $1 eq "super" ||
                          $1 eq "sub" ||
                          $1 eq "wide" ||
                          $1 eq "narrow" ||
                          $1 eq "compat" ||
                          $1 eq "small");
             $dst = hex $2;
         }
         elsif ($decomp =~ /^<compat>\s+0020\s+([0-9a-fA-F]+)/)
         {
             # decomposition "<compat> 0020 1234" -> combining accent
             $dst = hex $1;
         }
         elsif ($decomp =~ /^([0-9a-fA-F]+)/)
         {
             # decomposition contains only char values without prefix -> use first char
             $dst = hex $1;
         }
         else
         {
             next;
         }

         next if defined($unicode_defaults[$src]);  # may have been set in the defaults file

         # check for loops
         for ($i = $dst; ; $i = $unicode_defaults[$i])
         {
             die sprintf("loop detected for %04x -> %04x",$src,$dst) if $i == $src;
             last unless defined($unicode_defaults[$i]);
         }
         $unicode_defaults[$src] = $dst;
     }
 }


 ################################################################
 # parse the input file
 sub READ_FILE
 {
     my $name = shift;
     open INPUT,$name or die "Cannot open $name";
     @cp2uni = ();
     @lead_bytes = ();
     @uni2cp = ();

     while (<INPUT>)
     {
         next if /^\#/;  # skip comments
         next if /^$/;  # skip empty lines
         next if /\x1a/;  # skip ^Z
         if (/^0x([0-9a-fA-F]+)\s+\#UNDEFINED/)  # undefined char
         {
             $cp = hex $1;
             $uni = $cp;  # identity mapping
             $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
             $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
             next;
         }
         if (/^0x([0-9a-fA-F]+)\s+\#DBCS LEAD BYTE/)
         {
             $cp = hex $1;
             push @lead_bytes,$cp;
             $cp2uni[$cp] = 0;
             next;
         }
         if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
         {
             $cp = hex $1;
             $uni = hex $2;
             $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
             $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
             next;
         }
         die "$name: Unrecognized line $_\n";
     }

     # Apply aliases

     foreach $alias (@unicode_aliases)
     {
         my $target = undef;
         foreach $src (@$alias)
         {
             if (defined($uni2cp[$src]))
             {
                 $target = $uni2cp[$src];
                 last;
             }
         }
         next unless defined($target);

         # At least one char of the alias set is defined, set the others to the same value
         foreach $src (@$alias)
         {
             $uni2cp[$src] = $target unless defined($uni2cp[$src]);
         }
     }

     # For every src -> target mapping in the defaults table,
     # make uni2cp[src] = uni2cp[target] if uni2cp[target] is defined

     for ($src = 0; $src < 65536; $src++)
     {
         next if defined($uni2cp[$src]);  # source has a definition already
         next unless defined($unicode_defaults[$src]);  # no default for this char
         my $target = $unicode_defaults[$src];

         # do a recursive mapping until we find a target char that is defined
         while (!defined($uni2cp[$target]) &&
                defined($unicode_defaults[$target])) { $target = $unicode_defaults[$target]; }

         if (defined($uni2cp[$target])) { $uni2cp[$src] = $uni2cp[$target]; }
     }
 }


 ################################################################
 # dump an SBCS mapping table
 sub DUMP_SBCS_TABLE
 {
     my ($codepage, $name) = @_;
     my $x, $y;

     # output the ascii->unicode table

     printf OUTPUT "static const unsigned short cp2uni[256] =\n{\n    ";
     my $i = 0;
     for ($i = 0; $i < 256; $i++)
     {
         printf OUTPUT "0x%04x", (defined $cp2uni[$i] ? $cp2uni[$i] : $DEF_CHAR);
         if (($i % 8) != 7) { printf OUTPUT ", "; }
         else { print OUTPUT (($i < 255) ? ",\n    " : "\n};\n\n"); }
     }

     # count the number of unicode->ascii subtables that contain something

     my @filled = ();
     my $subtables = 1;
     for ($i = 0; $i < 65536; $i++)
     {
         next unless defined $uni2cp[$i];
         $filled[$i >> 8] = 1;
         $subtables++;
         $i = ($i & ~255) + 256;
     }

     # output all the subtables into a single array

     printf OUTPUT "static const unsigned char uni2cp_low[%d] =\n{\n   ", $subtables*256;
     for ($y = 0; $y < 256; $y++)
     {
         next unless $filled[$y];
         printf OUTPUT " /* 0x%02x00 .. 0x%02xff */\n   ", $y, $y;
         for ($x = 0; $x < 256; $x++)
         {
             printf OUTPUT " 0x%02x,", (defined $uni2cp[($y<<8)+$x] ?
                                      $uni2cp[($y<<8)+$x] : $DEF_CHAR);
             if (($x % 8) == 7) { printf OUTPUT "\n   "; }
         }
     }
     printf OUTPUT " /* defaults */\n   ";
     for ($x = 0; $x < 256; $x++)
     {
         printf OUTPUT " 0x%02x", $DEF_CHAR;
         if (($x % 8) != 7) { printf OUTPUT ","; }
         else { print OUTPUT (($x < 255) ? ",\n   " : "\n};\n\n"); }
     }

     # output a table of the offsets of the subtables in the previous array

     printf OUTPUT "static const unsigned short uni2cp_high[256] =\n{\n    ";
     my $pos = 0;
     for ($y = 0; $y < 256; $y++)
     {
         if ($filled[$y])
         {
             printf OUTPUT "0x%04x", $pos;
             $pos += 256;
         }
         else { printf OUTPUT "0x%04x", ($subtables-1) * 256; }
         if (($y % 8) != 7) { printf OUTPUT ", "; }
         else { print OUTPUT (($y < 255) ? ",\n    " : "\n};\n\n"); }
     }

     # output the code page descriptor

     printf OUTPUT "const struct sbcs_table cptable_%03d =\n{\n", $codepage;
     printf OUTPUT "    { %d, 1, 0x%04x, 0x%04x, \"%s\" },\n",
                   $codepage, $DEF_CHAR, $DEF_CHAR, $name;
     printf OUTPUT "    cp2uni,\n";
     printf OUTPUT "    uni2cp_low,\n";
     printf OUTPUT "    uni2cp_high\n};\n";
 }


 ################################################################
 # dump a DBCS mapping table
 sub DUMP_DBCS_TABLE
 {
     my ($codepage, $name) = @_;
     my $i, $x, $y;

     # build a list of lead bytes that are actually used

     my @lblist = ();
     LBLOOP: for ($y = 0; $y <= $#lead_bytes; $y++)
     {
         my $base = $lead_bytes[$y] << 8;
         for ($x = 0; $x < 256; $x++)
         {
             if (defined $cp2uni[$base+$x])
             {
                 push @lblist,$lead_bytes[$y];
                 next LBLOOP;
             }
         }
     }
     my $unused = ($#lead_bytes > $#lblist);

     # output the ascii->unicode table for the single byte chars

     printf OUTPUT "static const unsigned short cp2uni[%d] =\n{\n    ",
                   256 * ($#lblist + 2 + $unused);
     for ($x = 0; $x < 256; $x++)
     {
         printf OUTPUT "0x%04x", (defined $cp2uni[$x] ? $cp2uni[$x] : $DEF_CHAR);
         if (($x % 8) != 7) { printf OUTPUT ", "; }
         else { print OUTPUT ",\n    "; }
     }

     # output the default table for unused lead bytes

     if ($unused)
     {
         printf OUTPUT "/* unused lead bytes */\n    ";
         for ($x = 0; $x < 256; $x++)
         {
             printf OUTPUT "0x%04x", $DEF_CHAR;
             if (($x % 8) != 7) { printf OUTPUT ", "; }
             else { print OUTPUT ",\n    "; }
         }
     }

     # output the ascii->unicode table for each DBCS lead byte

     for ($y = 0; $y <= $#lblist; $y++)
     {
         my $base = $lblist[$y] << 8;
         printf OUTPUT "/* lead byte %02x */\n    ", $lblist[$y];
         for ($x = 0; $x < 256; $x++)
         {
             printf OUTPUT "0x%04x", (defined $cp2uni[$base+$x] ? $cp2uni[$base+$x] : $DEF_CHAR);
             if (($x % 8) != 7) { printf OUTPUT ", "; }
             else { print OUTPUT (($x < 255 || $y < $#lblist) ? ",\n    " : "\n};\n\n"); }
         }
     }

     # output the lead byte subtables offsets

     my @offsets = ();
     for ($x = 0; $x < 256; $x++) { $offsets[$x] = 0; }
     for ($x = 0; $x <= $#lblist; $x++) { $offsets[$lblist[$x]] = $x + 1; }
     if ($unused)
     {
         # increment all lead bytes offset to take into account the unused table
         for ($x = 0; $x <= $#lead_bytes; $x++) { $offsets[$lead_bytes[$x]]++; }
     }

     printf OUTPUT "static const unsigned char cp2uni_leadbytes[256] =\n{\n    ";
     for ($x = 0; $x < 256; $x++)
     {
         printf OUTPUT "0x%02x", $offsets[$x];
         if (($x % 8) != 7) { printf OUTPUT ", "; }
         else { print OUTPUT (($x < 255) ? ",\n    " : "\n};\n\n"); }
     }

     # count the number of unicode->ascii subtables that contain something

     my @filled = ();
     my $subtables = 1;
     for ($i = 0; $i < 65536; $i++)
     {
         next unless defined $uni2cp[$i];
         $filled[$i >> 8] = 1;
         $subtables++;
         $i = ($i & ~255) + 256;
     }

     # output all the subtables into a single array

     printf OUTPUT "static const unsigned short uni2cp_low[%d] =\n{\n   ", $subtables*256;
     for ($y = 0; $y < 256; $y++)
     {
         next unless $filled[$y];
         printf OUTPUT " /* 0x%02x00 .. 0x%02xff */\n   ", $y, $y;
         for ($x = 0; $x < 256; $x++)
         {
             printf OUTPUT " 0x%04x,", (defined $uni2cp[($y<<8)+$x] ?
                                      $uni2cp[($y<<8)+$x] : $DEF_CHAR);
             if (($x % 8) == 7) { printf OUTPUT "\n   "; }
         }
     }
     printf OUTPUT " /* defaults */\n   ";
     for ($x = 0; $x < 256; $x++)
     {
         printf OUTPUT " 0x%04x", $DEF_CHAR;
         if (($x % 8) != 7) { printf OUTPUT ","; }
         else { print OUTPUT (($x < 255) ? ",\n   " : "\n};\n\n"); }
     }

     # output a table of the offsets of the subtables in the previous array

     printf OUTPUT "static const unsigned short uni2cp_high[256] =\n{\n    ";
     my $pos = 0;
     for ($y = 0; $y < 256; $y++)
     {
         if ($filled[$y])
         {
             printf OUTPUT "0x%04x", $pos;
             $pos += 256;
         }
         else { printf OUTPUT "0x%04x", ($subtables-1) * 256; }

         if (($y % 8) != 7) { printf OUTPUT ", "; }
         else { print OUTPUT (($y < 255) ? ",\n    " : "\n};\n\n"); }
     }

     # output the code page descriptor

     printf OUTPUT "const struct dbcs_table cptable_%03d =\n{\n", $codepage;
     printf OUTPUT "    { %d, 2, 0x%04x, 0x%04x, \"%s\" },\n",
                   $codepage, $DEF_CHAR, $DEF_CHAR, $name;
     printf OUTPUT "    cp2uni,\n";
     printf OUTPUT "    cp2uni_leadbytes,\n";
     printf OUTPUT "    uni2cp_low,\n";
     printf OUTPUT "    uni2cp_high,\n";
     DUMP_LB_RANGES();
     printf OUTPUT "};\n";
 }


 ################################################################
 # dump the list of defined lead byte ranges
 sub DUMP_LB_RANGES
 {
     my @list = ();
     my $i = 0;
     foreach $i (@lead_bytes) { $list[$i] = 1; }
     my $on = 0;
     printf OUTPUT "    { ";
     for ($i = 0; $i < 256; $i++)
     {
         if ($on)
         {
             if (!defined $list[$i]) { printf OUTPUT "0x%02x, ", $i-1; $on = 0; }
         }
         else
         {
             if ($list[$i]) { printf OUTPUT "0x%02x, ", $i; $on = 1; }
         }
     }
     if ($on) { printf OUTPUT "0xff, "; }
     printf OUTPUT "0x00, 0x00 }\n";
 }


 ################################################################
 # read an input file and generate the corresponding .c file
 sub HANDLE_FILE
 {
     my ($codepage,$filename,$comment) = @_;

     READ_FILE($MAPPREFIX . $filename);

     my $output = sprintf "c_%03d.c", $codepage;
     open OUTPUT,">$output" or die "Cannot create $output";

     printf "Building %s from %s (%s)\n", $output, $filename, $comment;

     # dump all tables

     printf OUTPUT "/* code page %03d (%s) */\n", $codepage, $comment;
     printf OUTPUT "/* generated from %s */\n", $MAPPREFIX . $filename;
     printf OUTPUT "/* DO NOT EDIT!! */\n\n";
     printf OUTPUT "#include \"wine/unicode.h\"\n\n";

     if ($#lead_bytes == -1) { DUMP_SBCS_TABLE( $codepage, $comment ); }
     else { DUMP_DBCS_TABLE( $codepage, $comment ); }
 }


 ################################################################
 # output the list of codepage tables into the cptable.c file
 sub OUTPUT_CPTABLE
 {
     @tables_decl = ();

     foreach $file (@allfiles)
     {
         my ($codepage,$filename,$comment) = @$file;
         push @tables_decl, sprintf("extern union cptable cptable_%03d;\n",$codepage);
     }

     push @tables_decl, sprintf("\nstatic const union cptable * const cptables[%d] =\n{\n",$#allfiles+1);
     foreach $file (@allfiles)
     {
         my ($codepage,$filename,$comment) = @$file;
         push @tables_decl, sprintf("    &cptable_%03d,\n", $codepage);
     }
     push @tables_decl, "};";
     REPLACE_IN_FILE( "cptable.c", @tables_decl );
 }

 ################################################################
 # replace the contents of a file between ### cpmap ### marks

 sub REPLACE_IN_FILE
 {
     my $name = shift;
     my @data = @_;
     my @lines = ();
     open(FILE,$name) or die "Can't open $name";
     while (<FILE>)
     {
 	push @lines, $_;
 	last if /\#\#\# cpmap begin \#\#\#/;
     }
     push @lines, @data;
     while (<FILE>)
     {
 	if (/\#\#\# cpmap end \#\#\#/) { push @lines, "\n", $_; last; }
     }
     push @lines, <FILE>;
     open(FILE,">$name") or die "Can't modify $name";
     print FILE @lines;
     close(FILE);
 }
	#!/usr/bin/perl
	#
	# Generate code page .c files from ftp.unicode.org descriptions
	#
	# Copyright 2000 Alexandre Julliard
	#

	# base directory for ftp.unicode.org files
	$BASEDIR = "ftp.unicode.org/Public/";
	$MAPPREFIX = $BASEDIR . "MAPPINGS/";

	# UnicodeData file
	$UNICODEDATA = $BASEDIR . "UNIDATA/UnicodeData.txt";

	# Defaults mapping
	$DEFAULTS = "./defaults";

	# Default char for undefined mappings
	$DEF_CHAR = ord '?';

	@allfiles =
	(
	[ 37, "VENDORS/MICSFT/EBCDIC/CP037.TXT", "IBM EBCDIC US Canada" ],
	[ 424, "VENDORS/MISC/CP424.TXT", "IBM EBCDIC Hebrew" ],
	[ 437, "VENDORS/MICSFT/PC/CP437.TXT", "OEM United States" ],
	[ 500, "VENDORS/MICSFT/EBCDIC/CP500.TXT", "IBM EBCDIC International" ],
	[ 737, "VENDORS/MICSFT/PC/CP737.TXT", "OEM Greek 437G" ],
	[ 775, "VENDORS/MICSFT/PC/CP775.TXT", "OEM Baltic" ],
	[ 850, "VENDORS/MICSFT/PC/CP850.TXT", "OEM Multilingual Latin 1" ],
	[ 852, "VENDORS/MICSFT/PC/CP852.TXT", "OEM Slovak Latin 2" ],
	[ 855, "VENDORS/MICSFT/PC/CP855.TXT", "OEM Cyrillic" ],
	[ 856, "VENDORS/MISC/CP856.TXT", "Hebrew PC" ],
	[ 857, "VENDORS/MICSFT/PC/CP857.TXT", "OEM Turkish" ],
	[ 860, "VENDORS/MICSFT/PC/CP860.TXT", "OEM Portuguese" ],
	[ 861, "VENDORS/MICSFT/PC/CP861.TXT", "OEM Icelandic" ],
	[ 862, "VENDORS/MICSFT/PC/CP862.TXT", "OEM Hebrew" ],
	[ 863, "VENDORS/MICSFT/PC/CP863.TXT", "OEM Canadian French" ],
	[ 864, "VENDORS/MICSFT/PC/CP864.TXT", "OEM Arabic" ],
	[ 865, "VENDORS/MICSFT/PC/CP865.TXT", "OEM Nordic" ],
	[ 866, "VENDORS/MICSFT/PC/CP866.TXT", "OEM Russian" ],
	[ 869, "VENDORS/MICSFT/PC/CP869.TXT", "OEM Greek" ],
	[ 874, "VENDORS/MICSFT/PC/CP874.TXT", "ANSI/OEM Thai" ],
	[ 875, "VENDORS/MICSFT/EBCDIC/CP875.TXT", "IBM EBCDIC Greek" ],
	[ 878, "VENDORS/MISC/KOI8-R.TXT", "Russian KOI8" ],
	[ 932, "VENDORS/MICSFT/WINDOWS/CP932.TXT", "ANSI/OEM Japanese Shift-JIS" ],
	[ 936, "VENDORS/MICSFT/WINDOWS/CP936.TXT", "ANSI/OEM Simplified Chinese GBK" ],
	[ 949, "VENDORS/MICSFT/WINDOWS/CP949.TXT", "ANSI/OEM Korean Unified Hangul" ],
	[ 950, "VENDORS/MICSFT/WINDOWS/CP950.TXT", "ANSI/OEM Traditional Chinese Big5" ],
	[ 1006, "VENDORS/MISC/CP1006.TXT", "IBM Arabic" ],
	[ 1026, "VENDORS/MICSFT/EBCDIC/CP1026.TXT", "IBM EBCDIC Latin 5 Turkish" ],
	[ 1250, "VENDORS/MICSFT/WINDOWS/CP1250.TXT", "ANSI Eastern Europe" ],
	[ 1251, "VENDORS/MICSFT/WINDOWS/CP1251.TXT", "ANSI Cyrillic" ],
	[ 1252, "VENDORS/MICSFT/WINDOWS/CP1252.TXT", "ANSI Latin 1" ],
	[ 1253, "VENDORS/MICSFT/WINDOWS/CP1253.TXT", "ANSI Greek" ],
	[ 1254, "VENDORS/MICSFT/WINDOWS/CP1254.TXT", "ANSI Turkish" ],
	[ 1255, "VENDORS/MICSFT/WINDOWS/CP1255.TXT", "ANSI Hebrew" ],
	[ 1256, "VENDORS/MICSFT/WINDOWS/CP1256.TXT", "ANSI Arabic" ],
	[ 1257, "VENDORS/MICSFT/WINDOWS/CP1257.TXT", "ANSI Baltic" ],
	[ 1258, "VENDORS/MICSFT/WINDOWS/CP1258.TXT", "ANSI/OEM Viet Nam" ],
	[ 10000, "VENDORS/MICSFT/MAC/ROMAN.TXT", "Mac Roman" ],
	[ 10006, "VENDORS/MICSFT/MAC/GREEK.TXT", "Mac Greek" ],
	[ 10007, "VENDORS/MICSFT/MAC/CYRILLIC.TXT", "Mac Cyrillic" ],
	[ 10029, "VENDORS/MICSFT/MAC/LATIN2.TXT", "Mac Latin 2" ],
	[ 10079, "VENDORS/MICSFT/MAC/ICELAND.TXT", "Mac Icelandic" ],
	[ 10081, "VENDORS/MICSFT/MAC/TURKISH.TXT", "Mac Turkish" ],
	[ 20866, "VENDORS/MISC/KOI8-R.TXT", "Russian KOI8" ],
	[ 28591, "ISO8859/8859-1.TXT", "ISO 8859-1 Latin 1" ],
	[ 28592, "ISO8859/8859-2.TXT", "ISO 8859-2 Eastern Europe" ],
	[ 28593, "ISO8859/8859-3.TXT", "ISO 8859-3 Turkish" ],
	[ 28594, "ISO8859/8859-4.TXT", "ISO 8859-4 Baltic" ],
	[ 28595, "ISO8859/8859-5.TXT", "ISO 8859-5 Cyrillic" ],
	[ 28596, "ISO8859/8859-6.TXT", "ISO 8859-6 Arabic" ],
	[ 28597, "ISO8859/8859-7.TXT", "ISO 8859-7 Greek" ],
	[ 28598, "ISO8859/8859-8.TXT", "ISO 8859-8 Hebrew" ],
	[ 28599, "ISO8859/8859-9.TXT", "ISO 8859-9 Latin 5" ]
	);

	################################################################
	# main routine

	READ_DEFAULTS();

	foreach $file (@allfiles) { HANDLE_FILE( @$file ); }

	OUTPUT_CPTABLE();

	exit(0);


	################################################################
	# read in the defaults file
	sub READ_DEFAULTS
	{
	@unicode_defaults = ();
	@unicode_aliases = ();

	# first setup a few default mappings

	open DEFAULTS or die "Cannot open $DEFAULTS";
	print "Loading $DEFAULTS\n";
	while (<DEFAULTS>)
	{
	next if /^\#/; # skip comments
	next if /^$/; # skip empty lines
	if (/^(([0-9a-fA-F]+)(,[0-9a-fA-F]+))\s+([0-9a-fA-F]+\|'.'\|none)\s+(\#.)?/)
	{
	my @src = map hex, split /,/,$1;
	my $dst = $4;
	my $comment = $5;
	if ($#src > 0) { push @unicode_aliases, \@src; }
	next if ($dst eq "none");
	$dst = ($dst =~ /\'.\'/) ? ord substr($dst,1,1) : hex $dst;
	foreach $src (@src)
	{
	die "Duplicate value" if defined($unicode_defaults[$src]);
	$unicode_defaults[$src] = $dst;
	}
	next;
	}
	die "Unrecognized line $_\n";
	}

	# now build mappings from the decomposition field of the Unicode database

	open UNICODEDATA or die "Cannot open $UNICODEDATA";
	while (<UNICODEDATA>)
	{
	# Decode the fields ...
	($code, $name, $cat, $comb, $bidi,
	$decomp, $dec, $dig, $num, $mirror,
	$oldname, $comment, $upper, $lower, $title) = split /;/;

	next if $decomp eq ""; # no decomposition, skip it

	$src = hex $code;

	if ($decomp =~ /^<([a-zA-Z]+)>\s+([0-9a-fA-F]+)$/)
	{
	# decomposition of the form "<foo> 1234" -> use char if type is known
	next unless ($1 eq "font" \|\|
	$1 eq "noBreak" \|\|
	$1 eq "circle" \|\|
	$1 eq "super" \|\|
	$1 eq "sub" \|\|
	$1 eq "wide" \|\|
	$1 eq "narrow" \|\|
	$1 eq "compat" \|\|
	$1 eq "small");
	$dst = hex $2;
	}
	elsif ($decomp =~ /^<compat>\s+0020\s+([0-9a-fA-F]+)/)
	{
	# decomposition "<compat> 0020 1234" -> combining accent
	$dst = hex $1;
	}
	elsif ($decomp =~ /^([0-9a-fA-F]+)/)
	{
	# decomposition contains only char values without prefix -> use first char
	$dst = hex $1;
	}
	else
	{
	next;
	}

	next if defined($unicode_defaults[$src]); # may have been set in the defaults file

	# check for loops
	for ($i = $dst; ; $i = $unicode_defaults[$i])
	{
	die sprintf("loop detected for %04x -> %04x",$src,$dst) if $i == $src;
	last unless defined($unicode_defaults[$i]);
	}
	$unicode_defaults[$src] = $dst;
	}
	}


	################################################################
	# parse the input file
	sub READ_FILE
	{
	my $name = shift;
	open INPUT,$name or die "Cannot open $name";
	@cp2uni = ();
	@lead_bytes = ();
	@uni2cp = ();

	while (<INPUT>)
	{
	next if /^\#/; # skip comments
	next if /^$/; # skip empty lines
	next if /\x1a/; # skip ^Z
	if (/^0x([0-9a-fA-F]+)\s+\#UNDEFINED/) # undefined char
	{
	$cp = hex $1;
	$uni = $cp; # identity mapping
	$cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
	$uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
	next;
	}
	if (/^0x([0-9a-fA-F]+)\s+\#DBCS LEAD BYTE/)
	{
	$cp = hex $1;
	push @lead_bytes,$cp;
	$cp2uni[$cp] = 0;
	next;
	}
	if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
	{
	$cp = hex $1;
	$uni = hex $2;
	$cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
	$uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
	next;
	}
	die "$name: Unrecognized line $_\n";
	}

	# Apply aliases

	foreach $alias (@unicode_aliases)
	{
	my $target = undef;
	foreach $src (@$alias)
	{
	if (defined($uni2cp[$src]))
	{
	$target = $uni2cp[$src];
	last;
	}
	}
	next unless defined($target);

	# At least one char of the alias set is defined, set the others to the same value
	foreach $src (@$alias)
	{
	$uni2cp[$src] = $target unless defined($uni2cp[$src]);
	}
	}

	# For every src -> target mapping in the defaults table,
	# make uni2cp[src] = uni2cp[target] if uni2cp[target] is defined

	for ($src = 0; $src < 65536; $src++)
	{
	next if defined($uni2cp[$src]); # source has a definition already
	next unless defined($unicode_defaults[$src]); # no default for this char
	my $target = $unicode_defaults[$src];

	# do a recursive mapping until we find a target char that is defined
	while (!defined($uni2cp[$target]) &&
	defined($unicode_defaults[$target])) { $target = $unicode_defaults[$target]; }

	if (defined($uni2cp[$target])) { $uni2cp[$src] = $uni2cp[$target]; }
	}
	}


	################################################################
	# dump an SBCS mapping table
	sub DUMP_SBCS_TABLE
	{
	my ($codepage, $name) = @_;
	my $x, $y;

	# output the ascii->unicode table

	printf OUTPUT "static const unsigned short cp2uni[256] =\n{\n ";
	my $i = 0;
	for ($i = 0; $i < 256; $i++)
	{
	printf OUTPUT "0x%04x", (defined $cp2uni[$i] ? $cp2uni[$i] : $DEF_CHAR);
	if (($i % 8) != 7) { printf OUTPUT ", "; }
	else { print OUTPUT (($i < 255) ? ",\n " : "\n};\n\n"); }
	}

	# count the number of unicode->ascii subtables that contain something

	my @filled = ();
	my $subtables = 1;
	for ($i = 0; $i < 65536; $i++)
	{
	next unless defined $uni2cp[$i];
	$filled[$i >> 8] = 1;
	$subtables++;
	$i = ($i & ~255) + 256;
	}

	# output all the subtables into a single array

	printf OUTPUT "static const unsigned char uni2cp_low[%d] =\n{\n ", $subtables*256;
	for ($y = 0; $y < 256; $y++)
	{
	next unless $filled[$y];
	printf OUTPUT " /* 0x%02x00 .. 0x%02xff */\n ", $y, $y;
	for ($x = 0; $x < 256; $x++)
	{
	printf OUTPUT " 0x%02x,", (defined $uni2cp[($y<<8)+$x] ?
	$uni2cp[($y<<8)+$x] : $DEF_CHAR);
	if (($x % 8) == 7) { printf OUTPUT "\n "; }
	}
	}
	printf OUTPUT " /* defaults */\n ";
	for ($x = 0; $x < 256; $x++)
	{
	printf OUTPUT " 0x%02x", $DEF_CHAR;
	if (($x % 8) != 7) { printf OUTPUT ","; }
	else { print OUTPUT (($x < 255) ? ",\n " : "\n};\n\n"); }
	}

	# output a table of the offsets of the subtables in the previous array

	printf OUTPUT "static const unsigned short uni2cp_high[256] =\n{\n ";
	my $pos = 0;
	for ($y = 0; $y < 256; $y++)
	{
	if ($filled[$y])
	{
	printf OUTPUT "0x%04x", $pos;
	$pos += 256;
	}
	else { printf OUTPUT "0x%04x", ($subtables-1) * 256; }
	if (($y % 8) != 7) { printf OUTPUT ", "; }
	else { print OUTPUT (($y < 255) ? ",\n " : "\n};\n\n"); }
	}

	# output the code page descriptor

	printf OUTPUT "const struct sbcs_table cptable_%03d =\n{\n", $codepage;
	printf OUTPUT " { %d, 1, 0x%04x, 0x%04x, \"%s\" },\n",
	$codepage, $DEF_CHAR, $DEF_CHAR, $name;
	printf OUTPUT " cp2uni,\n";
	printf OUTPUT " uni2cp_low,\n";
	printf OUTPUT " uni2cp_high\n};\n";
	}


	################################################################
	# dump a DBCS mapping table
	sub DUMP_DBCS_TABLE
	{
	my ($codepage, $name) = @_;
	my $i, $x, $y;

	# build a list of lead bytes that are actually used

	my @lblist = ();
	LBLOOP: for ($y = 0; $y <= $#lead_bytes; $y++)
	{
	my $base = $lead_bytes[$y] << 8;
	for ($x = 0; $x < 256; $x++)
	{
	if (defined $cp2uni[$base+$x])
	{
	push @lblist,$lead_bytes[$y];
	next LBLOOP;
	}
	}
	}
	my $unused = ($#lead_bytes > $#lblist);

	# output the ascii->unicode table for the single byte chars

	printf OUTPUT "static const unsigned short cp2uni[%d] =\n{\n ",
	256 * ($#lblist + 2 + $unused);
	for ($x = 0; $x < 256; $x++)
	{
	printf OUTPUT "0x%04x", (defined $cp2uni[$x] ? $cp2uni[$x] : $DEF_CHAR);
	if (($x % 8) != 7) { printf OUTPUT ", "; }
	else { print OUTPUT ",\n "; }
	}

	# output the default table for unused lead bytes

	if ($unused)
	{
	printf OUTPUT "/* unused lead bytes */\n ";
	for ($x = 0; $x < 256; $x++)
	{
	printf OUTPUT "0x%04x", $DEF_CHAR;
	if (($x % 8) != 7) { printf OUTPUT ", "; }
	else { print OUTPUT ",\n "; }
	}
	}

	# output the ascii->unicode table for each DBCS lead byte

	for ($y = 0; $y <= $#lblist; $y++)
	{
	my $base = $lblist[$y] << 8;
	printf OUTPUT "/* lead byte %02x */\n ", $lblist[$y];
	for ($x = 0; $x < 256; $x++)
	{
	printf OUTPUT "0x%04x", (defined $cp2uni[$base+$x] ? $cp2uni[$base+$x] : $DEF_CHAR);
	if (($x % 8) != 7) { printf OUTPUT ", "; }
	else { print OUTPUT (($x < 255 \|\| $y < $#lblist) ? ",\n " : "\n};\n\n"); }
	}
	}

	# output the lead byte subtables offsets

	my @offsets = ();
	for ($x = 0; $x < 256; $x++) { $offsets[$x] = 0; }
	for ($x = 0; $x <= $#lblist; $x++) { $offsets[$lblist[$x]] = $x + 1; }
	if ($unused)
	{
	# increment all lead bytes offset to take into account the unused table
	for ($x = 0; $x <= $#lead_bytes; $x++) { $offsets[$lead_bytes[$x]]++; }
	}

	printf OUTPUT "static const unsigned char cp2uni_leadbytes[256] =\n{\n ";
	for ($x = 0; $x < 256; $x++)
	{
	printf OUTPUT "0x%02x", $offsets[$x];
	if (($x % 8) != 7) { printf OUTPUT ", "; }
	else { print OUTPUT (($x < 255) ? ",\n " : "\n};\n\n"); }
	}

	# count the number of unicode->ascii subtables that contain something

	my @filled = ();
	my $subtables = 1;
	for ($i = 0; $i < 65536; $i++)
	{
	next unless defined $uni2cp[$i];
	$filled[$i >> 8] = 1;
	$subtables++;
	$i = ($i & ~255) + 256;
	}

	# output all the subtables into a single array

	printf OUTPUT "static const unsigned short uni2cp_low[%d] =\n{\n ", $subtables*256;
	for ($y = 0; $y < 256; $y++)
	{
	next unless $filled[$y];
	printf OUTPUT " /* 0x%02x00 .. 0x%02xff */\n ", $y, $y;
	for ($x = 0; $x < 256; $x++)
	{
	printf OUTPUT " 0x%04x,", (defined $uni2cp[($y<<8)+$x] ?
	$uni2cp[($y<<8)+$x] : $DEF_CHAR);
	if (($x % 8) == 7) { printf OUTPUT "\n "; }
	}
	}
	printf OUTPUT " /* defaults */\n ";
	for ($x = 0; $x < 256; $x++)
	{
	printf OUTPUT " 0x%04x", $DEF_CHAR;
	if (($x % 8) != 7) { printf OUTPUT ","; }
	else { print OUTPUT (($x < 255) ? ",\n " : "\n};\n\n"); }
	}

	# output a table of the offsets of the subtables in the previous array

	printf OUTPUT "static const unsigned short uni2cp_high[256] =\n{\n ";
	my $pos = 0;
	for ($y = 0; $y < 256; $y++)
	{
	if ($filled[$y])
	{
	printf OUTPUT "0x%04x", $pos;
	$pos += 256;
	}
	else { printf OUTPUT "0x%04x", ($subtables-1) * 256; }

	if (($y % 8) != 7) { printf OUTPUT ", "; }
	else { print OUTPUT (($y < 255) ? ",\n " : "\n};\n\n"); }
	}

	# output the code page descriptor

	printf OUTPUT "const struct dbcs_table cptable_%03d =\n{\n", $codepage;
	printf OUTPUT " { %d, 2, 0x%04x, 0x%04x, \"%s\" },\n",
	$codepage, $DEF_CHAR, $DEF_CHAR, $name;
	printf OUTPUT " cp2uni,\n";
	printf OUTPUT " cp2uni_leadbytes,\n";
	printf OUTPUT " uni2cp_low,\n";
	printf OUTPUT " uni2cp_high,\n";
	DUMP_LB_RANGES();
	printf OUTPUT "};\n";
	}


	################################################################
	# dump the list of defined lead byte ranges
	sub DUMP_LB_RANGES
	{
	my @list = ();
	my $i = 0;
	foreach $i (@lead_bytes) { $list[$i] = 1; }
	my $on = 0;
	printf OUTPUT " { ";
	for ($i = 0; $i < 256; $i++)
	{
	if ($on)
	{
	if (!defined $list[$i]) { printf OUTPUT "0x%02x, ", $i-1; $on = 0; }
	}
	else
	{
	if ($list[$i]) { printf OUTPUT "0x%02x, ", $i; $on = 1; }
	}
	}
	if ($on) { printf OUTPUT "0xff, "; }
	printf OUTPUT "0x00, 0x00 }\n";
	}


	################################################################
	# read an input file and generate the corresponding .c file
	sub HANDLE_FILE
	{
	my ($codepage,$filename,$comment) = @_;

	READ_FILE($MAPPREFIX . $filename);

	my $output = sprintf "c_%03d.c", $codepage;
	open OUTPUT,">$output" or die "Cannot create $output";

	printf "Building %s from %s (%s)\n", $output, $filename, $comment;

	# dump all tables

	printf OUTPUT "/* code page %03d (%s) */\n", $codepage, $comment;
	printf OUTPUT "/* generated from %s */\n", $MAPPREFIX . $filename;
	printf OUTPUT "/* DO NOT EDIT!! */\n\n";
	printf OUTPUT "#include \"wine/unicode.h\"\n\n";

	if ($#lead_bytes == -1) { DUMP_SBCS_TABLE( $codepage, $comment ); }
	else { DUMP_DBCS_TABLE( $codepage, $comment ); }
	}


	################################################################
	# output the list of codepage tables into the cptable.c file
	sub OUTPUT_CPTABLE
	{
	@tables_decl = ();

	foreach $file (@allfiles)
	{
	my ($codepage,$filename,$comment) = @$file;
	push @tables_decl, sprintf("extern union cptable cptable_%03d;\n",$codepage);
	}

	push @tables_decl, sprintf("\nstatic const union cptable * const cptables[%d] =\n{\n",$#allfiles+1);
	foreach $file (@allfiles)
	{
	my ($codepage,$filename,$comment) = @$file;
	push @tables_decl, sprintf(" &cptable_%03d,\n", $codepage);
	}
	push @tables_decl, "};";
	REPLACE_IN_FILE( "cptable.c", @tables_decl );
	}

	################################################################
	# replace the contents of a file between ### cpmap ### marks

	sub REPLACE_IN_FILE
	{
	my $name = shift;
	my @data = @_;
	my @lines = ();
	open(FILE,$name) or die "Can't open $name";
	while (<FILE>)
	{
	push @lines, $_;
	last if /\#\#\# cpmap begin \#\#\#/;
	}
	push @lines, @data;
	while (<FILE>)
	{
	if (/\#\#\# cpmap end \#\#\#/) { push @lines, "\n", $_; last; }
	}
	push @lines, <FILE>;
	open(FILE,">$name") or die "Can't modify $name";
	print FILE @lines;
	close(FILE);
	}