4 title => 'charset cheat sheet',
7 "Reference sheet with all glyphs in common character encoding tables,",
8 "and an overview of Unicode ranges and UTF-8 bytes.",
11 charset codepage unicode ascii utf8 latin glyph character encoding
12 reference common overview table
14 stylesheet => [qw'light'],
15 data => [qw'charset-unicode.inc.pl charset-utf8.inc.pl'],
18 my @tablist = split m{/+}, $Request || 'default';
20 use List::Util qw( first pairmap pairfirst pairs );
23 <h1>Character encodings</h1>
27 if ($tablist[0] eq 'default') {
28 say "Overview of Unicode allocation and common latin code pages.";
29 say "Compare alternate charsets:";
32 say "Charset comparison:";
36 my ($title, $href, $selected) = @_;
38 $selected ? '<strong>%s</strong>' : '<a href="%2$s">%s</a>',
39 EscapeHTML($title), $href
45 join " ·\n", pairmap {
46 optionlink($b || ucfirst $a, '/charset'.($a && "/$a?compare"), $a eq $Request);
54 $tablist[0] eq 'default' ? () : ('' => 'common'),
59 norteur => 'North European',
72 use Shiar_Sheet::FormatChar;
73 my $glyphs = Shiar_Sheet::FormatChar->new;
77 # generate character table(s)
78 my $input = shift or return;
81 default => [qw( unicode+0-639 utf-8+realsize iso-8859-1 iso-8859-15 cp1252 cp437 cp850 )],
82 uc => [qw( U+cols=32 unicode+0-4095 unicode+4096-6319 unicode+6320-8191 )],
83 us => [qw( cp437 cp863 gsm0338 AdobeStandardEncoding )],
84 ebcdic => [qw( cp37 cp500 cp1047 posix-bc cp1026 cp875 )],
85 iso => [map {"iso-8859-$_"} 1 .. 11, 13 .. 16],
86 dos => [qw( cp437 cp865 cp861 cp860 cp863 cp850 cp857 cp852 cp775
87 cp737 cp869 cp866 cp855 cp862 cp864 )],
88 aix => [qw( cp1006 )],
89 win => [qw( cp1252 cp1250 cp1254 cp1257 cp1258 cp1253 cp1251 cp1255 cp1256 cp874 )],
90 mac => [qw( MacRoman MacRomanian MacRumanian MacCroatian MacCentralEurRoman MacTurkish MacIcelandic MacSami
91 MacGreek MacCyrillic MacHebrew MacArabic MacFarsi MacThai )],
94 westeur => [qw( iso-8859-1 iso-8859-15 cp1252 iso-8859-14 cp850 hp-roman8 nextstep MacRoman )],
95 centeur => [qw( iso-8859-2 iso-8859-16 cp1250 cp852 MacRomanian MacCroatian MacCentralEurRoman )], # MacRumanian only for DB
96 turkish => [qw( iso-8859-9 iso-8859-3 cp1254 cp857 MacTurkish )],
97 norteur => [qw( baltic nordic )],
98 baltic => [qw( iso-8859-4 iso-8859-13 cp1257 cp775 )],
99 nordic => [qw( iso-8859-10 cp865 cp861 MacIcelandic MacSami )],
100 cyrillic => [qw( koi8-r koi8-u koi8-f iso-8859-5 cp1251 MacCyrillic cp866 cp855
101 U4 U2DE-2DF UA64-A69 U50-52 )], # MacUkrainian is broken
102 arabic => [qw( iso-8859-6 cp1256 MacArabic cp864 cp1006 MacFarsi
103 U6 U8A-8F+0-31+64 U75-77 )],
104 greek => [qw( iso-8859-7 cp1253 MacGreek cp737 cp869 U37-3F U1F )],
105 hebrew => [qw( iso-8859-8 cp1255 MacHebrew cp862 U59-5F )],
106 thai => [qw( iso-8859-11 cp874 MacThai )],
107 vietnamese => [qw( viscii cp1258 MacVietnamese )],
108 symbol => [qw( symbol dingbats MacDingbats AdobeZdingbat AdobeSymbol )],
123 if (my $follow = $ALIAS->{$input}) {
124 tabinput($_) for ref $follow ? @{$follow} : $follow;
128 state $visible = {}; # all present tables
129 my %row = (offset => 0, cols => 16);
130 my $params = $input =~ s/[+](.*)\z// ? $1 : undef;
132 if (not defined $params) {
134 'cp437' => ['cp850' => 0, 'ascii' => '0-31+128'], # ascii range overridden later
135 'gsm0338' => ['ascii' => '0-127'],
137 'iso-8859-2' => ['iso-8859-1' => '160'],
138 'iso-8859-3' => ['iso-8859-1' => '160'], #TODO: also apply to iso-8859-9
139 'iso-8859-4' => ['iso-8859-2' => '160'],
140 'iso-8859-5' => ['iso-8859-1' => '160'],
141 'iso-8859-6' => ['cp1256' => '128', 'iso-8859-1' => '160'],
142 'iso-8859-7' => ['iso-8859-1' => '160'],
143 'iso-8859-8' => ['iso-8859-1' => '160'],
144 'iso-8859-9' => ['iso-8859-1' => '208-223+240'],
145 'iso-8859-10' => ['iso-8859-4' => '160'],
146 'iso-8859-11' => ['iso-8859-1' => '160'],
147 'iso-8859-13' => ['iso-8859-4' => '160'],
148 'iso-8859-14' => ['iso-8859-1' => '160'],
149 'iso-8859-15' => ['iso-8859-1' => '160-191'],
150 'iso-8859-16' => ['iso-8859-2' => '160'],
151 'hp-roman8' => ['iso-8859-1' => '160'],
153 'cp1252' => ['iso-8859-1' => '128-159'],
154 'cp1250' => ['iso-8859-2' => '128-191', 'cp1252' => '128'],
155 'cp1254' => ['iso-8859-9' => '128-159', 'cp1252' => '128-159+208'],
156 'cp874' => ['iso-8859-11' => '128-159', 'cp1252' => '128'], # windows-874 actually cp1162
157 'cp1257' => ['iso-8859-13' => '128-159+255', 'cp1252' => '128'],
158 'cp1251' => ['cp1252' => '128'],
159 'cp1253' => ['cp1252' => '128'],
160 'cp1255' => ['iso-8859-8' => '128-223', 'cp1252' => '128'],
161 'cp1256' => ['cp1252' => '128'],
162 'cp1258' => ['cp1252' => '128-159+192'],
164 'cp850' => ['cp437' => '144'],
165 'cp860' => ['cp437' => '128-175'],
166 'cp861' => ['cp865' => '128-175'],
167 'cp863' => ['cp437' => '128-175'],
168 'cp865' => ['cp437' => '144-175'],
169 'cp852' => ['cp850' => '128', 'cp437' => '128'],
170 'cp857' => ['cp850' => '128-175+208-239', 'cp437' => '128'],
171 'cp775' => ['cp850' => '128'], # partial cp437
172 'cp866' => ['cp437' => '128-175+224'],
173 'cp855' => ['cp437' => '128'],
174 'cp1006' => ['iso-8859-6' => '160', 'cp437' => '128'],
175 'cp737' => ['cp437' => '128-175+224'],
176 'cp869' => ['cp437' => '128'],
177 'cp862' => ['cp437' => '128-159'],
178 'cp864' => ['MacArabic' => '128', 'iso-8859-6' => '128', 'cp437' => '128'], #TODO: compare form variants
180 'koi8-u' => ['koi8-r' => '144-191'],
181 'koi8-f' => ['koi8-u' => '144-191'],
183 'MacRomanian' => ['MacRoman' => '160-191+208-223'],
184 'MacRumanian' => ['MacRomanian' => '160-191+208-223', 'MacRoman' => '160-191+208-223'],
185 'MacCroatian' => ['MacRoman' => '160'],
186 'MacCentralEurRoman' => ['MacRoman' => '128'],
187 'MacIcelandic'=> ['MacRoman' => '160-175+208-239'],
188 'MacTurkish' => ['MacRoman' => '208-223'], # F5 is unassigned
189 'MacSami' => ['MacIcelandic' => '144-191+208-223+240', 'MacRoman' => '144'],
190 'MacGreek' => ['MacRoman' => '128'],
191 'MacCyrillic' => ['MacRoman' => '128'],
192 'MacHebrew' => ['iso-8859-8' => '128', 'MacRoman' => '128-143+160'], # partial ascii
193 'MacArabic' => ['iso-8859-6' => '128', 'cp864' => '128', 'MacRoman' => '128'], #TODO: multiple parents
194 'MacFarsi' => ['MacArabic' => '176-191', 'MacRoman' => '128'],
196 'cp37' => ['posix-bc' => '0'],
197 'posix-bc' => ['cp1047' => '64'],
198 'cp500' => ['cp37' => '64-95+176-191'],
199 'cp1047' => ['cp37' => '16-47+80-95+160-191'],
200 'cp1026' => ['cp37' => '64'],
201 'cp875' => ['cp37' => '48'],
205 my @parents = @{ $INHERIT->{$input} || [] };
207 if (my ($parent, $part) = pairfirst { defined $visible->{$a} } @parents) {
208 $row{parent} = $parent;
210 $params = 128 unless $visible->{$parent}
211 or ($input eq 'MacCroatian' and defined $visible->{MacRomanian});
213 elsif (defined $visible->{ascii}) {
214 $row{parent} = $parents[0];
215 $params = $parents[1] // 128;
216 $params = 128 if $params >= 128; # ascii offset at most
219 $row{parent} = $parents[0];
220 $params = $parents[1] if $parents[1] == 0; # apply ascii end
222 $visible->{$_} //= 0 for $row{parent} || ();
225 for my $param (split /[+]+/, $params // '') {
226 if ($param eq 'realsize') {
229 elsif ($param =~ m{ \A cols = (\d+) \z }x) {
232 elsif ($param =~ m{ \A (?<offset> \d+) (?: [-] (?<endpoint> \d+) )? \z }x) {
233 if (defined $row{endpoint}) {
234 # extend earlier range
235 my $skip = int(($row{endpoint} || $row{offset}) / $row{cols});
236 for ($skip + 1 .. ($+{offset} / $row{cols}) - 1) {
237 $row{skip}->{ $_ * $row{cols} - $row{offset} }++;
241 $row{offset} = $+{offset};
243 $row{endpoint} = $+{endpoint} || 0;
246 Alert("Unknown option <q>$param</q> for charset $input");
250 if ($input =~ /^U([0-9a-fA-F]+)(?:-([0-9a-fA-F]+))?/) {
251 my $start = hex($1) << ($2 ? 4 : 8);
252 my $end = $2 ? (hex($2) << 4) + $row{cols} - 1 : $start + 255;
253 $row{table} = join '', map { chr } $start .. $end;
254 utf8::upgrade($row{table}); # prevent latin1 output
255 $row{endpoint} = $end - $start;
256 $row{set} = sprintf 'Unicode block U+%02Xxx', $start >> 8;
257 $row{offset} = $start % 256;
259 elsif ($input eq 'U') {
260 $row{set} = 'Unicode planes';
261 $row{cell} = do 'charset-ucplanes.inc.pl'
262 or Alert('Table data could not be read', $@ || $!);
263 $row{endpoint} = 1023 * $row{cell}->{colsize};
265 elsif ($row{set} = Encode::resolve_alias($input)) {
266 if ($row{set} eq 'Internal') {
267 $row{cell} = do 'charset-unicode.inc.pl'
268 or Alert('Table data could not be read', $@ || $!);
270 $row{endpoint} ||= 8191;
271 $row{endpoint} *= $row{cell}->{colsize};
272 $row{startpoint} = $row{cell}->{colsize} * $row{offset};
274 $row{set} = 'Unicode ' . (
275 $row{startpoint} < 0x10000 && $row{endpoint} < 0x10000 ? 'BMP' :
276 $row{startpoint} >= 0x10000 && $row{endpoint} < 0x20000 ? 'SMP' :
280 elsif ($row{set} eq 'utf-8-strict') {
282 $row{cell} = do 'charset-utf8.inc.pl'
283 or Alert('Table data could not be read', $@ || $!);
284 $row{endpoint} = 255;
287 $row{endpoint} ||= 255;
288 if ($row{set} eq 'MacHebrew' or $row{set} eq 'MacThai') {
289 # array of possibly multiple characters per code point
291 map { Encode::decode($row{set}, pack 'C*', $_) } $row{offset} .. $row{endpoint}
295 # ~16x faster than decoding in loop;
296 # substr strings is twice as fast as splitting to an array
297 $row{table} = Encode::decode($row{set}, pack 'C*', $row{offset} .. $row{endpoint});
299 $row{endpoint} -= $row{offset};
301 if ($row{set} eq 'cp437') {
302 for my $phipos (237 - $row{offset}) {
303 next if $phipos < 0 or $phipos > $row{endpoint};
305 substr($row{table}, $phipos, 1) = pack 'U*', 0x3D5;
307 if ($row{offset} == 0) {
308 # replace control characters by visible variants
309 substr($row{table}, 0, 32) = pack 'U*', map {hex} qw(
310 2007 263A 263B 2665 2666 2663 2660 2022
311 25D8 25CB 25D9 2642 2640 266A 266B 263C
312 25BA 25C4 2195 203C 00B6 00A7 25AC 21A8
313 2191 2193 2192 2190 221F 2194 25B2 25BC
318 $visible->{ascii} = # assume common base
319 $visible->{ $row{set} } = 1;
323 Alert("Encoding <q>$input</q> unknown");
326 push @request, \%row;
328 tabinput($_) for @tablist;
330 my $NOCHAR = chr 0xFFFD;
333 my ($info, $offset) = @_;
334 my $table = $info->{cell} or return;
335 my $def = $table->{$offset} or return;
336 my ($len, $class, $name, $title) = @{$def};
338 my $cols = $info->{cols};
339 my $colsize = $table->{colsize} || 1;
342 $name //= $len <= 2 ? 'res' : 'reserved';
344 if (my $part = $offset/$colsize % $cols) {
346 my $rest = $cols - $part; # remaining
347 $rest = $len if $len < $rest; #TODO: optimise
349 # continued on new row
350 my @next = ($len * $colsize, "$class joinu");
353 push @next, $name, $title;
358 # minority on next row
359 push @next, '"', $title || $name;
361 $table->{$offset + $colsize*$rest} //= \@next;
366 elsif (my $rows = int($len / $cols)) {
368 my $rowsize = $colsize * $cols;
369 if ($len -= $rows * $cols) {
370 # partial row remains
371 $table->{$offset + $rowsize * $rows} //= [$len*$colsize, "$class joinu", '', $title];
375 unless ($info->{realsize}) {
376 # coalesce multiple rows
378 $info->{skip}->{$offset += $rowsize}++;
382 $info->{skip}->{$offset += $rowsize} = 0;
386 $attr .= sprintf ' rowspan=%d', $rows;
390 $attr .= sprintf ' colspan=%d', $len unless $len == 1;
391 $attr .= $1 if $class and $class =~ s/( \w+="[^"]*")//;
392 $attr .= sprintf ' class="%s"', $class if $class;
393 $attr .= sprintf ' title="%s"', EscapeHTML($title) if $title;
394 return "<td$attr>$name";
397 for my $row (@request) {
398 my $cols = $row->{cols};
399 my $colsize = $row->{cell} && $row->{cell}->{colsize} || 1;
400 my $coldigits = ceil(log($colsize * $cols) / log(16)); # uniform length of hexadecimal header
401 my $rowdiv = 16 ** $coldigits; # row divide for column digits
402 $rowdiv = 1 if $rowdiv != $cols * $colsize; # divide only if all columns are matched
403 my $offset = $row->{startpoint} || 0;
405 printf '<div class="section"><table class="glyphs%s">', !$row->{cell} && ' charmap';
406 my $title = $row->{set};
407 $title .= " <aside>(over $_)</aside>"
408 for grep { $_ ne 'iso-8859-1' } $row->{parent} // ();
409 printf '<caption>%s</caption>', $title;
410 print '<col>' x ($cols + 1);
411 for my $section (qw{thead}) {
412 print "<$section><tr><th>", $rowdiv == 1 ? '+' : '↱';
413 printf '<th>%0*X', $coldigits, $_ * $colsize for 0 .. $cols - 1;
417 while ($offset < $row->{endpoint}) {
418 if ($row->{skip}->{$offset}) {
419 $offset += $cols * $colsize;
424 if (defined $row->{skip}->{$offset}) {
428 if (my $rowmod = $offset % $rowdiv) {
429 # offset in column units
430 printf '<small>+%X</small>', $rowmod;
434 printf '%X', ($offset + $row->{offset}) / $rowdiv;
439 print range_cell($row, $offset);
443 my $glyph = ref $row->{table} eq 'ARRAY' ? $row->{table}->[$offset] :
444 substr $row->{table}, $offset, 1;
445 if ($glyph eq $NOCHAR) {
450 if (exists $get{compare}) {
452 my $cp = $offset + $row->{offset};
453 printf '<td class="%s" title="%3$s">%2$s',
454 $cp == ord $glyph ? 'l4' :
455 $row->{parent} && $glyph eq
456 Encode::decode($row->{parent}, pack 'C', $cp) ? 'l3' :
457 $visible->{$glyph} ? 'l2' :
459 $glyphs->glyph_html($glyph);
460 $visible->{$glyph}++;
464 print "\n".$glyphs->glyph_cell($glyph);
471 say '</table></div>';
478 <table class="glyphs"><tr><: if (exists $get{compare}) { :>
479 <td class="X l4">unicode
480 <td class="X l3">inherited
481 <td class="X l2">existing
482 <td class="X l1">original
483 <td class="">unassigned
485 <td class="X Cc">control
486 <td class="X Zs"><span>whitespace</span>
487 <td class="X Mn">diacritic<table class="glyphs"><tr>
488 <td class="X Sk">letter
490 <td class="X Po">punctuation<table class="glyphs"><tr>
491 <td class="X Pf">quote
493 <td class="X So">symbol<table class="glyphs"><tr>
494 <td class="X Sm">math
495 <td class="X Sc">currency
497 <td class="X No">numeric
498 <td class="X Greek">greek<table class="glyphs"><tr>
499 <td class="X Latin">latin
500 <td class="X Cyrillic">cyrillic
502 <td class="X Aramaic">aramaic<table class="glyphs"><tr>
503 <td class="X Brahmic">brahmic
504 <td class="X Arabic">arabic
506 <td class="X Syllabic">syllabic<table class="glyphs"><tr>
507 <td class="X African">african
508 <td class="X Hiragana">japanese
509 <td class="X Han">cjk
510 <td class="X Bopomofo">chinese
512 <td class="X Alpha">alphabetic
515 <table class="glyphs"><tr>
516 <td class="X">unicode 7.0
517 <td class="X Xr">proposed
518 <td class="X Xd">deprecated
519 <td class="">unassigned
520 <td class="X Xi">invalid