4 title => 'charset cheat sheet',
7 "Reference sheet with all glyphs in common character encoding tables,",
8 "and an overview of Unicode ranges and UTF-8 bytes.",
11 charset codepage unicode ascii utf8 latin glyph character encoding
12 reference common overview table
14 stylesheet => [qw'light'],
15 data => [qw'charset-unicode.inc.pl charset-utf8.inc.pl'],
18 my @tablist = split m{/+}, $Request || 'default';
20 use List::Util qw( first pairmap pairfirst pairs );
23 <h1>Character encodings</h1>
27 if ($tablist[0] eq 'default') {
28 say "Overview of Unicode allocation and common latin code pages.";
29 say "Compare alternate charsets:";
32 say "Charset comparison:";
36 my ($title, $href, $selected) = @_;
38 $selected ? '<strong>%s</strong>' : '<a href="%2$s">%s</a>',
39 EscapeHTML($title), $href
45 join " ·\n", pairmap {
46 optionlink($b || ucfirst $a, '/charset'.($a && "/$a?compare"), $a eq $Request);
55 $tablist[0] eq 'default' ? () : ('' => 'common'),
60 norteur => 'North European',
72 use Shiar_Sheet::FormatChar;
73 my $glyphs = Shiar_Sheet::FormatChar->new;
77 # generate character table(s)
78 my $input = shift or return;
81 default => [qw( u+0-639 utf-8+realsize iso-8859-1 iso-8859-15 cp1252 cp437 cp850 )],
82 unicode => [qw( uu+cols=32+realsize u+0-4095 u+4096-6319 u+6320-8191 )],
83 us => [qw( cp437 cp863 gsm0338 AdobeStandardEncoding )],
84 ebcdic => [qw( cp37 cp500 cp1047 posix-bc cp1026 cp875 )],
85 iso => [map {"iso-8859-$_"} 1 .. 11, 13 .. 16],
86 dos => [qw( cp437 cp865 cp861 cp860 cp863 cp850 cp857 cp852 cp775
87 cp737 cp869 cp866 cp855 cp862 cp864 )],
88 aix => [qw( cp1006 )],
89 win => [qw( cp1252 cp1250 cp1254 cp1257 cp1258 cp1253 cp1251 cp1255 cp1256 cp874 )],
90 mac => [qw( MacRoman MacRomanian MacRumanian MacCroatian MacCentralEurRoman MacTurkish MacIcelandic MacSami
91 MacGreek MacCyrillic MacHebrew MacArabic MacFarsi MacThai )],
94 westeur => [qw( iso-8859-1 iso-8859-15 cp1252 iso-8859-14 cp850 hp-roman8 nextstep MacRoman )],
95 centeur => [qw( iso-8859-2 iso-8859-16 cp1250 cp852 MacRomanian MacCroatian MacCentralEurRoman )], # MacRumanian only for DB
96 turkish => [qw( iso-8859-9 iso-8859-3 cp1254 cp857 MacTurkish )],
97 norteur => [qw( baltic nordic )],
98 baltic => [qw( iso-8859-4 iso-8859-13 cp1257 cp775 )],
99 nordic => [qw( iso-8859-10 cp865 cp861 MacIcelandic MacSami )],
100 cyrillic => [qw( koi8-r koi8-u koi8-f iso-8859-5 cp1251 MacCyrillic cp866 cp855
101 U4 U2DE-2DF UA64-A69 U50-52 )], # MacUkrainian is broken
102 arabic => [qw( iso-8859-6 cp1256 MacArabic cp864 cp1006 MacFarsi
103 U6 U8A-8F+0-31+64 U75-77 )],
104 greek => [qw( iso-8859-7 cp1253 MacGreek cp737 cp869 U37-3F U1F )],
105 hebrew => [qw( iso-8859-8 cp1255 MacHebrew cp862 U59-5F )],
106 thai => [qw( iso-8859-11 cp874 MacThai )],
107 vietnamese => [qw( viscii cp1258 MacVietnamese )],
108 symbol => [qw( symbol dingbats MacDingbats AdobeZdingbat AdobeSymbol )],
123 if (my $follow = $ALIAS->{$input}) {
124 tabinput($_) for ref $follow ? @{$follow} : $follow;
128 state $visible = {}; # all present tables
129 my %row = (offset => 0, cols => 16);
130 my $params = $input =~ s/[+](.*)\z// ? $1 : undef;
132 if (not defined $params) {
134 'cp437' => ['cp850' => 0, 'ascii' => '0-31+128'], # ascii range overridden later
135 'gsm0338' => ['ascii' => '0-127'],
137 'iso-8859-2' => ['iso-8859-1' => '160'],
138 'iso-8859-3' => ['iso-8859-1' => '160'], #TODO: also apply to iso-8859-9
139 'iso-8859-4' => ['iso-8859-2' => '160'],
140 'iso-8859-5' => ['iso-8859-1' => '160'],
141 'iso-8859-6' => ['cp1256' => '128', 'iso-8859-1' => '160'],
142 'iso-8859-7' => ['iso-8859-1' => '160'],
143 'iso-8859-8' => ['iso-8859-1' => '160'],
144 'iso-8859-9' => ['iso-8859-1' => '208-223+240'],
145 'iso-8859-10' => ['iso-8859-4' => '160'],
146 'iso-8859-11' => ['iso-8859-1' => '160'],
147 'iso-8859-13' => ['iso-8859-4' => '160'],
148 'iso-8859-14' => ['iso-8859-1' => '160'],
149 'iso-8859-15' => ['iso-8859-1' => '160-191'],
150 'iso-8859-16' => ['iso-8859-2' => '160'],
151 'hp-roman8' => ['iso-8859-1' => '160'],
153 'cp1252' => ['iso-8859-1' => '128-159'],
154 'cp1250' => ['iso-8859-2' => '128-191', 'cp1252' => '128'],
155 'cp1254' => ['iso-8859-9' => '128-159', 'cp1252' => '128-159+208'],
156 'cp874' => ['iso-8859-11' => '128-159', 'cp1252' => '128'], # windows-874 actually cp1162
157 'cp1257' => ['iso-8859-13' => '128-159+255', 'cp1252' => '128'],
158 'cp1251' => ['cp1252' => '128'],
159 'cp1253' => ['cp1252' => '128'],
160 'cp1255' => ['iso-8859-8' => '128-223', 'cp1252' => '128'],
161 'cp1256' => ['cp1252' => '128'],
162 'cp1258' => ['cp1252' => '128-159+192'],
164 'cp850' => ['cp437' => '144'],
165 'cp860' => ['cp437' => '128-175'],
166 'cp861' => ['cp865' => '128-175'],
167 'cp863' => ['cp437' => '128-175'],
168 'cp865' => ['cp437' => '144-175'],
169 'cp852' => ['cp850' => '128', 'cp437' => '128'],
170 'cp857' => ['cp850' => '128-175+208-239', 'cp437' => '128'],
171 'cp775' => ['cp850' => '128'], # partial cp437
172 'cp866' => ['cp437' => '128-175+224'],
173 'cp855' => ['cp437' => '128'],
174 'cp1006' => ['iso-8859-6' => '160', 'cp437' => '128'],
175 'cp737' => ['cp437' => '128-175+224'],
176 'cp869' => ['cp437' => '128'],
177 'cp862' => ['cp437' => '128-159'],
178 'cp864' => ['MacArabic' => '128', 'iso-8859-6' => '128', 'cp437' => '128'], #TODO: compare form variants
180 'koi8-u' => ['koi8-r' => '144-191'],
181 'koi8-f' => ['koi8-u' => '144-191'],
183 'MacRomanian' => ['MacRoman' => '160-191+208-223'],
184 'MacRumanian' => ['MacRomanian' => '160-191+208-223', 'MacRoman' => '160-191+208-223'],
185 'MacCroatian' => ['MacRoman' => '160'],
186 'MacCentralEurRoman' => ['MacRoman' => '128'],
187 'MacIcelandic'=> ['MacRoman' => '160-175+208-239'],
188 'MacTurkish' => ['MacRoman' => '208-223'], # F5 is unassigned
189 'MacSami' => ['MacIcelandic' => '144-191+208-223+240', 'MacRoman' => '144'],
190 'MacGreek' => ['MacRoman' => '128'],
191 'MacCyrillic' => ['MacRoman' => '128'],
192 'MacHebrew' => ['iso-8859-8' => '128', 'MacRoman' => '128-143+160'], # partial ascii
193 'MacArabic' => ['iso-8859-6' => '128', 'cp864' => '128', 'MacRoman' => '128'], #TODO: multiple parents
194 'MacFarsi' => ['MacArabic' => '176-191', 'MacRoman' => '128'],
196 'cp37' => ['posix-bc' => '0'],
197 'posix-bc' => ['cp1047' => '64'],
198 'cp500' => ['cp37' => '64-95+176-191'],
199 'cp1047' => ['cp37' => '16-47+80-95+160-191'],
200 'cp1026' => ['cp37' => '64'],
201 'cp875' => ['cp37' => '48'],
205 my @parents = @{ $INHERIT->{$input} || [] };
207 if (my ($parent, $part) = pairfirst { defined $visible->{$a} } @parents) {
208 $row{parent} = $parent;
210 $params = 128 unless $visible->{$parent}
211 or ($input eq 'MacCroatian' and defined $visible->{MacRomanian});
213 elsif (defined $visible->{ascii}) {
214 $row{parent} = $parents[0];
215 $params = $parents[1] // 128;
216 $params = 128 if $params >= 128; # ascii offset at most
219 $row{parent} = $parents[0];
220 $params = $parents[1] if $parents[1] == 0; # apply ascii end
222 $visible->{$_} //= 0 for $row{parent} || ();
225 for my $param (split /[+]+/, $params // '') {
226 if ($param eq 'realsize') {
229 elsif ($param =~ m{ \A cols = (\d+) \z }x) {
232 elsif ($param =~ m{ \A (?<offset> \d+) (?: [-] (?<endpoint> \d+) )? \z }x) {
233 if (defined $row{endpoint}) {
234 # extend earlier range
235 my $skip = int(($row{endpoint} || $row{offset}) / $row{cols});
236 for ($skip + 1 .. ($+{offset} / $row{cols}) - 1) {
237 $row{skip}->{ $_ * $row{cols} - $row{offset} }++;
241 $row{offset} = $+{offset};
243 $row{endpoint} = $+{endpoint} || 0;
246 Alert("Unknown option <q>$param</q> for charset $input");
250 if ($input =~ m{ \A u ([0-9a-f]+) (?:-([0-9a-f]+))? \z }ix) {
251 my $start = hex($1) << ($2 ? 4 : 8);
252 my $end = $2 ? (hex($2) << 4) + $row{cols} - 1 : $start + 255;
253 $row{table} = join '', map { chr } $start .. $end;
254 utf8::upgrade($row{table}); # prevent latin1 output
255 $row{endpoint} = $end - $start;
256 $row{set} = sprintf 'Unicode block U+%02Xxx', $start >> 8;
257 $row{offset} = $start % 256;
259 elsif (lc $input eq 'uu') {
260 $row{set} = 'Unicode planes';
261 $row{cell} = do 'charset-ucplanes.inc.pl'
262 or Alert('Table data could not be read', $@ || $!);
263 $row{endpoint} = 1023 * $row{cell}->{colsize};
265 elsif (lc $input eq 'u') {
266 $row{cell} = do 'charset-unicode.inc.pl'
267 or Alert('Table data could not be read', $@ || $!);
269 $row{endpoint} ||= 8191;
270 $row{endpoint} *= $row{cell}->{colsize};
271 $row{startpoint} = $row{cell}->{colsize} * $row{offset};
273 $row{set} = 'Unicode ' . (
274 $row{startpoint} < 0x10000 && $row{endpoint} < 0x10000 ? 'BMP' :
275 $row{startpoint} >= 0x10000 && $row{endpoint} < 0x20000 ? 'SMP' :
279 elsif ($input =~ m/^utf-*8$/i) {
281 $row{cell} = do 'charset-utf8.inc.pl'
282 or Alert('Table data could not be read', $@ || $!);
283 $row{endpoint} = 255;
285 elsif ($row{set} = Encode::resolve_alias($input)) {
286 $row{endpoint} ||= 255;
287 if ($row{set} eq 'MacHebrew' or $row{set} eq 'MacThai') {
288 # array of possibly multiple characters per code point
290 map { Encode::decode($row{set}, pack 'C*', $_) } $row{offset} .. $row{endpoint}
294 # ~16x faster than decoding in loop;
295 # substr strings is twice as fast as splitting to an array
296 $row{table} = Encode::decode($row{set}, pack 'C*', $row{offset} .. $row{endpoint});
298 $row{endpoint} -= $row{offset};
300 if ($row{set} eq 'cp437') {
301 for my $phipos (237 - $row{offset}) {
302 next if $phipos < 0 or $phipos > $row{endpoint};
304 substr($row{table}, $phipos, 1) = pack 'U*', 0x3D5;
306 if ($row{offset} == 0) {
307 # replace control characters by visible variants
308 substr($row{table}, 0, 32) = pack 'U*', map {hex} qw(
309 2007 263A 263B 2665 2666 2663 2660 2022
310 25D8 25CB 25D9 2642 2640 266A 266B 263C
311 25BA 25C4 2195 203C 00B6 00A7 25AC 21A8
312 2191 2193 2192 2190 221F 2194 25B2 25BC
317 $visible->{ascii} = # assume common base
318 $visible->{ $row{set} } = 1;
321 Alert("Encoding <q>$input</q> unknown");
324 push @request, \%row;
326 tabinput($_) for @tablist;
328 my $NOCHAR = chr 0xFFFD;
331 my ($info, $offset) = @_;
332 my $table = $info->{cell} or return;
333 my $def = $table->{$offset} or return;
334 my ($len, $class, $name, $title) = @{$def};
336 my $cols = $info->{cols};
337 my $colsize = $table->{colsize} || 1;
340 $name //= $len <= 2 ? 'res' : 'reserved';
342 if (my $part = $offset/$colsize % $cols) {
344 my $rest = $cols - $part; # remaining
345 $rest = $len if $len < $rest; #TODO: optimise
347 # continued on new row
348 my @next = ($len * $colsize, "$class joinu");
351 push @next, $name, $title;
356 # minority on next row
357 push @next, '"', $title || $name;
359 $table->{$offset + $colsize*$rest} //= \@next;
364 elsif (my $rows = int($len / $cols)) {
366 my $rowsize = $colsize * $cols;
367 if ($len -= $rows * $cols) {
368 # partial row remains
369 $table->{$offset + $rowsize * $rows} //= [$len*$colsize, "$class joinu", '', $title];
373 unless ($info->{realsize}) {
374 # coalesce multiple rows
376 $info->{skip}->{$offset += $rowsize}++;
380 $info->{skip}->{$offset += $rowsize} = 0;
384 $attr .= sprintf ' rowspan=%d', $rows;
388 $attr .= sprintf ' colspan=%d', $len unless $len == 1;
389 $attr .= $1 if $class and $class =~ s/( \w+="[^"]*")//;
390 $attr .= sprintf ' class="%s"', $class if $class;
391 $attr .= sprintf ' title="%s"', EscapeHTML($title) if $title;
392 return "<td$attr>$name\n";
395 for my $row (@request) {
396 my $cols = $row->{cols};
397 my $colsize = $row->{cell} && $row->{cell}->{colsize} || 1;
398 my $coldigits = ceil(log($colsize * $cols) / log(16)); # uniform length of hexadecimal header
399 my $rowdiv = 16 ** $coldigits; # row divide for column digits
400 $rowdiv = 1 if $rowdiv != $cols * $colsize; # divide only if all columns are matched
401 my $offset = $row->{startpoint} || 0;
403 printf '<div class="section"><table class="glyphs%s">', !$row->{cell} && ' charmap';
404 my $title = $row->{set};
405 $title .= " <aside>(over $_)</aside>"
406 for grep { $_ ne 'iso-8859-1' } $row->{parent} // ();
407 printf '<caption>%s</caption>', $title;
408 print '<col>' x ($cols + 1);
409 for my $section (qw{thead}) {
410 print "<$section><tr><th>", $rowdiv == 1 ? '+' : '↱';
411 printf '<th>%0*X', $coldigits, $_ * $colsize for 0 .. $cols - 1;
415 while ($offset < $row->{endpoint}) {
416 if ($row->{skip}->{$offset}) {
417 $offset += $cols * $colsize;
422 if (defined $row->{skip}->{$offset}) {
426 if (my $rowmod = $offset % $rowdiv) {
427 # offset in column units
428 printf '<small>+%X</small>', $rowmod;
432 printf '%X', ($offset + $row->{offset}) / $rowdiv;
439 print range_cell($row, $offset);
443 my $cp = $offset + $row->{offset};
444 my $glyph = ref $row->{table} eq 'ARRAY' ? $row->{table}->[$offset] :
445 substr $row->{table}, $offset, 1;
446 my ($cell, $name, $class) = $glyph eq $NOCHAR ? () :
447 $glyphs->glyph_html($glyph);
449 if (exists $get{compare}) {
452 $cp == ord $glyph ? 'l4' :
453 $row->{parent} && $glyph eq
454 Encode::decode($row->{parent}, pack 'C', $cp) ? 'l3' :
456 $visible->{$glyph} ? 'l2' :
459 $visible->{$glyph}++;
462 say sprintf $class ? '<td title="%s" class="X %s">%s' : '<td title="%s">',
463 $name, $class, $cell;
469 say '</table></div>';
476 <table class="glyphs"><tr><: if (exists $get{compare}) { :>
477 <td class="X l4">unicode
478 <td class="X l3">inherited
479 <td class="X l2">existing
480 <td class="X l1">original
481 <td class="">unassigned
483 <td class="X Cc">control
484 <td class="X Zs"><span>whitespace</span>
485 <td class="X Mn">diacritic<table class="glyphs"><tr>
486 <td class="X Sk">letter
488 <td class="X Po">punctuation<table class="glyphs"><tr>
489 <td class="X Pf">quote
491 <td class="X So">symbol<table class="glyphs"><tr>
492 <td class="X Sm">math
493 <td class="X Sc">currency
495 <td class="X No">numeric
496 <td class="X Greek">greek<table class="glyphs"><tr>
497 <td class="X Latin">latin
498 <td class="X Cyrillic">cyrillic
500 <td class="X Aramaic">aramaic<table class="glyphs"><tr>
501 <td class="X Brahmic">brahmic
502 <td class="X Arabic">arabic
504 <td class="X Syllabic">syllabic<table class="glyphs"><tr>
505 <td class="X African">african
506 <td class="X Hiragana">japanese
507 <td class="X Han">cjk
508 <td class="X Bopomofo">chinese
510 <td class="X Alpha">alphabetic
513 <table class="glyphs"><tr>
514 <td class="X">unicode 7.0
515 <td class="X Xr">proposed
516 <td class="X Xd">deprecated
517 <td class="">unassigned
518 <td class="X Xi">invalid