4 title => 'charset cheat sheet',
7 "Reference sheet with all glyphs in common character encoding tables,",
8 "and an overview of Unicode ranges and UTF-8 bytes.",
11 charset codepage unicode ascii utf8 latin glyph character encoding
12 reference common overview table
14 stylesheet => [qw'light'],
15 data => [qw'charset-unicode.inc.pl charset-utf8.inc.pl'],
18 my @tablist = split m{/+}, $Request || 'default';
20 use List::Util qw( first pairmap pairfirst pairs );
23 <h1>Character encodings</h1>
27 if ($tablist[0] eq 'default') {
28 say "Overview of Unicode allocation and common latin code pages.";
29 say "Compare alternate charsets:";
32 say "Charset comparison:";
36 my ($title, $href, $selected) = @_;
38 $selected ? '<strong>%s</strong>' : '<a href="%2$s">%s</a>',
39 EscapeHTML($title), $href
45 join " ·\n", pairmap {
46 optionlink($b || ucfirst $a, '/charset'.($a && "/$a?compare"), $a eq $Request);
55 $tablist[0] eq 'default' ? () : ('' => 'common'),
60 norteur => 'North European',
72 use Shiar_Sheet::FormatChar;
73 my $glyphs = Shiar_Sheet::FormatChar->new;
77 # generate character table(s)
78 my $input = shift or return;
81 default => [qw( u+0-639 utf-8+realsize iso-8859-1 iso-8859-15 cp1252 cp437 cp850 )],
82 unicode => [qw( uu+cols=32+realsize u+0-4095 u+4096-6319 u+6320-8191 )],
83 us => [qw( cp437 cp863 gsm0338 AdobeStandardEncoding )],
84 ebcdic => [qw( cp37 cp500 cp1047 posix-bc cp1026 cp875 )],
85 iso => [map {"iso-8859-$_"} 1 .. 11, 13 .. 16],
86 dos => [qw( cp437 cp865 cp861 cp860 cp863 cp850 cp857 cp852 cp775
87 cp737 cp869 cp866 cp855 cp862 cp864 )],
88 aix => [qw( cp1006 )],
89 win => [qw( cp1252 cp1250 cp1254 cp1257 cp1258 cp1253 cp1251 cp1255 cp1256 cp874 )],
90 mac => [qw( MacRoman MacRomanian MacRumanian MacCroatian MacCentralEurRoman MacTurkish MacIcelandic MacSami
91 MacGreek MacCyrillic MacHebrew MacArabic MacFarsi MacThai )],
94 westeur => [qw( iso-8859-1 iso-8859-15 cp1252 iso-8859-14 cp850 hp-roman8 nextstep MacRoman )],
95 centeur => [qw( iso-8859-2 iso-8859-16 cp1250 cp852 MacRomanian MacCroatian MacCentralEurRoman )], # MacRumanian only for DB
96 turkish => [qw( iso-8859-9 iso-8859-3 cp1254 cp857 MacTurkish )],
97 norteur => [qw( baltic nordic )],
98 baltic => [qw( iso-8859-4 iso-8859-13 cp1257 cp775 )],
99 nordic => [qw( iso-8859-10 cp865 cp861 MacIcelandic MacSami )],
100 cyrillic => [qw( koi8-r koi8-u koi8-f iso-8859-5 cp1251 MacCyrillic cp866 cp855
101 U4 U2DE-2DF UA64-A69 U50-52 )], # MacUkrainian is broken
102 arabic => [qw( iso-8859-6 cp1256 MacArabic cp864 cp1006 MacFarsi
103 U6 U8A-8F+0-31+64 U75-77 )],
104 greek => [qw( iso-8859-7 cp1253 MacGreek cp737 cp869 U37-3F U1F )],
105 hebrew => [qw( iso-8859-8 cp1255 MacHebrew cp862 U59-5F )],
106 thai => [qw( iso-8859-11 cp874 MacThai )],
107 vietnamese => [qw( viscii cp1258 MacVietnamese )],
108 symbol => [qw( symbol dingbats MacDingbats AdobeZdingbat AdobeSymbol )],
123 if (my $follow = $ALIAS->{$input}) {
124 tabinput($_) for ref $follow ? @{$follow} : $follow;
128 state $visible = {}; # all present tables
129 my %row = (offset => 0, cols => 16);
130 my $params = $input =~ s/[+](.*)\z// ? $1 : undef;
132 if (not defined $params) {
134 'cp437' => ['cp850' => 0, 'ascii' => '0-31+128'], # ascii range overridden later
135 'gsm0338' => ['ascii' => '0-127'],
137 'iso-8859-2' => ['iso-8859-1' => '160'],
138 'iso-8859-3' => ['iso-8859-1' => '160'], #TODO: also apply to iso-8859-9
139 'iso-8859-4' => ['iso-8859-2' => '160'],
140 'iso-8859-5' => ['iso-8859-1' => '160'],
141 'iso-8859-6' => ['cp1256' => '128', 'iso-8859-1' => '160'],
142 'iso-8859-7' => ['iso-8859-1' => '160'],
143 'iso-8859-8' => ['iso-8859-1' => '160'],
144 'iso-8859-9' => ['iso-8859-1' => '208-223+240'],
145 'iso-8859-10' => ['iso-8859-4' => '160'],
146 'iso-8859-11' => ['iso-8859-1' => '160'],
147 'iso-8859-13' => ['iso-8859-4' => '160'],
148 'iso-8859-14' => ['iso-8859-1' => '160'],
149 'iso-8859-15' => ['iso-8859-1' => '160-191'],
150 'iso-8859-16' => ['iso-8859-2' => '160'],
151 'hp-roman8' => ['iso-8859-1' => '160'],
153 'cp1252' => ['iso-8859-1' => '128-159'],
154 'cp1250' => ['iso-8859-2' => '128-191', 'cp1252' => '128'],
155 'cp1254' => ['iso-8859-9' => '128-159', 'cp1252' => '128-159+208'],
156 'cp874' => ['iso-8859-11' => '128-159', 'cp1252' => '128'], # windows-874 actually cp1162
157 'cp1257' => ['iso-8859-13' => '128-159+255', 'cp1252' => '128'],
158 'cp1251' => ['cp1252' => '128'],
159 'cp1253' => ['cp1252' => '128'],
160 'cp1255' => ['iso-8859-8' => '128-223', 'cp1252' => '128'],
161 'cp1256' => ['cp1252' => '128'],
162 'cp1258' => ['cp1252' => '128-159+192'],
164 'cp850' => ['cp437' => '144'],
165 'cp860' => ['cp437' => '128-175'],
166 'cp861' => ['cp865' => '128-175'],
167 'cp863' => ['cp437' => '128-175'],
168 'cp865' => ['cp437' => '144-175'],
169 'cp852' => ['cp850' => '128', 'cp437' => '128'],
170 'cp857' => ['cp850' => '128-175+208-239', 'cp437' => '128'],
171 'cp775' => ['cp850' => '128'], # partial cp437
172 'cp866' => ['cp437' => '128-175+224'],
173 'cp855' => ['cp437' => '128'],
174 'cp1006' => ['iso-8859-6' => '160', 'cp437' => '128'],
175 'cp737' => ['cp437' => '128-175+224'],
176 'cp869' => ['cp437' => '128'],
177 'cp862' => ['cp437' => '128-159'],
178 'cp864' => ['MacArabic' => '128', 'iso-8859-6' => '128', 'cp437' => '128'], #TODO: compare form variants
180 'koi8-u' => ['koi8-r' => '144-191'],
181 'koi8-f' => ['koi8-u' => '144-191'],
183 'MacRomanian' => ['MacRoman' => '160-191+208-223'],
184 'MacRumanian' => ['MacRomanian' => '160-191+208-223', 'MacRoman' => '160-191+208-223'],
185 'MacCroatian' => ['MacRoman' => '160'],
186 'MacCentralEurRoman' => ['MacRoman' => '128'],
187 'MacIcelandic'=> ['MacRoman' => '160-175+208-239'],
188 'MacTurkish' => ['MacRoman' => '208-223'], # F5 is unassigned
189 'MacSami' => ['MacIcelandic' => '144-191+208-223+240', 'MacRoman' => '144'],
190 'MacGreek' => ['MacRoman' => '128'],
191 'MacCyrillic' => ['MacRoman' => '128'],
192 'MacHebrew' => ['iso-8859-8' => '128', 'MacRoman' => '128-143+160'], # partial ascii
193 'MacArabic' => ['iso-8859-6' => '128', 'cp864' => '128', 'MacRoman' => '128'], #TODO: multiple parents
194 'MacFarsi' => ['MacArabic' => '176-191', 'MacRoman' => '128'],
196 'cp37' => ['posix-bc' => '0'],
197 'posix-bc' => ['cp1047' => '64'],
198 'cp500' => ['cp37' => '64-95+176-191'],
199 'cp1047' => ['cp37' => '16-47+80-95+160-191'],
200 'cp1026' => ['cp37' => '64'],
201 'cp875' => ['cp37' => '48'],
205 my @parents = @{ $INHERIT->{$input} || [] };
207 if (my ($parent, $part) = pairfirst { defined $visible->{$a} } @parents) {
208 $row{parent} = $parent;
210 $params = 128 unless $visible->{$parent}
211 or ($input eq 'MacCroatian' and defined $visible->{MacRomanian});
213 elsif (defined $visible->{ascii}) {
214 $row{parent} = $parents[0];
215 $params = $parents[1] // 128;
216 $params = 128 if $params >= 128; # ascii offset at most
219 $row{parent} = $parents[0];
220 $params = $parents[1] if $parents[1] == 0; # apply ascii end
222 $visible->{$_} //= 0 for $row{parent} || ();
225 for my $param (split /[+]+/, $params // '') {
226 if ($param eq 'realsize') {
229 elsif ($param =~ m{ \A cols = (\d+) \z }x) {
232 elsif ($param =~ m{ \A (?<offset> \d+) (?: [-] (?<endpoint> \d+) )? \z }x) {
233 if (defined $row{endpoint}) {
234 # extend earlier range
235 my $skip = int(($row{endpoint} || $row{offset}) / $row{cols});
236 for ($skip + 1 .. ($+{offset} / $row{cols}) - 1) {
237 $row{skip}->{ $_ * $row{cols} - $row{offset} }++;
241 $row{offset} = $+{offset};
243 $row{endpoint} = $+{endpoint} || 0;
246 Alert("Unknown option <q>$param</q> for charset $input");
250 if ($input =~ m{ \A u ([0-9a-f]+) (?:-([0-9a-f]+))? \z }ix) {
251 my $start = hex($1) << ($2 ? 4 : 8);
252 my $end = $2 ? (hex($2) << 4) + $row{cols} - 1 : $start + 255;
253 $row{table} = join '', map { chr } $start .. $end;
254 utf8::upgrade($row{table}); # prevent latin1 output
255 $row{endpoint} = $end - $start;
256 $row{set} = sprintf 'Unicode block U+%02Xxx', $start >> 8;
257 $row{offset} = $start % 256;
259 elsif (lc $input eq 'uu') {
260 $row{set} = 'Unicode planes';
261 $row{cell} = do 'charset-ucplanes.inc.pl'
262 or Alert('Table data could not be read', $@ || $!);
263 $row{endpoint} = 1023 * $row{cell}->{colsize};
265 elsif (lc $input eq 'u') {
266 $row{cell} = do 'charset-unicode.inc.pl'
267 or Alert('Table data could not be read', $@ || $!);
269 $row{endpoint} ||= 8191;
270 $row{endpoint} *= $row{cell}->{colsize};
271 $row{startpoint} = $row{cell}->{colsize} * $row{offset};
273 $row{set} = 'Unicode ' . (
274 $row{startpoint} < 0x10000 && $row{endpoint} < 0x10000 ? 'BMP' :
275 $row{startpoint} >= 0x10000 && $row{endpoint} < 0x20000 ? 'SMP' :
279 elsif ($input =~ m/^utf-*8$/i) {
281 $row{cell} = do 'charset-utf8.inc.pl'
282 or Alert('Table data could not be read', $@ || $!);
283 $row{endpoint} = 255;
285 elsif ($row{set} = Encode::resolve_alias($input)) {
286 $row{endpoint} ||= 255;
287 if ($row{set} eq 'MacHebrew' or $row{set} eq 'MacThai') {
288 # array of possibly multiple characters per code point
290 map { Encode::decode($row{set}, pack 'C*', $_) } $row{offset} .. $row{endpoint}
294 # ~16x faster than decoding in loop;
295 # substr strings is twice as fast as splitting to an array
296 $row{table} = Encode::decode($row{set}, pack 'C*', $row{offset} .. $row{endpoint});
298 $row{endpoint} -= $row{offset};
300 if ($row{set} eq 'cp437') {
301 for my $phipos (237 - $row{offset}) {
302 next if $phipos < 0 or $phipos > $row{endpoint};
304 substr($row{table}, $phipos, 1) = pack 'U*', 0x3D5;
306 if ($row{offset} == 0) {
307 # replace control characters by visible variants
308 substr($row{table}, 0, 32) = pack 'U*', map {hex} qw(
309 2007 263A 263B 2665 2666 2663 2660 2022
310 25D8 25CB 25D9 2642 2640 266A 266B 263C
311 25BA 25C4 2195 203C 00B6 00A7 25AC 21A8
312 2191 2193 2192 2190 221F 2194 25B2 25BC
317 $visible->{ascii} = # assume common base
318 $visible->{ $row{set} } = 1;
321 Alert("Encoding <q>$input</q> unknown");
324 push @request, \%row;
326 tabinput($_) for @tablist;
328 my $NOCHAR = chr 0xFFFD;
331 my ($info, $offset) = @_;
332 my $table = $info->{cell} or return;
333 my $def = $table->{$offset} or return;
334 my ($len, $class, $name, $title) = @{$def};
336 my $cols = $info->{cols};
337 my $colsize = $table->{colsize} || 1;
340 $name //= $len <= 2 ? 'res' : 'reserved';
342 if (my $part = $offset/$colsize % $cols) {
344 my $rest = $cols - $part; # remaining
345 $rest = $len if $len < $rest; #TODO: optimise
347 # continued on new row
348 my @next = ($len * $colsize, "$class joinu");
349 my $separate = $cols - $len > $rest; # columns not on next row
352 push @next, $name, $title;
354 $name = $separate && '…';
357 # minority on next row
358 push @next, $separate && '"', $title || $name;
360 $table->{$offset + $colsize*$rest} //= \@next;
365 elsif (my $rows = int($len / $cols)) {
367 my $rowsize = $colsize * $cols;
368 if ($len -= $rows * $cols) {
369 # partial row remains
370 $table->{$offset + $rowsize * $rows} //= [$len*$colsize, "$class joinu", '', $title];
374 unless ($info->{realsize}) {
375 # coalesce multiple rows
377 $info->{skip}->{$offset += $rowsize}++;
381 $info->{skip}->{$offset += $rowsize} = 0;
385 $attr .= sprintf ' rowspan=%d', $rows;
389 $attr .= sprintf ' colspan=%d', $len unless $len == 1;
390 $attr .= $1 if $class and $class =~ s/( \w+="[^"]*")//;
391 $attr .= sprintf ' class="%s"', $class if $class;
392 $attr .= sprintf ' title="%s"', EscapeHTML($title) if $title;
393 return "<td$attr>$name\n";
396 for my $row (@request) {
397 my $cols = $row->{cols};
398 my $colsize = $row->{cell} && $row->{cell}->{colsize} || 1;
399 my $coldigits = ceil(log($colsize * $cols) / log(16)); # uniform length of hexadecimal header
400 my $rowdiv = 16 ** $coldigits; # row divide for column digits
401 $rowdiv = 1 if $rowdiv != $cols * $colsize; # divide only if all columns are matched
402 my $offset = $row->{startpoint} || 0;
404 printf '<div class="section"><table class="glyphs%s">', !$row->{cell} && ' charmap';
405 my $title = $row->{set};
406 $title .= " <aside>(over $_)</aside>"
407 for grep { $_ ne 'iso-8859-1' } $row->{parent} // ();
408 printf '<caption>%s</caption>', $title;
409 print '<col>' x ($cols + 1);
410 for my $section (qw{thead}) {
411 print "<$section><tr><th>", $rowdiv == 1 ? '+' : '↱';
412 printf '<th>%0*X', $coldigits, $_ * $colsize for 0 .. $cols - 1;
416 while ($offset < $row->{endpoint}) {
417 if ($row->{skip}->{$offset}) {
418 $offset += $cols * $colsize;
423 if (defined $row->{skip}->{$offset}) {
427 if (my $rowmod = $offset % $rowdiv) {
428 # offset in column units
429 printf '<small>+%X</small>', $rowmod;
433 printf '%X', ($offset + $row->{offset}) / $rowdiv;
440 print range_cell($row, $offset);
444 my $cp = $offset + $row->{offset};
445 my $glyph = ref $row->{table} eq 'ARRAY' ? $row->{table}->[$offset] :
446 substr $row->{table}, $offset, 1;
447 my ($cell, $name, $class) = $glyph eq $NOCHAR ? () :
448 $glyphs->glyph_html($glyph);
450 if (exists $get{compare}) {
453 $cp == ord $glyph ? 'l4' :
454 $row->{parent} && $glyph eq
455 Encode::decode($row->{parent}, pack 'C', $cp) ? 'l3' :
457 $visible->{$glyph} ? 'l2' :
460 $visible->{$glyph}++;
463 say sprintf $class ? '<td title="%s" class="X %s">%s' : '<td title="%s">',
464 $name, $class, $cell;
470 say '</table></div>';
477 <table class="glyphs"><tr><: if (exists $get{compare}) { :>
478 <td class="X l4">unicode
479 <td class="X l3">inherited
480 <td class="X l2">existing
481 <td class="X l1">original
482 <td class="">unassigned
484 <td class="X Cc">control
485 <td class="X Zs"><span>whitespace</span>
486 <td class="X Mn">diacritic<table class="glyphs"><tr>
487 <td class="X Sk">letter
489 <td class="X Po">punctuation<table class="glyphs"><tr>
490 <td class="X Pf">quote
492 <td class="X So">symbol<table class="glyphs"><tr>
493 <td class="X Sm">math
494 <td class="X Sc">currency
496 <td class="X No">numeric
497 <td class="X Greek">greek<table class="glyphs"><tr>
498 <td class="X Latin">latin
499 <td class="X Cyrillic">cyrillic
501 <td class="X Aramaic">aramaic<table class="glyphs"><tr>
502 <td class="X Brahmic">brahmic
503 <td class="X Arabic">arabic
505 <td class="X Syllabic">syllabic<table class="glyphs"><tr>
506 <td class="X African">african
507 <td class="X Hiragana">japanese
508 <td class="X Han">cjk
509 <td class="X Bopomofo">chinese
511 <td class="X Alpha">alphabetic
514 <table class="glyphs"><tr>
515 <td class="X">unicode 7.0
516 <td class="X Xr">proposed
517 <td class="X Xd">deprecated
518 <td class="">unassigned
519 <td class="X Xi">invalid