4 title => 'charset cheat sheet',
7 "Reference sheet with all glyphs in common character encoding tables,",
8 "and an overview of Unicode ranges and UTF-8 bytes.",
11 charset codepage unicode ascii utf8 latin glyph character encoding
12 reference common overview table
14 stylesheet => [qw'light'],
15 data => [qw'charset-unicode.inc.pl charset-utf8.inc.pl'],
18 my @tablist = split m{/+}, $Request || 'default';
21 <h1>Character encoding</h1>
25 use Shiar_Sheet::FormatChar;
26 my $glyphs = Shiar_Sheet::FormatChar->new;
30 # generate character table(s)
31 my $input = shift or return;
34 default => [qw( unicode+0-639 utf-8 iso-8859-1 iso-8859-15 cp1252 cp437 cp850 )],
35 us => [qw( cp437 cp863 gsm0338 AdobeStandardEncoding )],
36 ebcdic => [qw( cp37 cp500 cp1047 posix-bc cp1026 cp875 )],
37 iso => [map {"iso-8859-$_"} 1 .. 11, 13 .. 16],
38 dos => [qw( cp437 cp865 cp861 cp860 cp863 cp850 cp857 cp852 cp775
39 cp737 cp869 cp866 cp855 cp862 cp864 )],
40 aix => [qw( cp1006 )],
41 win => [qw( cp1252 cp1250 cp1254 cp1257 cp1258 cp1253 cp1251 cp1255 cp1256 cp874 )],
42 mac => [qw( MacRoman MacRomanian MacRumanian MacCroatian MacCentralEurRoman MacTurkish MacIcelandic MacSami
43 MacGreek MacCyrillic MacHebrew MacArabic MacFarsi MacThai )],
46 westeur => [qw( iso-8859-1 iso-8859-15 cp1252 iso-8859-14 cp850 hp-roman8 nextstep MacRoman )],
47 centeur => [qw( iso-8859-2 iso-8859-16 cp1250 cp852 MacRomanian MacCroatian MacCentralEurRoman )], # MacRumanian only for DB
48 turkish => [qw( iso-8859-9 iso-8859-3 cp1254 cp857 MacTurkish )],
49 baltic => [qw( iso-8859-4 iso-8859-13 cp1257 cp775 )],
50 nordic => [qw( iso-8859-10 cp865 cp861 MacIcelandic MacSami )],
51 cyrillic => [qw( koi8-r koi8-u koi8-f iso-8859-5 cp1251 MacCyrillic cp866 cp855 )], # MacUkrainian is broken
52 arabic => [qw( iso-8859-6 cp1256 MacArabic cp864 cp1006 MacFarsi )],
53 greek => [qw( iso-8859-7 cp1253 MacGreek cp737 cp869 )],
54 hebrew => [qw( iso-8859-8 cp1255 MacHebrew cp862 )],
55 thai => [qw( iso-8859-11 cp874 MacThai )],
56 vietnamese => [qw( viscii cp1258 MacVietnamese )],
57 symbol => [qw( symbol dingbats MacDingbats AdobeZdingbat AdobeSymbol )],
72 if (my $follow = $ALIAS->{$input}) {
73 tabinput($_) for ref $follow ? @{$follow} : $follow;
77 state $visible = {}; # all present tables
78 my %row = (offset => 0, cols => 16);
80 my $params = $input =~ s/[+](.*)\z// ? $1 : undef;
82 if (not defined $params) {
83 use List::Util qw( first pairfirst pairs );
86 'cp437' => ['cp850' => 0], # ascii range overridden later
87 'gsm0338' => ['ascii' => '0-127'],
89 'iso-8859-2' => ['iso-8859-1' => '160'],
90 'iso-8859-3' => ['iso-8859-1' => '160'], #TODO: also apply to iso-8859-9
91 'iso-8859-4' => ['iso-8859-2' => '160'],
92 'iso-8859-5' => ['iso-8859-1' => '160'],
93 'iso-8859-6' => ['cp1256' => '128', 'iso-8859-1' => '160'],
94 'iso-8859-7' => ['iso-8859-1' => '160'],
95 'iso-8859-8' => ['iso-8859-1' => '160'],
96 'iso-8859-9' => ['iso-8859-1' => '208-223+240'],
97 'iso-8859-10' => ['iso-8859-4' => '160'],
98 'iso-8859-11' => ['iso-8859-1' => '160'],
99 'iso-8859-13' => ['iso-8859-4' => '160'],
100 'iso-8859-14' => ['iso-8859-1' => '160'],
101 'iso-8859-15' => ['iso-8859-1' => '160-191'],
102 'iso-8859-16' => ['iso-8859-2' => '160'],
103 'hp-roman8' => ['iso-8859-1' => '160'],
105 'cp1252' => ['iso-8859-1' => '128-159'],
106 'cp1250' => ['iso-8859-2' => '128-191', 'cp1252' => '128'],
107 'cp1254' => ['iso-8859-9' => '128-159', 'cp1252' => '128-159+208'],
108 'cp874' => ['iso-8859-11' => '128-159', 'cp1252' => '128'], # windows-874 actually cp1162
109 'cp1257' => ['iso-8859-13' => '128-159+255', 'cp1252' => '128'],
110 'cp1251' => ['cp1252' => '128'],
111 'cp1253' => ['cp1252' => '128'],
112 'cp1255' => ['iso-8859-8' => '128-223', 'cp1252' => '128'],
113 'cp1256' => ['cp1252' => '128'],
114 'cp1258' => ['cp1252' => '128-159+192'],
116 'cp850' => ['cp437' => '144'],
117 'cp860' => ['cp437' => '128-175'],
118 'cp861' => ['cp865' => '128-175'],
119 'cp863' => ['cp437' => '128-175'],
120 'cp865' => ['cp437' => '144-175'],
121 'cp852' => ['cp850' => '128', 'cp437' => '128'],
122 'cp857' => ['cp850' => '128-175+208-239', 'cp437' => '128'],
123 'cp775' => ['cp850' => '128'], # partial cp437
124 'cp866' => ['cp437' => '128-175+224'],
125 'cp855' => ['cp437' => '128'],
126 'cp1006' => ['iso-8859-6' => '160', 'cp437' => '128'],
127 'cp737' => ['cp437' => '128-175+224'],
128 'cp869' => ['cp437' => '128'],
129 'cp862' => ['cp437' => '128-159'],
130 'cp864' => ['MacArabic' => '128', 'iso-8859-6' => '128', 'cp437' => '128'], #TODO: compare form variants
132 'koi8-u' => ['koi8-r' => 128],
133 'koi8-f' => ['koi8-u' => 128],
135 'MacRomanian' => ['MacRoman' => '160-191+208-223'],
136 'MacRumanian' => ['MacRomanian' => '160-191+208-223', 'MacRoman' => '160-191+208-223'],
137 'MacCroatian' => ['MacRoman' => '160'],
138 'MacCentralEurRoman' => ['MacRoman' => '128'],
139 'MacIcelandic'=> ['MacRoman' => '160-175+208-239'], #TODO: gaps at C/E
140 'MacTurkish' => ['MacRoman' => '208-223'], # F5 is unassigned
141 'MacSami' => ['MacIcelandic' => '144', 'MacRoman' => '144'],
142 'MacGreek' => ['MacRoman' => '128'],
143 'MacCyrillic' => ['MacRoman' => '128'],
144 'MacHebrew' => ['iso-8859-8' => '128', 'MacRoman' => '128-143+160'], # partial ascii
145 'MacArabic' => ['iso-8859-6' => '128', 'cp864' => '128', 'MacRoman' => '128'], #TODO: multiple parents
146 'MacFarsi' => ['MacArabic' => '176-191', 'MacRoman' => '128'],
148 'cp37' => ['posix-bc' => '0'],
149 'posix-bc' => ['cp1047' => '64'],
150 'cp500' => ['cp37' => '64-95+176-191'],
151 'cp1047' => ['cp37' => '16-95+160-191'], #TODO: gap at 3/4
152 'cp1026' => ['cp37' => '64'],
153 'cp875' => ['cp37' => '48'],
157 my @parents = @{ $INHERIT->{$input} || [] };
159 if (my ($parent, $part) = pairfirst { defined $visible->{$a} } @parents) {
160 $row{parent} = $parent;
162 $params = 128 unless $visible->{$parent}
163 or ($input eq 'MacCroatian' and defined $visible->{MacRomanian});
165 elsif (defined $visible->{ascii}) {
166 $row{parent} = $parents[0];
167 $params = $parents[1] || 128;
168 $params = 128 if $params >= 128; # ascii offset at most
171 $row{parent} = $parents[0];
173 $visible->{$_} //= 0 for $row{parent};
176 if (defined $params) {
179 (?: (?: [-] (?<stop> \d+) )? (?: [+] (?<restart> \d+) ) )?
180 (?: [-] (?<endpoint> \d+) )? \z
182 "Unknown range parameters for $input",
183 "<q>$params</q> is not in format start(-stop)(+restart(-end))",
186 $row{offset} = $+{offset};
187 $endpoint = $+{endpoint} if $+{endpoint};
188 if (my $restart = $+{restart}) {
189 my $skip = int(($+{stop} || $row{offset}) / $row{cols});
190 for ($skip + 1 .. ($restart / $row{cols}) - 1) {
191 $row{skip}->{ $_ * $row{cols} - $row{offset} }++;
196 if ($input =~ /^U([0-9a-f]+)(?:-([0-9a-f]+))?/) {
197 my $start = hex($1) << ($2 ? 4 : 8);
198 my $end = $2 ? hex($2) << 4 : $start + 240;
199 $row{table} = join '', map { chr } $start .. $end+15;
200 utf8::upgrade($row{table}); # prevent latin1 output
201 $row{endpoint} = $end + 14 - $start;
202 $row{set} = sprintf 'Unicode block U+%02Xxx', $start >> 8;
204 elsif ($input eq 'U') {
205 $row{set} = 'Unicode planes';
206 $row{cell} = do 'charset-ucplanes.inc.pl'
207 or Alert('Table data could not be read', $@ || $!);
209 $row{endpoint} = 1023 * $row{cell}->{colsize};
211 elsif ($row{set} = Encode::resolve_alias($input)) {
212 if ($row{set} eq 'Internal') {
213 $row{set} = 'Unicode BMP';
214 $row{cell} = do 'charset-unicode.inc.pl'
215 or Alert('Table data could not be read', $@ || $!);
216 $row{endpoint} = ($endpoint || 8191) * $row{cell}->{colsize};
218 elsif ($row{set} eq 'utf-8-strict') {
220 $row{cell} = do 'charset-utf8.inc.pl'
221 or Alert('Table data could not be read', $@ || $!);
222 $row{endpoint} = 255;
225 if ($row{set} eq 'MacHebrew' or $row{set} eq 'MacThai') {
226 # array of possibly multiple characters per code point
228 map { Encode::decode($row{set}, pack 'C*', $_) } $row{offset} .. $endpoint
232 # ~16x faster than decoding in loop;
233 # substr strings is twice as fast as splitting to an array
234 $row{table} = Encode::decode($row{set}, pack 'C*', $row{offset} .. $endpoint);
236 $row{endpoint} = $endpoint - $row{offset};
238 if ($row{set} eq 'cp437' and !$row{offset}) {
239 substr($row{table}, 237, 1) = pack 'U*', 0x3D5; # phi sign
240 substr($row{table}, 0, 32) = pack 'U*', map {hex} qw(
241 2007 263A 263B 2665 2666 2663 2660 2022
242 25D8 25CB 25D9 2642 2640 266A 266B 263C
243 25BA 25C4 2195 203C 00B6 00A7 25AC 21A8
244 2191 2193 2192 2190 221F 2194 25B2 25BC
248 $visible->{ascii} = # assume common base
249 $visible->{ $row{set} } = 1;
253 Alert("Encoding <q>$input</q> unknown");
256 push @request, \%row;
258 tabinput($_) for @tablist;
260 my $NOCHAR = chr 0xFFFD;
263 my ($info, $offset) = @_;
264 my $table = $info->{cell} or return;
265 my $def = $table->{$offset} or return;
266 my ($len, $class, $name, $title) = @{$def};
268 my $cols = $info->{cols};
269 my $colsize = $table->{colsize} || 1;
272 $name //= $len <= 2 ? 'res' : 'reserved';
274 if (my $part = $offset/$colsize % $cols) {
276 my $rest = $cols - $part; # remaining
277 $rest = $len if $len < $rest; #TODO: optimise
279 # continued on new row
280 my @next = ($len * $colsize, "$class joinu");
283 push @next, $name, $title;
288 # minority on next row
289 push @next, '"', $title || $name;
291 $table->{$offset + $colsize*$rest} //= \@next;
296 elsif (my $rows = int($len / $cols)) {
298 if ($len -= $rows * $cols) {
299 # partial row remains
300 $table->{$offset + $colsize*$rows * $cols} //= [$len*$colsize, "$class joinu", '', $title];
303 $attr .= sprintf ' rowspan=%d', $rows;
307 $attr .= sprintf ' colspan=%d', $len unless $len == 1;
308 $attr .= $1 if $class and $class =~ s/( \w+="[^"]*")//;
309 $attr .= sprintf ' class="%s"', $class if $class;
310 $attr .= sprintf ' title="%s"', EscapeHTML($title) if $title;
311 return "<td$attr>$name";
314 for my $row (@request) {
315 my $cols = $row->{cols};
316 my $colsize = $row->{cell} && $row->{cell}->{colsize} || 1;
317 my $coldigits = ceil(log($colsize * $cols) / log(16)); # uniform length of hexadecimal header
318 my $rowdiv = 16 ** $coldigits; # row divide for column digits
319 $rowdiv = 1 if $rowdiv != $cols * $colsize; # divide only if all columns are matched
322 printf '<div class="section"><table class="glyphs%s">', !$row->{cell} && ' charmap';
323 my $title = $row->{set};
324 $title .= " <aside>(over $_)</aside>"
325 for grep { $_ ne 'iso-8859-1' } $row->{parent} // ();
326 printf '<caption>%s</caption>', $title;
327 print '<col>' x ($cols + 1);
328 for my $section (qw{thead}) {
329 print "<$section><tr><th>", $rowdiv == 1 ? '+' : '↱';
330 printf '<th>%0*X', $coldigits, $_ * $colsize for 0 .. $cols - 1;
334 while ($offset < $row->{endpoint}) {
335 if ($row->{skip}->{$offset}) {
336 $offset += $cols * $colsize;
342 if (my $rowmod = $offset % $rowdiv) {
343 # offset in column units
344 printf '<small>+%X</small>', $rowmod;
348 printf '%X', ($offset + $row->{offset}) / $rowdiv;
353 print range_cell($row, $offset);
357 my $glyph = ref $row->{table} eq 'ARRAY' ? $row->{table}->[$offset] :
358 substr $row->{table}, $offset, 1;
359 if ($glyph eq $NOCHAR) {
364 if (exists $get{compare}) {
366 my $cp = $offset + $row->{offset};
367 printf '<td class="%s" title="%3$s">%2$s',
368 $cp == ord $glyph ? 'l4' :
369 $row->{parent} && $glyph eq
370 Encode::decode($row->{parent}, pack 'C', $cp) ? 'l3' :
371 $visible->{$glyph} ? 'l2' :
373 $glyphs->glyph_html($glyph);
374 $visible->{$glyph}++;
378 print "\n".$glyphs->glyph_cell($glyph);
385 say '</table></div>';
392 <table class="glyphs"><tr><: if (exists $get{compare}) { :>
393 <td class="X l4">unicode
394 <td class="X l3">inherited
395 <td class="X l2">existing
396 <td class="X l1">original
397 <td class="">unassigned
399 <td class="X Cc">control
400 <td class="X Zs"><span>whitespace</span>
401 <td class="X Mn">diacritic<table class="glyphs"><tr>
402 <td class="X Sk">letter
404 <td class="X Po">punctuation<table class="glyphs"><tr>
405 <td class="X Pf">quote
407 <td class="X So">symbol<table class="glyphs"><tr>
408 <td class="X Sm">math
409 <td class="X Sc">currency
411 <td class="X No">numeric
412 <td class="X Greek">greek<table class="glyphs"><tr>
413 <td class="X Latin">latin
414 <td class="X Cyrillic">cyrillic
416 <td class="X Aramaic">aramaic<table class="glyphs"><tr>
417 <td class="X Brahmic">brahmic
418 <td class="X Arabic">arabic
420 <td class="X Syllabic">syllabic<table class="glyphs"><tr>
421 <td class="X African">african
422 <td class="X Hiragana">japanese
423 <td class="X Han">cjk
424 <td class="X Bopomofo">chinese
426 <td class="X Alpha">alphabetic
429 <table class="glyphs"><tr>
430 <td class="X">unicode 7.0
431 <td class="X Xr">proposed
432 <td class="X Xd">deprecated
433 <td class="">unassigned
434 <td class="X Xi">invalid