4 title => 'charset cheat sheet',
7 "Reference sheet with all glyphs in common character encoding tables,",
8 "and an overview of Unicode ranges and UTF-8 bytes.",
11 charset codepage unicode ascii utf8 latin glyph character encoding
12 reference common overview table
14 stylesheet => [qw'light'],
15 data => [qw'charset-unicode.inc.pl charset-utf8.inc.pl'],
18 my @tablist = split /[^\w-]+/, $Request || 'default';
21 <h1>Character encoding</h1>
25 use Shiar_Sheet::FormatChar;
26 my $glyphs = Shiar_Sheet::FormatChar->new;
30 # generate character table(s)
31 my $input = shift or return;
34 default => [qw( unicode+0-192 utf-8 iso-8859-1 iso-8859-15 cp1252 cp437 cp850 )],
35 us => [qw( cp437 cp863 AdobeStandardEncoding gsm0338 )],
36 ebcdic => [qw( cp37 cp500 cp875 cp1026 cp1047 posix-bc )],
37 iso => [map {"iso-8859-$_"} 1 .. 11, 13 .. 16],
38 dos => [qw( cp437 cp865 cp861 cp860 cp863 cp850 cp857 cp852 cp775
39 cp737 cp869 cp866 cp855 cp862 cp864 )],
40 aix => [qw( cp1006 )],
41 win => [qw( cp1252 cp1250 cp1254 cp1257 cp1258 cp1253 cp1251 cp1255 cp1256 cp874 )],
43 westeur => [qw( iso-8859-1 iso-8859-15 cp1252 iso-8859-14 cp850 hp-roman8 nextstep MacRoman )],
44 centeur => [qw( iso-8859-2 iso-8859-16 cp1250 cp852 MacRomanian MacCroatian MacCentralEurRoman )], # MacRumanian only for DB
45 turkish => [qw( iso-8859-9 iso-8859-3 cp1254 cp857 MacTurkish )],
46 baltic => [qw( iso-8859-4 iso-8859-13 cp1257 cp775 )],
47 nordic => [qw( iso-8859-10 cp865 cp861 MacIcelandic MacSami )],
48 cyrillic => [qw( koi8-r koi8-u koi8-f iso-8859-5 cp1251 MacCyrillic cp866 cp855 )], # MacUkrainian is broken
49 arabic => [qw( iso-8859-6 cp1006 cp864 cp1256 MacArabic )], # MacFarsi same as MacArabic?
50 greek => [qw( iso-8859-7 cp1253 MacGreek cp737 cp869 )],
51 hebrew => [qw( iso-8859-8 cp1255 MacHebrew cp862 )],
52 thai => [qw( iso-8859-11 cp874 MacThai )],
53 vietnamese => [qw( viscii cp1258 MacVietnamese )],
54 symbol => [qw( symbol dingbats MacDingbats AdobeZdingbat AdobeSymbol )],
68 if (my $follow = $ALIAS->{$input}) {
69 tabinput($_) for ref $follow ? @{$follow} : $follow;
73 state $visible = {}; # all present tables
74 my %row = (offset => 0, cols => 16);
76 my $params = $input =~ s/[+](.*)\z// ? $1 : undef;
78 if (not defined $params) {
79 use List::Util qw( first pairfirst pairs );
82 'cp437' => ['cp850' => 0], # ascii range overridden later
83 'gsm0338' => ['ascii' => '0-127'],
85 'iso-8859-2' => ['iso-8859-1' => '160'],
86 'iso-8859-3' => ['iso-8859-1' => '160'],
87 'iso-8859-4' => ['iso-8859-2' => '160'],
88 'iso-8859-5' => ['iso-8859-1' => '160'],
89 'iso-8859-6' => ['iso-8859-1' => '160'],
90 'iso-8859-7' => ['iso-8859-1' => '160'],
91 'iso-8859-8' => ['iso-8859-1' => '160'],
92 'iso-8859-9' => ['iso-8859-1' => '208'],
93 'iso-8859-10' => ['iso-8859-4' => '160'],
94 'iso-8859-11' => ['iso-8859-1' => '160'],
95 'iso-8859-13' => ['iso-8859-4' => '160'],
96 'iso-8859-14' => ['iso-8859-1' => '160'],
97 'iso-8859-15' => ['iso-8859-1' => '160-191'],
98 'iso-8859-16' => ['iso-8859-2' => '160'],
99 'hp-roman8' => ['iso-8859-1' => '160'],
101 'cp1252' => ['iso-8859-1' => '128-159'],
102 'cp1250' => ['iso-8859-2' => '128-191'],
103 'cp1254' => ['iso-8859-9' => '128-159'],
104 'cp874' => ['iso-8859-11' => '128-159'], # windows-874 actually cp1162
105 'cp1257' => ['iso-8859-13' => '128'],
106 'cp1251' => ['cp1252' => '128'],
107 'cp1253' => ['cp1252' => '128'],
108 'cp1255' => ['iso-8859-8' => '128-223', 'cp1252' => '128'],
109 'cp1256' => ['cp1252' => '128'],
111 'cp850' => ['cp437' => '144'],
112 'cp860' => ['cp437' => '128-175'],
113 'cp861' => ['cp865' => '128-175'],
114 'cp863' => ['cp437' => '128-175'],
115 'cp865' => ['cp437' => '144-175'],
116 'cp852' => ['cp850' => '128', 'cp437' => '128'],
117 'cp857' => ['cp850' => '128-239', 'cp437' => '128'],
118 'cp775' => ['cp850' => '128'], # partial cp437
119 'cp866' => ['cp437' => '128'],
120 'cp855' => ['cp437' => '128'],
121 'cp1006' => ['cp437' => '128'],
122 'cp737' => ['cp437' => '128'],
123 'cp869' => ['cp437' => '128'],
124 'cp862' => ['cp437' => '128'],
125 'cp864' => ['cp437' => '128'],
127 'koi8-u' => ['koi8-r' => 128],
128 'koi8-f' => ['koi8-u' => 128],
130 'MacRomanian' => ['MacRoman' => '160'],
131 'MacCroatian' => ['MacRoman' => '160'],
132 'MacCentralEurRoman' => ['MacRoman' => '128'],
133 'MacTurkish' => ['MacRoman' => '208-223'], # F5 is unassigned
134 'MacCyrillic' => ['MacRoman' => '128'],
135 'MacHebrew' => ['MacRoman' => '128'],
138 my @parents = @{ $INHERIT->{$input} || [] };
140 if (my ($parent, $part) = pairfirst { defined $visible->{$a} } @parents) {
141 $row{parent} = $parent;
143 $params = 128 unless $visible->{$parent}
144 or ($input eq 'MacCroatian' and defined $visible->{MacRomanian});
146 elsif (defined $visible->{ascii}) {
147 $row{parent} = $parents[0];
148 $params = $parents[1] || 128;
149 $params = 128 if $params >= 128; # ascii offset at most
152 $row{parent} = $parents[0];
154 $visible->{$_} //= 0 for $row{parent};
157 if (defined $params and $params =~ m/^ (\d+) (-\d+)? /x) {
159 $endpoint = -$2 if $2;
162 if ($input =~ /^U([0-9a-f]+)(?:-([0-9a-f]+))?/) {
163 my $start = hex($1) << ($2 ? 4 : 8);
164 my $end = $2 ? hex($2) << 4 : $start + 240;
165 $row{table} = join '', map { chr } $start .. $end+15;
166 utf8::upgrade($row{table}); # prevent latin1 output
167 $row{len} = length $row{table};
168 $row{set} = sprintf 'Unicode block U+%02Xxx', $start >> 8;
170 elsif ($input eq 'U') {
172 $row{set} = 'Unicode planes';
173 $row{cell} = do 'charset-ucplanes.inc.pl'
174 or Alert('Table data could not be read', $@ || $!);
177 elsif ($row{set} = Encode::resolve_alias($input)) {
178 if ($row{set} eq 'Internal') {
179 $row{len} = $endpoint < 255 ? 640 : 8192;
180 $row{set} = 'Unicode BMP';
181 $row{cell} = do 'charset-unicode.inc.pl'
182 or Alert('Table data could not be read', $@ || $!);
184 elsif ($row{set} eq 'utf-8-strict') {
187 $row{cell} = do 'charset-utf8.inc.pl'
188 or Alert('Table data could not be read', $@ || $!);
191 if ($row{set} eq 'MacHebrew') {
192 # array of possibly multiple characters per code point
194 map { Encode::decode($row{set}, pack 'C*', $_) } $row{offset} .. $endpoint
198 # ~16x faster than decoding in loop;
199 # substr strings is twice as fast as splitting to an array
200 $row{table} = Encode::decode($row{set}, pack 'C*', $row{offset} .. $endpoint);
202 $row{len} = $endpoint - $row{offset};
204 if ($row{set} eq 'cp437' and !$row{offset}) {
205 substr($row{table}, 237, 1) = pack 'U*', 0x3D5; # phi sign
206 substr($row{table}, 0, 32) = pack 'U*', map {hex} qw(
207 2007 263A 263B 2665 2666 2663 2660 2022
208 25D8 25CB 25D9 2642 2640 266A 266B 263C
209 25BA 25C4 2195 203C 00B6 00A7 25AC 21A8
210 2191 2193 2192 2190 221F 2194 25B2 25BC
214 $visible->{ascii} = # assume common base
215 $visible->{ $row{set} } = 1;
219 Alert("Encoding <q>$input</q> unknown");
222 push @request, \%row;
224 tabinput($_) for @tablist;
226 my $NOCHAR = chr 0xFFFD;
229 my ($info, $offset) = @_;
230 my $table = $info->{cell} or return;
231 my $def = $table->{$offset} or return;
232 my ($len, $class, $name, $title) = @{$def};
234 my $cols = $info->{cols};
235 my $colsize = $table->{colsize} || 1;
238 $name //= $len <= 2 ? 'res' : 'reserved';
240 if (my $part = $offset/$colsize % $cols) {
242 my $rest = $cols - $part; # remaining
243 $rest = $len if $len < $rest; #TODO: optimise
245 # continued on new row
246 my @next = ($len * $colsize, "$class joinu");
249 push @next, $name, $title;
254 # minority on next row
255 push @next, '"', $title || $name;
257 $table->{$offset + $colsize*$rest} //= \@next;
262 elsif (my $rows = int($len / $cols)) {
264 if ($len -= $rows * $cols) {
265 # partial row remains
266 $table->{$offset + $colsize*$rows * $cols} //= [$len*$colsize, "$class joinu", '', $title];
269 $attr .= sprintf ' rowspan=%d', $rows;
273 $attr .= sprintf ' colspan=%d', $len unless $len == 1;
274 $attr .= $1 if $class and $class =~ s/( \w+="[^"]*")//;
275 $attr .= sprintf ' class="%s"', $class if $class;
276 $attr .= sprintf ' title="%s"', EscapeHTML($title) if $title;
277 return "<td$attr>$name";
280 for my $row (@request) {
281 my $cols = $row->{cols};
282 my $colsize = $row->{cell} && $row->{cell}->{colsize} || 1;
283 my $coldigits = ceil(log($colsize * $cols) / log(16)); # uniform length of hexadecimal header
284 my $rowdiv = 16 ** $coldigits; # row divide for column digits
285 $rowdiv = 1 if $rowdiv != $cols * $colsize; # divide only if all columns are matched
287 my $endpoint = $offset + $row->{len} * $colsize;
289 printf '<div class="section"><table class="glyphs%s">', !$row->{cell} && ' charmap';
290 my $title = $row->{set};
291 $title .= " <aside>(over $_)</aside>"
292 for grep { $_ ne 'iso-8859-1' } $row->{parent} // ();
293 printf '<caption>%s</caption>', $title;
294 print '<col>' x ($cols + 1);
295 for my $section (qw{thead}) {
296 print "<$section><tr><th>", $rowdiv == 1 ? '+' : '↱';
297 printf '<th>%0*X', $coldigits, $_ * $colsize for 0 .. $cols - 1;
301 while ($offset < $endpoint - 1) {
304 if (my $rowmod = $offset % $rowdiv) {
305 # offset in column units
306 printf '<small>+%X</small>', $rowmod;
310 printf '%X', ($offset + $row->{offset}) / $rowdiv;
315 print range_cell($row, $offset);
319 my $glyph = ref $row->{table} eq 'ARRAY' ? $row->{table}->[$offset] :
320 substr $row->{table}, $offset, 1;
321 if ($glyph eq $NOCHAR) {
326 if (exists $get{compare}) {
328 my $cp = $offset + $row->{offset};
329 printf '<td class="%s" title="%3$s">%2$s',
330 $cp == ord $glyph ? 'l4' :
331 $row->{parent} && $glyph eq
332 Encode::decode($row->{parent}, pack 'C', $cp) ? 'l3' :
333 $visible->{$glyph} ? 'l2' :
335 $glyphs->glyph_html($glyph);
336 $visible->{$glyph}++;
340 print "\n".$glyphs->glyph_cell($glyph);
347 say '</table></div>';
354 <table class="glyphs"><tr><: if (exists $get{compare}) { :>
355 <td class="X l4">unicode
356 <td class="X l3">inherited
357 <td class="X l2">existing
358 <td class="X l1">original
359 <td class="">unassigned
361 <td class="X Cc">control
362 <td class="X Zs"><span>whitespace</span>
363 <td class="X Mn">diacritic<table class="glyphs"><tr>
364 <td class="X Sk">letter
366 <td class="X Po">punctuation<table class="glyphs"><tr>
367 <td class="X Pf">quote
369 <td class="X So">symbol<table class="glyphs"><tr>
370 <td class="X Sm">math
371 <td class="X Sc">currency
373 <td class="X No">numeric
374 <td class="X Greek">greek<table class="glyphs"><tr>
375 <td class="X Latin">latin
376 <td class="X Cyrillic">cyrillic
378 <td class="X Aramaic">aramaic<table class="glyphs"><tr>
379 <td class="X Brahmic">brahmic
380 <td class="X Arabic">arabic
382 <td class="X Syllabic">syllabic<table class="glyphs"><tr>
383 <td class="X African">african
384 <td class="X Hiragana">japanese
385 <td class="X Han">cjk
386 <td class="X Bopomofo">chinese
388 <td class="X Alpha">alphabetic
391 <table class="glyphs"><tr>
392 <td class="X">unicode 7.0
393 <td class="X Xr">proposed
394 <td class="X Xd">deprecated
395 <td class="">unassigned
396 <td class="X Xi">invalid