t: move input files separate from output numbers

[barcat.git] / barcat

diff --git a/barcat b/barcat
index 1a1112c0ccdfa530e55b1792a7d25bf21f5381c1..431ed401489a1aa40aa9a1315966b2422734a2c5 100755 (executable)
--- a/barcat
+++ b/barcat
@@ -6,7 +6,7 @@ use List::Util qw( min max sum );
 use open qw( :std :utf8 );
 use re '/msx';
 
 use open qw( :std :utf8 );
 use re '/msx';
 

-our $VERSION = '1.08';
+our $VERSION = '1.09';

 
 my %opt;
 if (@ARGV) {
 
 my %opt;
 if (@ARGV) {


@@ -28,6 +28,7 @@ GetOptions(\%opt,
        },
        'human-readable|H!',
        'sexagesimal!',
        },
        'human-readable|H!',
        'sexagesimal!',

+       'reformat!',

        'interval|t:i',
        'trim|length|l=s' => sub {
                my ($optname, $optval) = @_;
        'interval|t:i',
        'trim|length|l=s' => sub {
                my ($optname, $optval) = @_;


@@ -53,6 +54,7 @@ GetOptions(\%opt,
                        " (range expected)\n"
                );
        },
                        " (range expected)\n"
                );
        },

+       'log|e!',

        'header!',
        'markers|m=s',
        'graph-format=s' => sub {
        'header!',
        'markers|m=s',
        'graph-format=s' => sub {


@@ -88,6 +90,7 @@ GetOptions(\%opt,
                };
        },
        'stat|s!',
                };
        },
        'stat|s!',

+       'report=s',

        'signal-stat=s',
        'unmodified|u!',
        'width|w=i',
        'signal-stat=s',
        'unmodified|u!',
        'width|w=i',


@@ -113,13 +116,18 @@ $opt{width} ||= $ENV{COLUMNS} || qx(tput cols) || 80 unless $opt{spark};
 $opt{color} //= $ENV{NO_COLOR} ? 0 : -t *STDOUT;  # enable on tty
 $opt{'graph-format'} //= '-';
 $opt{trim}   *= $opt{width} / 100 if $opt{trimpct};
 $opt{color} //= $ENV{NO_COLOR} ? 0 : -t *STDOUT;  # enable on tty
 $opt{'graph-format'} //= '-';
 $opt{trim}   *= $opt{width} / 100 if $opt{trimpct};

-$opt{units}   = [split //, ' kMGTPEZYyzafpn'.($opt{ascii} ? 'u' : 'μ').'m']
+$opt{units}   = [split //, ' kMGTPEZYRQqryzafpn'.($opt{ascii} ? 'u' : 'μ').'m']

        if $opt{'human-readable'};
 $opt{anchor} //= qr/\A/;
        if $opt{'human-readable'};
 $opt{anchor} //= qr/\A/;

-$opt{'value-length'} = 6 if $opt{units};
+$opt{'value-length'} = 4 if $opt{units};

 $opt{'value-length'} = 1 if $opt{unmodified};
 $opt{'signal-stat'} //= exists $SIG{INFO} ? 'INFO' : 'QUIT';
 $opt{markers} //= '=avg >31.73v <68.27v +50v |0';
 $opt{'value-length'} = 1 if $opt{unmodified};
 $opt{'signal-stat'} //= exists $SIG{INFO} ? 'INFO' : 'QUIT';
 $opt{markers} //= '=avg >31.73v <68.27v +50v |0';

+$opt{report} //= join(', ',
+       '${min; color(31)} min',
+       '${avg; $opt{reformat} or $_ = sprintf "%0.2f", $_; color(36)} avg',
+       '${max; color(32)} max',
+);

 $opt{palette} //= $opt{color} && [31, 90, 32];
 $opt{indicators} = [split //, $opt{indicators} ||
        ($opt{ascii} ? ' .oO' : $opt{spark} ? ' ▁▂▃▄▅▆▇█' : ' ▏▎▍▌▋▊▉█')
 $opt{palette} //= $opt{color} && [31, 90, 32];
 $opt{indicators} = [split //, $opt{indicators} ||
        ($opt{ascii} ? ' .oO' : $opt{spark} ? ' ▁▂▃▄▅▆▇█' : ' ▏▎▍▌▋▊▉█')


@@ -128,11 +136,10 @@ $opt{hidemin} = ($opt{hidemin} || 1) - 1;
 $opt{input} = (@ARGV && $ARGV[0] =~ m/\A[-0-9]/) ? \@ARGV : undef
        and undef $opt{interval};
 
 $opt{input} = (@ARGV && $ARGV[0] =~ m/\A[-0-9]/) ? \@ARGV : undef
        and undef $opt{interval};
 

-$opt{'sum-format'} = sub { sprintf '%.8g', $_[0] };

 $opt{'calc-format'} = sub { sprintf '%*.*f', 0, 2, $_[0] };
 $opt{'value-format'} = $opt{sexagesimal} ? sub {
 $opt{'calc-format'} = sub { sprintf '%*.*f', 0, 2, $_[0] };
 $opt{'value-format'} = $opt{sexagesimal} ? sub {

-       my $s = $_[0] + .5;
-       sprintf('%d:%02d:%02d', $s/3600, $s/60%60, $s%60);
+       my $s = abs($_[0]) + .5;
+       sprintf('%s%d:%02d:%02d', $_[0] < 0 && '-', $s/3600, $s/60%60, $s%60);

 } : $opt{units} && sub {
        my $unit = (
                log(abs $_[0] || 1) / log(10)
 } : $opt{units} && sub {
        my $unit = (
                log(abs $_[0] || 1) / log(10)


@@ -150,7 +157,8 @@ $opt{'value-format'} = $opt{sexagesimal} ? sub {
                $#{$opt{units}} * 1.5 < abs $unit ? sprintf('e%d', $unit) :
                        $opt{units}->[$unit/3]  # suffix
        );
                $#{$opt{units}} * 1.5 < abs $unit ? sprintf('e%d', $unit) :
                        $opt{units}->[$unit/3]  # suffix
        );

-};
+} and $opt{reformat}++;
+$opt{'value-format'} ||= sub { sprintf '%.8g', $_[0] };

 
 
 my (@lines, @values, @order);
 
 
 my (@lines, @values, @order);


@@ -172,9 +180,8 @@ if (defined $opt{interval}) {
        } or warn $@, "Expect slowdown with large datasets!\n";
 }
 
        } or warn $@, "Expect slowdown with large datasets!\n";
 }
 

-my $valmatch = qr<
-       $opt{anchor} ( \h* -? [0-9]* [.]? [0-9]+ (?: e[+-]?[0-9]+ )? |)
->x;
+my $float = qr<[0-9]* [.]? [0-9]+ (?: e[+-]?[0-9]+ )?>; # positive numberish
+my $valmatch = qr< $opt{anchor} ( \h* -? $float |) >x;

 while (defined ($_ = $opt{input} ? shift @{ $opt{input} } : readline)) {
        s/\r?\n\z//;
        s/\A\h*// unless $opt{unmodified};
 while (defined ($_ = $opt{input} ? shift @{ $opt{input} } : readline)) {
        s/\r?\n\z//;
        s/\A\h*// unless $opt{unmodified};


@@ -197,10 +204,6 @@ while (defined ($_ = $opt{input} ? shift @{ $opt{input} } : readline)) {
                and $. % $opt{interval} == 0;
 }
 
                and $. % $opt{interval} == 0;
 }
 

-if ($opt{'zero-missing'}) {
-       push @values, (0) x 10;
-}
-

 $SIG{INT} = 'DEFAULT';
 
 sub color {
 $SIG{INT} = 'DEFAULT';
 
 sub color {


@@ -233,40 +236,54 @@ my $maxval = $opt{maxval} // (
 ) // 0;
 my $minval = $opt{minval} // min $order[-1] // (), 0;
 my $range = $maxval - $minval;
 ) // 0;
 my $minval = $opt{minval} // min $order[-1] // (), 0;
 my $range = $maxval - $minval;

+$range &&= log $range if $opt{log};

 my $lenval = $opt{'value-length'} // max map { length } @order;
 my $len    = defined $opt{trim} && $opt{trim} <= 0 ? -$opt{trim} + 1 :
        max map { length $values[$_] && length $lines[$_] }
                0 .. min $#lines, $opt{hidemax} || ();  # left padding
 my $size   = defined $opt{width} && $range &&
 my $lenval = $opt{'value-length'} // max map { length } @order;
 my $len    = defined $opt{trim} && $opt{trim} <= 0 ? -$opt{trim} + 1 :
        max map { length $values[$_] && length $lines[$_] }
                0 .. min $#lines, $opt{hidemax} || ();  # left padding
 my $size   = defined $opt{width} && $range &&

-       ($opt{width} - $lenval - $len - !!$opt{indicators}) / $range;  # bar multiplication
+       ($opt{width} - $lenval - $len - !!$opt{indicators});  # bar multiplication

 
 my @barmark;
 if ($opt{markers} and $size > 0) {
        for my $markspec (split /\h/, $opt{markers}) {
                my ($char, $func) = split //, $markspec, 2;
 
 my @barmark;
 if ($opt{markers} and $size > 0) {
        for my $markspec (split /\h/, $opt{markers}) {
                my ($char, $func) = split //, $markspec, 2;

-               my $pos = eval {
+               my @pos = eval {

                        if ($func eq 'avg') {
                                return sum(@order) / @order;
                        }
                        elsif ($func =~ /\A([0-9.]+)v\z/) {
                        if ($func eq 'avg') {
                                return sum(@order) / @order;
                        }
                        elsif ($func =~ /\A([0-9.]+)v\z/) {

-                               die "Invalid marker $char: percentile $1 out of bounds\n" if $1 > 100;
+                               $1 <= 100 or die(
+                                       "Invalid marker $char: percentile $1 out of bounds\n"
+                               );

                                my $index = $#order * $1 / 100;
                                return ($order[$index] + $order[$index + .5]) / 2;
                        }
                        elsif ($func =~ /\A-?[0-9.]+\z/) {
                                return $func;
                        }
                                my $index = $#order * $1 / 100;
                                return ($order[$index] + $order[$index + .5]) / 2;
                        }
                        elsif ($func =~ /\A-?[0-9.]+\z/) {
                                return $func;
                        }

+                       elsif ($func =~ /\A\/($float)\z/) {
+                               my @range = my $multiple = my $next = $1;
+                               while ($next < $maxval) {
+                                       $multiple *= 10 if $opt{log};
+                                       push @range, $next += $multiple;
+                               }
+                               return @range;
+                       }

                        else {
                                die "Unknown marker $char: $func\n";
                        }
                };
                        else {
                                die "Unknown marker $char: $func\n";
                        }
                };

-               defined $pos or do {
+               @pos or do {

                        warn $@ if $@;
                        next;
                };
                        warn $@ if $@;
                        next;
                };

-               $pos -= $minval;
-               $pos >= 0 or next;
-               color(36) for $barmark[$pos * $size] = $char;
+               for my $pos (@pos) {
+                       $pos -= $minval;
+                       $pos &&= log $pos if $opt{log};
+                       $pos >= 0 or next;
+                       color(36) for $barmark[$pos / $range * $size] = $char;
+               }

        }
 
        state $lastmax = $maxval;
        }
 
        state $lastmax = $maxval;


@@ -274,10 +291,10 @@ if ($opt{markers} and $size > 0) {
                print ' ' x ($lenval + $len);
                printf color(90);
                printf '%-*s',
                print ' ' x ($lenval + $len);
                printf color(90);
                printf '%-*s',

-                       ($lastmax - $minval) * $size + .5,
-                       '-' x (($values[$nr - 1] - $minval) * $size);
+                       ($lastmax - $minval) * $size / $range + .5,
+                       '-' x (($values[$nr - 1] - $minval) * $size / $range);

                print color(92);
                print color(92);

-               say '+' x (($range - $lastmax) * $size + .5);
+               say '+' x (($range - $lastmax) * $size / $range + .5);

                print color(0);
                $lastmax = $maxval;
        }
                print color(0);
                $lastmax = $maxval;
        }


@@ -286,14 +303,19 @@ if ($opt{markers} and $size > 0) {
 say(
        color(31), sprintf('%*s', $lenval, $minval),
        color(90), '-', color(36), '+',
 say(
        color(31), sprintf('%*s', $lenval, $minval),
        color(90), '-', color(36), '+',

-       color(32), sprintf('%*s', $size * $range - 3, $maxval),
+       color(32), sprintf('%*s', $size - 3, $maxval),

        color(90), '-', color(36), '+',
        color(0),
 ) if $opt{header};
 
 while ($nr <= $limit) {
        my $val = $values[$nr];
        color(90), '-', color(36), '+',
        color(0),
 ) if $opt{header};
 
 while ($nr <= $limit) {
        my $val = $values[$nr];

-       my $rel = length $val && $range && min(1, ($val - $minval) / $range);
+       my $rel;
+       if (length $val) {
+               $rel = $val - $minval;
+               $rel &&= log $rel if $opt{log};
+               $rel = min(1, $rel / $range) if $range; # 0..1
+       }

        my $color = !length $val || !$opt{palette} ? undef :
                $val == $order[0] ? $opt{palette}->[-1] : # max
                $val == $order[-1] ? $opt{palette}->[0] : # min
        my $color = !length $val || !$opt{palette} ? undef :
                $val == $order[0] ? $opt{palette}->[-1] : # max
                $val == $order[-1] ? $opt{palette}->[0] : # min


@@ -313,8 +335,9 @@ while ($nr <= $limit) {
        print $indicator if defined $indicator;
 
        if (length $val) {
        print $indicator if defined $indicator;
 
        if (length $val) {

-               $val = $opt{'value-format'} ? $opt{'value-format'}->($val) :
-                       sprintf "%*s", $lenval, $val;
+               $val = sprintf("%*s", $lenval,
+                       $opt{reformat} ? $opt{'value-format'}->($val) : $val
+               );

                color($color) for $val;
        }
        my $line = $lines[$nr] =~ s/\n/$val/r;
                color($color) for $val;
        }
        my $line = $lines[$nr] =~ s/\n/$val/r;


@@ -323,8 +346,10 @@ while ($nr <= $limit) {
                next;
        }
        printf '%-*s', $len + length($val), $line;
                next;
        }
        printf '%-*s', $len + length($val), $line;

-       print $barmark[$_] // $opt{'graph-format'}
-               for 1 .. $size && (($values[$nr] || 0) - $minval) * $size + .5;
+       if ($rel and $size) {
+               print $barmark[$_] // $opt{'graph-format'}
+                       for 1 .. $rel * $size + .5;
+       }

        say '';
 }
 continue {
        say '';
 }
 continue {


@@ -343,25 +368,51 @@ sub show_stat {
                        $linemin += @lines;
                        $linemax = @lines - $linemax;
                }
                        $linemin += @lines;
                        $linemax = @lines - $linemax;
                }

-               printf '%.8g of ', $opt{'sum-format'}->(
-                       sum(grep {length} @values[$linemin .. $linemax]) // 0
-               );
+               print varfmt('${sum+} of ', {
+                       lines => $linemax - $linemin + 1,
+                       sum => sum(0, grep {length} @values[$linemin .. $linemax]),
+               });

        }
        if (@order) {
                my $total = sum @order;
        }
        if (@order) {
                my $total = sum @order;

-               printf '%s total', color(1) . $opt{'sum-format'}->($total) . color(0);
-               printf ' in %d values', scalar @order;
-               printf ' over %d lines', scalar @lines if @order != @lines;
-               printf(' (%s min, %s avg, %s max)',
-                       color(31) . ($opt{'value-format'} || sub {$_[0]})->($order[-1]) . color(0),
-                       color(36) . ($opt{'value-format'} || $opt{'calc-format'})->($total / @order) . color(0),
-                       color(32) . ($opt{'value-format'} || sub {$_[0]})->($order[0]) . color(0),
-               );
+               my $fmt = '${sum+;color(1)} total in ${count#} values';
+               $fmt .= ' over ${lines#} lines' if @order != @lines;
+               $fmt .= " ($_)" for $opt{report} || ();
+               print varfmt($fmt, {
+                       sum => $total,
+                       count => int @order,
+                       lines => int @lines,
+                       min => $order[-1],
+                       max => $order[0],
+                       avg => $total / @order,
+               });

        }
        say '';
        return 1;
 }
 
        }
        say '';
        return 1;
 }
 

+sub varfmt {
+       my ($fmt, $vars) = @_;
+       $fmt =~ s[\$\{ \h*+ ((?: [^{}]++ | \{(?1)\} )+) \}]{
+               my ($name, $cmd) = split /\s*;/, $1, 2;
+               my $format = $name =~ s/\+// || $name !~ s/\#// && $opt{reformat};
+               local $_ = $vars->{$name};
+               if (defined) {
+                       $_ = $opt{'value-format'}->($_) if $format;
+                       if ($cmd) {
+                               eval $cmd;
+                               warn "Error in \$$name report: $@" if $@;
+                       }
+                       $_;
+               }
+               else {
+                       warn "Unknown variable \$$name in report\n";
+                       "\$$name";
+               }
+       }eg;
+       return $fmt;
+}
+

 sub show_exit {
        show_lines();
        show_stat() if $opt{stat};
 sub show_exit {
        show_lines();
        show_stat() if $opt{stat};


@@ -391,6 +442,7 @@ Options:
   -l, --length=[-]SIZE[%]  Trim line contents (between number and bars)
   -L, --limit[=(N|-LAST|START-[END])]
                            Stop output after a number of lines
   -l, --length=[-]SIZE[%]  Trim line contents (between number and bars)
   -L, --limit[=(N|-LAST|START-[END])]
                            Stop output after a number of lines

+  -e, --log                Logarithmic (exponential) scale instead of linear

       --graph-format=CHAR  Glyph to repeat for the graph line
   -m, --markers=FORMAT     Statistical positions to indicate on bars
       --min=N, --max=N     Bars extend from 0 or the minimum value if lower
       --graph-format=CHAR  Glyph to repeat for the graph line
   -m, --markers=FORMAT     Statistical positions to indicate on bars
       --min=N, --max=N     Bars extend from 0 or the minimum value if lower


@@ -501,6 +553,11 @@ A specific range can be given by two values.
 All input is still counted and analyzed for statistics,
 but disregarded for padding and bar size.
 
 All input is still counted and analyzed for statistics,
 but disregarded for padding and bar size.
 

+=item -e, --log
+
+Logarithmic (I<e>xponential) scale instead of linear
+to compare orders of magnitude.
+

 =item --graph-format=<character>
 
 Glyph to repeat for the graph line.
 =item --graph-format=<character>
 
 Glyph to repeat for the graph line.


@@ -518,7 +575,12 @@ A single indicator glyph precedes each position:
 Exact value to match on the axis.
 A vertical bar at the zero crossing is displayed by I<|0>
 for negative values.
 Exact value to match on the axis.
 A vertical bar at the zero crossing is displayed by I<|0>
 for negative values.

-For example I<:3.14> would show a colon at pi.
+For example I<π3.14> would locate pi.
+
+=item I</><interval>
+
+Repeated at every multiple of a number.
+For example I<:/1> for a grid at every integer.

 
 =item <percentage>I<v>
 
 
 =item <percentage>I<v>
 


@@ -587,7 +649,9 @@ Reserved space for numbers.
 =item -w, --width=<columns>
 
 Override the maximum number of columns to use.
 =item -w, --width=<columns>
 
 Override the maximum number of columns to use.

-Appended graphics will extend to fill up the entire screen.
+Appended graphics will extend to fill up the entire screen,
+otherwise determined by the environment variable I<COLUMNS>
+or by running the C<tput> command.

 
 =item -h, --usage
 
 
 =item -h, --usage
 


@@ -650,19 +714,26 @@ Number of HTTP requests per day:
 
     cat httpd/access.log | cut -d\  -f4 | cut -d: -f1 | uniq -c | barcat
 
 
     cat httpd/access.log | cut -d\  -f4 | cut -d: -f1 | uniq -c | barcat
 

-Any kind of database query with counts, preserving returned alignment:
+Any kind of database query results, preserving returned alignment:

 
 

-    echo 'SELECT count(*),schemaname FROM pg_tables GROUP BY 2' |
+    echo 'SELECT sin(value * .1) FROM generate_series(0, 30) value' |

     psql -t | barcat -u
 
     psql -t | barcat -u
 

-In PostgreSQL from within the client:
+In PostgreSQL from within the client; a fancy C<\dt+> perhaps:
+
+    > SELECT schemaname, relname, pg_total_relation_size(relid)
+      FROM pg_statio_user_tables ORDER BY idx_blks_hit
+      \g |barcat -uHf+
+
+Same thing in SQLite (requires the sqlite3 client):

 
 

-    > SELECT sin(generate_series(0, 3, .1)) \g |barcat
+    > .once |barcat -Hf+
+    > SELECT name, sum(pgsize) FROM dbstat GROUP BY 1;

 
 Earthquakes worldwide magnitude 1+ in the last 24 hours:
 
     curl https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/1.0_day.csv |
 
 Earthquakes worldwide magnitude 1+ in the last 24 hours:
 
     curl https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/1.0_day.csv |

-    column -tns, | barcat -f4 -u -l80%
+    column -ts, -n | barcat -f4 -u -l80%

 
 External datasets, like movies per year:
 
 
 External datasets, like movies per year:
 


@@ -684,12 +755,12 @@ Total population history in XML from the World Bank:
 
     curl http://api.worldbank.org/v2/country/1W/indicator/SP.POP.TOTL |
     xmlstarlet sel -t -m '*/*' -v wb:date -o ' ' -v wb:value -n |
 
     curl http://api.worldbank.org/v2/country/1W/indicator/SP.POP.TOTL |
     xmlstarlet sel -t -m '*/*' -v wb:date -o ' ' -v wb:value -n |

-    barcat -f1 -H
+    barcat -f1 -H --markers=+/1e9

 
 Population and other information for all countries:
 
     curl http://download.geonames.org/export/dump/countryInfo.txt |
 
 Population and other information for all countries:
 
     curl http://download.geonames.org/export/dump/countryInfo.txt |

-    grep -v '^#\s' | column -tns$'\t' | barcat -f+2 -u -l150 -s
+    grep -v '^#\s' | column -ts$'\t' -n | barcat -f+2 -e -u -l150 -s

 
 And of course various Git statistics, such commit count by year:
 
 
 And of course various Git statistics, such commit count by year: