tools: validate utf8 encoding in stdin/out

[sheet.git] / tools / mkwordlist

diff --git a/tools/mkwordlist b/tools/mkwordlist
index b88cd1bdd1adf23a525f7c6715f541864d2e0617..78e36b6adb5d652e258ec1ad10a9df255d6da9e9 100755 (executable)
--- a/tools/mkwordlist
+++ b/tools/mkwordlist
@@ -1,5 +1,26 @@
-#!/bin/sh
-echo 'use utf8;'
-psql sheet -XAt -c "
-       SELECT json_object_agg(coalesce(form, ''), forms) FROM _cat_words
-" | sed 's/ : / => /g'
+#!/usr/bin/env perl
+use 5.014;
+use warnings;
+
+BEGIN { push @INC, '.' }
+use Shiar_Sheet::DB;
+use open ':std' => ':encoding(utf-8)';
+my $db = Shiar_Sheet::DB->connect;
+
+say 'use utf8;';
+
+use Data::Dump 'pp';
+my %rows;
+if (my $lang = shift @ARGV) {
+       my %filter = (lang => $lang);
+       my $cols = "ref, array_to_string(form || alt, '/'), prio + coalesce(grade, 90) / 100::float, id, cat";
+       %rows = $db->select(_word_ref => $cols, \%filter)->map_arrays;
+       say pp \%rows
+               =~ s/\\x\{([0-9A-F]+)\}/chr hex $1/ger;
+       exit;
+}
+else {
+       %rows = $db->select(_cat_words => "coalesce(id::text, ''), forms")->map;
+       $_ = [ map { [split /:/, $_, 3] } @{$_} ] for values %rows;
+       say pp \%rows;
+}