kanji 3.0.2 → 3.1.0

raw patch · 14 files changed

+740/−513 lines, 14 filesdep +HUnit-approxdep +aesondep +aeson-prettydep −bytestringdep ~basedep ~containersdep ~microlenssetup-changednew-component:exe:kanjiPVP ok

version bump matches the API change (PVP)

Dependencies added: HUnit-approx, aeson, aeson-pretty, criterion, deepseq, hashable, kanji, microlens-aeson, microlens-platform, optparse-applicative, protolude, tasty, tasty-hunit

Dependencies removed: bytestring

Dependency ranges changed: base, containers, microlens, text

API changes (from Hackage documentation)

Files

• CHANGELOG.md +12/−0
• Data/Kanji.hs +0/−150
• Data/Kanji/Levels.hs +0/−170
• Data/Kanji/Types.hs +0/−104
• Data/Kanji/Types/Internal.hs +0/−58
• README.md +92/−0
• Setup.hs +0/−2
• bench/Bench.hs +26/−0
• kanji.cabal +86/−29
• lib/Data/Kanji.hs +114/−0
• lib/Data/Kanji/Levels.hs +170/−0
• lib/Data/Kanji/Types.hs +92/−0
• nanq/nanq.hs +106/−0
• test/Test.hs +42/−0

@@ -0,0 +1,12 @@+## 3.1.0++- Overall simplification of library and reduction in boilerplate+  - `AsKanji` typeclass removed+  - `Rank` and `Level` are now just `Level`+  - More liberal use of `Map` and `Set`+- `nanq` renamed to `kanji` and moved inside same project++## 2.0.0++- New JSON-only output+- Revamped, modernized backend

@@ -1,150 +0,0 @@-{-# LANGUAGE TupleSections #-}-{-# LANGUAGE RankNTypes #-}---- |--- Module    : Data.Kanji--- Copyright : (c) Colin Woodbury, 2015, 2016--- License   : GPL3--- Maintainer: Colin Woodbury <colingw@gmail.com>------ A library for analysing the density of Kanji in given texts,--- according to their "Level" classification, as defined by the--- Japan Kanji Aptitude Testing Foundation (日本漢字能力検定協会).--module Data.Kanji-       (-         -- * Kanji-         AsKanji(..)-       , Kanji(..)-       , allKanji-       , isKanji-       , hasLevel-         -- * Levels-       , Level(..)-       , Rank(..)-       , level-       , levels-       , isKanjiInLevel-       , levelFromRank-         -- * Analysis-       , percentSpread-       , levelDist-       , averageLevel-       , uniques-         -- ** Densities-       , kanjiDensity-       , elementaryDen-       , middleDen-       , highDen-       , adultDen-       ) where--import           Control.Arrow hiding (second)-import           Data.Bool (bool)-import           Data.List (sort, group)-import qualified Data.Map.Lazy as M-import qualified Data.Set as S-import           Lens.Micro--import           Data.Kanji.Levels-import           Data.Kanji.Types--------- | All Japanese Kanji, grouped by their Level (級) in ascending order.--- Here, ascending order means from the lowest to the highest level,--- meaning from 10 to 1.-allKanji :: [S.Set Kanji]-allKanji =  map f ks-  where f = foldr (\k s -> bool s (S.insert (Kanji k) s) $ isKanji k) mempty-        ks = [tenth, ninth, eighth, seventh, sixth,-              fifth, fourth, third, preSecond, second]---- | Is the `Level` of a given `Kanji` known?-hasLevel :: Kanji -> Bool-hasLevel k = has _Just $ level k---- | What is the density @d@ of Kanji characters in a given String-like--- type, where @0 <= d <= 1@?-kanjiDensity :: Int -> [Kanji] -> Float-kanjiDensity len ks = fromIntegral (length ks) / fromIntegral len---- | How much of the Kanji found are learned in elementary school in Japan?------ > elementaryDen . levelDist :: [Kanji] -> Float-elementaryDen :: [(Rank,Float)] -> Float-elementaryDen dists = sum $ dists ^.. each . inRank [Five, Six ..]---- | How much of the Kanji found are learned by the end of middle school?------ > middleDen . levelDist :: [Kanji] -> Float-middleDen :: [(Rank,Float)] -> Float-middleDen dists = sum $ dists ^.. each . inRank [Three, Four ..]---- | How much of the Kanji found are learned by the end of high school?------ > highDen . levelDist :: [Kanji] -> Float-highDen :: [(Rank,Float)] -> Float-highDen dists = sum $ dists ^.. each . inRank [PreTwo, Three ..]---- | How much of the Kanji found should be able to read by the average person?------ > adultDen . levelDist :: [Kanji] -> Float-adultDen :: [(Rank,Float)] -> Float-adultDen dists = sum $ dists ^.. each . inRank [Two, PreTwo ..]--inRank :: [Rank] -> Traversal' (Rank,Float) Float-inRank rs f (r,n) | r `elem` rs = (r,) <$> f n-                  | otherwise = pure (r,n)---- | All `Level`s, with all their `Kanji`, ordered from Level-10 to Level-2.-levels :: [Level]-levels = zipWith Level allKanji [Ten ..]---- | What `Level` does a Kanji belong to?-level :: Kanji -> Maybe Level-level = level' levels-  where level' [] _     = Nothing-        level' (q:qs) k | isKanjiInLevel q k = Just q-                        | otherwise = level' qs k---- | Does a given `Kanji` belong to the given `Level`?-isKanjiInLevel :: Level -> Kanji -> Bool-isKanjiInLevel q k = S.member k $ _allKanji q---- | Is there a `Level` that corresponds with a given `Rank` value?-levelFromRank :: Rank -> Maybe Level-levelFromRank = levelFromRank' levels-  where levelFromRank' [] _      = Nothing-        levelFromRank' (q:qs) qn | _rank q == qn = Just q-                                 | otherwise     = levelFromRank' qs qn---- | Find the average `Level` of a given set of `Kanji`.-averageLevel :: [Kanji] -> Float-averageLevel ks = average ranks-  where ranks = ks ^.. each . to level . _Just . to _rank . to fromRank-        average ns = sum ns / fromIntegral (length ns)---- | How much of each `Level` is represented by a group of Kanji?-levelDist :: [Kanji] -> [(Rank,Float)]-levelDist ks = map toNumPercentPair $ group sortedRanks-  where sortedRanks = sort $ ks ^.. each . to level . _Just . to _rank-        toNumPercentPair qns = (head qns, length' qns / length' ks)-        length' n = fromIntegral $ length n---- | The distribution of each `Kanji` in a set of them.--- The distribution values must sum to 1.-percentSpread :: [Kanji] -> [(Kanji,Float)]-percentSpread ks = map getPercent kQuants-    where getPercent (k,q) = (k, fromIntegral q / totalKanji)-          kQuants    = kanjiQuantities ks-          totalKanji = fromIntegral $ foldl (\acc (_,q) -> q + acc) 0 kQuants---- | Determines how many times each `Kanji` appears in given set of them.-kanjiQuantities :: [Kanji] -> [(Kanji,Int)]-kanjiQuantities = map (head &&& length) . group . sort---- | Which Kanji appeared from each Level in the text?-uniques :: [Kanji] -> [(Rank,[Kanji])]-uniques = M.toList . S.foldl h M.empty . S.fromList-  where h a k = maybe a (\l -> M.insertWith (++) (_rank l) [k] a) $ level k

@@ -1,170 +0,0 @@--- |--- Module    : Data.Kanji.Levels--- Copyright : (c) Colin Woodbury, 2015, 2016--- License   : GPL3--- Maintainer: Colin Woodbury <colingw@gmail.com>------ All Kanji from levels 10 to Pre-2.--module Data.Kanji.Levels where--import qualified Data.Set as S--------- | The Kanji unique to Level-10, studied by the end of 1st grade in--- Japanese elementary schools.-tenth :: S.Set Char-tenth = S.fromDistinctAscList $-  "一七三上下中九二五人休先入八六円出力十千" ++-  "口右名四土夕大天女子字学小山川左年手文日" ++-  "早月木本村林校森正気水火犬玉王生田男町白" ++-  "百目石空立竹糸耳花草虫見貝赤足車金雨青音"---- | The Kanji unique to Level-9, studied by the end of 2nd grade in--- Japanese elementary schools.-ninth :: S.Set Char-ninth = S.fromDistinctAscList $-  "万丸交京今会体何作元兄光公内冬刀分切前北" ++-  "午半南原友古台合同回図国園地場声売夏外多" ++-  "夜太妹姉室家寺少岩工市帰広店弓引弟弱強当" ++-  "形後心思戸才教数新方明星春昼時晴曜書朝来" ++-  "東楽歌止歩母毎毛池汽活海点父牛理用画番直" ++-  "矢知社秋科答算米紙細組絵線羽考聞肉自船色" ++-  "茶行西親角言計記話語読谷買走近通週道遠里" ++-  "野長門間雪雲電頭顔風食首馬高魚鳥鳴麦黄黒"---- | The Kanji unique to Level-8, studied by the end of 3rd grade in--- Japanese elementary schools.-eighth :: S.Set Char-eighth = S.fromDistinctAscList $-  "丁世両主乗予事仕他代住使係倍全具写列助勉" ++-  "動勝化区医去反取受号向君味命和品員商問坂" ++-  "央始委守安定実客宮宿寒対局屋岸島州帳平幸" ++-  "度庫庭式役待急息悪悲想意感所打投拾持指放" ++-  "整旅族昔昭暑暗曲有服期板柱根植業様横橋次" ++-  "歯死氷決油波注泳洋流消深温港湖湯漢炭物球" ++-  "由申界畑病発登皮皿相県真着短研礼神祭福秒" ++-  "究章童笛第筆等箱級終緑練羊美習者育苦荷落" ++-  "葉薬血表詩調談豆負起路身転軽農返追送速進" ++-  "遊運部都配酒重鉄銀開院陽階集面題飲館駅鼻"---- | The Kanji unique to Level-7, studied by the end of 4th grade in--- Japanese elementary schools.-seventh :: S.Set Char-seventh = S.fromDistinctAscList $-  "不争付令以仲伝位低例便信倉候借停健側働億" ++-  "兆児共兵典冷初別利刷副功加努労勇包卒協単" ++-  "博印参史司各告周唱喜器囲固型堂塩士変夫失" ++-  "好季孫完官害察巣差希席帯底府康建径徒得必" ++-  "念愛成戦折挙改救敗散料旗昨景最望未末札材" ++-  "束松果栄案梅械極標機欠歴残殺毒氏民求治法" ++-  "泣浅浴清満漁灯無然焼照熱牧特産的省祝票種" ++-  "積競笑管節粉紀約結給続置老胃脈腸臣航良芸" ++-  "芽英菜街衣要覚観訓試説課議象貨貯費賞軍輪" ++-  "辞辺連達選郡量録鏡関陸隊静順願類飛飯養験"---- | The Kanji unique to Level-6, studied by the end of 5th grade in--- Japanese elementary schools.-sixth :: S.Set Char-sixth = S.fromDistinctAscList $-  "久仏仮件任似余価保修俵個備像再刊判制券則" ++-  "効務勢厚句可営因団圧在均基報境墓増夢妻婦" ++-  "容寄富導居属布師常幹序弁張往復徳志応快性" ++-  "恩情態慣承技招授採接提損支政故敵断旧易暴" ++-  "条枝査格桜検構武比永河液混減測準演潔災燃" ++-  "版犯状独率現留略益眼破確示祖禁移程税築精" ++-  "素経統絶綿総編績織罪群義耕職肥能興舌舎術" ++-  "衛製複規解設許証評講謝識護豊財貧責貸貿賀" ++-  "資賛質輸述迷退逆造過適酸鉱銅銭防限険際雑" ++-  "非預領額飼"---- | The Kanji unique to Level-5, studied by the end of 6th grade in--- Japanese elementary schools.-fifth :: S.Set Char-fifth = S.fromDistinctAscList $-  "並乱乳亡仁供俳値傷優党冊処刻割創劇勤危卵" ++-  "厳収后否吸呼善困垂城域奏奮姿存孝宅宇宗宙" ++-  "宝宣密寸専射将尊就尺届展層己巻幕干幼庁座" ++-  "延律従忘忠憲我批担拝拡捨探推揮操敬映晩暖" ++-  "暮朗机枚染株棒模権樹欲段沿泉洗派済源潮激" ++-  "灰熟片班異疑痛皇盛盟看砂磁私秘穀穴窓筋策" ++-  "簡糖系紅納純絹縦縮署翌聖肺背胸脳腹臓臨至" ++-  "若著蒸蔵蚕衆裁装裏補視覧討訪訳詞誌認誕誠" ++-  "誤論諸警貴賃遺郵郷針鋼閉閣降陛除障難革頂" ++-  "骨"---- | The Kanji unique to Level-4, studied during middle school in Japan.-fourth :: S.Set Char-fourth = S.fromDistinctAscList $-  "丈与丘丹乾互井介仰伺依侵俗倒偉傍傾僧儀兼" ++-  "冒凡凶刈到刺剣剤劣勧匹占即却及叫召吐含吹" ++-  "咲唐嘆噴圏坊執堅堤塔壁壊壱奇奥奴妙姓威娘" ++-  "婚寂寝尋尽尾屈峠峰巡巨帽幅幾床弐弾彩影彼" ++-  "征御微徴忙怒怖恋恐恒恥恵悩惑惨慎慢慮憶戒" ++-  "戯扇払扱抗抜抱抵押拍拓拠振捕掘描握援搬摘" ++-  "撃攻敏敷斜旨旬是普暇暦曇更替朱朽杯枯柄柔" ++-  "桃欄歓歳殖殿汗汚沈沖沢沼況泊浜浮浸涙淡添" ++-  "渡溶滴漫澄濁濃為烈煙煮燥爆狂狩狭猛獣獲玄" ++-  "珍環甘畳疲療皆盆盗監盤盾眠瞬矛砲祈秀称稲" ++-  "稿突端箇範粒紋紫紹絡継維網緯縁繁繰罰翼耐" ++-  "肩肪胴脂脚脱腐腕腰膚致舗舞舟般芋芝茂荒菓" ++-  "蓄薄薪被襲触訴詰詳誇誉謡豪販賦贈越趣距跡" ++-  "跳踊踏躍軒較載輝輩込迎迫逃透途遅違遣避郎" ++-  "釈鈍鉛鋭鎖鑑闘陣陰隠隣隷雄雅雌離雷需震霧" ++-  "露響項頼飾香駆騒驚髪鬼鮮麗黙鼓齢"---- | The Kanji unique to Level-3, studied by the end of middle school--- in Japan.-third :: S.Set Char-third = S.fromDistinctAscList $-  "乏乙了企伏伐伴伸佳侍促倣倹偶催債克免冗冠" ++-  "凍凝刑削励勘募匠匿卑卓卸厘又双吉吏哀哲啓" ++-  "喚喫嘱坑埋塊塗墜墨墳墾壇奉契奪如妨姫娯婆" ++-  "婿嫁嬢孔孤宴審寿封尿岐岳峡崩巧帆帝幻幽廉" ++-  "廊弧彫徐忌怠怪恨悔悟悦惜愚慈慌慕慨慰憂憎" ++-  "憩房抑択抽拘掃掌排掛控措掲揚換揺携搾摂撮" ++-  "擁擦敢斗斤斥施既昇晶暫架某桑棄棋楼概欧欺" ++-  "殊殴没泌浪湾湿滅滑滝滞漂漏潜潤濫瀬炉炊炎" ++-  "焦牲犠猟獄甲畔畜疾痘癖硬碑礎祉稚穂穏穫窒" ++-  "符篤簿籍粋粗粘糧紛紺絞綱緊締緩縛縫繕翻聴" ++-  "肝胆胎胞脅膜膨芳苗菊華葬藩虐虚蛮衝衰袋裂" ++-  "裸覆訂託詠該誘請諮諾謀譲豚貫賊賢赦赴超軌" ++-  "軸辛辱逮遂遇遭遵邦邪郊郭酔酵鋳錠錬錯鍛鎮" ++-  "鐘閲阻陪陳陵陶隆随隔隻雇零霊顧飽餓駐騎髄" ++-  "魂魅魔鯨鶏"---- | The Kanji unique to Level-Pre2, considerend "mid high school" level.-preSecond :: S.Set Char-preSecond = S.fromDistinctAscList $-  "且丙亜享亭仙伯但佐併侮侯俊俸倫偏偵偽傑傘" ++-  "僕僚儒償充准凸凹刃剖剛剰劾勅勲升厄叔叙吟" ++-  "呈呉唆唇唯喝喪嗣嚇囚坪垣培堀堕堪塀塁塑塚" ++-  "塾壌壮奔奨妃妄妊妥姻娠媒嫌嫡宜宰宵寛寡寧" ++-  "寮尉尚尼履屯岬崇崎帥幣庶庸廃廷弊弔弦彰循" ++-  "徹忍恭悠患悼惰愁愉慶憤憾懇懐懲懸戻扉扶抄" ++-  "把披抹拐拒拙括拷挑挟挿捜据搭摩撤撲擬斉斎" ++-  "旋昆暁曹朕朴杉析枠枢柳栓核栽桟棚棟棺槽款" ++-  "殉殻汁江沸泡泥泰洞津洪浄浦涯涼淑渇渉渋渓" ++-  "渦溝滋漆漠漬漸潟濯煩爵猫献猶猿珠琴璽瓶甚" ++-  "畝疎疫症痢痴癒盲眺睡督矯砕硝硫碁磨礁祥禅" ++-  "禍租秩稼窃窮窯竜筒粛粧糾紡索累紳緒縄繊繭" ++-  "缶罷羅翁耗肌肖肢肯臭舶艇艦茎荘菌薦薫藻虜" ++-  "虞蚊蛇蛍融衡衷裕褐褒襟覇訟診詐詔誓諭謁謄" ++-  "謙謹譜貞貢賄賓賜賠購践軟轄迅迭逐逓逝逸遍" ++-  "遮遷還邸酌酢酪酬酷醜醸釣鈴鉢銃銘閑閥附陥" ++-  "隅雰霜靴韻頑頒頻顕飢駄騰麻"---- | The Kanji unique to Level-2, considered "standard adult" level.-second :: S.Set Char-second = S.fromDistinctAscList $-  "串丼乞亀伎侶俺傲僅冥冶凄刹剥勃勾匂叱呂呪" ++-  "咽哺唄唾喉喩嗅嘲埼堆塞填奈妖妬媛嫉宛尻岡" ++-  "崖嵐巾弄弥彙怨恣惧慄憧憬戚戴拉拭拳拶挨挫" ++-  "捉捗捻摯斑斬旦旺昧曖曽枕柵柿栃桁梗梨椅椎" ++-  "楷毀氾汎汰沃沙淫湧溺潰煎熊爪爽牙狙玩瑠璃" ++-  "璧瓦畏畿痕痩瘍眉睦瞭瞳稽窟箋箸籠綻緻罵羞" ++-  "羨肘股脇脊腎腫腺膝膳臆臼舷艶芯苛茨萎葛蓋" ++-  "蔑蔽藍藤虎虹蜂蜜袖裾訃詣詮誰諦諧謎貌貪貼" ++-  "賂賭踪蹴辣遜遡那酎醒采釜錦錮鍋鍵鎌闇阜阪" ++-  "隙韓頃須頓頬顎餅餌駒骸鬱鶴鹿麓麺"

@@ -1,104 +0,0 @@-{-# LANGUAGE FlexibleInstances #-}---- |--- Module    : Data.Kanji.Types--- Copyright : (c) Colin Woodbury, 2015, 2016--- License   : GPL3--- Maintainer: Colin Woodbury <colingw@gmail.com>------ Types for this library. Note that typeclass instances for `ByteString`s--- are not provided, as `SB.pack` garbles characters above 0xFF.--module Data.Kanji.Types where--import           Data.Char (ord)-import           Data.Kanji.Types.Internal ()-import           Data.Maybe (fromJust)-import qualified Data.Set as S-import qualified Data.Text as ST-import qualified Data.Text.Lazy as LT-import           Lens.Micro--------- | Anything that can be transformed into a list of Kanji.-class AsKanji a where--  -- | Traverse into this type to find 0 or more Kanji.-  ---  -- Despite what the Haddock documentation says, this is part of the-  -- minimal complete definition.-  _Kanji :: Traversal' a Kanji--  -- | Transform this string type into a list of Kanji. The source string-  -- and the resulting list might not have the same length, if there-  -- were `Char` in the source that did not fall within the legal-  -- UTF8 range for Kanji.-  asKanji :: a -> [Kanji]-  asKanji a = a ^.. _Kanji--instance AsKanji Char where-  _Kanji f c = if isKanji c then _kanji <$> f (Kanji c) else pure c-  {-# INLINE _Kanji #-}--instance AsKanji [Char] where-  _Kanji = traverse . _Kanji-  {-# INLINE _Kanji #-}--instance AsKanji ST.Text where-  _Kanji = each . _Kanji-  {-# INLINE _Kanji #-}--instance AsKanji LT.Text where-  _Kanji = each . _Kanji-  {-# INLINE _Kanji #-}---- | A single symbol of Kanji. Japanese Kanji were borrowed from China--- over several waves during the past millenium. Japan names 2136 of--- these as their standard set, with rarer characters being the domain--- of academia and esoteric writers.------ Japanese has several Japan-only Kanji, including:------ * 畑 (a type of rice field)--- * 峠 (a narrow mountain pass)--- * 働 (to do physical labour)-newtype Kanji = Kanji { _kanji :: Char } deriving (Eq, Ord, Show)---- | A Level or "Kyuu" (級) of Japanese Kanji ranking. There are 12 of these,--- from 10 to 1, including intermediate levels between 3 and 2, and 2 and 1.------ Japanese students will typically have Level-5 ability by the time they--- finish elementary school. Level-5 accounts for 1006 characters.------ By the end of middle school, they would have covered up to Level-3--- (1607 Kanji) in their Japanese class curriculum.------ While Level-2 (2136 Kanji) is considered "standard adult" ability,--- many adults could not pass the Level-2, or even the Level-Pre2 (1940 Kanji)--- exam without considerable study.------ Level data for Kanji above Level-2 is currently not provided by--- this library.-data Level = Level { _allKanji :: S.Set Kanji-                   , _rank :: Rank-                   } deriving (Eq, Show)---- | A numeric representation of a `Level`.-data Rank = Ten | Nine | Eight | Seven | Six | Five | Four | Three | PreTwo-  | Two | PreOne | One deriving (Eq, Ord, Enum, Read, Show)---- | Discover a Rank's numeric representation, as a `Float`.-fromRank :: Rank -> Float-fromRank = fromJust . flip lookup rankMap---- | A mapping of Ranks to their numeric representation.-rankMap :: [(Rank,Float)]-rankMap = zip [Ten ..] [10,9,8,7,6,5,4,3,2.5,2,1.5,1]---- | Legal Kanji appear between UTF8 characters 19968 and 40959.-isKanji :: Char -> Bool-isKanji c = lowLimit <= c' && c' <= highLimit-    where c' = ord c-          lowLimit  = 19968  -- This is `一`-          highLimit = 40959  -- I don't have the right fonts to display this.

@@ -1,58 +0,0 @@-{-# LANGUAGE FlexibleInstances #-}-{-# LANGUAGE MultiParamTypeClasses #-}-{-# LANGUAGE TypeFamilies #-}-{-# LANGUAGE UndecidableInstances #-}---- |--- Module    : Data.Kanji.Types.Internal--- Copyright : (c) Colin Woodbury, 2015, 2016--- License   : GPL3--- Maintainer: Colin Woodbury <colingw@gmail.com>------ Instances needed for using `each` over `Text` types.--- This code is borrowed from `microlens-platform`:--- http://hackage.haskell.org/package/microlens-platform--module Data.Kanji.Types.Internal () where--import           Data.Monoid (Endo)-import qualified Data.Text as T-import qualified Data.Text.Lazy as TL-import           Lens.Micro-import           Lens.Micro.Internal-------instance (a ~ Char, b ~ Char) => Each T.Text T.Text a b where-  each = strictText-  {-# INLINE each #-}--instance (a ~ Char, b ~ Char) => Each TL.Text TL.Text a b where-  each = lazyText-  {-# INLINE each #-}--strictUnpacked :: Lens' T.Text String-strictUnpacked f t = T.pack <$> f (T.unpack t)-{-# INLINE strictUnpacked #-}--strictText :: Traversal' T.Text Char-strictText = strictUnpacked . traversed-{-# INLINE [0] strictText #-}--{-# RULES-"strict text -> map"    strictText = sets T.map        :: ASetter' T.Text Char;-"strict text -> foldr"  strictText = foldring T.foldr  :: Getting (Endo r) T.Text Char;-  #-}--lazyUnpacked :: Lens' TL.Text String-lazyUnpacked f t = TL.pack <$> f (TL.unpack t)-{-# INLINE lazyUnpacked #-}--lazyText :: Traversal' TL.Text Char-lazyText = lazyUnpacked . traversed-{-# INLINE [0] lazyText #-}--{-# RULES-"lazy text -> map"    lazyText = sets TL.map        :: ASetter' TL.Text Char;-"lazy text -> foldr"  lazyText = foldring TL.foldr  :: Getting (Endo r) TL.Text Char;-  #-}

@@ -0,0 +1,92 @@+Kanji+====++`kanji` is a Japanese Kanji library and analysation program written in Haskell. Its main+function is to tell what Kanji belong to what Level of the Japanese National+Kanji Examination (漢字検定).++`kanji` can be used to:+ - determine what Level individual Kanji belong to+ - determine the average Level (difficulty, in other words) of a group of Kanji+ - apply the above to whole files of Japanese++INSTALLING `kanji`+---------------+First, get the source files from:++https://github.com/fosskers/kanji++`kanji` is written in Haskell and uses the+[stack](http://docs.haskellstack.org/en/stable/README.html) tool. Once+`stack` is installed, move to the source directory and perform:++    stack install++USAGE+-----+Assuming you've made it so that you can run the executable, the following+command-line options are available:++```+Usage: kanji [-u|--unknowns] [-d|--density] [-e|--elementary] [-l|--leveldist]+             [-a|--average] [-s|--splits] ((-f|--file ARG) | JAPANESE)++Available options:+  -h,--help                Show this help text+  -u,--unknowns            Find Kanji whose Level couldn't be determined+  -d,--density             Find how much of the input is made of Kanji+  -e,--elementary          Find density of Kanji learnt in elementary school+  -l,--leveldist           Find the distribution of Kanji levels+  -a,--average             Find the average Level of all Kanji present+  -s,--splits              Show which Level each Kanji belongs to+  -f,--file ARG            Take input from a file+```++#### NOTES ON CLOs+ * All options above can be mixed to include their analysis result+ in the output JSON.+ * `-h` will over-ride any other options or arguments, discarding them and+   printing a help message.++#### Examples+*Single Kanji*+```+$> kanji -s 日+{+    "levelSplit": {+        "Ten": [+            "日"+        ]+    }+}+```++*A Japanese sentence*+```+$> kanji -s これは日本語+{+    "levelSplit": {+        "Nine": ["語"],+        "Ten": ["本", "日"]+    }+}+```++*All options*+```+$> kanji -leadus これは日本語+{+    "levelSplit": {+        "Nine": ["語"],+        "Ten": ["本", "日"]+    },+    "elementary": 1,+    "average": 9.666667,+    "density": 0.5,+    "unknowns": [],+    "distributions": {+        "Nine": 0.33333334,+        "Ten": 0.6666667+    }+}+```

@@ -1,2 +0,0 @@-import Distribution.Simple-main = defaultMain

@@ -0,0 +1,26 @@+module Main where++import           Criterion.Main+import           Data.Foldable (fold)+import           Data.Kanji+import           Data.Kanji.Types+import qualified Data.Set as S++---++main :: IO ()+main = defaultMain+  [ bgroup "Analysis"+    [+      bench "testKanji - uniques" $ nf uniques testKanji+    , bench "bigKanji - uniques" $ nf uniques bigKanji+    , bench "level 一" $ nf level (Kanji '一')+    , bench "level 串" $ nf level (Kanji '串')+    ]+  ]++testKanji :: [Kanji]+testKanji = map Kanji "本猫机鉛筆紙帽子包丁床天井壁光電気棚箱靴家階段戸屋根地面"++bigKanji :: [Kanji]+bigKanji = S.toList $ fold allKanji

@@ -1,35 +1,92 @@-name:                kanji-version:             3.0.2-synopsis:            Perform 漢字検定 (Japan Kanji Aptitude Test) level analysis on Japanese Kanji.-description:         Perform 漢字検定 (Japan Kanji Aptitude Test) level analysis on Japanese Kanji.-license:             GPL-3-license-file:        LICENSE-author:              Colin Woodbury-maintainer:          colingw@gmail.com--homepage:            https://github.com/fosskers/nanq--- copyright:-category:            Data-build-type:          Simple--- extra-source-files:-cabal-version:       >=1.10+-- This file has been generated from package.yaml by hpack version 0.20.0.+--+-- see: https://github.com/sol/hpack+--+-- hash: c5331657544f2fd01e194d0fe17da254a4f957bf458c13dca0c5da45ea3566db -library-  exposed-modules:     Data.Kanji-                     , Data.Kanji.Types-                     , Data.Kanji.Levels+name:           kanji+version:        3.1.0+synopsis:       Perform 漢字検定 (Japan Kanji Aptitude Test) level analysis on Japanese Kanji+description:    Perform 漢字検定 (Japan Kanji Aptitude Test) level analysis on Japanese Kanji.+category:       Data+homepage:       https://github.com/fosskers/kanji+author:         Colin Woodbury+maintainer:     colingw@gmail.com+copyright:      2011 - 2018 Colin Woodbury+license:        BSD3+license-file:   LICENSE+build-type:     Simple+cabal-version:  >= 1.10 -  other-modules:       Data.Kanji.Types.Internal+extra-source-files:+    CHANGELOG.md+    README.md -  other-extensions:    FlexibleInstances+library+  hs-source-dirs:+      lib+  ghc-options: -fwarn-unused-imports -fwarn-unused-binds -fwarn-name-shadowing -fwarn-unused-matches -fwarn-incomplete-patterns -fwarn-incomplete-uni-patterns+  build-depends:+      aeson >=1.2 && <1.3+    , base >=4.7 && <5+    , containers+    , deepseq+    , hashable+    , text+  exposed-modules:+      Data.Kanji+      Data.Kanji.Levels+      Data.Kanji.Types+  default-language: Haskell2010 -  build-depends:       base >= 4.8 && < 4.11-                     , containers >= 0.5 && < 0.6-                     , bytestring >= 0.10 && < 0.11-                     , microlens >= 0.4.1 && < 0.5-                     , text >= 1.2 && < 1.3+executable kanji+  main-is: nanq.hs+  hs-source-dirs:+      nanq+  default-extensions: NoImplicitPrelude OverloadedStrings+  ghc-options: -fwarn-unused-imports -fwarn-unused-binds -fwarn-name-shadowing -fwarn-unused-matches -fwarn-incomplete-patterns -fwarn-incomplete-uni-patterns -O2+  build-depends:+      aeson >=1.2 && <1.3+    , aeson-pretty+    , base >=4.7 && <5+    , containers+    , kanji+    , microlens >=0.4 && <0.5+    , microlens-aeson >=2.2 && <2.3+    , microlens-platform+    , optparse-applicative >=0.14 && <0.15+    , protolude >=0.2 && <0.3+    , text+  default-language: Haskell2010 -  -- hs-source-dirs:-  default-language:    Haskell2010+test-suite kanji-test+  type: exitcode-stdio-1.0+  main-is: Test.hs+  hs-source-dirs:+      test+  ghc-options: -fwarn-unused-imports -fwarn-unused-binds -fwarn-name-shadowing -fwarn-unused-matches -fwarn-incomplete-patterns -fwarn-incomplete-uni-patterns -threaded+  build-depends:+      HUnit-approx >=1.1 && <1.2+    , aeson >=1.2 && <1.3+    , base >=4.7 && <5+    , containers+    , kanji+    , tasty >=0.11 && <1.1+    , tasty-hunit >=0.9 && <0.11+    , text+  default-language: Haskell2010 -  ghc-options: -O2+benchmark kanji-bench+  type: exitcode-stdio-1.0+  main-is: Bench.hs+  hs-source-dirs:+      bench+  ghc-options: -fwarn-unused-imports -fwarn-unused-binds -fwarn-name-shadowing -fwarn-unused-matches -fwarn-incomplete-patterns -fwarn-incomplete-uni-patterns -threaded -O2+  build-depends:+      aeson >=1.2 && <1.3+    , base >=4.7 && <5+    , containers+    , criterion >=1.1 && <1.3+    , kanji+    , text+  default-language: Haskell2010

@@ -0,0 +1,114 @@+-- |+-- Module    : Data.Kanji+-- Copyright : (c) Colin Woodbury, 2015 - 2018+-- License   : GPL3+-- Maintainer: Colin Woodbury <colingw@gmail.com>+--+-- A library for analysing the density of Kanji in given texts,+-- according to their "Level" classification, as defined by the+-- Japan Kanji Aptitude Testing Foundation (日本漢字能力検定協会).++module Data.Kanji+       (+         -- * Kanji+         Kanji+       , kanji, _kanji+       , allKanji+       , isKanji+         -- * Levels+       , Level(..)+       , level+         -- * Analysis+       , percentSpread+       , levelDist+       , averageLevel+       , uniques+         -- ** Densities+       , kanjiDensity+       , elementaryDen+       , middleDen+       , highDen+       , adultDen+       ) where++import           Control.Applicative ((<|>))+import           Control.Arrow hiding (second)+import           Data.Bool (bool)+import           Data.Foldable (foldl')+import           Data.Kanji.Levels+import           Data.Kanji.Types+import           Data.List (sort, group)+import qualified Data.Map.Strict as M+import           Data.Maybe (catMaybes)+import           Data.Semigroup ((<>))+import qualified Data.Set as S++---++-- | All Japanese Kanji, grouped by their Level (級).+allKanji :: M.Map Level (S.Set Kanji)+allKanji =  M.fromList . zip [ Ten .. ] $ map (S.map Kanji) ks+  where ks = [ tenth, ninth, eighth, seventh, sixth+             , fifth, fourth, third, preSecond, second ]++-- | What `Level` does a Kanji belong to?+level :: Kanji -> Maybe Level+level k = M.foldlWithKey' (\acc l ks -> acc <|> bool Nothing (Just l) (S.member k ks)) Nothing allKanji++-- | Given the length of some String-like type and a list of `Kanji` found therein,+-- what percentage of them were Kanji?+kanjiDensity :: Int -> [Kanji] -> Float+kanjiDensity len ks = fromIntegral (length ks) / fromIntegral len++-- | How much of the Kanji found are learnt in elementary school in Japan?+--+-- > elementaryDen . levelDist :: [Kanji] -> Float+elementaryDen :: M.Map Level Float -> Float+elementaryDen m = M.foldl' (+) 0 . M.restrictKeys m $ S.fromList [ Five, Six .. ]++-- | How much of the Kanji found are learnt by the end of middle school?+--+-- > middleDen . levelDist :: [Kanji] -> Float+middleDen :: M.Map Level Float -> Float+middleDen m = M.foldl' (+) 0 . M.restrictKeys m $ S.fromList [ Three, Four .. ]++-- | How much of the Kanji found are learnt by the end of high school?+--+-- > highDen . levelDist :: [Kanji] -> Float+highDen :: M.Map Level Float -> Float+highDen m = M.foldl' (+) 0 . M.restrictKeys m $ S.fromList [ PreTwo, Three .. ]++-- | How much of the Kanji found should be able to be read by the average person?+--+-- > adultDen . levelDist :: [Kanji] -> Float+adultDen :: M.Map Level Float -> Float+adultDen m = M.foldl' (+) 0 . M.restrictKeys m $ S.fromList [ Two, PreTwo .. ]++-- | Find the average `Level` of a given set of `Kanji`.+averageLevel :: [Kanji] -> Float+averageLevel ks = average . map numericLevel . catMaybes $ map level ks+  where average ns = foldl' (+) 0 ns / fromIntegral (length ns)++-- | How much of each `Level` is represented by a group of Kanji?+-- The distribution values must sum to 1.+levelDist :: [Kanji] -> M.Map Level Float+levelDist ks = M.fromList . map percentPair . group . sort . catMaybes $ map level ks+  where percentPair qns = (head qns, fromIntegral (length qns) / totalKs)+        totalKs = fromIntegral $ length ks++-- | The distribution of each `Kanji` in a set of them.+-- The distribution values must sum to 1.+percentSpread :: [Kanji] -> M.Map Kanji Float+percentSpread ks = getPercent <$> kQuants+    where getPercent q = fromIntegral q / totalKanji+          kQuants      = kanjiQuantities ks+          totalKanji   = fromIntegral $ M.foldl' (+) 0 kQuants++-- | Determines how many times each `Kanji` appears in given set of them.+kanjiQuantities :: [Kanji] -> M.Map Kanji Int+kanjiQuantities = M.fromList . map (head &&& length) . group . sort++-- | Which Kanji appeared from each Level in the text?+uniques :: [Kanji] -> M.Map Level (S.Set Kanji)+uniques = S.foldl' h M.empty . S.fromList+  where h a k = maybe a (\l -> M.insertWith (<>) l (S.singleton k) a) $ level k

@@ -0,0 +1,170 @@+-- |+-- Module    : Data.Kanji.Levels+-- Copyright : (c) Colin Woodbury, 2015 - 2018+-- License   : GPL3+-- Maintainer: Colin Woodbury <colingw@gmail.com>+--+-- All Kanji from levels 10 to Pre-2.++module Data.Kanji.Levels where++import qualified Data.Set as S++---++-- | The Kanji unique to Level-10, studied by the end of 1st grade in+-- Japanese elementary schools.+tenth :: S.Set Char+tenth = S.fromDistinctAscList $+  "一七三上下中九二五人休先入八六円出力十千" +++  "口右名四土夕大天女子字学小山川左年手文日" +++  "早月木本村林校森正気水火犬玉王生田男町白" +++  "百目石空立竹糸耳花草虫見貝赤足車金雨青音"++-- | The Kanji unique to Level-9, studied by the end of 2nd grade in+-- Japanese elementary schools.+ninth :: S.Set Char+ninth = S.fromDistinctAscList $+  "万丸交京今会体何作元兄光公内冬刀分切前北" +++  "午半南原友古台合同回図国園地場声売夏外多" +++  "夜太妹姉室家寺少岩工市帰広店弓引弟弱強当" +++  "形後心思戸才教数新方明星春昼時晴曜書朝来" +++  "東楽歌止歩母毎毛池汽活海点父牛理用画番直" +++  "矢知社秋科答算米紙細組絵線羽考聞肉自船色" +++  "茶行西親角言計記話語読谷買走近通週道遠里" +++  "野長門間雪雲電頭顔風食首馬高魚鳥鳴麦黄黒"++-- | The Kanji unique to Level-8, studied by the end of 3rd grade in+-- Japanese elementary schools.+eighth :: S.Set Char+eighth = S.fromDistinctAscList $+  "丁世両主乗予事仕他代住使係倍全具写列助勉" +++  "動勝化区医去反取受号向君味命和品員商問坂" +++  "央始委守安定実客宮宿寒対局屋岸島州帳平幸" +++  "度庫庭式役待急息悪悲想意感所打投拾持指放" +++  "整旅族昔昭暑暗曲有服期板柱根植業様横橋次" +++  "歯死氷決油波注泳洋流消深温港湖湯漢炭物球" +++  "由申界畑病発登皮皿相県真着短研礼神祭福秒" +++  "究章童笛第筆等箱級終緑練羊美習者育苦荷落" +++  "葉薬血表詩調談豆負起路身転軽農返追送速進" +++  "遊運部都配酒重鉄銀開院陽階集面題飲館駅鼻"++-- | The Kanji unique to Level-7, studied by the end of 4th grade in+-- Japanese elementary schools.+seventh :: S.Set Char+seventh = S.fromDistinctAscList $+  "不争付令以仲伝位低例便信倉候借停健側働億" +++  "兆児共兵典冷初別利刷副功加努労勇包卒協単" +++  "博印参史司各告周唱喜器囲固型堂塩士変夫失" +++  "好季孫完官害察巣差希席帯底府康建径徒得必" +++  "念愛成戦折挙改救敗散料旗昨景最望未末札材" +++  "束松果栄案梅械極標機欠歴残殺毒氏民求治法" +++  "泣浅浴清満漁灯無然焼照熱牧特産的省祝票種" +++  "積競笑管節粉紀約結給続置老胃脈腸臣航良芸" +++  "芽英菜街衣要覚観訓試説課議象貨貯費賞軍輪" +++  "辞辺連達選郡量録鏡関陸隊静順願類飛飯養験"++-- | The Kanji unique to Level-6, studied by the end of 5th grade in+-- Japanese elementary schools.+sixth :: S.Set Char+sixth = S.fromDistinctAscList $+  "久仏仮件任似余価保修俵個備像再刊判制券則" +++  "効務勢厚句可営因団圧在均基報境墓増夢妻婦" +++  "容寄富導居属布師常幹序弁張往復徳志応快性" +++  "恩情態慣承技招授採接提損支政故敵断旧易暴" +++  "条枝査格桜検構武比永河液混減測準演潔災燃" +++  "版犯状独率現留略益眼破確示祖禁移程税築精" +++  "素経統絶綿総編績織罪群義耕職肥能興舌舎術" +++  "衛製複規解設許証評講謝識護豊財貧責貸貿賀" +++  "資賛質輸述迷退逆造過適酸鉱銅銭防限険際雑" +++  "非預領額飼"++-- | The Kanji unique to Level-5, studied by the end of 6th grade in+-- Japanese elementary schools.+fifth :: S.Set Char+fifth = S.fromDistinctAscList $+  "並乱乳亡仁供俳値傷優党冊処刻割創劇勤危卵" +++  "厳収后否吸呼善困垂城域奏奮姿存孝宅宇宗宙" +++  "宝宣密寸専射将尊就尺届展層己巻幕干幼庁座" +++  "延律従忘忠憲我批担拝拡捨探推揮操敬映晩暖" +++  "暮朗机枚染株棒模権樹欲段沿泉洗派済源潮激" +++  "灰熟片班異疑痛皇盛盟看砂磁私秘穀穴窓筋策" +++  "簡糖系紅納純絹縦縮署翌聖肺背胸脳腹臓臨至" +++  "若著蒸蔵蚕衆裁装裏補視覧討訪訳詞誌認誕誠" +++  "誤論諸警貴賃遺郵郷針鋼閉閣降陛除障難革頂" +++  "骨"++-- | The Kanji unique to Level-4, studied during middle school in Japan.+fourth :: S.Set Char+fourth = S.fromDistinctAscList $+  "丈与丘丹乾互井介仰伺依侵俗倒偉傍傾僧儀兼" +++  "冒凡凶刈到刺剣剤劣勧匹占即却及叫召吐含吹" +++  "咲唐嘆噴圏坊執堅堤塔壁壊壱奇奥奴妙姓威娘" +++  "婚寂寝尋尽尾屈峠峰巡巨帽幅幾床弐弾彩影彼" +++  "征御微徴忙怒怖恋恐恒恥恵悩惑惨慎慢慮憶戒" +++  "戯扇払扱抗抜抱抵押拍拓拠振捕掘描握援搬摘" +++  "撃攻敏敷斜旨旬是普暇暦曇更替朱朽杯枯柄柔" +++  "桃欄歓歳殖殿汗汚沈沖沢沼況泊浜浮浸涙淡添" +++  "渡溶滴漫澄濁濃為烈煙煮燥爆狂狩狭猛獣獲玄" +++  "珍環甘畳疲療皆盆盗監盤盾眠瞬矛砲祈秀称稲" +++  "稿突端箇範粒紋紫紹絡継維網緯縁繁繰罰翼耐" +++  "肩肪胴脂脚脱腐腕腰膚致舗舞舟般芋芝茂荒菓" +++  "蓄薄薪被襲触訴詰詳誇誉謡豪販賦贈越趣距跡" +++  "跳踊踏躍軒較載輝輩込迎迫逃透途遅違遣避郎" +++  "釈鈍鉛鋭鎖鑑闘陣陰隠隣隷雄雅雌離雷需震霧" +++  "露響項頼飾香駆騒驚髪鬼鮮麗黙鼓齢"++-- | The Kanji unique to Level-3, studied by the end of middle school+-- in Japan.+third :: S.Set Char+third = S.fromDistinctAscList $+  "乏乙了企伏伐伴伸佳侍促倣倹偶催債克免冗冠" +++  "凍凝刑削励勘募匠匿卑卓卸厘又双吉吏哀哲啓" +++  "喚喫嘱坑埋塊塗墜墨墳墾壇奉契奪如妨姫娯婆" +++  "婿嫁嬢孔孤宴審寿封尿岐岳峡崩巧帆帝幻幽廉" +++  "廊弧彫徐忌怠怪恨悔悟悦惜愚慈慌慕慨慰憂憎" +++  "憩房抑択抽拘掃掌排掛控措掲揚換揺携搾摂撮" +++  "擁擦敢斗斤斥施既昇晶暫架某桑棄棋楼概欧欺" +++  "殊殴没泌浪湾湿滅滑滝滞漂漏潜潤濫瀬炉炊炎" +++  "焦牲犠猟獄甲畔畜疾痘癖硬碑礎祉稚穂穏穫窒" +++  "符篤簿籍粋粗粘糧紛紺絞綱緊締緩縛縫繕翻聴" +++  "肝胆胎胞脅膜膨芳苗菊華葬藩虐虚蛮衝衰袋裂" +++  "裸覆訂託詠該誘請諮諾謀譲豚貫賊賢赦赴超軌" +++  "軸辛辱逮遂遇遭遵邦邪郊郭酔酵鋳錠錬錯鍛鎮" +++  "鐘閲阻陪陳陵陶隆随隔隻雇零霊顧飽餓駐騎髄" +++  "魂魅魔鯨鶏"++-- | The Kanji unique to Level-Pre2, considerend "mid high school" level.+preSecond :: S.Set Char+preSecond = S.fromDistinctAscList $+  "且丙亜享亭仙伯但佐併侮侯俊俸倫偏偵偽傑傘" +++  "僕僚儒償充准凸凹刃剖剛剰劾勅勲升厄叔叙吟" +++  "呈呉唆唇唯喝喪嗣嚇囚坪垣培堀堕堪塀塁塑塚" +++  "塾壌壮奔奨妃妄妊妥姻娠媒嫌嫡宜宰宵寛寡寧" +++  "寮尉尚尼履屯岬崇崎帥幣庶庸廃廷弊弔弦彰循" +++  "徹忍恭悠患悼惰愁愉慶憤憾懇懐懲懸戻扉扶抄" +++  "把披抹拐拒拙括拷挑挟挿捜据搭摩撤撲擬斉斎" +++  "旋昆暁曹朕朴杉析枠枢柳栓核栽桟棚棟棺槽款" +++  "殉殻汁江沸泡泥泰洞津洪浄浦涯涼淑渇渉渋渓" +++  "渦溝滋漆漠漬漸潟濯煩爵猫献猶猿珠琴璽瓶甚" +++  "畝疎疫症痢痴癒盲眺睡督矯砕硝硫碁磨礁祥禅" +++  "禍租秩稼窃窮窯竜筒粛粧糾紡索累紳緒縄繊繭" +++  "缶罷羅翁耗肌肖肢肯臭舶艇艦茎荘菌薦薫藻虜" +++  "虞蚊蛇蛍融衡衷裕褐褒襟覇訟診詐詔誓諭謁謄" +++  "謙謹譜貞貢賄賓賜賠購践軟轄迅迭逐逓逝逸遍" +++  "遮遷還邸酌酢酪酬酷醜醸釣鈴鉢銃銘閑閥附陥" +++  "隅雰霜靴韻頑頒頻顕飢駄騰麻"++-- | The Kanji unique to Level-2, considered "standard adult" level.+second :: S.Set Char+second = S.fromDistinctAscList $+  "串丼乞亀伎侶俺傲僅冥冶凄刹剥勃勾匂叱呂呪" +++  "咽哺唄唾喉喩嗅嘲埼堆塞填奈妖妬媛嫉宛尻岡" +++  "崖嵐巾弄弥彙怨恣惧慄憧憬戚戴拉拭拳拶挨挫" +++  "捉捗捻摯斑斬旦旺昧曖曽枕柵柿栃桁梗梨椅椎" +++  "楷毀氾汎汰沃沙淫湧溺潰煎熊爪爽牙狙玩瑠璃" +++  "璧瓦畏畿痕痩瘍眉睦瞭瞳稽窟箋箸籠綻緻罵羞" +++  "羨肘股脇脊腎腫腺膝膳臆臼舷艶芯苛茨萎葛蓋" +++  "蔑蔽藍藤虎虹蜂蜜袖裾訃詣詮誰諦諧謎貌貪貼" +++  "賂賭踪蹴辣遜遡那酎醒采釜錦錮鍋鍵鎌闇阜阪" +++  "隙韓頃須頓頬顎餅餌駒骸鬱鶴鹿麓麺"

@@ -0,0 +1,92 @@+{-# LANGUAGE DeriveAnyClass, DeriveGeneric #-}++-- |+-- Module    : Data.Kanji.Types+-- Copyright : (c) Colin Woodbury, 2015, 2016+-- License   : GPL3+-- Maintainer: Colin Woodbury <colingw@gmail.com>+--+-- Types for this library. While a constructor for `Kanji` is made available+-- here, you should prefer the `kanji` "smart constructor" unless you know+-- for sure that the `Char` in question falls within the correct UTF8 range.++module Data.Kanji.Types where++import           Control.DeepSeq (NFData)+import           Data.Aeson+import           Data.Aeson.Encoding (text)+import           Data.Bool (bool)+import           Data.Char (ord)+import           Data.Hashable+import qualified Data.Map.Strict as M+import           Data.Maybe (fromJust)+import qualified Data.Text as T+import           GHC.Generics++---++-- | A single symbol of Kanji. Japanese Kanji were borrowed from China+-- over several waves during the last 1,500 years. Japan names 2,136 of+-- these as their standard set, with rarer characters being the domain+-- of academia and esoteric writers.+--+-- Japanese has several Japan-only Kanji, including:+--+-- * 畑 (a type of rice field)+-- * 峠 (a narrow mountain pass)+-- * 働 (to do physical labour)+newtype Kanji = Kanji {+  _kanji :: Char -- ^ The original `Char` of a `Kanji`.+  } deriving (Eq, Ord, Show, Generic, Hashable, NFData)++instance ToJSON Kanji where+  toJSON (Kanji c) = String $ T.singleton c++instance FromJSON Kanji where+  parseJSON = withText "Kanji" $ \t -> case T.uncons t of+    Nothing -> fail "No Kanji given"+    Just (c, t') | not (T.null t') -> fail "More than one Kanji given in a single String"+                 | otherwise -> pure $ Kanji c++-- | Construct a `Kanji` value from some `Char` if it falls in the correct UTF8 range.+kanji :: Char -> Maybe Kanji+kanji c = bool Nothing (Just $ Kanji c) $ isKanji c++-- | A Level or "Kyuu" (級) of Japanese Kanji ranking. There are 12 of these,+-- from 10 to 1, including intermediate levels between 3 and 2, and 2 and 1.+--+-- Japanese students will typically have Level-5 ability by the time they+-- finish elementary school. Level-5 accounts for 1,006 characters.+--+-- By the end of middle school, they would have covered up to Level-3+-- (1607 Kanji) in their Japanese class curriculum.+--+-- While Level-2 (2,136 Kanji) is considered "standard adult" ability,+-- many adults could not pass the Level-2, or even the Level-Pre2 (1940 Kanji)+-- exam without considerable study.+--+-- Level data for Kanji above Level-2 is currently not provided by+-- this library.+data Level = Ten | Nine | Eight | Seven | Six | Five | Four | Three | PreTwo+           | Two | PreOne | One+           deriving (Eq, Ord, Enum, Show, Generic, Hashable, NFData, ToJSON, FromJSON)++instance ToJSONKey Level where+  toJSONKey = ToJSONKeyText f g+    where f = T.pack . show+          g = text . T.pack . show++-- | Discover a `Level`'s numeric representation, as a `Float`.+numericLevel :: Level -> Float+numericLevel = fromJust . flip M.lookup rankMap++-- | A mapping of Ranks to their numeric representation.+rankMap :: M.Map Level Float+rankMap = M.fromList $ zip [Ten ..] [10,9,8,7,6,5,4,3,2.5,2,1.5,1]++-- | Legal Kanji appear between UTF8 characters 19968 and 40959.+isKanji :: Char -> Bool+isKanji c = lowLimit <= c' && c' <= highLimit+    where c' = ord c+          lowLimit  = 19968  -- This is `一`+          highLimit = 40959  -- I don't have the right fonts to display this.

@@ -0,0 +1,106 @@+{-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE TupleSections #-}++module Main ( main ) where++import           Data.Aeson+import           Data.Aeson.Encode.Pretty+import           Data.Kanji+import qualified Data.Map.Strict as M+import           Data.Maybe (isNothing)+import qualified Data.Set as S+import qualified Data.Text as T+import qualified Data.Text.IO as TIO+import           Data.Text.Lazy.Builder (toLazyText)+import           Lens.Micro+import           Lens.Micro.Aeson+import           Lens.Micro.Platform ()+import           Options.Applicative+import           Protolude hiding (to)++---++data Flags = Flags [Operation] (Either FilePath Text) deriving (Eq)++data Operation = Unknowns | Density | Elementary+               | Distribution | Average | Splits deriving (Eq)++data Env = Env { _allKs :: [Kanji]+               , _original :: Text } deriving Eq++-- | Long, Short, Help+lsh l s h = long l <> short s <> help h++flags :: Parser Flags+flags = Flags <$> operations <*> (file <|> japanese)+  where file = Left <$> strOption (lsh "file" 'f' "Take input from a file")+        japanese = Right <$> argument str (metavar "JAPANESE")++operations :: Parser [Operation]+operations = catMaybes <$> ops+  where ops = traverse optional+          [ flag' Unknowns $+            lsh "unknowns" 'u' "Find Kanji whose Level couldn't be determined"+          , flag' Density $+            lsh "density" 'd' "Find how much of the input is made of Kanji"+          , flag' Elementary $+            lsh "elementary" 'e' "Find density of Kanji learnt in elementary school"+          , flag' Distribution $+            lsh "leveldist" 'l' "Find the distribution of Kanji levels"+          , flag' Average $+            lsh "average" 'a' "Find the average Level of all Kanji present"+          , flag' Splits $+            lsh "splits" 's' "Show which Level each Kanji belongs to" ]++-- | Shortcut for singleton objects+ob :: ToJSON v => Text -> v -> Value+ob k v = object [ k .= v ]++averageLev :: Reader Env Value+averageLev = ob "average" . averageLevel <$> asks _allKs++splits :: Reader Env Value+splits = ob "levelSplit" . S.foldl' f mempty . S.fromList <$> asks _allKs+  where f a k = maybe a (\l -> M.insertWith (++) l [k] a) $ level k++unknowns :: Reader Env Value+unknowns = ob "unknowns" . S.map _kanji . S.filter (isNothing . level) . S.fromList <$> asks _allKs++distribution :: Reader Env Value+distribution = ob "distributions" . levelDist <$> asks _allKs++density :: Reader Env Value+density = do+  d <- kanjiDensity <$> asks (T.length . _original) <*> asks _allKs+  pure $ ob "density" d++elementaryDensity :: Reader Env Value+elementaryDensity = ob "elementary" . elementaryDen . levelDist <$> asks _allKs++-- | All operations return JSON, to be aggregated into a master Object.+execOp :: Operation -> Reader Env Value+execOp Unknowns = unknowns+execOp Density = density+execOp Elementary = elementaryDensity+execOp Distribution = distribution+execOp Average = averageLev+execOp Splits = splits++output :: Value -> IO ()+output = putLText . toLazyText . encodePrettyToTextBuilder++-- | Dispatch on each `Operation` given. Aggregates the resulting JSON.+work :: (Env, [Operation]) -> Value+work (e, os) = Object $ vals ^. each . _Object+  where vals = runReader (traverse execOp os) e++env :: Flags -> IO (Env, [Operation])+env (Flags os inp) = case inp of+  Right t -> pure (e t, os)+  Left  f -> (, os) . e <$> TIO.readFile f+  where e t = Env (t ^.. each . to kanji . _Just) t++main :: IO ()+main = execParser opts >>= env >>= output . work+  where opts = info (helper <*> flags)+          (fullDesc <> header "nanq - Kanji analysis of Japanese text")

@@ -0,0 +1,42 @@+{-# LANGUAGE ImplicitParams #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++module Main ( main ) where++import           Data.Aeson+import           Data.Kanji+import           Data.Kanji.Types+import qualified Data.Map.Strict as M+import           Test.HUnit.Approx+import           Test.Tasty+import           Test.Tasty.HUnit++---++main :: IO ()+main = defaultMain suite++suite :: TestTree+suite = testGroup "Unit Tests"+  [ testGroup "Analysis"+    [+      testCase "percentSpread totals 1.0" $ M.foldl' (+) 0 (percentSpread testKanji) @?~ 1.0+    , testCase "levelDist totals 1.0"     $ M.foldl' (+) 0 (levelDist testKanji) @?~ 1.0+    , testCase "levelDist == adultDen"    $ M.foldl' (+) 0 (levelDist testKanji) @?~ adultDen (levelDist testKanji)+    ]+  , testGroup "JSON"+    [+      testCase "Empty String should fail to parse"    $ assert . isError $ fromJSON @Kanji (String "")+    , testCase "String w/ multiple Kanji should fail" $ assert . isError $ fromJSON @Kanji (String "泥棒猫")+    , testCase "String w/ one Kanji should succeed"   $ fromJSON (String "猫") @?= Success (Kanji '猫')+    ]+  ]+  where ?epsilon = 0.001++testKanji :: [Kanji]+testKanji = map Kanji "本猫机鉛筆紙帽子包丁床天井壁光電気棚箱靴家階段戸屋根地面"++isError :: Result a -> Bool+isError (Error _) = True+isError _ = False