kanji 3.2.1 → 3.3.0

raw patch · 7 files changed

+91/−76 lines, 7 filesdep +transformersdep −microlens-platformdep −protoludePVP ok

version bump matches the API change (PVP)

Dependencies added: transformers

Dependencies removed: microlens-platform, protolude

API changes (from Hackage documentation)

Files

• CHANGELOG.md +8/−0
• README.md +11/−11
• kanji.cabal +4/−5
• lib/Data/Kanji.hs +18/−21
• lib/Data/Kanji/Types.hs +37/−16
• nanq/nanq.hs +11/−22
• test/Test.hs +2/−1

@@ -1,3 +1,11 @@+## 3.3.0++- New `Unknown` level for Kanji above level `Two`+- `level` is now a total function+- Removed `averageLevel`, as it wasn't correct nor useful+- New `densities` function with associated type `CharCat` for getting more+  interesting character density information+ ## 3.2.1  - `ToJSON` and `FromJSON` instances for `Kanji` derived via `Generic`

@@ -28,16 +28,14 @@ command-line options are available:  ```-Usage: kanji [-u|--unknowns] [-d|--density] [-e|--elementary] [-l|--leveldist]-             [-a|--average] [-s|--splits] ((-f|--file ARG) | JAPANESE)+Usage: kanji [-d|--density] [-e|--elementary] [-l|--leveldist] [-s|--splits]+             ((-f|--file ARG) | JAPANESE)  Available options:   -h,--help                Show this help text-  -u,--unknowns            Find Kanji whose Level couldn't be determined   -d,--density             Find how much of the input is made of Kanji   -e,--elementary          Find density of Kanji learnt in elementary school   -l,--leveldist           Find the distribution of Kanji levels-  -a,--average             Find the average Level of all Kanji present   -s,--splits              Show which Level each Kanji belongs to   -f,--file ARG            Take input from a file ```@@ -74,19 +72,21 @@  *All options* ```-$> kanji -leadus これは日本語+$> kanji -sled これは日本語。串と糞 {     "levelSplit": {         "Nine": ["語"],-        "Ten": ["本", "日"]+        "Ten": ["本", "日"],+        "Unknown": ["糞"],+        "Two": ["串"]     },-    "elementary": 1,-    "average": 9.666667,+    "elementary": 0.6,     "density": 0.5,-    "unknowns": [],     "distributions": {-        "Nine": 0.33333334,-        "Ten": 0.6666667+        "Nine": 0.2,+        "Ten": 0.4,+        "Unknown": 0.2,+        "Two": 0.2     } } ```

@@ -2,10 +2,10 @@ -- -- see: https://github.com/sol/hpack ----- hash: 151854e92d22569c786a39e97c1a9922bfe2a86908bb691351fe7c16d8095572+-- hash: 540350499b57a0eb12fb2b4340b009e5daf99fef4eba303fd0e645a289e75063  name:           kanji-version:        3.2.1+version:        3.3.0 synopsis:       Perform 漢字検定 (Japan Kanji Aptitude Test) level analysis on Japanese Kanji description:    Perform 漢字検定 (Japan Kanji Aptitude Test) level analysis on Japanese Kanji. category:       Data@@ -43,7 +43,7 @@   main-is: nanq.hs   hs-source-dirs:       nanq-  default-extensions: NoImplicitPrelude OverloadedStrings+  default-extensions: OverloadedStrings   ghc-options: -fwarn-unused-imports -fwarn-unused-binds -fwarn-name-shadowing -fwarn-unused-matches -fwarn-incomplete-patterns -fwarn-incomplete-uni-patterns -O2   build-depends:       aeson >=1.2 && <1.4@@ -53,10 +53,9 @@     , kanji     , microlens >=0.4 && <0.5     , microlens-aeson >=2.2 && <2.4-    , microlens-platform     , optparse-applicative >=0.14 && <0.15-    , protolude >=0.2 && <0.3     , text+    , transformers   default-language: Haskell2010  test-suite kanji-test

@@ -16,17 +16,19 @@          Kanji        , kanji, _kanji        , allKanji-       , isKanji+       , isKanji, isHiragana, isKatakana+         -- * Character Categories+       , CharCat(..)+       , category          -- * Levels        , Level(..)        , level          -- * Analysis        , percentSpread        , levelDist-       , averageLevel        , uniques          -- ** Densities-       , kanjiDensity+       , densities        , elementaryDen        , middleDen        , highDen@@ -34,14 +36,14 @@        ) where  import           Control.Arrow hiding (second)-import           Data.Foldable (foldl', fold)+import           Data.Foldable (fold) import           Data.Kanji.Levels import           Data.Kanji.Types import           Data.List (sort, group) import qualified Data.Map.Strict as M-import           Data.Maybe (catMaybes) import           Data.Semigroup ((<>)) import qualified Data.Set as S+import qualified Data.Text as T  --- @@ -55,15 +57,16 @@ allKanji' :: M.Map Kanji Level allKanji' = M.fromList . S.toList . fold $ M.mapWithKey (\k v -> S.map (,k) v) allKanji --- | What `Level` does a Kanji belong to?-level :: Kanji -> Maybe Level-level k = M.lookup k allKanji'+-- | What `Level` does a Kanji belong to? `Unknown` for Kanji above level `Two`.+level :: Kanji -> Level+level k = maybe Unknown id $ M.lookup k allKanji' {-# INLINE level #-} --- | Given the length of some String-like type and a list of `Kanji` found therein,--- what percentage of them were Kanji?-kanjiDensity :: Int -> [Kanji] -> Float-kanjiDensity len ks = fromIntegral (length ks) / fromIntegral len+-- | Percentage of appearance of each `CharCat` in the source text.+-- The percentages will sum to 1.0.+densities :: T.Text -> M.Map CharCat Float+densities t = M.fromList . map (head &&& f) . group . sort . map category $ T.unpack t+  where f xs = fromIntegral (length xs) / fromIntegral (T.length t)  -- | How much of the Kanji found are learnt in elementary school in Japan? --@@ -89,16 +92,10 @@ adultDen :: M.Map Level Float -> Float adultDen m = M.foldl' (+) 0 . M.restrictKeys m $ S.fromList [ Two, PreTwo .. ] --- | Find the average `Level` of a given set of `Kanji`.-averageLevel :: [Kanji] -> Maybe Float-averageLevel [] = Nothing-averageLevel ks = Just . average . map numericLevel . catMaybes $ map level ks-  where average ns = foldl' (+) 0 ns / fromIntegral (length ns)- -- | How much of each `Level` is represented by a group of Kanji?--- The distribution values will sum to <= 1.+-- The distribution values will sum to 1. levelDist :: [Kanji] -> M.Map Level Float-levelDist ks = M.fromList . map percentPair . group . sort . catMaybes $ map level ks+levelDist ks = M.fromList . map percentPair . group . sort $ map level ks   where percentPair qns = (head qns, fromIntegral (length qns) / totalKs)         totalKs = fromIntegral $ length ks @@ -117,4 +114,4 @@ -- | Which Kanji appeared from each Level in the text? uniques :: [Kanji] -> M.Map Level (S.Set Kanji) uniques = S.foldl' h M.empty . S.fromList-  where h a k = maybe a (\l -> M.insertWith (<>) l (S.singleton k) a) $ level k+  where h a k = (\l -> M.insertWith (<>) l (S.singleton k) a) $ level k

@@ -1,4 +1,5 @@ {-# LANGUAGE DeriveAnyClass, DeriveGeneric #-}+{-# LANGUAGE ViewPatterns #-}  -- | -- Module    : Data.Kanji.Types@@ -16,10 +17,8 @@ import           Data.Aeson import           Data.Aeson.Encoding (text) import           Data.Bool (bool)-import           Data.Char (ord)+import           Data.Char (ord, isLetter, isNumber, isPunctuation) import           Data.Hashable-import qualified Data.Map.Strict as M-import           Data.Maybe (fromJust) import qualified Data.Text as T import           GHC.Generics @@ -61,7 +60,7 @@ -- Level data for Kanji above Level-2 is currently not provided by -- this library. data Level = Ten | Nine | Eight | Seven | Six | Five | Four | Three | PreTwo-           | Two | PreOne | One+           | Two | PreOne | One | Unknown            deriving (Eq, Ord, Enum, Show, Generic, Hashable, NFData, ToJSON, FromJSON)  instance ToJSONKey Level where@@ -69,17 +68,39 @@     where f = T.pack . show           g = text . T.pack . show --- | Discover a `Level`'s numeric representation, as a `Float`.-numericLevel :: Level -> Float-numericLevel = fromJust . flip M.lookup rankMap+-- | Legal Kanji appear between UTF-8 characters 19968 and 40959.+isKanji :: Char -> Bool+isKanji (ord -> c) = 19968 <= c && c <= 40959+{-# INLINE isKanji #-} --- | A mapping of Ranks to their numeric representation.-rankMap :: M.Map Level Float-rankMap = M.fromList $ zip [Ten ..] [10,9,8,7,6,5,4,3,2.5,2,1.5,1]+-- | あ to ん.+isHiragana :: Char -> Bool+isHiragana (ord -> c) = 0x3040 <= c && c <= 0x309f+{-# INLINE isHiragana #-} --- | Legal Kanji appear between UTF8 characters 19968 and 40959.-isKanji :: Char -> Bool-isKanji c = lowLimit <= c' && c' <= highLimit-    where c' = ord c-          lowLimit  = 19968  -- This is `一`-          highLimit = 40959  -- I don't have the right fonts to display this.+-- | ア to ン.+isKatakana :: Char -> Bool+isKatakana (ord -> c) = 0x30a0 <= c && c <= 0x30ff+{-# INLINE isKatakana #-}++-- | General categories for characters, at least as is useful for thinking about+-- Japanese.+--+-- Japanese "full-width" numbers and letters will be counted as `Numeral`+-- and `RomanLetter` respectively, alongside their usual ASCII forms.+data CharCat = Hanzi | Hiragana | Katakana | Numeral | RomanLetter | Punctuation | Other+  deriving (Eq, Ord, Show, Generic, Hashable, NFData, ToJSON, FromJSON)++category :: Char -> CharCat+category c | isKanji c       = Hanzi+           | isHiragana c    = Hiragana+           | isKatakana c    = Katakana+           | isLetter c      = RomanLetter+           | isNumber c      = Numeral+           | isPunctuation c = Punctuation+           | otherwise       = Other++instance ToJSONKey CharCat where+  toJSONKey = ToJSONKeyText f g+    where f = T.pack . show+          g = text . T.pack . show

@@ -3,27 +3,28 @@  module Main ( main ) where +import           Control.Monad.Trans.Reader import           Data.Aeson import           Data.Aeson.Encode.Pretty import           Data.Kanji import qualified Data.Map.Strict as M-import           Data.Maybe (isNothing)+import           Data.Maybe (catMaybes, mapMaybe)+import           Data.Monoid ((<>)) import qualified Data.Set as S+import           Data.Text (Text) import qualified Data.Text as T import qualified Data.Text.IO as TIO import           Data.Text.Lazy.Builder (toLazyText)+import           Data.Text.Lazy.IO as TLIO import           Lens.Micro import           Lens.Micro.Aeson-import           Lens.Micro.Platform () import           Options.Applicative-import           Protolude hiding (to)  ---  data Flags = Flags [Operation] (Either FilePath Text) deriving (Eq) -data Operation = Unknowns | Density | Elementary-               | Distribution | Average | Splits deriving (Eq)+data Operation = Density | Elementary | Distribution | Splits deriving (Eq)  data Env = Env { _allKs :: [Kanji]                , _original :: Text } deriving Eq@@ -39,16 +40,12 @@ operations :: Parser [Operation] operations = catMaybes <$> ops   where ops = traverse optional-          [ flag' Unknowns $-            lsh "unknowns" 'u' "Find Kanji whose Level couldn't be determined"-          , flag' Density $+          [ flag' Density $             lsh "density" 'd' "Find how much of the input is made of Kanji"           , flag' Elementary $             lsh "elementary" 'e' "Find density of Kanji learnt in elementary school"           , flag' Distribution $             lsh "leveldist" 'l' "Find the distribution of Kanji levels"-          , flag' Average $-            lsh "average" 'a' "Find the average Level of all Kanji present"           , flag' Splits $             lsh "splits" 's' "Show which Level each Kanji belongs to" ] @@ -56,22 +53,16 @@ ob :: ToJSON v => Text -> v -> Value ob k v = object [ k .= v ] -averageLev :: Reader Env Value-averageLev = ob "average" . averageLevel <$> asks _allKs- splits :: Reader Env Value splits = ob "levelSplit" . S.foldl' f mempty . S.fromList <$> asks _allKs-  where f a k = maybe a (\l -> M.insertWith (++) l [k] a) $ level k--unknowns :: Reader Env Value-unknowns = ob "unknowns" . S.map _kanji . S.filter (isNothing . level) . S.fromList <$> asks _allKs+  where f a k = (\l -> M.insertWith (++) l [k] a) $ level k  distribution :: Reader Env Value distribution = ob "distributions" . levelDist <$> asks _allKs  density :: Reader Env Value density = do-  d <- kanjiDensity <$> asks (T.length . _original) <*> asks _allKs+  d <- maybe 0 id . M.lookup Hanzi . densities <$> asks _original   pure $ ob "density" d  elementaryDensity :: Reader Env Value@@ -79,15 +70,13 @@  -- | All operations return JSON, to be aggregated into a master Object. execOp :: Operation -> Reader Env Value-execOp Unknowns = unknowns execOp Density = density execOp Elementary = elementaryDensity execOp Distribution = distribution-execOp Average = averageLev execOp Splits = splits  output :: Value -> IO ()-output = putLText . toLazyText . encodePrettyToTextBuilder+output = TLIO.putStrLn . toLazyText . encodePrettyToTextBuilder  -- | Dispatch on each `Operation` given. Aggregates the resulting JSON. work :: (Env, [Operation]) -> Value@@ -98,7 +87,7 @@ env (Flags os inp) = case inp of   Right t -> pure (e t, os)   Left  f -> (, os) . e <$> TIO.readFile f-  where e t = Env (t ^.. each . to kanji . _Just) t+  where e t = Env (mapMaybe kanji $ T.unpack t) t  main :: IO () main = execParser opts >>= env >>= output . work

@@ -22,6 +22,7 @@   [ testGroup "Analysis"     [       testCase "percentSpread totals 1.0" $ M.foldl' (+) 0 (percentSpread testKanji) @?~ 1.0+    , testCase "levelDist totals 1.0"     $ M.foldl' (+) 0 (levelDist testKanji) @?~ 1.0     , testCase "levelDist == adultDen"    $ M.foldl' (+) 0 (levelDist testKanji) @?~ adultDen (levelDist testKanji)     ]   , testGroup "JSON"@@ -38,7 +39,7 @@   where ?epsilon = 0.001  testKanji :: [Kanji]-testKanji = map Kanji "本猫机鉛筆紙帽子包丁床天井壁光電気棚箱靴家階段戸屋根地面糞"+testKanji = map Kanji "本猫机鉛筆紙帽子包丁床天井壁光電気棚箱靴家階段戸屋根地面"  isError :: Result a -> Bool isError (Error _) = True