ENIG 0.0.0.2 → 0.0.1.0

raw patch · 8 files changed

+176/−61 lines, 8 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• ChangeLog.md +20/−2
• ENIG.cabal +2/−3
• README.md +14/−0
• src/Data/Text/ENIG.hs +71/−7
• src/Data/Text/ENIG/Config.hs +26/−3
• src/Data/Text/ENIG/Detect.hs +0/−2
• test/Test/ENIG.hs +43/−6
• test/Test/ENIG/Config.hs +0/−38

@@ -2,12 +2,30 @@  ## [Unreleased] +* Optimized `enigAuto`+  * Current version is just a proof of concept. This should be optimized.+  * Use `Builder`+* Enough test cases for `enigAuto`++## [0.0.1.0] -- 2019-05-14++### Added+* Add `enigAuto` which takes just string(`Text`) and process automatically. It returns just string(`Text`)++## [0.0.0.2] -- 2019-03-16++### Changed+* Could handle `아/야` and `으/null`++### Added+* Add Haddock documentation comments+ ## [0.0.0.1] -- 2018-06-20  ### Changed * Use `unicode-transforms` instead of `text-icu`  ### Added-* Reimplement ENIG with features 2016 and 2018 only-  * Reimplementing features 2017 is postponed+* Re-implement ENIG with features 2016 and 2018 only+  * Re-implementing features 2017 is postponed * Implement handy functions

@@ -4,10 +4,10 @@ -- -- see: https://github.com/sol/hpack ----- hash: 42ae09380483cc6cd54e76091436d679146fefdf25a2e8cdfe6bb0f73feaf94e+-- hash: b9401ca815f587c7d68dfcbc534b46dadf4523546b4d76189a1ca5b6f429b563  name:           ENIG-version:        0.0.0.2+version:        0.0.1.0 synopsis:       Auto Korean conjugator/adjustor/adopter/converter description:    Please see the README on GitHub at <https://github.com/QuietJoon/ENIG#readme> category:       Natural Language Processing@@ -50,7 +50,6 @@   main-is: Spec.hs   other-modules:       Test.ENIG-      Test.ENIG.Config       Test.ENIG.Detect       Test.ENIG.Premise       Paths_ENIG

@@ -9,6 +9,20 @@  ENIG focuses on programmers to handle Korean strings. +## How to use++### Core functions++#### enigPPP++You can get proper Korean particle by giving `enigPPP` post word and a Korean postposition particle category `++### Handy functions (Not yet implemented)++You can use ENIG without any integration on your code by `enigAuto`.+When you give a text like "과자이(가) 두개입니다." to `enigAuto` like `enigAuto "과자이(가) 두개입니다."`, the function returns `"과자가 두개입니다."`++ ## Long-range Plans  ### Korean

@@ -26,15 +26,14 @@ -- >>> enigPPP "무엇" WG -- "과" enigPPP :: Text -> PPPCategory -> Text-enigPPP inputStr pppCa =-  if isHangul lastComponent-    then tShowPPPId . (toEnum :: Int -> PPPIdentity) $+enigPPP inputStr pppCa+  | isHangul lastComponent =+    tShowPPPId . (toEnum :: Int -> PPPIdentity) $       if isSecondType         then snd selectedPPPIPair         else fst selectedPPPIPair-    else if isDigit lastComponent-      then error "enigPPPByDigit is not implemented"-      else tShowPPPCa pppCa+  | isDigit lastComponent = error "enigPPPByDigit is not implemented"+  | otherwise = tShowPPPCa pppCa   where     isSecondType = isLastVowel lastComponent || (pppCa == EuX && isLastR lastComponent)     selectedPPPIPair = pppidVector V.! fromEnum pppCa@@ -53,4 +52,69 @@ -- >>> enigPPP "무엇" EuX "로" -- "으로" enigPPPWithPost :: Text -> PPPCategory -> Text -> Text-enigPPPWithPost inputStr pppCa postStr = T.append (enigPPP inputStr pppCa) postStr+enigPPPWithPost inputStr pppCa = T.append (enigPPP inputStr pppCa)+++{-+++# Pattern++* 무엇(으)로+* 무엇을(를)++# Problems++* How to Identify pattern without false positive+  * Case #1: Not 'IX'/'EuX'+  * Case #2: 'IX'/'EuX'++# Tasks++* Try to use 'Builder' or producer/consumer model++# Development Steps++1. Implement with 'List'+2. Implement with tail recursive with 'Char' and 'Text'+-}+-- | Find replacing pattern and apply enigPPP from given text automatically+-- TODO: Now working by reverse order.+enigAuto :: Text -> Text+-- ERROR: Qualified name in binding position: T.empty+-- enigAuto T.empty = T.empty+enigAuto "" = T.empty+enigAuto input =+  maybe+    (T.append target (enigAuto rest))+    (\(found, ppp, post) -> T.concat [found, enigPPP found ppp, post, enigAuto rest])+    result+  where+    (target,rest) = getTarget "" input+    result = findingPPPPattern target++getTarget :: Text -> Text -> (Text,Text)+getTarget prior "" = (prior,"")+getTarget prior input = if T.null target+  then getTarget (T.append prior (T.take 1 input)) (T.drop 1 input)+  else (T.append prior target,rest)+  where+    (target,rest) = T.break isDelimiter input++findingPPPPattern :: Text -> Maybe (Text,PPPCategory,Text)+findingPPPPattern input =+  maybe+    Nothing+    (\x -> Just (T.dropEnd (T.length . tFst $ x) input, tSnd x, tTrd x))+    (match' (\x -> T.isSuffixOf (tFst x) input) autoPatternList)+  where+   tFst (a,_,_) = a+   tSnd (_,b,_) = b+   tTrd (_,_,c) = c++match' _ [] = Nothing+match' condition (x:xs) = if condition x then Just x else match' condition xs++isDelimiter '(' = False+isDelimiter ')' = False+isDelimiter x = Prelude.any (\f -> f x) [isSpace, isPunctuation, isMark, isSeparator]

@@ -1,3 +1,5 @@+{-# LANGUAGE OverloadedStrings #-}+ module Data.Text.ENIG.Config where  @@ -5,7 +7,9 @@   import Data.Char-import qualified Data.Vector.Unboxed as V+import Data.Text (Text)+import qualified Data.Vector.Unboxed as VU+import qualified Data.Vector as V  -- | List of the last consonants' code (NFKDed) --@@ -45,8 +49,8 @@ -- | PPP corresponding list -- --   조사 대응 리스트-pppidVector :: V.Vector (Code,Code)-pppidVector = V.fromList+pppidVector :: VU.Vector (Code,Code)+pppidVector = VU.fromList   [(fromEnum Eun,fromEnum Neun)   ,(fromEnum Eul,fromEnum Leul)   ,(fromEnum I,fromEnum Ga)@@ -54,4 +58,23 @@   ,(fromEnum A,fromEnum Ya)   ,(fromEnum Ix,fromEnum X)   ,(fromEnum Eux,fromEnum X)+  ]++-- autoPatternList :: V.Vector Text+-- autoPatternList = V.fromList+autoPatternList :: [(Text,PPPCategory,Text)]+autoPatternList =+  [ ("은(는)",EN,"")+  , ("을(를)",EL,"")+  , ("이(가)",IG,"")+  , ("와(과)",WG,"")+  , ("아(야)",AY,"")+  , ("(이)나",IX,"나")+  , ("(이)든",IX,"든")+  , ("(이)란",IX,"란")+  , ("(이)랑",IX,"랑")+  , ("(이)면",IX,"면")+  , ("(이)여",IX,"여")+  , ("(으)로",EuX,"로")+  -- , "(이)다" - Not yet defined   ]

@@ -1,5 +1,3 @@-{-# LANGUAGE OverloadedStrings #-}- module Data.Text.ENIG.Detect where  import Data.Text.ENIG.Config

@@ -20,16 +20,35 @@   test_UsualCases =-  [ testCase "과자을(를) = 과자를" c_01-  , testCase "무엇을(를) = 무엇을" c_02-  , testCase "한글을(를) = 한글을" c_03+  [ testCase "과자을(를) => 과자를" c_01+  , testCase "무엇을(를) => 무엇을" c_02+  , testCase "한글을(를) => 한글을" c_03   ]+ test_EuXCases =-  [ testCase "과자(으)로 = 과자로" c_11-  , testCase "무엇(으)로 = 무엇으로" c_12-  , testCase "한글(으)로 = 한글로" c_13+  [ testCase "과자(으)로 => 과자로" c_11+  , testCase "무엇(으)로 => 무엇으로" c_12+  , testCase "한글(으)로 => 한글로" c_13   ] +test_AutoBaseCases =+  [ testCase "enigAuto 과자을(를) = 과자를" c_a01+  , testCase "enigAuto 무엇을(를) = 무엇을" c_a02+  , testCase "enigAuto 한글을(를) = 한글을" c_a03+  , testCase "enigAuto 과자(으)로 = 과자로" c_a11+  , testCase "enigAuto 무엇(으)로 = 무엇으로" c_a12+  , testCase "enigAuto 한글(으)로 = 한글로" c_a13+  ]++test_AutoComplexPatternCases =+  [ testCase "enigAuto 과자을(를) = 과자를" c_acp01+  , testCase "enigAuto 무엇을(를) = 무엇을" c_acp02+  , testCase "enigAuto 한글을(를) = 한글을" c_acp03+  , testCase "enigAuto 과자(으)로 = 과자로" c_acp11+  , testCase "enigAuto 무엇(으)로 = 무엇으로" c_acp12+  , testCase "enigAuto 한글(으)로 = 한글로" c_acp13+  ]+ preHangulStrList = ["과자", "무엇", "한글", "ㄱ", "ㅏ"] preNotHangulStrList = ["1", "2", "test", "harm"] @@ -46,3 +65,21 @@   where arg = preHangulStrList !! 1 c_13 = tShowPPPId X @?= (enigPPP  arg EuX)   where arg = preHangulStrList !! 2++c_a01 = enigAuto "과자을(를)" @?= "과자를"+c_a02 = enigAuto "무엇을(를)" @?= "무엇을"+c_a03 = enigAuto "한글을(를)"  @?= "한글을"++c_a11 = enigAuto "과자(으)로" @?= "과자로"+c_a12 = enigAuto "무엇(으)로" @?= "무엇으로"+c_a13 = enigAuto "한글(으)로" @?= "한글로"++c_acp01 = enigAuto "나은(는) 과자을(를) 먹었다." @?= "나는 과자를 먹었다."+c_acp02 = enigAuto "너은(는) 무엇을(를) 하였느냐?" @?= "너는 무엇을 하였느냐?"+c_acp03 = enigAuto "네, 한글을(를) 다루었습니다."  @?= "네, 한글을 다루었습니다."++c_acp11 = enigAuto "배고픔을 과자(으)로 때웠다" @?= "배고픔을 과자로 때웠다"+c_acp12 = enigAuto "이것은, 무엇(으)로 만들었는가?" @?= "이것은, 무엇으로 만들었는가?"+c_acp13 = enigAuto "예(Yes)! 한글(으)로 썼습니다" @?= "예(Yes)! 한글로 썼습니다"++-- TODO: Add enigAuto test code

@@ -1,38 +0,0 @@-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TemplateHaskell #-}--module Test.ENIG.Config where--import Test.Framework-import Test.Framework.Providers.HUnit-import Test.Framework.TH-import Test.HUnit.Base--import Data.Char-import qualified Data.Text as T-import Data.Text.Normalize-import qualified Data.Vector.Unboxed as V--import Data.Text.ENIG.Data-import Data.Text.ENIG.Detect---lastConsonantCodeList = [4520..4607]-lastConsonantList = map chr lastConsonantCodeList--vowelCodeList = [4449..4519]-vowelList = map chr vowelCodeList--hangulComponentCodeList = [4352..4607]-hangulComponentList = map chr hangulComponentCodeList----- Not good generation code. Need to be refactored.-pppidVector = V.fromList-  [(fromEnum Eun,fromEnum Neun)-  ,(fromEnum Eul,fromEnum Leul)-  ,(fromEnum I,fromEnum Ga)-  ,(fromEnum Gwa,fromEnum Wa)-  ,(fromEnum Ix,fromEnum X)-  ,(fromEnum Eux,fromEnum X)-  ]