ENIG (empty) → 0.0.0.2

raw patch · 14 files changed

+602/−0 lines, 14 filesdep +ENIGdep +HUnitdep +QuickCheck

Dependencies added: ENIG, HUnit, QuickCheck, base, test-framework, test-framework-hunit, test-framework-quickcheck2, test-framework-th, text, unicode-transforms, vector

Files

• ChangeLog.md +13/−0
• ENIG.cabal +72/−0
• LICENSE +30/−0
• README.md +31/−0
• src/Data/Text/ENIG.hs +56/−0
• src/Data/Text/ENIG/Config.hs +57/−0
• src/Data/Text/ENIG/Data.hs +37/−0
• src/Data/Text/ENIG/Detect.hs +44/−0
• src/Data/Text/ENIG/Show.hs +43/−0
• test/Spec.hs +14/−0
• test/Test/ENIG.hs +48/−0
• test/Test/ENIG/Config.hs +38/−0
• test/Test/ENIG/Detect.hs +62/−0
• test/Test/ENIG/Premise.hs +57/−0

@@ -0,0 +1,13 @@+# Changelog for ENIG++## [Unreleased]++## [0.0.0.1] -- 2018-06-20++### Changed+* Use `unicode-transforms` instead of `text-icu`++### Added+* Reimplement ENIG with features 2016 and 2018 only+  * Reimplementing features 2017 is postponed+* Implement handy functions

@@ -0,0 +1,72 @@+cabal-version: 1.12
++-- This file has been generated from package.yaml by hpack version 0.31.1.+--+-- see: https://github.com/sol/hpack+--+-- hash: 42ae09380483cc6cd54e76091436d679146fefdf25a2e8cdfe6bb0f73feaf94e++name:           ENIG+version:        0.0.0.2+synopsis:       Auto Korean conjugator/adjustor/adopter/converter+description:    Please see the README on GitHub at <https://github.com/QuietJoon/ENIG#readme>+category:       Natural Language Processing+homepage:       https://github.com/QuietJoon/ENIG#readme+bug-reports:    https://github.com/QuietJoon/ENIG/issues+author:         YongJoon Joe+maintainer:     developer+ENIG@quietjoon.net+copyright:      2016-2019 YongJoon Joe+license:        BSD3+license-file:   LICENSE+build-type:     Simple+extra-source-files:+    README.md+    ChangeLog.md++source-repository head+  type: git+  location: https://github.com/QuietJoon/ENIG++library+  exposed-modules:+      Data.Text.ENIG+      Data.Text.ENIG.Config+      Data.Text.ENIG.Data+      Data.Text.ENIG.Detect+      Data.Text.ENIG.Show+  other-modules:+      Paths_ENIG+  hs-source-dirs:+      src+  build-depends:+      base >=4.7 && <5+    , text+    , unicode-transforms+    , vector+  default-language: Haskell2010++test-suite ENIG-test+  type: exitcode-stdio-1.0+  main-is: Spec.hs+  other-modules:+      Test.ENIG+      Test.ENIG.Config+      Test.ENIG.Detect+      Test.ENIG.Premise+      Paths_ENIG+  hs-source-dirs:+      test+  default-extensions: TemplateHaskell+  ghc-options: -threaded -rtsopts -with-rtsopts=-N+  build-depends:+      ENIG+    , HUnit+    , QuickCheck+    , base+    , test-framework+    , test-framework-hunit+    , test-framework-quickcheck2+    , test-framework-th+    , text+    , unicode-transforms+  default-language: Haskell2010

@@ -0,0 +1,30 @@+Copyright YongJoon Joe (c) 2016-2019++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++    * Neither the name of YongJoon Joe nor the names of other+      contributors may be used to endorse or promote products derived+      from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

@@ -0,0 +1,31 @@+ENIG+====++ENIG is an acronym of typical Korean particles "Eun Neun I Ga (은는이가)".+Korean postposition particles change their form depending on the pronounciation of the previous consonant.+However, in trouble, because figuring out the previous consonant is not an easy problem.+Therefore, most of the programmers who handle Korean treat Korean by starting with "SOMETHING은(는) ~~".+This seems to be awkward in a sentence like "SOMETHING is(are), Two apple(s)."++ENIG focuses on programmers to handle Korean strings.++## Long-range Plans++### Korean++* Managing Korean string which involves Korean particle.++### English++* Managing English string which involves plural/singular form.++## Known Issues++### When you get error like `invalid argument` or `invalid character`++When you get a message during test the library like `invalid argument` or `invalid character`, check your language configuration.++This is not program error.++Because of this library handling Hangul itself, the test code should contains Korean characters.+Therefore, You should set your shell's encoding by `chcp 65001` for Windows or `export LC_ALL=en_US.UTF-8` for *nix platforms.

@@ -0,0 +1,56 @@+{-# LANGUAGE OverloadedStrings #-}++module Data.Text.ENIG where++import Data.Text.ENIG.Config+import Data.Text.ENIG.Data+import Data.Text.ENIG.Detect+import Data.Text.ENIG.Show+++import Data.Char++import           Data.Text (Text)+import qualified Data.Text as T++import Data.Text.Normalize++import qualified Data.Vector.Unboxed as V+++-- | Return proper and minimal PPP about given text+--   주어진 단어와 조사의 종류에 대해서 최소한의 적절한 조사의 문자열을 반환함+--+-- >>> enigPPP "과자" WG+-- "와"+-- >>> enigPPP "무엇" WG+-- "과"+enigPPP :: Text -> PPPCategory -> Text+enigPPP inputStr pppCa =+  if isHangul lastComponent+    then tShowPPPId . (toEnum :: Int -> PPPIdentity) $+      if isSecondType+        then snd selectedPPPIPair+        else fst selectedPPPIPair+    else if isDigit lastComponent+      then error "enigPPPByDigit is not implemented"+      else tShowPPPCa pppCa+  where+    isSecondType = isLastVowel lastComponent || (pppCa == EuX && isLastR lastComponent)+    selectedPPPIPair = pppidVector V.! fromEnum pppCa+    -- preprocessForPPP for handling parenthesis, quatation, etc.+    preprocessForPPP = id+    preprocessed = preprocessForPPP inputStr+    lastComponent = getLastComponentCode preprocessed+    -- handling PPP with digit is not implemented+    isDigit _ = False++-- | Return proper PPP about given text with post text+--   주어진 단어와 조사의 종류에 대해서 적절한 조사의 문자열을 반환함+--+-- >>> enigPPP "과자" EuX "로"+-- "로"+-- >>> enigPPP "무엇" EuX "로"+-- "으로"+enigPPPWithPost :: Text -> PPPCategory -> Text -> Text+enigPPPWithPost inputStr pppCa postStr = T.append (enigPPP inputStr pppCa) postStr

@@ -0,0 +1,57 @@+module Data.Text.ENIG.Config where+++import Data.Text.ENIG.Data+++import Data.Char+import qualified Data.Vector.Unboxed as V++-- | List of the last consonants' code (NFKDed)+--+--   받침의 문자코드 리스트+lastConsonantCodeList :: [Code]+lastConsonantCodeList = [4520..4607]+-- | List of the last consonants+--+--   받침의 문자 리스트+lastConsonantList :: [Char]+lastConsonantList = map chr lastConsonantCodeList++-- | List of the vowel consonants' code (NFKDed)+--+--   모음의 문자코드 리스트+vowelCodeList :: [Code]+vowelCodeList = [4449..4519]+-- | List of the vowel consonants+--+--   모음의 문자 리스트+vowelList :: [Char]+vowelList = map chr vowelCodeList++-- | List of Hangul components' code+--+--   한글 자소 코드 리스트+hangulComponentCodeList :: [Code]+hangulComponentCodeList = [4352..4607]+-- | List of Hangul components+--+--   한글 자소 리스트+hangulComponentList :: [Char]+hangulComponentList = map chr hangulComponentCodeList+++-- TODO: Not good generation code. Need to be refactored.+-- | PPP corresponding list+--+--   조사 대응 리스트+pppidVector :: V.Vector (Code,Code)+pppidVector = V.fromList+  [(fromEnum Eun,fromEnum Neun)+  ,(fromEnum Eul,fromEnum Leul)+  ,(fromEnum I,fromEnum Ga)+  ,(fromEnum Gwa,fromEnum Wa)+  ,(fromEnum A,fromEnum Ya)+  ,(fromEnum Ix,fromEnum X)+  ,(fromEnum Eux,fromEnum X)+  ]

@@ -0,0 +1,37 @@+module Data.Text.ENIG.Data where+++import           Data.Text               (Text)+import qualified Data.Text               as T+++-- TODO: Optimize order of PPPI by Hangul statistical data+-- | Category of PostPosition Particle in Hangul+data PPPCategory+  = EN  -- ^ __은/는__+  | EL  -- ^ __을/를__+  | IG  -- ^ __이/가__+  | WG  -- ^ __와/과__+  | AY  -- ^ __아/야__+  | IX  -- ^ __이/null__+  | EuX -- ^ __으/null__+  deriving (Eq,Ord,Enum,Bounded,Show,Read)++-- | Identity of PostPosition Particle in Hangul+data PPPIdentity+  = Eun  -- ^ __/은/__+  | Neun -- ^ __/는/__+  | Eul  -- ^ __/을/__+  | Leul -- ^ __/를/__+  | I    -- ^ __/이/__+  | Ga   -- ^ __/가/__+  | Wa   -- ^ __/와/__+  | Gwa  -- ^ __/과/__+  | A    -- ^ __/아/__+  | Ya   -- ^ __/야/__+  | Ix   -- ^ Continuous __/이/__+  | Eux  -- ^ __/으/__+  | X    -- ^ __/null/__+  deriving (Eq,Ord,Enum,Bounded,Show,Read)++type Code = Int

@@ -0,0 +1,44 @@+{-# LANGUAGE OverloadedStrings #-}++module Data.Text.ENIG.Detect where++import Data.Text.ENIG.Config+import Data.Text.ENIG.Data+++import Data.Char++import           Data.Text (Text)+import qualified Data.Text as T++import Data.Text.Normalize++import qualified Data.Vector.Unboxed as V+++-- | Detect whether the last component is an last consonant or not+isLastConsonant :: Int -> Bool+isLastConsonant hangulCode =+  (head lastConsonantCodeList <= hangulCode) && (hangulCode <= last lastConsonantCodeList)++-- | Detect whether the last component is an vowel or not+--   To avoid the issue with the first consonant, like 'ㄱ'.+isLastVowel :: Int -> Bool+isLastVowel hangulCode =+  (head vowelCodeList <= hangulCode) && (hangulCode <= last vowelCodeList)++-- | Detect whether the last component is the last consonant 'ㄹ'+isLastR :: Int -> Bool+isLastR hangulCode = hangulCode == 4527++-- | Detect whether the given code is an Hangul component or not+isHangul :: Int -> Bool+isHangul mHangulCode =+  (head hangulComponentCodeList <= mHangulCode) && (mHangulCode <= last hangulComponentCodeList)++-- | Get the last component's code of given Hangel+getLastComponentCode :: Text -> Int+getLastComponentCode str = ord aComponent+  where+    lastChar = T.singleton . T.last $ str+    aComponent = T.last . normalize NFKD $ lastChar

@@ -0,0 +1,43 @@+{-# LANGUAGE OverloadedStrings #-}++module Data.Text.ENIG.Show where++import Data.Text.ENIG.Data++import Data.Text (Text)+import qualified Data.Text as T+++-- | Show function for 'PPPCategory' as 'Text'+tShowPPPCa :: PPPCategory -> Text+tShowPPPCa EN  = "은(는)"+tShowPPPCa EL  = "을(를)"+tShowPPPCa IG  = "이(가)"+tShowPPPCa WG  = "와(과)"+tShowPPPCa AY  = "아(야)"+tShowPPPCa IX  = "(이)"+tShowPPPCa EuX = "(으)"++-- | Show function for 'PPPCategory' as 'String'+sShowPPPCa :: PPPCategory -> String+sShowPPPCa = T.unpack . tShowPPPCa++-- | Show function for 'PPPIdentity' as 'Text'+tShowPPPId :: PPPIdentity -> Text+tShowPPPId Eun  = "은"+tShowPPPId Neun = "는"+tShowPPPId Eul  = "을"+tShowPPPId Leul = "를"+tShowPPPId I    = "이"+tShowPPPId Ga   = "가"+tShowPPPId Wa   = "와"+tShowPPPId Gwa  = "과"+tShowPPPId A    = "아"+tShowPPPId Ya   = "야"+tShowPPPId Ix   = "이"+tShowPPPId Eux  = "으"+tShowPPPId X    = ""++-- | Show function for 'PPPIdentity' as 'String'+sShowPPPId :: PPPIdentity -> String+sShowPPPId = T.unpack . tShowPPPId

@@ -0,0 +1,14 @@+import qualified Test.Framework as Test++import Test.ENIG+import Test.ENIG.Detect+import Test.ENIG.Premise+++main :: IO ()+main = do+  Test.defaultMain+    [ Test.ENIG.Premise.tests+    , Test.ENIG.Detect.tests+    , Test.ENIG.tests+    ]

@@ -0,0 +1,48 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TemplateHaskell #-}++module Test.ENIG where++import Test.Framework+import Test.Framework.Providers.HUnit+import Test.Framework.TH+import Test.HUnit.Base++import qualified Data.Text as T+import Data.Text.Normalize++import Data.Text.ENIG+import Data.Text.ENIG.Data+import Data.Text.ENIG.Show+++tests = $(testGroupGenerator)+++test_UsualCases =+  [ testCase "과자을(를) = 과자를" c_01+  , testCase "무엇을(를) = 무엇을" c_02+  , testCase "한글을(를) = 한글을" c_03+  ]+test_EuXCases =+  [ testCase "과자(으)로 = 과자로" c_11+  , testCase "무엇(으)로 = 무엇으로" c_12+  , testCase "한글(으)로 = 한글로" c_13+  ]++preHangulStrList = ["과자", "무엇", "한글", "ㄱ", "ㅏ"]+preNotHangulStrList = ["1", "2", "test", "harm"]++c_01 = tShowPPPId Leul @?= (enigPPP  arg EL)+  where arg = preHangulStrList !! 0+c_02 = tShowPPPId Eul @?= (enigPPP  arg EL)+  where arg = preHangulStrList !! 1+c_03 = tShowPPPId Eul @?= (enigPPP  arg EL)+  where arg = preHangulStrList !! 3++c_11 = tShowPPPId X @?= (enigPPP  arg EuX)+  where arg = preHangulStrList !! 0+c_12 = tShowPPPId Eux @?= (enigPPP  arg EuX)+  where arg = preHangulStrList !! 1+c_13 = tShowPPPId X @?= (enigPPP  arg EuX)+  where arg = preHangulStrList !! 2

@@ -0,0 +1,38 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TemplateHaskell #-}++module Test.ENIG.Config where++import Test.Framework+import Test.Framework.Providers.HUnit+import Test.Framework.TH+import Test.HUnit.Base++import Data.Char+import qualified Data.Text as T+import Data.Text.Normalize+import qualified Data.Vector.Unboxed as V++import Data.Text.ENIG.Data+import Data.Text.ENIG.Detect+++lastConsonantCodeList = [4520..4607]+lastConsonantList = map chr lastConsonantCodeList++vowelCodeList = [4449..4519]+vowelList = map chr vowelCodeList++hangulComponentCodeList = [4352..4607]+hangulComponentList = map chr hangulComponentCodeList+++-- Not good generation code. Need to be refactored.+pppidVector = V.fromList+  [(fromEnum Eun,fromEnum Neun)+  ,(fromEnum Eul,fromEnum Leul)+  ,(fromEnum I,fromEnum Ga)+  ,(fromEnum Gwa,fromEnum Wa)+  ,(fromEnum Ix,fromEnum X)+  ,(fromEnum Eux,fromEnum X)+  ]

@@ -0,0 +1,62 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TemplateHaskell #-}++module Test.ENIG.Detect where++import Test.Framework+import Test.Framework.Providers.HUnit+import Test.Framework.TH+import Test.HUnit.Base++import Data.Char+import qualified Data.Text as T+import Data.Text.Normalize++import Data.Text.ENIG.Data+import Data.Text.ENIG.Detect++tests = $(testGroupGenerator)+++preHangulStrList = ["과자", "무엇", "한글", "ㄱ", "ㅏ"]+++c_11 = 4449 @?= getLastComponentCode arg+  where arg = preHangulStrList !! 0+c_12 = 4538 @?= getLastComponentCode arg+  where arg = preHangulStrList !! 1+c_13 = 4527 @?= getLastComponentCode arg+  where arg = preHangulStrList !! 2++test_ComponentCode =+  [ testCase "lastComponent(과자) = ㅏ" c_11+  , testCase "lastComponent(무엇) = ㅅ" c_12+  , testCase "lastComponent(한글) = ㄹ" c_13+  ]++c_21 = assertBool "" (not . isLastConsonant . getLastComponentCode $ arg)+  where arg = preHangulStrList !! 0+c_22 = assertBool "" (isLastConsonant . getLastComponentCode $ arg)+  where arg = preHangulStrList !! 1+c_23 = assertBool "" (isLastConsonant . getLastComponentCode $ arg)+  where arg = preHangulStrList !! 2++test_isLastConsonant =+  [ testCase "isLastConsonant(과자): False" c_21+  , testCase "isLastConsonant(무엇): True"  c_22+  , testCase "isLastConsonant(한글): True"  c_23+  ]++c_31 = assertBool "" (not . isLastR . getLastComponentCode $ arg)+  where arg = preHangulStrList !! 0+c_32 = assertBool "" (not . isLastR . getLastComponentCode $ arg)+  where arg = preHangulStrList !! 1+c_33 = assertBool "" (isLastR . getLastComponentCode $ arg)+  where arg = preHangulStrList !! 2++test_isLastR =+  [ testCase "isLastR(과자): False" c_31+  , testCase "isLastR(무엇): False" c_32+  , testCase "isLastR(한글): True"  c_33+  ]+

@@ -0,0 +1,57 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TemplateHaskell #-}++module Test.ENIG.Premise where++import Test.Framework+import Test.Framework.Providers.HUnit+import Test.Framework.TH+import Test.HUnit.Base++import qualified Data.Text as T+import Data.Text.Normalize++import Data.Text.ENIG.Data++tests = $(testGroupGenerator)++ga   = "가"+gag  = "각"+ha   = "하"+heuh = "헣"+g    = "ㄱ"+_g   = T.singleton . T.index (nfd gag) $ 0+__g  = T.singleton . T.index (nfd gag) $ 2+_a   = "ㅏ"+__a  = T.singleton . T.index (nfd gag) $ 1++sampleHangul = [g, _g, __g, _a, __a, ga, gag, ha, heuh] :: [T.Text]++nfc  = normalize NFC+nfkc = normalize NFKC+nfd  = normalize NFD+nfkd = normalize NFKD++nfcd   = nfc  . nfd+nfkcd  = nfkc . nfd+nfckd  = nfc  . nfkd+nfkckd = nfkc . nfkd++-- # Result+-- * NFC/NFD does not handle "ㅏ" or "ㄱ" as a component.+-- * When handling them different, NFC/NFD should be used.+-- * However, because of their pronouces, do not need to handle then differently.++c_normalizationNFCD+  = assertBool+      "nfc .  nfd /= nfkc .  nfd"+      (not $ all  (\x ->  nfcd x ==  nfkcd x) sampleHangul)+c_normalizationNFCKD+  = assertBool+      "nfc . nfkd == nfkc . nfkd"+      (all  (\x -> nfckd x == nfkckd x) sampleHangul)++test_NormalizationComparison =+  [ testCase ("nfc .  nfd == nfkc . nfkd with " ++ (T.unpack $ T.intercalate ", " sampleHangul)) c_normalizationNFCD+  , testCase ("nfc . nfkd == nfkc . nfkd with " ++ (T.unpack $ T.intercalate ", " sampleHangul)) c_normalizationNFCKD+  ]