diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,13 @@ +# Changelog for ENIG + +## [Unreleased] + +## [0.0.0.1] -- 2018-06-20 + +### Changed +* Use `unicode-transforms` instead of `text-icu` + +### Added +* Reimplement ENIG with features 2016 and 2018 only + * Reimplementing features 2017 is postponed +* Implement handy functions diff --git a/ENIG.cabal b/ENIG.cabal new file mode 100644 --- /dev/null +++ b/ENIG.cabal @@ -0,0 +1,72 @@ +cabal-version: 1.12 + +-- This file has been generated from package.yaml by hpack version 0.31.1. +-- +-- see: https://github.com/sol/hpack +-- +-- hash: 42ae09380483cc6cd54e76091436d679146fefdf25a2e8cdfe6bb0f73feaf94e + +name: ENIG +version: 0.0.0.2 +synopsis: Auto Korean conjugator/adjustor/adopter/converter +description: Please see the README on GitHub at +category: Natural Language Processing +homepage: https://github.com/QuietJoon/ENIG#readme +bug-reports: https://github.com/QuietJoon/ENIG/issues +author: YongJoon Joe +maintainer: developer+ENIG@quietjoon.net +copyright: 2016-2019 YongJoon Joe +license: BSD3 +license-file: LICENSE +build-type: Simple +extra-source-files: + README.md + ChangeLog.md + +source-repository head + type: git + location: https://github.com/QuietJoon/ENIG + +library + exposed-modules: + Data.Text.ENIG + Data.Text.ENIG.Config + Data.Text.ENIG.Data + Data.Text.ENIG.Detect + Data.Text.ENIG.Show + other-modules: + Paths_ENIG + hs-source-dirs: + src + build-depends: + base >=4.7 && <5 + , text + , unicode-transforms + , vector + default-language: Haskell2010 + +test-suite ENIG-test + type: exitcode-stdio-1.0 + main-is: Spec.hs + other-modules: + Test.ENIG + Test.ENIG.Config + Test.ENIG.Detect + Test.ENIG.Premise + Paths_ENIG + hs-source-dirs: + test + default-extensions: TemplateHaskell + ghc-options: -threaded -rtsopts -with-rtsopts=-N + build-depends: + ENIG + , HUnit + , QuickCheck + , base + , test-framework + , test-framework-hunit + , test-framework-quickcheck2 + , test-framework-th + , text + , unicode-transforms + default-language: Haskell2010 diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright YongJoon Joe (c) 2016-2019 + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of YongJoon Joe nor the names of other + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +ENIG +==== + +ENIG is an acronym of typical Korean particles "Eun Neun I Ga (은는이가)". +Korean postposition particles change their form depending on the pronounciation of the previous consonant. +However, in trouble, because figuring out the previous consonant is not an easy problem. +Therefore, most of the programmers who handle Korean treat Korean by starting with "SOMETHING은(는) ~~". +This seems to be awkward in a sentence like "SOMETHING is(are), Two apple(s)." + +ENIG focuses on programmers to handle Korean strings. + +## Long-range Plans + +### Korean + +* Managing Korean string which involves Korean particle. + +### English + +* Managing English string which involves plural/singular form. + +## Known Issues + +### When you get error like `invalid argument` or `invalid character` + +When you get a message during test the library like `invalid argument` or `invalid character`, check your language configuration. + +This is not program error. + +Because of this library handling Hangul itself, the test code should contains Korean characters. +Therefore, You should set your shell's encoding by `chcp 65001` for Windows or `export LC_ALL=en_US.UTF-8` for *nix platforms. diff --git a/src/Data/Text/ENIG.hs b/src/Data/Text/ENIG.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/ENIG.hs @@ -0,0 +1,56 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Data.Text.ENIG where + +import Data.Text.ENIG.Config +import Data.Text.ENIG.Data +import Data.Text.ENIG.Detect +import Data.Text.ENIG.Show + + +import Data.Char + +import Data.Text (Text) +import qualified Data.Text as T + +import Data.Text.Normalize + +import qualified Data.Vector.Unboxed as V + + +-- | Return proper and minimal PPP about given text +-- 주어진 단어와 조사의 종류에 대해서 최소한의 적절한 조사의 문자열을 반환함 +-- +-- >>> enigPPP "과자" WG +-- "와" +-- >>> enigPPP "무엇" WG +-- "과" +enigPPP :: Text -> PPPCategory -> Text +enigPPP inputStr pppCa = + if isHangul lastComponent + then tShowPPPId . (toEnum :: Int -> PPPIdentity) $ + if isSecondType + then snd selectedPPPIPair + else fst selectedPPPIPair + else if isDigit lastComponent + then error "enigPPPByDigit is not implemented" + else tShowPPPCa pppCa + where + isSecondType = isLastVowel lastComponent || (pppCa == EuX && isLastR lastComponent) + selectedPPPIPair = pppidVector V.! fromEnum pppCa + -- preprocessForPPP for handling parenthesis, quatation, etc. + preprocessForPPP = id + preprocessed = preprocessForPPP inputStr + lastComponent = getLastComponentCode preprocessed + -- handling PPP with digit is not implemented + isDigit _ = False + +-- | Return proper PPP about given text with post text +-- 주어진 단어와 조사의 종류에 대해서 적절한 조사의 문자열을 반환함 +-- +-- >>> enigPPP "과자" EuX "로" +-- "로" +-- >>> enigPPP "무엇" EuX "로" +-- "으로" +enigPPPWithPost :: Text -> PPPCategory -> Text -> Text +enigPPPWithPost inputStr pppCa postStr = T.append (enigPPP inputStr pppCa) postStr diff --git a/src/Data/Text/ENIG/Config.hs b/src/Data/Text/ENIG/Config.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/ENIG/Config.hs @@ -0,0 +1,57 @@ +module Data.Text.ENIG.Config where + + +import Data.Text.ENIG.Data + + +import Data.Char +import qualified Data.Vector.Unboxed as V + +-- | List of the last consonants' code (NFKDed) +-- +-- 받침의 문자코드 리스트 +lastConsonantCodeList :: [Code] +lastConsonantCodeList = [4520..4607] +-- | List of the last consonants +-- +-- 받침의 문자 리스트 +lastConsonantList :: [Char] +lastConsonantList = map chr lastConsonantCodeList + +-- | List of the vowel consonants' code (NFKDed) +-- +-- 모음의 문자코드 리스트 +vowelCodeList :: [Code] +vowelCodeList = [4449..4519] +-- | List of the vowel consonants +-- +-- 모음의 문자 리스트 +vowelList :: [Char] +vowelList = map chr vowelCodeList + +-- | List of Hangul components' code +-- +-- 한글 자소 코드 리스트 +hangulComponentCodeList :: [Code] +hangulComponentCodeList = [4352..4607] +-- | List of Hangul components +-- +-- 한글 자소 리스트 +hangulComponentList :: [Char] +hangulComponentList = map chr hangulComponentCodeList + + +-- TODO: Not good generation code. Need to be refactored. +-- | PPP corresponding list +-- +-- 조사 대응 리스트 +pppidVector :: V.Vector (Code,Code) +pppidVector = V.fromList + [(fromEnum Eun,fromEnum Neun) + ,(fromEnum Eul,fromEnum Leul) + ,(fromEnum I,fromEnum Ga) + ,(fromEnum Gwa,fromEnum Wa) + ,(fromEnum A,fromEnum Ya) + ,(fromEnum Ix,fromEnum X) + ,(fromEnum Eux,fromEnum X) + ] diff --git a/src/Data/Text/ENIG/Data.hs b/src/Data/Text/ENIG/Data.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/ENIG/Data.hs @@ -0,0 +1,37 @@ +module Data.Text.ENIG.Data where + + +import Data.Text (Text) +import qualified Data.Text as T + + +-- TODO: Optimize order of PPPI by Hangul statistical data +-- | Category of PostPosition Particle in Hangul +data PPPCategory + = EN -- ^ __은/는__ + | EL -- ^ __을/를__ + | IG -- ^ __이/가__ + | WG -- ^ __와/과__ + | AY -- ^ __아/야__ + | IX -- ^ __이/null__ + | EuX -- ^ __으/null__ + deriving (Eq,Ord,Enum,Bounded,Show,Read) + +-- | Identity of PostPosition Particle in Hangul +data PPPIdentity + = Eun -- ^ __/은/__ + | Neun -- ^ __/는/__ + | Eul -- ^ __/을/__ + | Leul -- ^ __/를/__ + | I -- ^ __/이/__ + | Ga -- ^ __/가/__ + | Wa -- ^ __/와/__ + | Gwa -- ^ __/과/__ + | A -- ^ __/아/__ + | Ya -- ^ __/야/__ + | Ix -- ^ Continuous __/이/__ + | Eux -- ^ __/으/__ + | X -- ^ __/null/__ + deriving (Eq,Ord,Enum,Bounded,Show,Read) + +type Code = Int diff --git a/src/Data/Text/ENIG/Detect.hs b/src/Data/Text/ENIG/Detect.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/ENIG/Detect.hs @@ -0,0 +1,44 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Data.Text.ENIG.Detect where + +import Data.Text.ENIG.Config +import Data.Text.ENIG.Data + + +import Data.Char + +import Data.Text (Text) +import qualified Data.Text as T + +import Data.Text.Normalize + +import qualified Data.Vector.Unboxed as V + + +-- | Detect whether the last component is an last consonant or not +isLastConsonant :: Int -> Bool +isLastConsonant hangulCode = + (head lastConsonantCodeList <= hangulCode) && (hangulCode <= last lastConsonantCodeList) + +-- | Detect whether the last component is an vowel or not +-- To avoid the issue with the first consonant, like 'ㄱ'. +isLastVowel :: Int -> Bool +isLastVowel hangulCode = + (head vowelCodeList <= hangulCode) && (hangulCode <= last vowelCodeList) + +-- | Detect whether the last component is the last consonant 'ㄹ' +isLastR :: Int -> Bool +isLastR hangulCode = hangulCode == 4527 + +-- | Detect whether the given code is an Hangul component or not +isHangul :: Int -> Bool +isHangul mHangulCode = + (head hangulComponentCodeList <= mHangulCode) && (mHangulCode <= last hangulComponentCodeList) + +-- | Get the last component's code of given Hangel +getLastComponentCode :: Text -> Int +getLastComponentCode str = ord aComponent + where + lastChar = T.singleton . T.last $ str + aComponent = T.last . normalize NFKD $ lastChar diff --git a/src/Data/Text/ENIG/Show.hs b/src/Data/Text/ENIG/Show.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/ENIG/Show.hs @@ -0,0 +1,43 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Data.Text.ENIG.Show where + +import Data.Text.ENIG.Data + +import Data.Text (Text) +import qualified Data.Text as T + + +-- | Show function for 'PPPCategory' as 'Text' +tShowPPPCa :: PPPCategory -> Text +tShowPPPCa EN = "은(는)" +tShowPPPCa EL = "을(를)" +tShowPPPCa IG = "이(가)" +tShowPPPCa WG = "와(과)" +tShowPPPCa AY = "아(야)" +tShowPPPCa IX = "(이)" +tShowPPPCa EuX = "(으)" + +-- | Show function for 'PPPCategory' as 'String' +sShowPPPCa :: PPPCategory -> String +sShowPPPCa = T.unpack . tShowPPPCa + +-- | Show function for 'PPPIdentity' as 'Text' +tShowPPPId :: PPPIdentity -> Text +tShowPPPId Eun = "은" +tShowPPPId Neun = "는" +tShowPPPId Eul = "을" +tShowPPPId Leul = "를" +tShowPPPId I = "이" +tShowPPPId Ga = "가" +tShowPPPId Wa = "와" +tShowPPPId Gwa = "과" +tShowPPPId A = "아" +tShowPPPId Ya = "야" +tShowPPPId Ix = "이" +tShowPPPId Eux = "으" +tShowPPPId X = "" + +-- | Show function for 'PPPIdentity' as 'String' +sShowPPPId :: PPPIdentity -> String +sShowPPPId = T.unpack . tShowPPPId diff --git a/test/Spec.hs b/test/Spec.hs new file mode 100644 --- /dev/null +++ b/test/Spec.hs @@ -0,0 +1,14 @@ +import qualified Test.Framework as Test + +import Test.ENIG +import Test.ENIG.Detect +import Test.ENIG.Premise + + +main :: IO () +main = do + Test.defaultMain + [ Test.ENIG.Premise.tests + , Test.ENIG.Detect.tests + , Test.ENIG.tests + ] diff --git a/test/Test/ENIG.hs b/test/Test/ENIG.hs new file mode 100644 --- /dev/null +++ b/test/Test/ENIG.hs @@ -0,0 +1,48 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} + +module Test.ENIG where + +import Test.Framework +import Test.Framework.Providers.HUnit +import Test.Framework.TH +import Test.HUnit.Base + +import qualified Data.Text as T +import Data.Text.Normalize + +import Data.Text.ENIG +import Data.Text.ENIG.Data +import Data.Text.ENIG.Show + + +tests = $(testGroupGenerator) + + +test_UsualCases = + [ testCase "과자을(를) = 과자를" c_01 + , testCase "무엇을(를) = 무엇을" c_02 + , testCase "한글을(를) = 한글을" c_03 + ] +test_EuXCases = + [ testCase "과자(으)로 = 과자로" c_11 + , testCase "무엇(으)로 = 무엇으로" c_12 + , testCase "한글(으)로 = 한글로" c_13 + ] + +preHangulStrList = ["과자", "무엇", "한글", "ㄱ", "ㅏ"] +preNotHangulStrList = ["1", "2", "test", "harm"] + +c_01 = tShowPPPId Leul @?= (enigPPP arg EL) + where arg = preHangulStrList !! 0 +c_02 = tShowPPPId Eul @?= (enigPPP arg EL) + where arg = preHangulStrList !! 1 +c_03 = tShowPPPId Eul @?= (enigPPP arg EL) + where arg = preHangulStrList !! 3 + +c_11 = tShowPPPId X @?= (enigPPP arg EuX) + where arg = preHangulStrList !! 0 +c_12 = tShowPPPId Eux @?= (enigPPP arg EuX) + where arg = preHangulStrList !! 1 +c_13 = tShowPPPId X @?= (enigPPP arg EuX) + where arg = preHangulStrList !! 2 diff --git a/test/Test/ENIG/Config.hs b/test/Test/ENIG/Config.hs new file mode 100644 --- /dev/null +++ b/test/Test/ENIG/Config.hs @@ -0,0 +1,38 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} + +module Test.ENIG.Config where + +import Test.Framework +import Test.Framework.Providers.HUnit +import Test.Framework.TH +import Test.HUnit.Base + +import Data.Char +import qualified Data.Text as T +import Data.Text.Normalize +import qualified Data.Vector.Unboxed as V + +import Data.Text.ENIG.Data +import Data.Text.ENIG.Detect + + +lastConsonantCodeList = [4520..4607] +lastConsonantList = map chr lastConsonantCodeList + +vowelCodeList = [4449..4519] +vowelList = map chr vowelCodeList + +hangulComponentCodeList = [4352..4607] +hangulComponentList = map chr hangulComponentCodeList + + +-- Not good generation code. Need to be refactored. +pppidVector = V.fromList + [(fromEnum Eun,fromEnum Neun) + ,(fromEnum Eul,fromEnum Leul) + ,(fromEnum I,fromEnum Ga) + ,(fromEnum Gwa,fromEnum Wa) + ,(fromEnum Ix,fromEnum X) + ,(fromEnum Eux,fromEnum X) + ] diff --git a/test/Test/ENIG/Detect.hs b/test/Test/ENIG/Detect.hs new file mode 100644 --- /dev/null +++ b/test/Test/ENIG/Detect.hs @@ -0,0 +1,62 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} + +module Test.ENIG.Detect where + +import Test.Framework +import Test.Framework.Providers.HUnit +import Test.Framework.TH +import Test.HUnit.Base + +import Data.Char +import qualified Data.Text as T +import Data.Text.Normalize + +import Data.Text.ENIG.Data +import Data.Text.ENIG.Detect + +tests = $(testGroupGenerator) + + +preHangulStrList = ["과자", "무엇", "한글", "ㄱ", "ㅏ"] + + +c_11 = 4449 @?= getLastComponentCode arg + where arg = preHangulStrList !! 0 +c_12 = 4538 @?= getLastComponentCode arg + where arg = preHangulStrList !! 1 +c_13 = 4527 @?= getLastComponentCode arg + where arg = preHangulStrList !! 2 + +test_ComponentCode = + [ testCase "lastComponent(과자) = ㅏ" c_11 + , testCase "lastComponent(무엇) = ㅅ" c_12 + , testCase "lastComponent(한글) = ㄹ" c_13 + ] + +c_21 = assertBool "" (not . isLastConsonant . getLastComponentCode $ arg) + where arg = preHangulStrList !! 0 +c_22 = assertBool "" (isLastConsonant . getLastComponentCode $ arg) + where arg = preHangulStrList !! 1 +c_23 = assertBool "" (isLastConsonant . getLastComponentCode $ arg) + where arg = preHangulStrList !! 2 + +test_isLastConsonant = + [ testCase "isLastConsonant(과자): False" c_21 + , testCase "isLastConsonant(무엇): True" c_22 + , testCase "isLastConsonant(한글): True" c_23 + ] + +c_31 = assertBool "" (not . isLastR . getLastComponentCode $ arg) + where arg = preHangulStrList !! 0 +c_32 = assertBool "" (not . isLastR . getLastComponentCode $ arg) + where arg = preHangulStrList !! 1 +c_33 = assertBool "" (isLastR . getLastComponentCode $ arg) + where arg = preHangulStrList !! 2 + +test_isLastR = + [ testCase "isLastR(과자): False" c_31 + , testCase "isLastR(무엇): False" c_32 + , testCase "isLastR(한글): True" c_33 + ] + diff --git a/test/Test/ENIG/Premise.hs b/test/Test/ENIG/Premise.hs new file mode 100644 --- /dev/null +++ b/test/Test/ENIG/Premise.hs @@ -0,0 +1,57 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} + +module Test.ENIG.Premise where + +import Test.Framework +import Test.Framework.Providers.HUnit +import Test.Framework.TH +import Test.HUnit.Base + +import qualified Data.Text as T +import Data.Text.Normalize + +import Data.Text.ENIG.Data + +tests = $(testGroupGenerator) + +ga = "가" +gag = "각" +ha = "하" +heuh = "헣" +g = "ㄱ" +_g = T.singleton . T.index (nfd gag) $ 0 +__g = T.singleton . T.index (nfd gag) $ 2 +_a = "ㅏ" +__a = T.singleton . T.index (nfd gag) $ 1 + +sampleHangul = [g, _g, __g, _a, __a, ga, gag, ha, heuh] :: [T.Text] + +nfc = normalize NFC +nfkc = normalize NFKC +nfd = normalize NFD +nfkd = normalize NFKD + +nfcd = nfc . nfd +nfkcd = nfkc . nfd +nfckd = nfc . nfkd +nfkckd = nfkc . nfkd + +-- # Result +-- * NFC/NFD does not handle "ㅏ" or "ㄱ" as a component. +-- * When handling them different, NFC/NFD should be used. +-- * However, because of their pronouces, do not need to handle then differently. + +c_normalizationNFCD + = assertBool + "nfc . nfd /= nfkc . nfd" + (not $ all (\x -> nfcd x == nfkcd x) sampleHangul) +c_normalizationNFCKD + = assertBool + "nfc . nfkd == nfkc . nfkd" + (all (\x -> nfckd x == nfkckd x) sampleHangul) + +test_NormalizationComparison = + [ testCase ("nfc . nfd == nfkc . nfkd with " ++ (T.unpack $ T.intercalate ", " sampleHangul)) c_normalizationNFCD + , testCase ("nfc . nfkd == nfkc . nfkd with " ++ (T.unpack $ T.intercalate ", " sampleHangul)) c_normalizationNFCKD + ]