seonbi 0.2.2 → 0.2.3
raw patch · 6 files changed
+109/−37 lines, 6 files
Files
- CHANGES.md +12/−0
- README.md +1/−1
- seonbi.cabal +1/−1
- src/Text/Seonbi/Punctuation.hs +83/−25
- test/Text/Seonbi/Html/TextNormalizerSpec.hs +4/−4
- test/Text/Seonbi/PunctuationSpec.hs +8/−6
CHANGES.md view
@@ -1,6 +1,18 @@ Seonbi changelog ================ +Version 0.2.3+-------------++Released on September 26, 2021.++ - Fixed stops normalizer's bug where trailing spaces following stops had been+ trimmed after normalized.+ - Fixed stops normalizer's buf where unnecessary trailing spaces following+ stops had been inserted after normalized. In particular, unnecessary+ spaces between stops and closing parentheses/brackets are no more inserted.++ Version 0.2.2 -------------
README.md view
@@ -128,7 +128,7 @@ HTTP/1.1 200 OK Content-Type: application/json- Server: Seonbi/0.2.2+ Server: Seonbi/0.2.3 { "success": true,
seonbi.cabal view
@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: seonbi-version: 0.2.2+version: 0.2.3 synopsis: SmartyPants for Korean language description: Please see the README.md on GitHub at <https://github.com/dahlia/seonbi>. category: Text
src/Text/Seonbi/Punctuation.hs view
@@ -36,7 +36,6 @@ import Prelude hiding (takeWhile) -import Control.Applicative ((<|>)) import Control.Monad import Data.Char (isSpace) import Data.Either@@ -316,47 +315,106 @@ , Data.Text.singleton <$> anyChar ] endOfInput- return $ Data.Text.dropWhileEnd (' ' ==) $ Data.Text.concat chunks+ return $ Data.Text.concat chunks stops' :: Parser Text stops' = choice- [ period' >> return (toEntity $ period stops)- , comma' >> return (toEntity $ comma stops)- , interpunct' >> return (toEntity $ interpunct stops)+ [ do { ending <- period'+ ; return (toEntity $ adjustEnding ending $ period stops)+ }+ , do { ending <- comma'+ ; return (toEntity $ adjustEnding ending $ comma stops)+ }+ , do { ending <- interpunct'+ ; return (toEntity $ adjustEnding ending $ interpunct stops)+ } ]+ adjustEnding :: Ending -> Text -> Text+ adjustEnding ending text+ | Data.Text.length text > 0 && isSpace (Data.Text.last text) =+ stripEnd text <> case ending of { TrailingChars c -> c+ ; TrailingSpaces s -> s+ ; Ending -> Data.Text.empty+ }+ | otherwise = text <> case ending of { TrailingChars c -> c+ ; _ -> Data.Text.empty+ } toEntity :: Text -> Text toEntity = Data.Text.concatMap $ \ c -> if c < '\x80' -- ASCII compatible characters then Data.Text.singleton c else Data.Text.concat ["&#x", pack $ showHex (fromEnum c) "", ";"]- period' :: Parser ()- period' = void $ choice- [ char '.' >> takeWhile isSpace >> return ""- , char '。' >> return ""- , string "."- , string "."- , string "。"- , asciiCI "."- , asciiCI "。"+ period' :: Parser Ending+ period' = choice+ [ char '.' >> boundary+ , char '。' >> trailingSpaces+ , string "." >> boundary+ , string "." >> boundary+ , string "。" >> trailingSpaces+ , asciiCI "." >> boundary+ , asciiCI "。" >> trailingSpaces ]- comma' :: Parser ()- comma' = void $ choice- [ char '、' >> return ""- , string "," >> (takeWhile isSpace <|> (endOfInput >> return ""))- , string ", "- , string ", "- , string "、"- , asciiCI ", "- , asciiCI "、"+ comma' :: Parser Ending+ comma' = choice+ [ char '、' >> trailingSpaces+ , string "," >> boundary+ , string "," >> boundary+ , string "," >> boundary+ , string "、" >> trailingSpaces+ , asciiCI "," >> boundary+ , asciiCI "、" >> trailingSpaces ]- interpunct' :: Parser ()- interpunct' = void $ choice+ interpunct' :: Parser Ending+ interpunct' = choice [ char '·' >> return "" , string "·" , string "·" , string "·" , string "·" , asciiCI "·"+ ] >> return Ending+ closingChars :: String+ closingChars =+ [ '"', '”', '\'', '’', ')', ']', '}', '」', '』', '〉', '》', '）', '〕'+ , '］', '｝', '｠', '】', '〗', '〙', '〛', '›', '»' ]+ closingEntities :: [Text]+ closingEntities =+ [ """, """ -- "+ , "'" -- '+ , ")" -- )+ , "]", "]" -- ]+ , "}", "}" -- }+ , "»" -- »+ , "’", "’", "’" -- ’+ , "”", "”", "”" -- ”+ , "›" -- ›+ ]+ closing :: Parser Text+ closing = choice $+ [string [c] | c <- closingChars] +++ [string e | e <- closingEntities] +++ [asciiCI $ pack $ "&#x" ++ showHex (fromEnum c) "" ++ ";"+ | c <- closingChars+ ] +++ [string $ "&#" <> pack (show c) <> ";" | c <- closingChars]+ ending' :: Parser Ending+ ending' = choice+ [ endOfInput >> return Ending+ , TrailingChars <$> closing+ ]+ boundary :: Parser Ending+ boundary = choice+ [ ending'+ , TrailingSpaces <$> takeWhile1 isSpace+ ]+ trailingSpaces :: Parser Ending+ trailingSpaces = choice+ [ boundary+ , return $ TrailingSpaces " "+ ]+++data Ending = TrailingChars Text | TrailingSpaces Text | Ending -- | Substitution options for 'transformArrow' function. These options can
test/Text/Seonbi/Html/TextNormalizerSpec.hs view
@@ -17,21 +17,21 @@ normalizeText [ HtmlText { tagStack = [], rawText = "foo " } , HtmlText { tagStack = [], rawText = "& bar" }- , HtmlCdata { tagStack = [], text = " & baz" }+ , HtmlCdata { tagStack = [], text = " & baz " } , HtmlStartTag { tagStack = [], tag = P, rawAttributes = "" } , HtmlText { tagStack = [P], rawText = "qux " } , HtmlCdata { tagStack = [P], text = "& \"quux\"" } , HtmlEndTag { tagStack = [], tag = P }- , HtmlCdata { tagStack = [], text = "<end>" }+ , HtmlCdata { tagStack = [], text = " <end>" } ] `shouldBe`- [ HtmlText { tagStack = [], rawText = "foo & bar & baz" }+ [ HtmlText { tagStack = [], rawText = "foo & bar & baz " } , HtmlStartTag { tagStack = [], tag = P, rawAttributes = "" } , HtmlText { tagStack = [P] , rawText = "qux & "quux"" } , HtmlEndTag { tagStack = [], tag = P }- , HtmlText { tagStack = [], rawText = "<end>" }+ , HtmlText { tagStack = [], rawText = " <end>" } ] describe "normalizeCdata" $ do
test/Text/Seonbi/PunctuationSpec.hs view
@@ -464,7 +464,7 @@ describe "normalizeStops" $ do let periods =- [ ". ", ".", ".", "."+ [ ". ", ". ", ". ", ". " , "。", "。", "。" ] :: [Text] let commas =@@ -475,9 +475,10 @@ [ "·", "·", "·", "·" , "·", "·", "·" ] :: [Text]+ let s = stripEnd let examples =- [ [qc|봄{i1}여름{i2}가을{i3}겨울{p1}어제{c}오늘{p2}|]- | p1 <- periods, p2 <- periods+ [ [qc|봄{i1}여름{i2}가을{i3}겨울{p1}(括弧{s p3}) 어제{c}오늘{s p2}|]+ | p1 <- periods, p2 <- periods, p3 <- periods , c <- commas , i1 <- interpuncts, i2 <- interpuncts, i3 <- interpuncts ] :: [Text]@@ -498,18 +499,19 @@ normalizeStops horizontalStops input `shouldBe` [ HtmlStartTag [] P "" , HtmlText [P]- "봄·여름·가을·겨울. 어제, 오늘."+ "봄·여름·가을·겨울. (括弧.) 어제, 오늘." , HtmlEndTag [] P ] normalizeStops verticalStops input `shouldBe` [ HtmlStartTag [] P "" , HtmlText [P]- "봄·여름·가을·겨울。어제、오늘。"+ ("봄·여름·가을·겨울。(括弧。) " <>+ "어제、오늘。") , HtmlEndTag [] P ] normalizeStops horizontalStopsWithSlashes input `shouldBe` [ HtmlStartTag [] P ""- , HtmlText [P] "봄/여름/가을/겨울. 어제, 오늘."+ , HtmlText [P] "봄/여름/가을/겨울. (括弧.) 어제, 오늘." , HtmlEndTag [] P ] it "normalizes stops followed by boundaries as well" $ do