diff --git a/CHANGES.md b/CHANGES.md --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,18 @@ Seonbi changelog ================ +Version 0.2.3 +------------- + +Released on September 26, 2021. + + - Fixed stops normalizer's bug where trailing spaces following stops had been + trimmed after normalized. + - Fixed stops normalizer's buf where unnecessary trailing spaces following + stops had been inserted after normalized. In particular, unnecessary + spaces between stops and closing parentheses/brackets are no more inserted. + + Version 0.2.2 ------------- diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ HTTP/1.1 200 OK Content-Type: application/json - Server: Seonbi/0.2.2 + Server: Seonbi/0.2.3 { "success": true, diff --git a/seonbi.cabal b/seonbi.cabal --- a/seonbi.cabal +++ b/seonbi.cabal @@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: seonbi -version: 0.2.2 +version: 0.2.3 synopsis: SmartyPants for Korean language description: Please see the README.md on GitHub at . category: Text diff --git a/src/Text/Seonbi/Punctuation.hs b/src/Text/Seonbi/Punctuation.hs --- a/src/Text/Seonbi/Punctuation.hs +++ b/src/Text/Seonbi/Punctuation.hs @@ -36,7 +36,6 @@ import Prelude hiding (takeWhile) -import Control.Applicative ((<|>)) import Control.Monad import Data.Char (isSpace) import Data.Either @@ -316,47 +315,106 @@ , Data.Text.singleton <$> anyChar ] endOfInput - return $ Data.Text.dropWhileEnd (' ' ==) $ Data.Text.concat chunks + return $ Data.Text.concat chunks stops' :: Parser Text stops' = choice - [ period' >> return (toEntity $ period stops) - , comma' >> return (toEntity $ comma stops) - , interpunct' >> return (toEntity $ interpunct stops) + [ do { ending <- period' + ; return (toEntity $ adjustEnding ending $ period stops) + } + , do { ending <- comma' + ; return (toEntity $ adjustEnding ending $ comma stops) + } + , do { ending <- interpunct' + ; return (toEntity $ adjustEnding ending $ interpunct stops) + } ] + adjustEnding :: Ending -> Text -> Text + adjustEnding ending text + | Data.Text.length text > 0 && isSpace (Data.Text.last text) = + stripEnd text <> case ending of { TrailingChars c -> c + ; TrailingSpaces s -> s + ; Ending -> Data.Text.empty + } + | otherwise = text <> case ending of { TrailingChars c -> c + ; _ -> Data.Text.empty + } toEntity :: Text -> Text toEntity = Data.Text.concatMap $ \ c -> if c < '\x80' -- ASCII compatible characters then Data.Text.singleton c else Data.Text.concat ["&#x", pack $ showHex (fromEnum c) "", ";"] - period' :: Parser () - period' = void $ choice - [ char '.' >> takeWhile isSpace >> return "" - , char '。' >> return "" - , string "." - , string "." - , string "。" - , asciiCI "." - , asciiCI "。" + period' :: Parser Ending + period' = choice + [ char '.' >> boundary + , char '。' >> trailingSpaces + , string "." >> boundary + , string "." >> boundary + , string "。" >> trailingSpaces + , asciiCI "." >> boundary + , asciiCI "。" >> trailingSpaces ] - comma' :: Parser () - comma' = void $ choice - [ char '、' >> return "" - , string "," >> (takeWhile isSpace <|> (endOfInput >> return "")) - , string ", " - , string ", " - , string "、" - , asciiCI ", " - , asciiCI "、" + comma' :: Parser Ending + comma' = choice + [ char '、' >> trailingSpaces + , string "," >> boundary + , string "," >> boundary + , string "," >> boundary + , string "、" >> trailingSpaces + , asciiCI "," >> boundary + , asciiCI "、" >> trailingSpaces ] - interpunct' :: Parser () - interpunct' = void $ choice + interpunct' :: Parser Ending + interpunct' = choice [ char '·' >> return "" , string "·" , string "·" , string "·" , string "·" , asciiCI "·" + ] >> return Ending + closingChars :: String + closingChars = + [ '"', '”', '\'', '’', ')', ']', '}', '」', '』', '〉', '》', '）', '〕' + , '］', '｝', '｠', '】', '〗', '〙', '〛', '›', '»' ] + closingEntities :: [Text] + closingEntities = + [ """, """ -- " + , "'" -- ' + , ")" -- ) + , "]", "]" -- ] + , "}", "}" -- } + , "»" -- » + , "’", "’", "’" -- ’ + , "”", "”", "”" -- ” + , "›" -- › + ] + closing :: Parser Text + closing = choice $ + [string [c] | c <- closingChars] ++ + [string e | e <- closingEntities] ++ + [asciiCI $ pack $ "&#x" ++ showHex (fromEnum c) "" ++ ";" + | c <- closingChars + ] ++ + [string $ "&#" <> pack (show c) <> ";" | c <- closingChars] + ending' :: Parser Ending + ending' = choice + [ endOfInput >> return Ending + , TrailingChars <$> closing + ] + boundary :: Parser Ending + boundary = choice + [ ending' + , TrailingSpaces <$> takeWhile1 isSpace + ] + trailingSpaces :: Parser Ending + trailingSpaces = choice + [ boundary + , return $ TrailingSpaces " " + ] + + +data Ending = TrailingChars Text | TrailingSpaces Text | Ending -- | Substitution options for 'transformArrow' function. These options can diff --git a/test/Text/Seonbi/Html/TextNormalizerSpec.hs b/test/Text/Seonbi/Html/TextNormalizerSpec.hs --- a/test/Text/Seonbi/Html/TextNormalizerSpec.hs +++ b/test/Text/Seonbi/Html/TextNormalizerSpec.hs @@ -17,21 +17,21 @@ normalizeText [ HtmlText { tagStack = [], rawText = "foo " } , HtmlText { tagStack = [], rawText = "& bar" } - , HtmlCdata { tagStack = [], text = " & baz" } + , HtmlCdata { tagStack = [], text = " & baz " } , HtmlStartTag { tagStack = [], tag = P, rawAttributes = "" } , HtmlText { tagStack = [P], rawText = "qux " } , HtmlCdata { tagStack = [P], text = "& \"quux\"" } , HtmlEndTag { tagStack = [], tag = P } - , HtmlCdata { tagStack = [], text = "" } + , HtmlCdata { tagStack = [], text = " " } ] `shouldBe` - [ HtmlText { tagStack = [], rawText = "foo & bar & baz" } + [ HtmlText { tagStack = [], rawText = "foo & bar & baz " } , HtmlStartTag { tagStack = [], tag = P, rawAttributes = "" } , HtmlText { tagStack = [P] , rawText = "qux & "quux"" } , HtmlEndTag { tagStack = [], tag = P } - , HtmlText { tagStack = [], rawText = "<end>" } + , HtmlText { tagStack = [], rawText = " <end>" } ] describe "normalizeCdata" $ do diff --git a/test/Text/Seonbi/PunctuationSpec.hs b/test/Text/Seonbi/PunctuationSpec.hs --- a/test/Text/Seonbi/PunctuationSpec.hs +++ b/test/Text/Seonbi/PunctuationSpec.hs @@ -464,7 +464,7 @@ describe "normalizeStops" $ do let periods = - [ ". ", ".", ".", "." + [ ". ", ". ", ". ", ". " , "。", "。", "。" ] :: [Text] let commas = @@ -475,9 +475,10 @@ [ "·", "·", "·", "·" , "·", "·", "·" ] :: [Text] + let s = stripEnd let examples = - [ [qc|봄{i1}여름{i2}가을{i3}겨울{p1}어제{c}오늘{p2}|] - | p1 <- periods, p2 <- periods + [ [qc|봄{i1}여름{i2}가을{i3}겨울{p1}(括弧{s p3}) 어제{c}오늘{s p2}|] + | p1 <- periods, p2 <- periods, p3 <- periods , c <- commas , i1 <- interpuncts, i2 <- interpuncts, i3 <- interpuncts ] :: [Text] @@ -498,18 +499,19 @@ normalizeStops horizontalStops input `shouldBe` [ HtmlStartTag [] P "" , HtmlText [P] - "봄·여름·가을·겨울. 어제, 오늘." + "봄·여름·가을·겨울. (括弧.) 어제, 오늘." , HtmlEndTag [] P ] normalizeStops verticalStops input `shouldBe` [ HtmlStartTag [] P "" , HtmlText [P] - "봄·여름·가을·겨울。어제、오늘。" + ("봄·여름·가을·겨울。(括弧。) " <> + "어제、오늘。") , HtmlEndTag [] P ] normalizeStops horizontalStopsWithSlashes input `shouldBe` [ HtmlStartTag [] P "" - , HtmlText [P] "봄/여름/가을/겨울. 어제, 오늘." + , HtmlText [P] "봄/여름/가을/겨울. (括弧.) 어제, 오늘." , HtmlEndTag [] P ] it "normalizes stops followed by boundaries as well" $ do