diff --git a/hw-conduit.cabal b/hw-conduit.cabal --- a/hw-conduit.cabal +++ b/hw-conduit.cabal @@ -1,5 +1,5 @@ name: hw-conduit -version: 0.0.0.9 +version: 0.0.0.10 synopsis: Conduits for tokenizing streams. description: Please see README.md homepage: http://github.com/haskell-works/hw-conduit#readme @@ -35,6 +35,7 @@ , conduit , hw-bits , resourcet + , word8 default-language: Haskell2010 ghc-options: -rtsopts -with-rtsopts=-N -Wall -O2 -Wall -msse4.2 diff --git a/src/HaskellWorks/Data/Conduit/Json.hs b/src/HaskellWorks/Data/Conduit/Json.hs --- a/src/HaskellWorks/Data/Conduit/Json.hs +++ b/src/HaskellWorks/Data/Conduit/Json.hs @@ -17,13 +17,13 @@ import Data.Conduit import Data.Int import Data.Word +import Data.Word8 import HaskellWorks.Data.Bits.BitWise -import HaskellWorks.Data.Conduit.Json.Words import Prelude as P interestingWord8s :: A.UArray Word8 Word8 interestingWord8s = A.array (0, 255) [ - (w, if w == wOpenBracket || w == wOpenBrace || w == wOpenParen || w == wt || w == wf || w == wn || w == w1 + (w, if w == _bracketleft || w == _braceleft || w == _parenleft || w == _t || w == _f || w == _n || w == _1 then 1 else 0) | w <- [0 .. 255]] @@ -69,16 +69,16 @@ blankedJsonToBalancedParens' bs = case BS.uncons bs of Just (c, cs) -> do case c of - d | d == wOpenBrace -> yield True - d | d == wCloseBrace -> yield False - d | d == wOpenBracket -> yield True - d | d == wCloseBracket -> yield False - d | d == wOpenParen -> yield True - d | d == wCloseParen -> yield False - d | d == wt -> yield True >> yield False - d | d == wf -> yield True >> yield False - d | d == w1 -> yield True >> yield False - d | d == wn -> yield True >> yield False + d | d == _braceleft -> yield True + d | d == _braceright -> yield False + d | d == _bracketleft -> yield True + d | d == _bracketright -> yield False + d | d == _parenleft -> yield True + d | d == _parenright -> yield False + d | d == _t -> yield True >> yield False + d | d == _f -> yield True >> yield False + d | d == _1 -> yield True >> yield False + d | d == _n -> yield True >> yield False _ -> return () blankedJsonToBalancedParens' cs Nothing -> return () @@ -107,30 +107,30 @@ yield cs Nothing -> return () where gen :: (Maybe Bool, ByteString) -> Maybe (Word8, (Maybe Bool, ByteString)) - gen (Just True , bs) = Just (wFF, (Nothing, bs)) - gen (Just False , bs) = Just (w00, (Nothing, bs)) + gen (Just True , bs) = Just (0xFF, (Nothing, bs)) + gen (Just False , bs) = Just (0x00, (Nothing, bs)) gen (Nothing , bs) = case BS.uncons bs of Just (c, cs) -> case balancedParensOf c of MiniN -> gen (Nothing , cs) - MiniT -> Just (wFF, (Nothing , cs)) - MiniF -> Just (w00, (Nothing , cs)) - MiniTF -> Just (wFF, (Just False , cs)) + MiniT -> Just (0xFF, (Nothing , cs)) + MiniF -> Just (0x00, (Nothing , cs)) + MiniTF -> Just (0xFF, (Just False , cs)) Nothing -> Nothing data MiniBP = MiniN | MiniT | MiniF | MiniTF balancedParensOf :: Word8 -> MiniBP balancedParensOf c = case c of - d | d == wOpenBrace -> MiniT - d | d == wCloseBrace -> MiniF - d | d == wOpenBracket -> MiniT - d | d == wCloseBracket -> MiniF - d | d == wOpenParen -> MiniT - d | d == wCloseParen -> MiniF - d | d == wt -> MiniTF - d | d == wf -> MiniTF - d | d == w1 -> MiniTF - d | d == wn -> MiniTF + d | d == _braceleft -> MiniT + d | d == _braceright -> MiniF + d | d == _bracketleft -> MiniT + d | d == _bracketright -> MiniF + d | d == _parenleft -> MiniT + d | d == _parenright -> MiniF + d | d == _t -> MiniTF + d | d == _f -> MiniTF + d | d == _1 -> MiniTF + d | d == _n -> MiniTF _ -> MiniN yieldBitsOfWord8 :: Monad m => Word8 -> Conduit BS.ByteString m Bool diff --git a/src/HaskellWorks/Data/Conduit/Json/Blank.hs b/src/HaskellWorks/Data/Conduit/Json/Blank.hs --- a/src/HaskellWorks/Data/Conduit/Json/Blank.hs +++ b/src/HaskellWorks/Data/Conduit/Json/Blank.hs @@ -10,6 +10,7 @@ import Data.ByteString as BS import Data.Conduit import Data.Word +import Data.Word8 import HaskellWorks.Data.Conduit.Json.Words import Prelude as P @@ -35,28 +36,28 @@ where blankByteString :: (BlankState, ByteString) -> Maybe (Word8, (BlankState, ByteString)) blankByteString (InJson, bs) = case BS.uncons bs of - Just (!c, !cs) | isLeadingDigit c -> Just (w1 , (InNumber , cs)) - Just (!c, !cs) | c == wDoubleQuote -> Just (wOpenParen , (InString , cs)) - Just (!c, !cs) | isAlphabetic c -> Just (c , (InIdent , cs)) - Just (!c, !cs) -> Just (c , (InJson , cs)) + Just (!c, !cs) | isLeadingDigit c -> Just (_1 , (InNumber , cs)) + Just (!c, !cs) | c == _quotedbl -> Just (_parenleft , (InString , cs)) + Just (!c, !cs) | isAlphabetic c -> Just (c , (InIdent , cs)) + Just (!c, !cs) -> Just (c , (InJson , cs)) Nothing -> Nothing blankByteString (InString, bs) = case BS.uncons bs of - Just (!c, !cs) | c == wBackslash -> Just (wSpace , (Escaped , cs)) - Just (!c, !cs) | c == wDoubleQuote -> Just (wCloseParen, (InJson , cs)) - Just (_ , !cs) -> Just (wSpace , (InString , cs)) + Just (!c, !cs) | c == _backslash -> Just (_space , (Escaped , cs)) + Just (!c, !cs) | c == _quotedbl -> Just (_parenright , (InJson , cs)) + Just (_ , !cs) -> Just (_space , (InString , cs)) Nothing -> Nothing blankByteString (Escaped, bs) = case BS.uncons bs of - Just (_, !cs) -> Just (wSpace, (InString, cs)) + Just (_, !cs) -> Just (_space, (InString, cs)) Nothing -> Nothing blankByteString (InNumber, bs) = case BS.uncons bs of - Just (!c, !cs) | isTrailingDigit c -> Just (w0 , (InNumber , cs)) - Just (!c, !cs) | c == wDoubleQuote -> Just (wOpenParen , (InString , cs)) - Just (!c, !cs) | isAlphabetic c -> Just (c , (InIdent , cs)) - Just (!c, !cs) -> Just (c , (InJson , cs)) + Just (!c, !cs) | isTrailingDigit c -> Just (_0 , (InNumber , cs)) + Just (!c, !cs) | c == _quotedbl -> Just (_parenleft , (InString , cs)) + Just (!c, !cs) | isAlphabetic c -> Just (c , (InIdent , cs)) + Just (!c, !cs) -> Just (c , (InJson , cs)) Nothing -> Nothing blankByteString (InIdent, bs) = case BS.uncons bs of - Just (!c, !cs) | isAlphabetic c -> Just (wUnderscore, (InIdent , cs)) - Just (!c, !cs) | isLeadingDigit c -> Just (w1 , (InNumber , cs)) - Just (!c, !cs) | c == wDoubleQuote -> Just (wOpenParen , (InString , cs)) - Just (!c, !cs) -> Just (c , (InJson , cs)) + Just (!c, !cs) | isAlphabetic c -> Just (_underscore , (InIdent , cs)) + Just (!c, !cs) | isLeadingDigit c -> Just (_1 , (InNumber , cs)) + Just (!c, !cs) | c == _quotedbl -> Just (_parenleft , (InString , cs)) + Just (!c, !cs) -> Just (c , (InJson , cs)) Nothing -> Nothing diff --git a/src/HaskellWorks/Data/Conduit/Json/Words.hs b/src/HaskellWorks/Data/Conduit/Json/Words.hs --- a/src/HaskellWorks/Data/Conduit/Json/Words.hs +++ b/src/HaskellWorks/Data/Conduit/Json/Words.hs @@ -1,124 +1,13 @@ module HaskellWorks.Data.Conduit.Json.Words where -import Data.Char import Data.Word - -wBackslash :: Word8 -wBackslash = fromIntegral (ord '\\') - -wBackspace :: Word8 -wBackspace = fromIntegral (ord '\b') - -wDoubleQuote :: Word8 -wDoubleQuote = fromIntegral (ord '"') - -wUnderscore :: Word8 -wUnderscore = fromIntegral (ord '_') - -wNewline :: Word8 -wNewline = fromIntegral (ord '\n') - -wLinefeed :: Word8 -wLinefeed = fromIntegral (ord '\f') - -wCarriageReturn :: Word8 -wCarriageReturn = fromIntegral (ord '\r') - -wTab :: Word8 -wTab = fromIntegral (ord '\t') - -wQuote :: Word8 -wQuote = fromIntegral (ord '\'') - -wSlash :: Word8 -wSlash = fromIntegral (ord '/') - -wSpace :: Word8 -wSpace = fromIntegral (ord ' ') - -wOpenParen :: Word8 -wOpenParen = fromIntegral (ord '(') - -wCloseParen :: Word8 -wCloseParen = fromIntegral (ord ')') - -wOpenBracket :: Word8 -wOpenBracket = fromIntegral (ord '[') - -wCloseBracket :: Word8 -wCloseBracket = fromIntegral (ord ']') - -wOpenBrace :: Word8 -wOpenBrace = fromIntegral (ord '{') - -wCloseBrace :: Word8 -wCloseBrace = fromIntegral (ord '}') - -wPlus :: Word8 -wPlus = fromIntegral (ord '+') - -wA :: Word8 -wA = fromIntegral (ord 'A') - -wa :: Word8 -wa = fromIntegral (ord 'a') - -wB :: Word8 -wB = fromIntegral (ord 'B') - -wb :: Word8 -wb = fromIntegral (ord 'b') - -we :: Word8 -we = fromIntegral (ord 'e') - -wE :: Word8 -wE = fromIntegral (ord 'E') - -wf :: Word8 -wf = fromIntegral (ord 'f') - -wn :: Word8 -wn = fromIntegral (ord 'n') - -wr :: Word8 -wr = fromIntegral (ord 'r') - -wt :: Word8 -wt = fromIntegral (ord 't') - -wz :: Word8 -wz = fromIntegral (ord 'z') - -wZ :: Word8 -wZ = fromIntegral (ord 'Z') - -wDot :: Word8 -wDot = fromIntegral (ord '.') - -wMinus :: Word8 -wMinus = fromIntegral (ord '-') - -w0 :: Word8 -w0 = fromIntegral (ord '0') - -w1 :: Word8 -w1 = fromIntegral (ord '1') - -w9 :: Word8 -w9 = fromIntegral (ord '9') - -w00 :: Word8 -w00 = fromIntegral (ord '0') - -wFF :: Word8 -wFF = fromIntegral (ord '\255') +import Data.Word8 isLeadingDigit :: Word8 -> Bool -isLeadingDigit w = w == wMinus || (w >= w0 && w <= w9) +isLeadingDigit w = w == _hyphen || (w >= _0 && w <= _9) isTrailingDigit :: Word8 -> Bool -isTrailingDigit w = w == wPlus || w == wMinus || (w >= w0 && w <= w9) || w == wDot || w == wE || w == we +isTrailingDigit w = w == _plus || w == _hyphen || (w >= _0 && w <= _9) || w == _period || w == _E || w == _e isAlphabetic :: Word8 -> Bool -isAlphabetic w = (w >= wA && w <= wZ) || (w >= wa && w <= wz) +isAlphabetic w = (w >= _A && w <= _Z) || (w >= _a && w <= _z)