packages feed

enumerator 0.4.17 → 0.4.18

raw patch · 10 files changed

+198/−26 lines, 10 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

enumerator.cabal view
@@ -1,5 +1,5 @@ name: enumerator-version: 0.4.17+version: 0.4.18 synopsis: Reliable, high-performance processing with left-fold enumerators license: MIT license-file: license.txt@@ -67,6 +67,10 @@   scripts/run-tests   --   tests/enumerator-tests.cabal+  tests/data/ascii-crlf.txt+  tests/data/ascii-lf.txt+  tests/data/utf8-crlf.txt+  tests/data/utf8-lf.txt   tests/EnumeratorTests.hs   tests/EnumeratorTests/Binary.hs   tests/EnumeratorTests/Binary/Consume.hs@@ -131,7 +135,7 @@ source-repository this   type: bazaar   location: https://john-millikin.com/branches/enumerator/0.4/-  tag: enumerator_0.4.17+  tag: enumerator_0.4.18  library   ghc-options: -Wall -O2
lib/Data/Enumerator/Text.hs view
@@ -1,3 +1,5 @@+{-# LANGUAGE CPP #-}+ -- | -- Module: Data.Enumerator.Text -- Copyright: 2010-2011 John Millikin@@ -751,32 +753,65 @@ lines :: Monad m => Enumeratee T.Text T.Text m b lines = splitWhen (== '\n') --- | Read lines of text from the handle, and stream them to an 'Iteratee'.+-- | Read lines of text from a handle, and stream them to an 'Iteratee'. -- If an exception occurs during file IO, enumeration will stop and 'Error' -- will be returned. Exceptions from the iteratee are not caught. -- -- The handle should be opened with an appropriate text encoding, and -- in 'IO.ReadMode' or 'IO.ReadWriteMode'. --+-- This function may be significantly slower than using+-- @Data.Enumerator.Binary.enumHandle@, due to the additional overhead of+-- decoding input data to Unicode. Users who can depend on their input files+-- being in a certain encoding (such as UTF8) are encouraged to use binary+-- input and 'decode'.+--+-- Changed in 0.4.18: Lines streamed from 'enumHandle' and 'enumFile' now+-- include their trailing newline.+-- -- Since: 0.2 enumHandle :: MonadIO m => IO.Handle            -> Enumerator T.Text m b enumHandle h = checkContinue0 $ \loop k -> do-	let getText = Exc.catch-		(Just `fmap` TIO.hGetLine h)-		(\err -> if isEOFError err-			then return Nothing-			else Exc.throwIO err)-	-	maybeText <- tryIO getText+	maybeText <- tryIO (textGetLine h) 	case maybeText of 		Nothing -> continue k 		Just text -> k (Chunks [text]) >>== loop-	 +textGetLine :: IO.Handle -> IO (Maybe T.Text)+textGetLine h = loop [] where+#if MIN_VERSION_base(4,2,0)+	pack = T.pack+#else+	pack = TE.decodeUtf8 . B8.pack+#endif+	loop acc = Exc.catch+		(do+			c <- IO.hGetChar h+			if c == '\n'+				then return (Just (pack (reverse (c:acc))))+				else loop (c:acc))+		(\err -> if isEOFError err+			then case acc of+				[] -> return Nothing+				_ -> return (Just (pack (reverse acc)))+			else Exc.throwIO err) --- | Opens a file path in text mode, and passes the handle to 'enumHandle'.--- The file will be closed when the 'Iteratee' finishes.+-- | Read lines of text from a file, and stream them to an 'Iteratee'.+-- If an exception occurs during file IO, enumeration will stop and 'Error'+-- will be returned. Exceptions from the iteratee are not caught.+--+-- The file will be opened in text mode, and will be closed when the+-- 'Iteratee' finishes.+--+-- This function may be significantly slower than using+-- @Data.Enumerator.Binary.enumFile@, due to the additional overhead of+-- decoding input data to Unicode. Users who can depend on their input files+-- being in a certain encoding (such as UTF8) are encouraged to use binary+-- input and 'decode'.+--+-- Changed in 0.4.18: Lines streamed from 'enumHandle' and 'enumFile' now+-- include their trailing newline. -- -- Since: 0.2 enumFile :: FilePath -> Enumerator T.Text IO b
scripts/run-tests view
@@ -10,10 +10,10 @@  require_cabal_dev -clean_dev_install+# clean_dev_install  pushd tests-rm -rf dist+# rm -rf dist $CABAL_DEV -s ../cabal-dev install || exit 1 popd
tests/EnumeratorTests/Text.hs view
@@ -34,6 +34,7 @@ 	, test_ConcatMapAccumM 	, test_Drop 	, test_DropWhile+	, test_EnumFile 	, test_EnumHandle 	, test_Filter 	, test_FilterM
tests/EnumeratorTests/Text/Handle.hs view
@@ -7,14 +7,20 @@ -- See license.txt for details module EnumeratorTests.Text.Handle 	( test_EnumHandle+	, test_EnumFile 	, test_IterHandle 	) where  import           Test.Chell  #ifdef MIN_VERSION_knob- import           Data.Knob+#else+import           EnumeratorTests.Util (todo)+#endif++import           Control.Monad.IO.Class (liftIO)+import           Data.Text (Text, replace) import qualified System.IO as IO  import qualified Data.Enumerator as E@@ -22,13 +28,111 @@ import qualified Data.Enumerator.List as EL import qualified Data.Enumerator.Text as ET +import           Paths_enumerator_tests (getDataFileName)+ test_EnumHandle :: Suite-test_EnumHandle = assertions "enumHandle" $ do-	knob <- newKnob "0123\n\n4567"-	chunks <- withFileHandle knob "" IO.ReadMode $ \h -> do-		E.run_ (ET.enumHandle h $$ EL.consume)-	$expect (equal chunks ["0123", "", "4567"])+test_EnumHandle = suite "enumHandle"+	[ test_EnumHandle_AsciiLF+	, test_EnumHandle_AsciiCRLF+	, test_EnumHandle_Utf8LF+	, test_EnumHandle_Utf8CRLF+	] +--- define locally, because it's not present in GHC 6.10+data Newline = LF | CRLF+	deriving (Show, Eq)++nativeNewline :: Newline+#if MIN_VERSION_base(4,2,0)+nativeNewline = case IO.nativeNewline of+	IO.LF -> LF+	IO.CRLF -> CRLF+#else+#ifdef CABAL_OS_WINDOWS+nativeNewline = CRLF+#else+nativeNewline = LF+#endif+#endif++runEnumHandle :: String -> Assertions [Text]+runEnumHandle name = do+	path <- liftIO (getDataFileName name)+	chunks <- liftIO (IO.withFile path IO.ReadMode (\h -> do+#if MIN_VERSION_base(4,2,0)+		IO.hSetEncoding h IO.utf8+#endif+		E.run_ (ET.enumHandle h $$ EL.consume)))+	return chunks++test_EnumHandle_AsciiLF :: Suite+test_EnumHandle_AsciiLF = assertions "ascii-lf" $ do+	chunks <- runEnumHandle "data/ascii-lf.txt"+	$expect (equalItems+		[ "hello\n"+		, "\n"+		, "world\n"+		] chunks)++test_EnumHandle_AsciiCRLF :: Suite+test_EnumHandle_AsciiCRLF = assertions "ascii-crlf" $ do+	chunks <- runEnumHandle "data/ascii-crlf.txt"+	let rawLines =+		[ "hello\r\n"+		, "\r\n"+		, "world\r\n"+		]+	let expected = case nativeNewline of+		CRLF -> map (replace "\r\n" "\n") rawLines+		LF -> rawLines+	$expect (equalItems expected chunks)++test_EnumHandle_Utf8LF :: Suite+test_EnumHandle_Utf8LF = assertions "utf8-lf" $ do+	chunks <- runEnumHandle "data/utf8-lf.txt"+	$expect (equalItems+		[ "hello world\n"+		, "\n"+		, "\20320\22909\19990\30028\n"+		, "\n"+		, "\1605\1585\1581\1576\1575 \1575\1604\1593\1575\1604\1605\n"+		, "\n"+		, "\12371\12435\12395\12385\12399\19990\30028\n"+		, "\n"+		, "\2997\2979\2965\3021\2965\2990\3021\n"+		] chunks)++test_EnumHandle_Utf8CRLF :: Suite+test_EnumHandle_Utf8CRLF = assertions "utf8-crlf" $ do+	chunks <- runEnumHandle "data/utf8-crlf.txt"+	let rawLines =+		[ "hello world\r\n"+		, "\r\n"+		, "\20320\22909\19990\30028\r\n"+		, "\r\n"+		, "\1605\1585\1581\1576\1575 \1575\1604\1593\1575\1604\1605\r\n"+		, "\r\n"+		, "\12371\12435\12395\12385\12399\19990\30028\r\n"+		, "\r\n"+		, "\2997\2979\2965\3021\2965\2990\3021\r\n"+		]+	let expected = case nativeNewline of+		CRLF -> map (replace "\r\n" "\n") rawLines+		LF -> rawLines+	$expect (equalItems expected chunks)++test_EnumFile :: Suite+test_EnumFile = assertions "enumFile" $ do+	path <- liftIO (getDataFileName "data/ascii-lf.txt")+	chunks <- liftIO (E.run_ (ET.enumFile path $$ EL.consume))+	$expect (equal+		[ "hello\n"+		, "\n"+		, "world\n"+		] chunks)++#ifdef MIN_VERSION_knob+ test_IterHandle :: Suite test_IterHandle = assertions "iterHandle" $ do 	knob <- newKnob ""@@ -38,11 +142,6 @@ 	$expect (equal bytes "ABC")  #else--import           EnumeratorTests.Util (todo)--test_EnumHandle :: Suite-test_EnumHandle = todo "enumHandle"  test_IterHandle :: Suite test_IterHandle = todo "iterHandle"
+ tests/data/ascii-crlf.txt view
@@ -0,0 +1,3 @@+hello
+
+world
+ tests/data/ascii-lf.txt view
@@ -0,0 +1,3 @@+hello++world
+ tests/data/utf8-crlf.txt view
@@ -0,0 +1,9 @@+hello world
+
+你好世界
+
+مرحبا العالم
+
+こんにちは世界
+
+வணக்கம்
+ tests/data/utf8-lf.txt view
@@ -0,0 +1,9 @@+hello world++你好世界++مرحبا العالم++こんにちは世界++வணக்கம்
tests/enumerator-tests.cabal view
@@ -3,6 +3,12 @@ build-type: Simple cabal-version: >= 1.6 +data-files:+  data/ascii-crlf.txt+  data/ascii-lf.txt+  data/utf8-crlf.txt+  data/utf8-lf.txt+ flag coverage   default: False   manual: True@@ -29,6 +35,9 @@   else     build-depends:         base >= 4.0 && < 5.0++  if os(windows)+    cpp-options: -DCABAL_OS_WINDOWS    build-depends:       bytestring