buffer-builder 0.2.3.0 → 0.2.4.0
raw patch · 5 files changed
+170/−28 lines, 5 filesdep +quickcheck-instancesPVP ok
version bump matches the API change (PVP)
Dependencies added: quickcheck-instances
API changes (from Hackage documentation)
+ Data.BufferBuilder: calculateLength :: BufferBuilder a -> Int
Files
- buffer-builder.cabal +3/−2
- cbits/buffer.cpp +98/−16
- changelog.md +4/−0
- src/Data/BufferBuilder.hs +23/−5
- test/BufferTest.hs +42/−5
buffer-builder.cabal view
@@ -1,5 +1,5 @@ name: buffer-builder-version: 0.2.3.0+version: 0.2.4.0 synopsis: Library for efficiently building up buffers, one piece at a time description: @@ -28,7 +28,7 @@ Using BufferBuilder is very simple: . > import qualified Data.BufferBuilder as BB- > + > > let byteString = BB.runBufferBuilder $ do > BB.appendBS "http" > BB.appendChar8 '/'@@ -85,6 +85,7 @@ , tasty-hunit , tasty-th , tasty-quickcheck+ , quickcheck-instances , HUnit , text , vector
cbits/buffer.cpp view
@@ -25,9 +25,12 @@ #define BW_UNLIKELY(x) x #endif -#define BW_ENSURE_CAPACITY(bw, cap) \+#define BW_ENSURE_CAPACITY(bw, cap, count_size) \ do { \ if (!ensureCapacity((bw), (cap))) { \+ if (bw->counting_size) { \+ bw->counted_size += count_size; \+ } \ return; \ } \ } while (0)@@ -39,7 +42,12 @@ unsigned char* capacity; unsigned char* data; + bool counting_size;+ size_t counted_size;+ // invariants:+ // if (counting_size), end, capacity, and data are null+ // else, counted_size is zero and: // if (end <= capacity), then data is non-null and end >= data // if (capacity == 0), then data and end are null, and object is dead @@ -61,7 +69,7 @@ // TODO: detect overflow? newCapacity *= 2; } while (newCapacity < size + amount);- + unsigned char* newData = reinterpret_cast<unsigned char*>(realloc(bw->data, newCapacity)); if (newData) { bw->data = newData;@@ -105,9 +113,25 @@ bw->end = bw->data; bw->capacity = bw->data + initialCapacity;+ bw->counting_size = false;+ bw->counted_size = 0; return bw; } +extern "C" BufferWriter* bw_new_length_calculator() {+ BufferWriter* bw = reinterpret_cast<BufferWriter*>(malloc(sizeof(BufferWriter)));+ if (!bw) {+ return 0;+ }++ bw->data = 0;+ bw->end = 0;+ bw->capacity = 0;+ bw->counting_size = true;+ bw->counted_size = 0;+ return bw;+}+ extern "C" void bw_free(BufferWriter* bw) { free(bw->data); free(bw);@@ -134,7 +158,11 @@ } extern "C" size_t bw_get_size(BufferWriter* bw) {- return bw->end - bw->data;+ if (bw->counting_size) {+ return bw->counted_size;+ } else {+ return bw->end - bw->data;+ } } extern "C" unsigned char* bw_release_address(BufferWriter* bw) {@@ -143,18 +171,35 @@ bw->end = 0; bw->capacity = 0; bw->data = 0;+ bw->counting_size = false;+ bw->counted_size = 0; return data; // null if dead } extern "C" void bw_append_byte(BufferWriter* bw, unsigned char byte) {- BW_ENSURE_CAPACITY(bw, 1);+ BW_ENSURE_CAPACITY(bw, 1, 1); bw->end[0] = byte; bw->end += 1; } +inline size_t utf8_char_length(uint32_t c) {+ if (c <= 0x7F) {+ return 1;+ } else if (c <= 0x7FF) {+ return 2;+ } else if (c <= 0xFFFF) {+ return 3;+ } else if (c <= 0x1FFFFF) {+ return 4;+ } else {+ // Unicode characters out of this range are illegal...+ return 0;+ }+}+ extern "C" void bw_append_char_utf8(BufferWriter* bw, uint32_t c) {- BW_ENSURE_CAPACITY(bw, 4);+ BW_ENSURE_CAPACITY(bw, 4, utf8_char_length(c)); unsigned char* data = bw->end; @@ -184,7 +229,7 @@ } extern "C" void bw_append_bs(BufferWriter* bw, size_t size, const unsigned char* data) {- BW_ENSURE_CAPACITY(bw, size);+ BW_ENSURE_CAPACITY(bw, size, size); memcpy(bw->end, data, size); bw->end += size;@@ -195,14 +240,14 @@ } extern "C" void bw_append_byte7(BufferWriter* bw, unsigned char byte) {- BW_ENSURE_CAPACITY(bw, 1);+ BW_ENSURE_CAPACITY(bw, 1, 1); bw->end[0] = byte & 0x7F; bw->end += 1; } extern "C" void bw_append_bs7(BufferWriter* bw, size_t size, const unsigned char* data) {- BW_ENSURE_CAPACITY(bw, size);+ BW_ENSURE_CAPACITY(bw, size, size); unsigned char* out = bw->end; for (size_t i = 0; i < size; ++i) {@@ -215,8 +260,23 @@ bw_append_bs7(bw, strlen(reinterpret_cast<const char*>(data)), data); } +inline size_t count_escaped_json_length(size_t size, const unsigned char* data) {+ size_t rv = 2;+ for (size_t i = 0; i < size; ++i) {+ switch (data[i]) {+ case '\n': rv += 2; break;+ case '\r': rv += 2; break;+ case '\t': rv += 2; break;+ case '\\': rv += 2; break;+ case '\"': rv += 2; break;+ default: rv += 1; break;+ }+ }+ return rv;+}+ extern "C" void bw_append_json_escaped(BufferWriter* bw, size_t size, const unsigned char* data) {- BW_ENSURE_CAPACITY(bw, 2 + size * 2);+ BW_ENSURE_CAPACITY(bw, 2 + size * 2, count_escaped_json_length(size, data)); unsigned char* dest = bw->end; *dest++ = '\"';@@ -229,7 +289,7 @@ case '\t': *dest++ = '\\'; *dest++ = 't'; break; case '\\': *dest++ = '\\'; *dest++ = '\\'; break; case '\"': *dest++ = '\\'; *dest++ = '\"'; break;- default: *dest++ = data[i];+ default: *dest++ = data[i]; break; } ++i; }@@ -238,15 +298,25 @@ bw->end = dest; } +inline size_t count_decimal_signed_int_length(int64_t i) {+ char buf[32]; // enough+ return i64toa_branchlut(i, buf);+}+ extern "C" void bw_append_decimal_signed_int(BufferWriter* bw, int64_t i) {- BW_ENSURE_CAPACITY(bw, 32); // enough+ BW_ENSURE_CAPACITY(bw, 32, count_decimal_signed_int_length(i)); // enough unsigned used = i64toa_branchlut(i, reinterpret_cast<char*>(bw->end)); bw->end += used; } +inline size_t count_decimal_double_length(double d) {+ char buf[318];+ return sprintf(buf, "%f", d);+}+ extern "C" void bw_append_decimal_double(BufferWriter* bw, double d) {- BW_ENSURE_CAPACITY(bw, 318); // sprintf("%f", -DBL_MAX);+ BW_ENSURE_CAPACITY(bw, 318, count_decimal_double_length(d)); // sprintf("%f", -DBL_MAX); size_t used = sprintf(reinterpret_cast<char*>(bw->end), "%f", d); bw->end += used;@@ -349,17 +419,17 @@ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,1,1,0, 1,1,1,1, 1,1,1,1, 1,1,0,0, 0,0,0,0,- + 0,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,0,1, 0,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,1,0,- + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,- + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,@@ -370,9 +440,21 @@ '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f' }; +inline size_t count_url_encoded_length(size_t size, const unsigned char* data) {+ size_t rv = 0;+ for (size_t i = 0; i < size; ++i) {+ if (UNRESERVED[data[i]]) {+ rv += 1;+ } else {+ rv += 3;+ }+ }+ return rv;+}+ extern "C" void bw_append_url_encoded(BufferWriter* bw, size_t size, const unsigned char* data) { // worst case- BW_ENSURE_CAPACITY(bw, size * 3);+ BW_ENSURE_CAPACITY(bw, size * 3, count_url_encoded_length(size, data)); const unsigned char* src = data; unsigned char* dst = bw->end;
changelog.md view
@@ -1,3 +1,7 @@+0.2.4.0++* Add the ability to calculate the output length of a BufferBuilder without allocating or writing bytes+ 0.2.3.0 * Add the ability to query the current buffer size
src/Data/BufferBuilder.hs view
@@ -17,7 +17,8 @@ , runBufferBuilderWithOptions , runBufferBuilderWithOptions' - -- * Query current state+ -- * Query builder+ , calculateLength , currentLength -- * Appending bytes and byte strings@@ -76,6 +77,7 @@ foreign import ccall unsafe "strlen" c_strlen :: Ptr Word8 -> IO Int foreign import ccall unsafe "bw_new" bw_new :: Int -> IO Handle+foreign import ccall unsafe "bw_new_length_calculator" bw_new_length_calculator :: IO Handle foreign import ccall unsafe "&bw_free" bw_free :: FunPtr (Handle -> IO ()) foreign import ccall unsafe "bw_trim" bw_trim :: Handle -> IO () foreign import ccall unsafe "bw_get_size" bw_get_size :: Handle -> IO Int@@ -100,7 +102,7 @@ -- | BufferBuilder is the type of a monadic action that appends to an implicit, -- growable buffer. Use 'runBufferBuilder' to extract the resulting--- buffer as a 'BS.ByteString'. +-- buffer as a 'BS.ByteString'. newtype BufferBuilder a = BB (Handle -> IO a) --deriving (Applicative, Monad, MonadReader Handle) @@ -167,7 +169,7 @@ handle <- bw_new initialCapacity when (handle == nullPtr) $ do throw BufferOutOfMemoryError- + handleFP <- newForeignPtr bw_free handle rv <- bw handle @@ -188,7 +190,23 @@ touchForeignPtr handleFP return (rv, bs) --- | Reads current length of BufferBuilder+-- | Given a BufferBuilder, calculate its length. This runs every BufferBuilder action+-- in a mode that simply accumulates the number of bytes without copying any data into an+-- output buffer.+calculateLength :: BufferBuilder a -> Int+calculateLength !(BB bw) = unsafeDupablePerformIO $ do+ handle <- bw_new_length_calculator+ when (handle == nullPtr) $ do+ throw BufferOutOfMemoryError++ handleFP <- newForeignPtr bw_free handle+ _ <- bw handle+ size <- bw_get_size handle+ touchForeignPtr handleFP+ return size++-- | Reads current length of BufferBuilder. If memory allocation has failed at any point, this returns zero.+-- In the future, currentLength may throw an exception upon memory allocation failure. currentLength :: BufferBuilder Int currentLength = withHandle bw_get_size @@ -245,7 +263,7 @@ -- Per byte, this is the fastest append function. It amounts to a C function call -- with two constant arguments. The C function checks to see if it needs to grow -- the buffer and then it simply calls memcpy.--- +-- -- __WARNING__: passing an incorrect length value is likely to cause an access -- violation or worse. unsafeAppendLiteralN :: Int -> Addr# -> BufferBuilder ()
test/BufferTest.hs view
@@ -1,4 +1,5 @@-{-# LANGUAGE MagicHash, OverloadedStrings, TemplateHaskell #-}+{-# LANGUAGE MagicHash, OverloadedStrings, TemplateHaskell, FlexibleInstances #-}+{-# OPTIONS_GHC -fno-warn-orphans #-} module BufferTest (tests, main) where @@ -8,6 +9,7 @@ import Test.Tasty.QuickCheck import Test.Tasty.HUnit import Data.BufferBuilder+import Test.QuickCheck.Instances () case_append_bytes :: Assertion case_append_bytes = do@@ -47,20 +49,55 @@ appendUrlEncoded safe assertEqual "matches" "%00%01%20%2f%5c%24%25" result -case_query_size :: Assertion-case_query_size = do+case_query_current_length :: Assertion+case_query_current_length = do let ((f, s), result) = runBufferBuilder' $ do appendBS "foo" first <- currentLength appendBS "bar" second <- currentLength return (first, second)- assertEqual "first size" 3 f- assertEqual "second size" 6 s+ assertEqual "first length" 3 f+ assertEqual "second length" 6 s assertEqual "result" "foobar" result +case_calculate_length :: Assertion+case_calculate_length = do+ let first = appendBS "foo"+ let second = first >> appendBS "bar"++ assertEqual "first length" 3 $ calculateLength first+ assertEqual "second length" 6 $ calculateLength second+ prop_match_intDec :: Int -> Bool prop_match_intDec i = runBufferBuilder (appendDecimalSignedInt i) == (BSC.pack $ show i)++instance Arbitrary (BufferBuilder String) where+ arbitrary = oneof+ [ a "appendBS" appendBS+ , a "appendChar8" appendChar8+ , a "appendCharUtf8" appendCharUtf8+ , a "appendStringUtf8" appendStringUtf8+ , a "appendEscapedJson" appendEscapedJson+ , a "appendEscapedJsonText" appendEscapedJsonText+ , a "appendDecimalSignedInt" appendDecimalSignedInt+ , a "appendDecimalDouble" appendDecimalDouble+ , a "appendUrlEncoded" appendUrlEncoded+ ]+ where a :: Arbitrary arg => String -> (arg -> BufferBuilder ()) -> Gen (BufferBuilder String)+ a name action = do+ arg <- arbitrary+ return $ do+ action arg+ return name++instance Show (BufferBuilder String) where+ show = fst . runBufferBuilder'++prop_length_calculation_matches_actual_length :: BufferBuilder String -> Bool+prop_length_calculation_matches_actual_length bb =+ let cc = bb >> return () in+ BSC.length (runBufferBuilder cc) == calculateLength cc tests :: TestTree tests = $(testGroupGenerator)