diff --git a/buffer-builder.cabal b/buffer-builder.cabal --- a/buffer-builder.cabal +++ b/buffer-builder.cabal @@ -1,5 +1,5 @@ name: buffer-builder -version: 0.2.3.0 +version: 0.2.4.0 synopsis: Library for efficiently building up buffers, one piece at a time description: @@ -28,7 +28,7 @@ Using BufferBuilder is very simple: . > import qualified Data.BufferBuilder as BB - > + > > let byteString = BB.runBufferBuilder $ do > BB.appendBS "http" > BB.appendChar8 '/' @@ -85,6 +85,7 @@ , tasty-hunit , tasty-th , tasty-quickcheck + , quickcheck-instances , HUnit , text , vector diff --git a/cbits/buffer.cpp b/cbits/buffer.cpp --- a/cbits/buffer.cpp +++ b/cbits/buffer.cpp @@ -25,9 +25,12 @@ #define BW_UNLIKELY(x) x #endif -#define BW_ENSURE_CAPACITY(bw, cap) \ +#define BW_ENSURE_CAPACITY(bw, cap, count_size) \ do { \ if (!ensureCapacity((bw), (cap))) { \ + if (bw->counting_size) { \ + bw->counted_size += count_size; \ + } \ return; \ } \ } while (0) @@ -39,7 +42,12 @@ unsigned char* capacity; unsigned char* data; + bool counting_size; + size_t counted_size; + // invariants: + // if (counting_size), end, capacity, and data are null + // else, counted_size is zero and: // if (end <= capacity), then data is non-null and end >= data // if (capacity == 0), then data and end are null, and object is dead @@ -61,7 +69,7 @@ // TODO: detect overflow? newCapacity *= 2; } while (newCapacity < size + amount); - + unsigned char* newData = reinterpret_cast(realloc(bw->data, newCapacity)); if (newData) { bw->data = newData; @@ -105,9 +113,25 @@ bw->end = bw->data; bw->capacity = bw->data + initialCapacity; + bw->counting_size = false; + bw->counted_size = 0; return bw; } +extern "C" BufferWriter* bw_new_length_calculator() { + BufferWriter* bw = reinterpret_cast(malloc(sizeof(BufferWriter))); + if (!bw) { + return 0; + } + + bw->data = 0; + bw->end = 0; + bw->capacity = 0; + bw->counting_size = true; + bw->counted_size = 0; + return bw; +} + extern "C" void bw_free(BufferWriter* bw) { free(bw->data); free(bw); @@ -134,7 +158,11 @@ } extern "C" size_t bw_get_size(BufferWriter* bw) { - return bw->end - bw->data; + if (bw->counting_size) { + return bw->counted_size; + } else { + return bw->end - bw->data; + } } extern "C" unsigned char* bw_release_address(BufferWriter* bw) { @@ -143,18 +171,35 @@ bw->end = 0; bw->capacity = 0; bw->data = 0; + bw->counting_size = false; + bw->counted_size = 0; return data; // null if dead } extern "C" void bw_append_byte(BufferWriter* bw, unsigned char byte) { - BW_ENSURE_CAPACITY(bw, 1); + BW_ENSURE_CAPACITY(bw, 1, 1); bw->end[0] = byte; bw->end += 1; } +inline size_t utf8_char_length(uint32_t c) { + if (c <= 0x7F) { + return 1; + } else if (c <= 0x7FF) { + return 2; + } else if (c <= 0xFFFF) { + return 3; + } else if (c <= 0x1FFFFF) { + return 4; + } else { + // Unicode characters out of this range are illegal... + return 0; + } +} + extern "C" void bw_append_char_utf8(BufferWriter* bw, uint32_t c) { - BW_ENSURE_CAPACITY(bw, 4); + BW_ENSURE_CAPACITY(bw, 4, utf8_char_length(c)); unsigned char* data = bw->end; @@ -184,7 +229,7 @@ } extern "C" void bw_append_bs(BufferWriter* bw, size_t size, const unsigned char* data) { - BW_ENSURE_CAPACITY(bw, size); + BW_ENSURE_CAPACITY(bw, size, size); memcpy(bw->end, data, size); bw->end += size; @@ -195,14 +240,14 @@ } extern "C" void bw_append_byte7(BufferWriter* bw, unsigned char byte) { - BW_ENSURE_CAPACITY(bw, 1); + BW_ENSURE_CAPACITY(bw, 1, 1); bw->end[0] = byte & 0x7F; bw->end += 1; } extern "C" void bw_append_bs7(BufferWriter* bw, size_t size, const unsigned char* data) { - BW_ENSURE_CAPACITY(bw, size); + BW_ENSURE_CAPACITY(bw, size, size); unsigned char* out = bw->end; for (size_t i = 0; i < size; ++i) { @@ -215,8 +260,23 @@ bw_append_bs7(bw, strlen(reinterpret_cast(data)), data); } +inline size_t count_escaped_json_length(size_t size, const unsigned char* data) { + size_t rv = 2; + for (size_t i = 0; i < size; ++i) { + switch (data[i]) { + case '\n': rv += 2; break; + case '\r': rv += 2; break; + case '\t': rv += 2; break; + case '\\': rv += 2; break; + case '\"': rv += 2; break; + default: rv += 1; break; + } + } + return rv; +} + extern "C" void bw_append_json_escaped(BufferWriter* bw, size_t size, const unsigned char* data) { - BW_ENSURE_CAPACITY(bw, 2 + size * 2); + BW_ENSURE_CAPACITY(bw, 2 + size * 2, count_escaped_json_length(size, data)); unsigned char* dest = bw->end; *dest++ = '\"'; @@ -229,7 +289,7 @@ case '\t': *dest++ = '\\'; *dest++ = 't'; break; case '\\': *dest++ = '\\'; *dest++ = '\\'; break; case '\"': *dest++ = '\\'; *dest++ = '\"'; break; - default: *dest++ = data[i]; + default: *dest++ = data[i]; break; } ++i; } @@ -238,15 +298,25 @@ bw->end = dest; } +inline size_t count_decimal_signed_int_length(int64_t i) { + char buf[32]; // enough + return i64toa_branchlut(i, buf); +} + extern "C" void bw_append_decimal_signed_int(BufferWriter* bw, int64_t i) { - BW_ENSURE_CAPACITY(bw, 32); // enough + BW_ENSURE_CAPACITY(bw, 32, count_decimal_signed_int_length(i)); // enough unsigned used = i64toa_branchlut(i, reinterpret_cast(bw->end)); bw->end += used; } +inline size_t count_decimal_double_length(double d) { + char buf[318]; + return sprintf(buf, "%f", d); +} + extern "C" void bw_append_decimal_double(BufferWriter* bw, double d) { - BW_ENSURE_CAPACITY(bw, 318); // sprintf("%f", -DBL_MAX); + BW_ENSURE_CAPACITY(bw, 318, count_decimal_double_length(d)); // sprintf("%f", -DBL_MAX); size_t used = sprintf(reinterpret_cast(bw->end), "%f", d); bw->end += used; @@ -349,17 +419,17 @@ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,1,1,0, 1,1,1,1, 1,1,1,1, 1,1,0,0, 0,0,0,0, - + 0,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,0,1, 0,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,1,0, - + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, @@ -370,9 +440,21 @@ '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f' }; +inline size_t count_url_encoded_length(size_t size, const unsigned char* data) { + size_t rv = 0; + for (size_t i = 0; i < size; ++i) { + if (UNRESERVED[data[i]]) { + rv += 1; + } else { + rv += 3; + } + } + return rv; +} + extern "C" void bw_append_url_encoded(BufferWriter* bw, size_t size, const unsigned char* data) { // worst case - BW_ENSURE_CAPACITY(bw, size * 3); + BW_ENSURE_CAPACITY(bw, size * 3, count_url_encoded_length(size, data)); const unsigned char* src = data; unsigned char* dst = bw->end; diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,7 @@ +0.2.4.0 + +* Add the ability to calculate the output length of a BufferBuilder without allocating or writing bytes + 0.2.3.0 * Add the ability to query the current buffer size diff --git a/src/Data/BufferBuilder.hs b/src/Data/BufferBuilder.hs --- a/src/Data/BufferBuilder.hs +++ b/src/Data/BufferBuilder.hs @@ -17,7 +17,8 @@ , runBufferBuilderWithOptions , runBufferBuilderWithOptions' - -- * Query current state + -- * Query builder + , calculateLength , currentLength -- * Appending bytes and byte strings @@ -76,6 +77,7 @@ foreign import ccall unsafe "strlen" c_strlen :: Ptr Word8 -> IO Int foreign import ccall unsafe "bw_new" bw_new :: Int -> IO Handle +foreign import ccall unsafe "bw_new_length_calculator" bw_new_length_calculator :: IO Handle foreign import ccall unsafe "&bw_free" bw_free :: FunPtr (Handle -> IO ()) foreign import ccall unsafe "bw_trim" bw_trim :: Handle -> IO () foreign import ccall unsafe "bw_get_size" bw_get_size :: Handle -> IO Int @@ -100,7 +102,7 @@ -- | BufferBuilder is the type of a monadic action that appends to an implicit, -- growable buffer. Use 'runBufferBuilder' to extract the resulting --- buffer as a 'BS.ByteString'. +-- buffer as a 'BS.ByteString'. newtype BufferBuilder a = BB (Handle -> IO a) --deriving (Applicative, Monad, MonadReader Handle) @@ -167,7 +169,7 @@ handle <- bw_new initialCapacity when (handle == nullPtr) $ do throw BufferOutOfMemoryError - + handleFP <- newForeignPtr bw_free handle rv <- bw handle @@ -188,7 +190,23 @@ touchForeignPtr handleFP return (rv, bs) --- | Reads current length of BufferBuilder +-- | Given a BufferBuilder, calculate its length. This runs every BufferBuilder action +-- in a mode that simply accumulates the number of bytes without copying any data into an +-- output buffer. +calculateLength :: BufferBuilder a -> Int +calculateLength !(BB bw) = unsafeDupablePerformIO $ do + handle <- bw_new_length_calculator + when (handle == nullPtr) $ do + throw BufferOutOfMemoryError + + handleFP <- newForeignPtr bw_free handle + _ <- bw handle + size <- bw_get_size handle + touchForeignPtr handleFP + return size + +-- | Reads current length of BufferBuilder. If memory allocation has failed at any point, this returns zero. +-- In the future, currentLength may throw an exception upon memory allocation failure. currentLength :: BufferBuilder Int currentLength = withHandle bw_get_size @@ -245,7 +263,7 @@ -- Per byte, this is the fastest append function. It amounts to a C function call -- with two constant arguments. The C function checks to see if it needs to grow -- the buffer and then it simply calls memcpy. --- +-- -- __WARNING__: passing an incorrect length value is likely to cause an access -- violation or worse. unsafeAppendLiteralN :: Int -> Addr# -> BufferBuilder () diff --git a/test/BufferTest.hs b/test/BufferTest.hs --- a/test/BufferTest.hs +++ b/test/BufferTest.hs @@ -1,4 +1,5 @@ -{-# LANGUAGE MagicHash, OverloadedStrings, TemplateHaskell #-} +{-# LANGUAGE MagicHash, OverloadedStrings, TemplateHaskell, FlexibleInstances #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} module BufferTest (tests, main) where @@ -8,6 +9,7 @@ import Test.Tasty.QuickCheck import Test.Tasty.HUnit import Data.BufferBuilder +import Test.QuickCheck.Instances () case_append_bytes :: Assertion case_append_bytes = do @@ -47,20 +49,55 @@ appendUrlEncoded safe assertEqual "matches" "%00%01%20%2f%5c%24%25" result -case_query_size :: Assertion -case_query_size = do +case_query_current_length :: Assertion +case_query_current_length = do let ((f, s), result) = runBufferBuilder' $ do appendBS "foo" first <- currentLength appendBS "bar" second <- currentLength return (first, second) - assertEqual "first size" 3 f - assertEqual "second size" 6 s + assertEqual "first length" 3 f + assertEqual "second length" 6 s assertEqual "result" "foobar" result +case_calculate_length :: Assertion +case_calculate_length = do + let first = appendBS "foo" + let second = first >> appendBS "bar" + + assertEqual "first length" 3 $ calculateLength first + assertEqual "second length" 6 $ calculateLength second + prop_match_intDec :: Int -> Bool prop_match_intDec i = runBufferBuilder (appendDecimalSignedInt i) == (BSC.pack $ show i) + +instance Arbitrary (BufferBuilder String) where + arbitrary = oneof + [ a "appendBS" appendBS + , a "appendChar8" appendChar8 + , a "appendCharUtf8" appendCharUtf8 + , a "appendStringUtf8" appendStringUtf8 + , a "appendEscapedJson" appendEscapedJson + , a "appendEscapedJsonText" appendEscapedJsonText + , a "appendDecimalSignedInt" appendDecimalSignedInt + , a "appendDecimalDouble" appendDecimalDouble + , a "appendUrlEncoded" appendUrlEncoded + ] + where a :: Arbitrary arg => String -> (arg -> BufferBuilder ()) -> Gen (BufferBuilder String) + a name action = do + arg <- arbitrary + return $ do + action arg + return name + +instance Show (BufferBuilder String) where + show = fst . runBufferBuilder' + +prop_length_calculation_matches_actual_length :: BufferBuilder String -> Bool +prop_length_calculation_matches_actual_length bb = + let cc = bb >> return () in + BSC.length (runBufferBuilder cc) == calculateLength cc tests :: TestTree tests = $(testGroupGenerator)