base64 0.3.0.0 → 0.3.1.0

raw patch · 11 files changed

+154/−56 lines, 11 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• CHANGELOG.md +7/−0
• README.md +6/−5
• base64.cabal +3/−4
• benchmarks/Base64Bench.hs +30/−4
• benchmarks/PERFORMANCE.md +0/−23
• src/Data/ByteString/Base64.hs +15/−4
• src/Data/ByteString/Base64/Internal.hs +25/−0
• src/Data/ByteString/Base64/URL.hs +15/−4
• src/Data/Text/Encoding/Base64.hs +13/−6
• src/Data/Text/Encoding/Base64/URL.hs +13/−6
• test/Base64Tests.hs +27/−0

@@ -1,5 +1,12 @@ # Revision history for base64 +## 0.3.1.0 -- 2020-01-08++* Bug fix for `isBase64` and `isBase64Url` - wrong alphabet was used+* Added `isValidBase64` and `isValidBase64Url` for alphabet conformity. The `isBase64*` functions now tell if it's *correct* base64 now in the sense that it's decodable and valid.+* Dropped Cabal version to 2.0 for backcompat with Stack+* Better documentation+ ## 0.3.0.0 -- 2020-01-07  * After a discussion with lexilambda, we're making 'encodeBase64' be `ByteString -> Text` by default, offering `ByteString -> ByteString` as

@@ -3,21 +3,22 @@ [![Build Status](https://travis-ci.com/emilypi/base64.svg?branch=master)](https://travis-ci.com/emilypi/base64) [![Hackage](https://img.shields.io/hackage/v/base64.svg)](https://hackage.haskell.org/package/base64) -Padded and unpadded base64 and base64url encodings for `Text` and `ByteString` values, along with their optics.+Padded and unpadded base64 and base64url encoding and decoding for `Text` and `ByteString` values. -For the companion `Prism`s and pattern synonyms, see [base64-lens](https://hackage.haskell.org/package/base64-lens).+For the companion optics and pattern synonyms, see [base64-lens](https://hackage.haskell.org/package/base64-lens).   ### Summary  What does this library provide? Here is the summary: -- Better performance over existing Base64 libraries (2x and 3x for most use-cases - see [PERFORMANCE.md](benchmarks/PERFORMANCE.md))-- Support for unpadded Base64 and Base64-url+- Good performance compared to existing Base64 libraries (see [PERFORMANCE.md](benchmarks/PERFORMANCE.md))+- Support for padded and unpadded Base64 and Base64url - Support for `Text` encodings and decodings - Optics for handling more complex structures with Base64 representations via the `base64-lens` package+- Checks for both valid Base64 and correct Base64 and Base64url encodings -There are no dependencies aside from `base`:+There are no dependencies aside from those bundled with GHC:  ![base64 dependencies](https://i.imgur.com/qynI5HM.png) 

@@ -1,13 +1,13 @@-cabal-version:       2.4+cabal-version:       2.0  name:                base64-version:             0.3.0.0+version:             0.3.1.0 synopsis:            RFC 4648-compliant padded and unpadded base64 and base64url encodings description:   RFC 4648-compliant padded and unpadded base64 and base64url encoding and decoding. homepage:            https://github.com/emilypi/base64 bug-reports:         https://github.com/emilypi/base64/issues-license:             BSD-3-Clause+license:             BSD3 license-file:        LICENSE author:              Emily Pillmore maintainer:          emilypi@cohomolo.gy@@ -17,7 +17,6 @@ extra-source-files:   CHANGELOG.md   README.md-  benchmarks/PERFORMANCE.md  tested-with:         GHC ==8.8.1 || ==8.6.5 || ==8.6.3 || ==8.4.4 || ==8.4.3 || ==8.2.2 

@@ -32,12 +32,15 @@   main :: IO ()-main = defaultMain-    $ fmap (benchN random encode_) sizes-    ++ fmap (benchN (fmap B64.encodeBase64' . random) decode_) sizes+main = defaultMain $ bench' stepwise random encode_+    ++ bench' stepwise (fmap B64.encodeBase64' . random) decode_+    ++ bench' stepwise (fmap B64.encodeBase64' . random) lenient_+    bench' chonkers random mbPerSec    where-    sizes = [25,100,1000,10000,100000]+    bench' sz f b = fmap (benchN f b) sz+    stepwise = [25,100,1000,10000,100000]+    chonkers = fmap (* 1000000) [1..5]     benchN f bs n = env (f n) $ bgroup (show n) . bs      encode_ e =@@ -56,12 +59,29 @@         ]       ] +    lenient_ e =+      [ bgroup "base64 decode-lenient"+        [ lenientBench @'Bos e+        , lenientBench @'B64 e+        ]+      ]++    mbPerSec e =+      [ bgroup "base64 MB/s benches"+        [ encodeBench @'Mem e+        , encodeBench @'Bos e+        , encodeBench @'B64 e+        ]+      ]+ encodeBench :: forall a. Harness a => Base64 a -> Benchmark encodeBench = bench (label @a) . nf (encoder @a)  decodeBench :: forall a. Harness a => Base64 a -> Benchmark decodeBench = bench (label @a) . nf (decoder @a) +lenientBench :: forall a. Harness a => Base64 a -> Benchmark+lenientBench = bench (label @a) . nf (lenient @a)  data Bench where   Mem :: Bench@@ -69,12 +89,14 @@   B64 :: Bench   T64 :: Bench + class (NFData (Base64 a), NFData (Err a)) => Harness (a :: Bench) where     type Base64 a :: Type     type Err a :: Type     label :: String     encoder :: Base64 a -> Base64 a     decoder :: Base64 a -> Either (Err a) (Base64 a)+    lenient :: Base64 a -> Base64 a  instance Harness 'Mem where     type Base64 'Mem = ByteString@@ -82,6 +104,7 @@     label = "memory"     encoder = Mem.convertToBase Mem.Base64     decoder = Mem.convertFromBase Mem.Base64+    lenient = id  instance Harness 'Bos where     type Base64 'Bos = ByteString@@ -89,6 +112,7 @@     label = "base64-bytestring"     encoder = Bos.encode     decoder = Bos.decode+    lenient = Bos.decodeLenient  instance Harness 'B64 where     type Base64 'B64 = ByteString@@ -96,6 +120,7 @@     label = "base64"     encoder = B64.encodeBase64'     decoder = B64.decodeBase64+    lenient = B64.decodeBase64Lenient  instance Harness 'T64 where     type Base64 'T64 = Text@@ -103,3 +128,4 @@     label = "base64-text"     encoder = B64T.encodeBase64     decoder = B64T.decodeBase64+    lenient = B64T.decodeBase64Lenient

@@ -1,23 +0,0 @@-## Performance--The story so far:--- 3x encoding/decoding performance for bytestrings ∊ \[0, 100\[-- 2x encoding/decoding performance for bytestrings ∊ ]100, 1000\[-- 1-2μs improvement in encoding/decoding performance for bytestrings ∊ ]1000, 10,000[-- 2-3μs improvement in encoding/decoding performance for bytestrings ∊ ]10,000, 1,000,000]-- Smaller heap footprint in general.--Most of this performance increase for smaller bytestrings is due to optimization of the building of the encoding tables. Additionally, I've factored out several read/writes and unnecessary IO done in `bytestring-base64`. The inner loop is as optimized as it can be for `Word16`-based optimizations (3 8-Byte reads, 2 12-Byte writes), and the tail completion is optimized by elimating 4 unnecessary reads, and using one big `if-then-else` branch to eliminate 1 read and 1 write per case. Smaller improvements have been the elimination of unnecessary wrappers (i.e. not using `BS.pack` and simply `malloc`'ing and rolling my own pointers), as well as properly inlining auxiliary functions which failed to inline in Bos' version. Additionally, using unpacked `Addr#`'s for the alphabet means we don't have to use a `ForeignPtr` box or touch it unnecessarily. See the outputs in the benchmarks directory for more detail..--### Improvements--My suspicion is that we can get away with 1 `Word32` read, discard the upper 8 bytes, do some bitshifting magic, and then do a single `Word32` (with only the lower 24 Bytes filled) write to the output pointer in the inner loop, eliminating 2 reads and 1 write to make it single read/write. Additionally, if we want to impose size constraints, we should be able to fill the bottom 48 Bytes of a `Word64` and still be on the right side of the cache line, eliminating 2x loops per iteration. Additionally, factoring out the encoding tables to a static `CString` sitting in `cbits` would eliminate the need to construct it each time - there are only 2 alphabets, and we can maintain two static tables. We can do the same with the alphabets, but this is dubious since unboxed strlit `Addr#` is cheap and if we can get away with not calling to the FFI, that would be preferable. In the end, why not just do the whole thing in C and call it a day though? (Thoughts?)--### To come later--Daniel Lemire has a great [blog post](https://lemire.me/blog/2018/01/17/ridiculously-fast-base64-encoding-and-decoding/) detailing how he and Wojciech Mula implemented fast AVX and SSE base64 vectorization with a nifty library to boot. This has all been incorporated into Alfred Klomp's [base64](https://github.com/aklomp/base64) library, which is very up to date. The speed up people see is *massive*, and is BSD-2. We can inline the useful portions of the library along with the copyright notice in `cbits` and be very happy.--### Update--Using Wojciech Mula's implementation as Cbits, i'm happy to report that it may be a useful gain, but only for bytestrings greater than 1,000,000B in size, as the cost of an FFI call is too great for smaller bytestrings. The 12-bit lookup table for >=32-bit word architectures is still optimal. In the future, I'd like to implement Alfred Klomp's rounds in Haskell, optimizing for 64B words.

@@ -1,3 +1,4 @@+{-# LANGUAGE BangPatterns #-} {-# LANGUAGE OverloadedStrings #-} -- | -- Module       : Data.ByteString.Base64@@ -21,12 +22,14 @@ , decodeBase64Unpadded , decodeBase64Lenient , isBase64+, isValidBase64 ) where   import Data.ByteString (ByteString) import qualified Data.ByteString as BS import Data.ByteString.Base64.Internal+import Data.Either (isRight) import Data.Text (Text) import qualified Data.Text.Encoding as T @@ -110,10 +113,18 @@ decodeBase64Lenient = decodeBase64Lenient_ decodeB64Table {-# INLINE decodeBase64Lenient #-} --- | Tell whether a 'ByteString' value is Base64-encoded+-- | Tell whether a 'ByteString' value is base64 encoded. -- isBase64 :: ByteString -> Bool-isBase64 = BS.all (`BS.elem` alphabet)-  where-    alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"+isBase64 bs = isValidBase64 bs && isRight (decodeBase64 bs) {-# INLINE isBase64 #-}++-- | Tell whether a 'ByteString' value is a valid Base64 format.+--+-- This will not tell you whether or not this is a correct Base64url representation,+-- only that it conforms to the correct shape. To check whether it is a true+-- Base64 encoded 'ByteString' value, use 'isBase64'.+--+isValidBase64 :: ByteString -> Bool+isValidBase64 = validateBase64 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"+{-# INLINE isValidBase64 #-}

@@ -35,6 +35,9 @@    -- ** Base64-url , base64UrlTable++  -- * Validating Base64+, validateBase64 ) where  @@ -102,6 +105,28 @@ base64Table = packTable "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"# {-# NOINLINE base64Table #-} ++-- -------------------------------------------------------------------------- --+-- Validating Base64++validateBase64 :: ByteString -> ByteString -> Bool+validateBase64 !alphabet (PS fp off l) =+    accursedUnutterablePerformIO $ withForeignPtr fp $ \p ->+      go (plusPtr p off) (plusPtr p (l + off))+  where+    go !p !end+      | p == end = return True+      | otherwise = do+        w <- peek p++        let f a+              | a == 0x3d, plusPtr p 1 == end = True+              | a == 0x3d, plusPtr p 2 == end = True+              | a == 0x3d = False+              | otherwise = BS.elem a alphabet++        if f w then go (plusPtr p 1) end else return False+{-# INLINE validateBase64 #-}  -- -------------------------------------------------------------------------- -- -- Encode Base64

@@ -1,3 +1,4 @@+{-# LANGUAGE BangPatterns #-} {-# LANGUAGE OverloadedStrings #-} -- | -- Module       : Data.ByteString.Base64.URL@@ -21,11 +22,13 @@ , decodeBase64Unpadded , decodeBase64Lenient , isBase64Url+, isValidBase64Url ) where  import Data.ByteString (ByteString) import qualified Data.ByteString as BS import Data.ByteString.Base64.Internal+import Data.Either (isRight) import Data.Text (Text) import qualified Data.Text.Encoding as T @@ -101,10 +104,18 @@ decodeBase64Lenient = decodeBase64Lenient_ decodeB64UrlTable {-# INLINE decodeBase64Lenient #-} --- | Tell whether a bytestring is Base64-encoded+-- | Tell whether a 'ByteString' is Base64url-encoded. -- isBase64Url :: ByteString -> Bool-isBase64Url = BS.all (`BS.elem` alphabet)-  where-    alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"+isBase64Url bs = isValidBase64Url bs && isRight (decodeBase64 bs) {-# INLINE isBase64Url #-}++-- | Tell whether a 'ByteString' is a valid Base64url format.+--+-- This will not tell you whether or not this is a correct Base64url representation,+-- only that it conforms to the correct shape. To check whether it is a true+-- Base64 encoded 'ByteString' value, use 'isBase64Url'.+--+isValidBase64Url :: ByteString -> Bool+isValidBase64Url = validateBase64 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"+{-# INLINE isValidBase64Url #-}

@@ -19,14 +19,13 @@ , decodeBase64Unpadded , decodeBase64Lenient , isBase64+, isValidBase64 ) where   import qualified Data.ByteString.Base64 as B64 -import Data.Maybe (isJust) import Data.Text (Text)-import qualified Data.Text as T import qualified Data.Text.Encoding as T  -- | Encode a 'Text' value in Base64 with padding.@@ -85,10 +84,18 @@ {-# INLINE decodeBase64Lenient #-}  --- | Tell whether a 'Text' value is Base64-encoded+-- | Tell whether a 'Text' value is Base64-encoded. -- isBase64 :: Text -> Bool-isBase64 = T.all (isJust . flip T.find alphabet . (==))-  where-    alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"+isBase64 = B64.isBase64 . T.encodeUtf8 {-# INLINE isBase64 #-}++-- | Tell whether a 'Text' value is a valid Base64 format.+--+-- This will not tell you whether or not this is a correct Base64 representation,+-- only that it conforms to the correct shape. To check whether it is a true+-- Base64 encoded 'Text' value, use 'isBase64'.+--+isValidBase64 :: Text -> Bool+isValidBase64 = B64.isValidBase64 . T.encodeUtf8+{-# INLINE isValidBase64 #-}

@@ -18,14 +18,13 @@ , decodeBase64Unpadded , decodeBase64Lenient , isBase64Url+, isValidBase64Url ) where   import qualified Data.ByteString.Base64.URL as B64U -import Data.Maybe (isJust) import Data.Text (Text)-import qualified Data.Text as T import qualified Data.Text.Encoding as T  -- | Encode a 'Text' value in Base64url with padding.@@ -80,10 +79,18 @@     . T.encodeUtf8 {-# INLINE decodeBase64Lenient #-} --- | Tell whether a 'Text' value is Base64url-encoded+-- | Tell whether a 'Text' value is Base64url-encoded. -- isBase64Url :: Text -> Bool-isBase64Url = T.all (isJust . flip T.find alphabet . (==))-  where-    alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"+isBase64Url = B64U.isBase64Url . T.encodeUtf8 {-# INLINE isBase64Url #-}++-- | Tell whether a 'Text' value is a valid Base64url format.+--+-- This will not tell you whether or not this is a correct Base64url representation,+-- only that it conforms to the correct shape. To check whether it is a true+-- Base64 encoded 'Text' value, use 'isBase64Url'.+--+isValidBase64Url :: Text -> Bool+isValidBase64Url = B64U.isValidBase64Url . T.encodeUtf8+{-# INLINE isValidBase64Url #-}

@@ -24,6 +24,7 @@ tests = testGroup "Base64 Tests"     [ testVectors     , sanityTests+    , alphabetTests     ]  testVectors :: TestTree@@ -76,3 +77,29 @@       bs <- random n       B64.decodeBase64 (B64.encodeBase64' bs) @=? Right bs       B64U.decodeBase64 (B64U.encodeBase64' bs) @=? Right bs++alphabetTests :: TestTree+alphabetTests = testGroup "Alphabet tests"+    [ base64Tests 0+    , base64Tests 4+    , base64Tests 5+    , base64Tests 6+    , base64Tests 100+    , base64UrlTests 0+    , base64UrlTests 4+    , base64UrlTests 5+    , base64UrlTests 6+    , base64UrlTests 100+    ]+  where+    base64Tests n = testCase ("Conforms to Base64 alphabet: " ++ show n) $ do+      bs <- random n+      let b = B64.encodeBase64' bs+      assertBool ("failed validity: " ++ show b) $ B64.isValidBase64 b+      assertBool ("failed correctness: " ++ show b) $ B64.isBase64 b++    base64UrlTests n = testCase ("Conforms to Base64url alphabet: " ++ show n) $ do+      bs <- random n+      let b = B64U.encodeBase64' bs+      assertBool ("failed validity: " ++ show b) $ B64U.isValidBase64Url b+      assertBool ("failed correctness: " ++ show b) $ B64U.isBase64Url b