diff --git a/Data/BloomFilter.hs b/Data/BloomFilter.hs --- a/Data/BloomFilter.hs +++ b/Data/BloomFilter.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE Rank2Types, TypeOperators #-} +{-# LANGUAGE BangPatterns, Rank2Types, TypeOperators #-} {-# OPTIONS_GHC -fglasgow-exts #-} -- | @@ -89,6 +89,7 @@ import Control.Monad (liftM, forM_) import Control.Monad.ST (ST, runST) +import Control.Parallel.Strategies (NFData(..)) import Data.Array.Base (unsafeAt, unsafeRead, unsafeWrite) import Data.Array.ST (STUArray, thaw, unsafeFreeze) import Data.Array.Unboxed (UArray) @@ -135,6 +136,9 @@ instance Show (Bloom a) where show ub = "Bloom { " ++ show (lengthB ub) ++ " bits } " + +instance NFData (Bloom a) where + rnf !_ = () -- | Create a new mutable Bloom filter. For efficiency, the number of -- bits used may be larger than the number requested. It is always diff --git a/Data/BloomFilter/Easy.hs b/Data/BloomFilter/Easy.hs --- a/Data/BloomFilter/Easy.hs +++ b/Data/BloomFilter/Easy.hs @@ -27,7 +27,7 @@ , suggestSizing ) where -import Data.BloomFilter (Bloom, elemB, emptyB, fromListB, lengthB, notElemB) +import Data.BloomFilter (Bloom, elemB, fromListB, lengthB, notElemB) import Data.BloomFilter.Hash (Hashable, cheapHashes) import Data.BloomFilter.Util (nextPowerOfTwo) import qualified Data.ByteString as SB @@ -43,7 +43,6 @@ {-# SPECIALIZE easyList :: Double -> [String] -> Bloom String #-} {-# SPECIALIZE easyList :: Double -> [LB.ByteString] -> Bloom LB.ByteString #-} {-# SPECIALIZE easyList :: Double -> [SB.ByteString] -> Bloom SB.ByteString #-} -{-# INLINE easyList #-} easyList errRate xs = fromListB (cheapHashes numHashes) numBits xs where capacity = length xs (numBits, numHashes) diff --git a/Data/BloomFilter/Hash.hs b/Data/BloomFilter/Hash.hs --- a/Data/BloomFilter/Hash.hs +++ b/Data/BloomFilter/Hash.hs @@ -42,12 +42,15 @@ import Data.List (unfoldr) import Data.Int (Int8, Int16, Int32, Int64) import Data.Word (Word8, Word16, Word32, Word64) +import Foreign.C.String (CString) import Foreign.C.Types (CInt, CSize) -import Foreign.Marshal.Array (withArrayLen) -import Foreign.Marshal.Utils (with) -import Foreign.Ptr (Ptr, castPtr, plusPtr) -import Foreign.Storable (Storable, peek, sizeOf) +import Foreign.ForeignPtr (withForeignPtr) +import Foreign.Marshal.Alloc (alloca) +import Foreign.Marshal.Array (allocaArray, withArrayLen) +import Foreign.Ptr (Ptr, castPtr, nullPtr, plusPtr) +import Foreign.Storable (Storable, peek, poke, sizeOf) import System.IO.Unsafe (unsafePerformIO) +import Data.ByteString.Internal (ByteString(..)) import qualified Data.ByteString as SB import qualified Data.ByteString.Lazy.Internal as LB import qualified Data.ByteString.Lazy as LB @@ -101,18 +104,14 @@ -> a -- ^ value to hash -> Word32 {-# INLINE hashSalt32 #-} -hashSalt32 salt k = - let !r = unsafePerformIO $ hashIO32 k salt - in r +hashSalt32 salt k = unsafePerformIO $ hashIO32 k salt -- | Compute a salted 64-bit hash. hashSalt64 :: Hashable a => Word64 -- ^ salt -> a -- ^ value to hash -> Word64 {-# INLINE hashSalt64 #-} -hashSalt64 salt k = - let !r = unsafePerformIO $ hashIO64 k salt - in r +hashSalt64 salt k = unsafePerformIO $ hashIO64 k salt -- | Compute a list of 32-bit hashes. The value to hash may be -- inspected as many times as there are hashes requested. @@ -230,38 +229,46 @@ cast32 :: CInt -> IO Word32 cast32 = return . fromIntegral +-- Inlined from Foreign.Marshal.Utils, for performance reasons. +with :: Storable a => a -> (Ptr a -> IO b) -> IO b +with val f = + alloca $ \ptr -> do + poke ptr val + f ptr + alignedHash2 :: Ptr a -> CSize -> Word64 -> IO Word64 alignedHash2 ptr bytes salt = with (fromIntegral salt) $ \sp -> do let p1 = castPtr sp p2 = castPtr sp `plusPtr` 4 - go p1 p2 + doubleHash ptr bytes p1 p2 peek sp - where go p1 p2 + +doubleHash :: Ptr a -> CSize -> Ptr CInt -> Ptr CInt -> IO () +doubleHash ptr bytes p1 p2 | bytes .&. 3 == 0 = hashWord2 (castPtr ptr) (div4 bytes) p1 p2 | otherwise = hashLittle2 ptr bytes p1 p2 instance Hashable SB.ByteString where - hashIO32 bs salt = SB.useAsCStringLen bs $ \(ptr, len) -> do - alignedHash ptr (fromIntegral len) salt + hashIO32 bs salt = unsafeUseAsCStringLen bs $ \ptr len -> + alignedHash ptr (fromIntegral len) salt {-# INLINE hashIO64 #-} - hashIO64 bs salt = SB.useAsCStringLen bs $ \(ptr, len) -> do - alignedHash2 ptr (fromIntegral len) salt + hashIO64 bs salt = unsafeUseAsCStringLen bs $ \ptr len -> + alignedHash2 ptr (fromIntegral len) salt rechunk :: LB.ByteString -> [SB.ByteString] rechunk s | LB.null s = [] | otherwise = let (pre,suf) = LB.splitAt chunkSize s in repack pre : rechunk suf - where repack = SB.concat . LB.toChunks - chunkSize = fromIntegral LB.defaultChunkSize + where repack = SB.concat . LB.toChunks + chunkSize = fromIntegral LB.defaultChunkSize instance Hashable LB.ByteString where hashIO32 bs salt = foldM (flip hashIO32) salt (rechunk bs) {-# INLINE hashIO64 #-} - hashIO64 bs salt = foldM go salt (rechunk bs) - where go a s = hashIO64 s a + hashIO64 = hashChunks instance Hashable a => Hashable (Maybe a) where hashIO32 Nothing salt = return salt @@ -319,3 +326,55 @@ hashList64 xs salt = withArrayLen xs $ \len ptr -> alignedHash2 ptr (fromIntegral (len * sizeOf (head xs))) salt + +unsafeUseAsCStringLen :: SB.ByteString -> (CString -> Int -> IO a) -> IO a +unsafeUseAsCStringLen (PS fp o l) action = + withForeignPtr fp $ \p -> action (p `plusPtr` o) l + +type HashState = Ptr Word32 + +foreign import ccall unsafe "lookup3.h _jenkins_little2_begin" c_begin + :: Ptr Word32 -> Ptr Word32 -> HashState -> IO () + +foreign import ccall unsafe "lookup3.h _jenkins_little2_frag" c_frag + :: Ptr a -> CSize -> HashState -> CSize -> IO CSize + +foreign import ccall unsafe "lookup3.h _jenkins_little2_step" c_step + :: Ptr a -> CSize -> HashState -> IO CSize + +foreign import ccall unsafe "lookup3.h _jenkins_little2_end" c_end + :: CInt -> Ptr Word32 -> Ptr Word32 -> HashState -> IO () + +unsafeAdjustCStringLen :: SB.ByteString -> Int -> (CString -> Int -> IO a) + -> IO a +unsafeAdjustCStringLen (PS fp o l) d action + | d > l = action nullPtr 0 + | otherwise = withForeignPtr fp $ \p -> action (p `plusPtr` (o + d)) (l - d) + +hashChunks :: LB.ByteString -> Word64 -> IO Word64 +hashChunks s salt = do + with (fromIntegral salt) $ \sp -> do + let p1 = castPtr sp + p2 = castPtr sp `plusPtr` 4 + allocaArray 3 $ \st -> do + let step :: LB.ByteString -> Int -> IO Int + step (LB.Chunk x xs) off = do + unread <- unsafeAdjustCStringLen x off $ \ptr len -> + c_step ptr (fromIntegral len) st + if unread > 0 + then frag xs unread + else step xs 0 + step _ _ = return 0 + + frag :: LB.ByteString -> CSize -> IO Int + frag c@(LB.Chunk x xs) stoff = do + nstoff <- unsafeUseAsCStringLen x $ \ptr len -> do + c_frag ptr (fromIntegral len) st stoff + if nstoff == 12 + then step c (fromIntegral (nstoff - stoff)) + else frag xs nstoff + frag LB.Empty stoff = return (fromIntegral (12 - stoff)) + c_begin p1 p2 st + unread <- step s 0 + c_end (fromIntegral unread) p1 p2 st + peek sp diff --git a/bloomfilter.cabal b/bloomfilter.cabal --- a/bloomfilter.cabal +++ b/bloomfilter.cabal @@ -1,5 +1,5 @@ name: bloomfilter -version: 1.2.3 +version: 1.2.4 license: BSD3 license-file: LICENSE author: Bryan O'Sullivan @@ -28,7 +28,7 @@ build-depends: base < 2.0 || >= 3, bytestring >= 0.9 if flag(split-base) - build-depends: base >= 3.0, containers, array + build-depends: base >= 3.0, containers, array, parallel else build-depends: base < 3.0 diff --git a/cbits/lookup3.c b/cbits/lookup3.c --- a/cbits/lookup3.c +++ b/cbits/lookup3.c @@ -51,119 +51,8 @@ #include "lookup3.h" -/* - * My best guess at if you are big-endian or little-endian. This may - * need adjustment. - */ -#if (defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \ - __BYTE_ORDER == __LITTLE_ENDIAN) || \ - (defined(i386) || defined(__i386__) || defined(__i486__) || \ - defined(__i586__) || defined(__i686__) || defined(vax) || defined(MIPSEL)) -# define HASH_LITTLE_ENDIAN 1 -# define HASH_BIG_ENDIAN 0 -#elif (defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && \ - __BYTE_ORDER == __BIG_ENDIAN) || \ - (defined(sparc) || defined(POWERPC) || defined(mc68000) || defined(sel)) -# define HASH_LITTLE_ENDIAN 0 -# define HASH_BIG_ENDIAN 1 -#else -# define HASH_LITTLE_ENDIAN 0 -# define HASH_BIG_ENDIAN 0 -#endif - #define hashsize(n) ((uint32_t)1<<(n)) #define hashmask(n) (hashsize(n)-1) -#define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) - -/* -------------------------------------------------------------------------------- -mix -- mix 3 32-bit values reversibly. - -This is reversible, so any information in (a,b,c) before mix() is -still in (a,b,c) after mix(). - -If four pairs of (a,b,c) inputs are run through mix(), or through -mix() in reverse, there are at least 32 bits of the output that -are sometimes the same for one pair and different for another pair. -This was tested for: -* pairs that differed by one bit, by two bits, in any combination - of top bits of (a,b,c), or in any combination of bottom bits of - (a,b,c). -* "differ" is defined as +, -, ^, or ~^. For + and -, I transformed - the output delta to a Gray code (a^(a>>1)) so a string of 1's (as - is commonly produced by subtraction) look like a single 1-bit - difference. -* the base values were pseudorandom, all zero but one bit set, or - all zero plus a counter that starts at zero. - -Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that -satisfy this are - 4 6 8 16 19 4 - 9 15 3 18 27 15 - 14 9 3 7 17 3 -Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing -for "differ" defined as + with a one-bit base and a two-bit delta. I -used http://burtleburtle.net/bob/hash/avalanche.html to choose -the operations, constants, and arrangements of the variables. - -This does not achieve avalanche. There are input bits of (a,b,c) -that fail to affect some output bits of (a,b,c), especially of a. The -most thoroughly mixed value is c, but it doesn't really even achieve -avalanche in c. - -This allows some parallelism. Read-after-writes are good at doubling -the number of bits affected, so the goal of mixing pulls in the opposite -direction as the goal of parallelism. I did what I could. Rotates -seem to cost as much as shifts on every machine I could lay my hands -on, and rotates are much kinder to the top and bottom bits, so I used -rotates. -------------------------------------------------------------------------------- -*/ -#define mix(a,b,c) \ -{ \ - a -= c; a ^= rot(c, 4); c += b; \ - b -= a; b ^= rot(a, 6); a += c; \ - c -= b; c ^= rot(b, 8); b += a; \ - a -= c; a ^= rot(c,16); c += b; \ - b -= a; b ^= rot(a,19); a += c; \ - c -= b; c ^= rot(b, 4); b += a; \ -} - -/* -------------------------------------------------------------------------------- -final -- final mixing of 3 32-bit values (a,b,c) into c - -Pairs of (a,b,c) values differing in only a few bits will usually -produce values of c that look totally different. This was tested for -* pairs that differed by one bit, by two bits, in any combination - of top bits of (a,b,c), or in any combination of bottom bits of - (a,b,c). -* "differ" is defined as +, -, ^, or ~^. For + and -, I transformed - the output delta to a Gray code (a^(a>>1)) so a string of 1's (as - is commonly produced by subtraction) look like a single 1-bit - difference. -* the base values were pseudorandom, all zero but one bit set, or - all zero plus a counter that starts at zero. - -These constants passed: - 14 11 25 16 4 14 24 - 12 14 25 16 4 14 24 -and these came close: - 4 8 15 26 3 22 24 - 10 8 15 26 3 22 24 - 11 8 15 26 3 22 24 -------------------------------------------------------------------------------- -*/ -#define final(a,b,c) \ -{ \ - c ^= b; c -= rot(b,14); \ - a ^= c; a -= rot(c,11); \ - b ^= a; b -= rot(a,25); \ - c ^= b; c -= rot(b,16); \ - a ^= c; a -= rot(c,4); \ - b ^= a; b -= rot(a,14); \ - c ^= b; c -= rot(b,24); \ -} /* -------------------------------------------------------------------- diff --git a/cbits/lookup3.h b/cbits/lookup3.h --- a/cbits/lookup3.h +++ b/cbits/lookup3.h @@ -4,6 +4,118 @@ #include #include +/* + * My best guess at if you are big-endian or little-endian. This may + * need adjustment. + */ +#if (defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \ + __BYTE_ORDER == __LITTLE_ENDIAN) || \ + (defined(i386) || defined(__i386__) || defined(__i486__) || \ + defined(__i586__) || defined(__i686__) || defined(vax) || defined(MIPSEL)) +# define HASH_LITTLE_ENDIAN 1 +# define HASH_BIG_ENDIAN 0 +#elif (defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && \ + __BYTE_ORDER == __BIG_ENDIAN) || \ + (defined(sparc) || defined(POWERPC) || defined(mc68000) || defined(sel)) +# define HASH_LITTLE_ENDIAN 0 +# define HASH_BIG_ENDIAN 1 +#else +# define HASH_LITTLE_ENDIAN 0 +# define HASH_BIG_ENDIAN 0 +#endif + +#define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) + +/* +------------------------------------------------------------------------------- +mix -- mix 3 32-bit values reversibly. + +This is reversible, so any information in (a,b,c) before mix() is +still in (a,b,c) after mix(). + +If four pairs of (a,b,c) inputs are run through mix(), or through +mix() in reverse, there are at least 32 bits of the output that +are sometimes the same for one pair and different for another pair. +This was tested for: +* pairs that differed by one bit, by two bits, in any combination + of top bits of (a,b,c), or in any combination of bottom bits of + (a,b,c). +* "differ" is defined as +, -, ^, or ~^. For + and -, I transformed + the output delta to a Gray code (a^(a>>1)) so a string of 1's (as + is commonly produced by subtraction) look like a single 1-bit + difference. +* the base values were pseudorandom, all zero but one bit set, or + all zero plus a counter that starts at zero. + +Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that +satisfy this are + 4 6 8 16 19 4 + 9 15 3 18 27 15 + 14 9 3 7 17 3 +Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing +for "differ" defined as + with a one-bit base and a two-bit delta. I +used http://burtleburtle.net/bob/hash/avalanche.html to choose +the operations, constants, and arrangements of the variables. + +This does not achieve avalanche. There are input bits of (a,b,c) +that fail to affect some output bits of (a,b,c), especially of a. The +most thoroughly mixed value is c, but it doesn't really even achieve +avalanche in c. + +This allows some parallelism. Read-after-writes are good at doubling +the number of bits affected, so the goal of mixing pulls in the opposite +direction as the goal of parallelism. I did what I could. Rotates +seem to cost as much as shifts on every machine I could lay my hands +on, and rotates are much kinder to the top and bottom bits, so I used +rotates. +------------------------------------------------------------------------------- +*/ +#define mix(a,b,c) \ +{ \ + a -= c; a ^= rot(c, 4); c += b; \ + b -= a; b ^= rot(a, 6); a += c; \ + c -= b; c ^= rot(b, 8); b += a; \ + a -= c; a ^= rot(c,16); c += b; \ + b -= a; b ^= rot(a,19); a += c; \ + c -= b; c ^= rot(b, 4); b += a; \ +} + +/* +------------------------------------------------------------------------------- +final -- final mixing of 3 32-bit values (a,b,c) into c + +Pairs of (a,b,c) values differing in only a few bits will usually +produce values of c that look totally different. This was tested for +* pairs that differed by one bit, by two bits, in any combination + of top bits of (a,b,c), or in any combination of bottom bits of + (a,b,c). +* "differ" is defined as +, -, ^, or ~^. For + and -, I transformed + the output delta to a Gray code (a^(a>>1)) so a string of 1's (as + is commonly produced by subtraction) look like a single 1-bit + difference. +* the base values were pseudorandom, all zero but one bit set, or + all zero plus a counter that starts at zero. + +These constants passed: + 14 11 25 16 4 14 24 + 12 14 25 16 4 14 24 +and these came close: + 4 8 15 26 3 22 24 + 10 8 15 26 3 22 24 + 11 8 15 26 3 22 24 +------------------------------------------------------------------------------- +*/ +#define final(a,b,c) \ +{ \ + c ^= b; c -= rot(b,14); \ + a ^= c; a -= rot(c,11); \ + b ^= a; b -= rot(a,25); \ + c ^= b; c -= rot(b,16); \ + a ^= c; a -= rot(c,4); \ + b ^= a; b -= rot(a,14); \ + c ^= b; c -= rot(b,24); \ +} + uint32_t _jenkins_hashword(const uint32_t *k, size_t length, uint32_t initval); uint32_t _jenkins_hashlittle(const void *key, size_t length, uint32_t initval); @@ -13,5 +125,224 @@ void _jenkins_hashlittle2(const void *key, size_t length, uint32_t *pc, uint32_t *pb); + +static inline void _jenkins_little2_begin(const uint32_t *pc, + const uint32_t *pb, + uint32_t st[3]) +{ + uint32_t a,b,c; + + /* Set up the internal state */ + a = b = c = 0xdeadbeef + *pc; + c += *pb; + + st[0] = a; + st[1] = b; + st[2] = c; +} + +static inline size_t _jenkins_little2_frag(const void *key, + size_t length, + uint32_t st[4], + size_t offset) +{ + const uint8_t *k = key; + size_t i; + + for (i = 0; i < length && offset < 12; i++, offset++) { + st[offset >> 2] += k[i] << (8 * (offset & 3)); + } + + if (offset == 12) { + uint32_t a = st[0], b = st[1], c = st[2]; + + mix(a,b,c); + + st[0] = a; + st[1] = b; + st[2] = c; + } + + return offset; +} + +static inline size_t _jenkins_little2_step(const void *key, + size_t length, + uint32_t st[3]) +{ + uint32_t a = st[0], b = st[1], c = st[2]; /* internal state */ + union { const void *ptr; size_t i; } u; /* needed for Mac Powerbook G4 */ + + u.ptr = key; + if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) { + const uint32_t *k = (const uint32_t *)key; /* read 32-bit chunks */ + + /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ + while (length >= 12) + { + a += k[0]; + b += k[1]; + c += k[2]; + mix(a,b,c); + length -= 12; + k += 3; + } + + /*----------------------------- handle the last (probably partial) block */ + /* + * "k[2]&0xffffff" actually reads beyond the end of the string, but + * then masks off the part it's not allowed to read. Because the + * string is aligned, the masked-off tail is in the same word as the + * rest of the string. Every machine with memory protection I've seen + * does it on word boundaries, so is OK with this. But VALGRIND will + * still catch it and complain. The masking trick does make the hash + * noticably faster for short strings (like English words). + */ +#ifndef VALGRIND + + switch(length) + { + case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; + case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break; + case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break; + case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break; + case 8 : b+=k[1]; a+=k[0]; break; + case 7 : b+=k[1]&0xffffff; a+=k[0]; break; + case 6 : b+=k[1]&0xffff; a+=k[0]; break; + case 5 : b+=k[1]&0xff; a+=k[0]; break; + case 4 : a+=k[0]; break; + case 3 : a+=k[0]&0xffffff; break; + case 2 : a+=k[0]&0xffff; break; + case 1 : a+=k[0]&0xff; break; + } + +#else /* make valgrind happy */ + + k8 = (const uint8_t *)k; + switch(length) + { + case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; + case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ + case 10: c+=((uint32_t)k8[9])<<8; /* fall through */ + case 9 : c+=k8[8]; /* fall through */ + case 8 : b+=k[1]; a+=k[0]; break; + case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ + case 6 : b+=((uint32_t)k8[5])<<8; /* fall through */ + case 5 : b+=k8[4]; /* fall through */ + case 4 : a+=k[0]; break; + case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ + case 2 : a+=((uint32_t)k8[1])<<8; /* fall through */ + case 1 : a+=k8[0]; break; + } + +#endif /* !valgrind */ + + } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) { + const uint16_t *k = (const uint16_t *)key; /* read 16-bit chunks */ + const uint8_t *k8; + + /*--------------- all but last block: aligned reads and different mixing */ + while (length >= 12) + { + a += k[0] + (((uint32_t)k[1])<<16); + b += k[2] + (((uint32_t)k[3])<<16); + c += k[4] + (((uint32_t)k[5])<<16); + mix(a,b,c); + length -= 12; + k += 6; + } + + /*----------------------------- handle the last (probably partial) block */ + k8 = (const uint8_t *)k; + switch(length) + { + case 12: c+=k[4]+(((uint32_t)k[5])<<16); + b+=k[2]+(((uint32_t)k[3])<<16); + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ + case 10: c+=k[4]; + b+=k[2]+(((uint32_t)k[3])<<16); + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 9 : c+=k8[8]; /* fall through */ + case 8 : b+=k[2]+(((uint32_t)k[3])<<16); + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ + case 6 : b+=k[2]; + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 5 : b+=k8[4]; /* fall through */ + case 4 : a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ + case 2 : a+=k[0]; + break; + case 1 : a+=k8[0]; + break; + } + + } else { /* need to read the key one byte at a time */ + const uint8_t *k = (const uint8_t *)key; + + /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ + while (length >= 12) + { + a += k[0]; + a += ((uint32_t)k[1])<<8; + a += ((uint32_t)k[2])<<16; + a += ((uint32_t)k[3])<<24; + b += k[4]; + b += ((uint32_t)k[5])<<8; + b += ((uint32_t)k[6])<<16; + b += ((uint32_t)k[7])<<24; + c += k[8]; + c += ((uint32_t)k[9])<<8; + c += ((uint32_t)k[10])<<16; + c += ((uint32_t)k[11])<<24; + mix(a,b,c); + length -= 12; + k += 12; + } + + /*-------------------------------- last block: affect all 32 bits of (c) */ + switch(length) /* all the case statements fall through */ + { + case 12: c+=((uint32_t)k[11])<<24; + case 11: c+=((uint32_t)k[10])<<16; + case 10: c+=((uint32_t)k[9])<<8; + case 9 : c+=k[8]; + case 8 : b+=((uint32_t)k[7])<<24; + case 7 : b+=((uint32_t)k[6])<<16; + case 6 : b+=((uint32_t)k[5])<<8; + case 5 : b+=k[4]; + case 4 : a+=((uint32_t)k[3])<<24; + case 3 : a+=((uint32_t)k[2])<<16; + case 2 : a+=((uint32_t)k[1])<<8; + case 1 : a+=k[0]; + break; + } + } + + st[0] = a; + st[1] = b; + st[2] = c; + + return length; +} + + +static inline void _jenkins_little2_end(int nonempty, + uint32_t *pc, + uint32_t *pb, + const uint32_t st[3]) +{ + uint32_t a = st[0], b = st[1], c = st[2]; + + if (nonempty) + final(a,b,c); + *pc=c; *pb=b; +} #endif /* _lookup3_h */ diff --git a/examples/Words.hs b/examples/Words.hs --- a/examples/Words.hs +++ b/examples/Words.hs @@ -2,11 +2,13 @@ -- builds a Bloom filter from a list of words, one per line, and -- queries it exhaustively. +module Main () where + import Control.Monad (forM_, mapM_) import Data.BloomFilter (Bloom, fromListB, elemB, lengthB) import Data.BloomFilter.Hash (cheapHashes) import Data.BloomFilter.Easy (easyList, suggestSizing) -import qualified Data.ByteString.Char8 as B +import qualified Data.ByteString.Lazy.Char8 as B import Data.Time.Clock (diffUTCTime, getCurrentTime) import System.Environment (getArgs)