rocksdb-haskell-jprupp 2.7.1 → 2.8.0
raw patch · 15 files changed
+4880/−1140 lines, 15 filesdep +filepathsetup-changedPVP ok
version bump matches the API change (PVP)
Dependencies added: filepath
API changes (from Hackage documentation)
+ Database.RocksDB: ChunkConfig :: !Int -> !Int -> !ByteString -> !Bool -> ChunkConfig
+ Database.RocksDB: Merge :: !ByteString -> !ByteString -> BatchOp
+ Database.RocksDB: MergeCF :: !ColumnFamily -> !ByteString -> !ByteString -> BatchOp
+ Database.RocksDB: MergeFunctions :: !String -> !FunPtr FullMerge -> !FunPtr PartialMerge -> MergeOperator
+ Database.RocksDB: PackedDel :: PackedOp
+ Database.RocksDB: PackedMerge :: PackedOp
+ Database.RocksDB: PackedPut :: PackedOp
+ Database.RocksDB: UInt64Add :: MergeOperator
+ Database.RocksDB: [chunkBackwards] :: ChunkConfig -> !Bool
+ Database.RocksDB: [chunkBytes] :: ChunkConfig -> !Int
+ Database.RocksDB: [chunkEntries] :: ChunkConfig -> !Int
+ Database.RocksDB: [chunkPrefix] :: ChunkConfig -> !ByteString
+ Database.RocksDB: [disableAutoCompactions] :: Config -> !Bool
+ Database.RocksDB: [fullMerge] :: MergeOperator -> !FunPtr FullMerge
+ Database.RocksDB: [maxBackgroundJobs] :: Config -> !Maybe Int
+ Database.RocksDB: [mergeName] :: MergeOperator -> !String
+ Database.RocksDB: [mergeOperator] :: Config -> !Maybe MergeOperator
+ Database.RocksDB: [partialMerge] :: MergeOperator -> !FunPtr PartialMerge
+ Database.RocksDB: [partitionIndexAndFilters] :: Config -> !Bool
+ Database.RocksDB: compactRange :: DB -> Maybe ColumnFamily -> Maybe ByteString -> Maybe ByteString -> IO ()
+ Database.RocksDB: data ChunkConfig
+ Database.RocksDB: data MergeOperator
+ Database.RocksDB: data PackedOp
+ Database.RocksDB: flush :: DB -> Maybe ColumnFamily -> IO ()
+ Database.RocksDB: iterChunk :: Iterator -> ChunkConfig -> IO ([(ByteString, ByteString)], Bool)
+ Database.RocksDB: iterEntries :: Iterator -> Int -> IO [(ByteString, ByteString)]
+ Database.RocksDB: iterEntriesBack :: Iterator -> Int -> IO [(ByteString, ByteString)]
+ Database.RocksDB: iterEntriesPrefix :: Iterator -> ByteString -> Int -> IO [(ByteString, ByteString)]
+ Database.RocksDB: iterEntriesPrefixBack :: Iterator -> ByteString -> Int -> IO [(ByteString, ByteString)]
+ Database.RocksDB: merge :: DB -> ByteString -> ByteString -> IO ()
+ Database.RocksDB: mergeCF :: DB -> ColumnFamily -> ByteString -> ByteString -> IO ()
+ Database.RocksDB: packOps :: [BatchOp] -> ByteString
+ Database.RocksDB: packedHeaderSize :: Int
+ Database.RocksDB: pokePackedHeader :: Ptr a -> Maybe ColumnFamily -> PackedOp -> Int -> Int -> IO ()
+ Database.RocksDB: type FullMerge = Ptr () -> CString -> CSize -> CString -> CSize -> Ptr CString -> Ptr CSize -> CInt -> Ptr CUChar -> Ptr CSize -> IO CString
+ Database.RocksDB: type PartialMerge = Ptr () -> CString -> CSize -> Ptr CString -> Ptr CSize -> CInt -> Ptr CUChar -> Ptr CSize -> IO CString
+ Database.RocksDB: writePacked :: DB -> [ByteString] -> IO ()
- Database.RocksDB: Config :: !Bool -> !Bool -> !Bool -> !Bool -> !Maybe Int -> !Maybe Int -> !Bool -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Bool -> !Maybe Int -> Config
+ Database.RocksDB: Config :: !Bool -> !Bool -> !Bool -> !Bool -> !Maybe Int -> !Maybe Int -> !Bool -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Bool -> !Bool -> !Maybe Int -> !Maybe Int -> !Maybe MergeOperator -> !Bool -> Config
Files
- CHANGELOG.md +135/−0
- README.md +316/−9
- Setup.hs +1/−0
- bench/Batch.hs +187/−0
- bench/Scan.hs +164/−0
- cbits/rocksdb_haskell.c +379/−0
- cbits/rocksdb_haskell.h +228/−0
- rocksdb-haskell-jprupp.cabal +43/−1
- src/Database/RocksDB.hs +8/−0
- src/Database/RocksDB/Base.hs +967/−551
- src/Database/RocksDB/C.hs +617/−258
- src/Database/RocksDB/Internal.hs +341/−160
- src/Database/RocksDB/Iterator.hs +422/−160
- test/Spec.hs +938/−1
- test/cbits/merge_test.c +134/−0
CHANGELOG.md view
@@ -1,3 +1,138 @@+## 2.8.0++ * `maxBackgroundJobs`, a new field of `Config`: how many threads RocksDB may+ use at once to flush memtables and to compact files, where its default is+ two, one of each. A database that has a backlog of compactions, after a+ bulk load for instance, works through it with one thread unless it is+ given more. More threads only help where the disk and the cores have room+ for them: a compaction that already waits for a slow disk is not made+ faster by a second one, and both take from the reads.+ * `partitionIndexAndFilters`, another new field of `Config`: the index and+ the filter of each table file are written in pieces of 4 KiB under a small+ index of their own. Each is otherwise one block that grows with the file,+ megabytes for a file of 64 MiB with a filter, and a database that has more+ of them than its block cache holds loads and drops them at nearly every+ lookup. Measured on a database of 1.6 TB with a cache of 1 GiB: the cache+ held the index and filter of 400 of 32,000 files and three megabytes of+ data, and a lookup took 0.8 ms of processor time on an idle disk. With+ pieces, a lookup loads the two that it needs. It applies to files written+ from then on; existing files keep their layout until compacted.+ * `compactRange` writes again every table file of a range of keys, those of+ the last level included, so that the files a database already has get the+ layout of its present `Config`: a Bloom filter, or+ `partitionIndexAndFilters`. RocksDB's own compactions rarely reach the+ last level, where most of the data is.+ * Many entries of an iterator in one foreign call. `iterNext` and `iterPrev`+ can wait for the disk, so each is a call for which the Haskell thread+ gives up its capability and has to get one back, and a scan made one for+ every entry. `iterChunk` walks the iterator in C, in a small function+ that now comes with the package (`cbits/rocksdb_haskell.c`), and returns+ up to 256 entries or 1 MiB from one call, forwards or backwards, within a+ prefix if one is given, with a flag that tells whether more follow. A+ `ChunkConfig` sets those limits. `iterEntries`, `iterEntriesBack`,+ `iterEntriesPrefix` and `iterEntriesPrefixBack` are short forms for a+ known number of entries. All of them leave the iterator on the first+ entry that they did not return, respect the bounds of an `IterConfig`,+ and throw if the iterator stopped at an error. The single steps are+ unchanged.++ Seconds for every thread to read 200,000 entries of 52 bytes, on four+ capabilities and four cores (`stack bench`):++ | Scan | 1 thread | 16 threads |+ | ------------------------------------- | -------- | ---------- |+ | Single steps, loop not tail recursive | 0.78 | 3.9 |+ | Single steps, loop tail recursive | 0.090 | 0.50 |+ | Chunks of 256 entries | 0.048 | 0.25 |++ With all 24 hardware threads of the machine free, the sixteen threads+ took 4.4, 0.83 and 0.12 seconds: threads inside the call hold no+ capability, so chunks gain more where there are more cores than+ capabilities. The first row is a loop that returns its list with+ `(entry :) <$> go`; the run-time system walks the stack at every such+ call, and that loop leaves a frame on it for every entry.+ * A large batch in a few foreign calls. `write` hands RocksDB the+ operations of a batch one call at a time, each a call for which the+ Haskell thread gives up its capability and has to get one back. For the+ 65,000 keys of a Bitcoin block that was more than half of the time of+ the batch, and nearly all of it in a program whose other threads were+ busy. `writePacked` takes the batch as chunks of packed records, and a+ second function of `cbits/rocksdb_haskell.c` adds the operations of a+ chunk to the batch in one call. The batch is then written as before:+ atomically, with one write, however many chunks it came in.++ `packOps` packs a list of `BatchOp` into a chunk. A program that+ serialises its keys and values itself can write records straight into+ its own buffers instead, with no `ByteString` for each: it leaves+ `packedHeaderSize` bytes, serialises the key and the value after them,+ and fills in the header with `pokePackedHeader` once the lengths are+ known. Records are not aligned, so the next one starts where the last+ one ended.++ A chunk is checked in C before anything of it is used. One that is cut+ short, that has a length reaching past its end, an operation that does+ not exist or a delete with a value, makes `writePacked` throw, with+ nothing written.++ A record can also ask for a merge (`PackedMerge`), which is handed to+ `rocksdb_writebatch_merge`. The binding still has no way to give a+ column family a merge operator, so RocksDB refuses such a batch; the+ format has it so that it need not change later.++ The format is this package's own. It is not RocksDB's internal encoding+ of a batch (`rocksdb_writebatch_create_from`), which would save a copy+ and tie the binding to the insides of RocksDB.++ Milliseconds to write a batch of 65,000 operations and 6 MB over four+ column families, on four capabilities and four cores+ (`stack bench rocksdb-haskell-jprupp:bench:batch`):++ | Way to write the batch | Alone | Next to 8 busy threads |+ | ----------------------------------------- | ----- | ---------------------- |+ | `write`: a call for each operation | 48 | 420 |+ | `writePacked`, with `packOps` in the time | 28 | 115 |+ | `writePacked`, chunks packed beforehand | 22 | 28 |++ All three end with the same write, of about 20 ms. Alone, the calls of+ `write` cost 400 ns an operation. Next to threads that compute, the+ thread waits for a capability after each of its 65,000 calls. With all+ 24 hardware threads of the machine free, that took `write` 1,160 ms,+ and packed chunks 31.+ * Merge records and merge operators. A merge record is a third kind of+ write, next to a put and a delete: it says how a value changes, and+ RocksDB stores it without reading the value. The merge operator of the+ column family works out the value when the key is read, and when a+ compaction brings the records and the value together. A program that+ adds to counters or to lists by the thousand no longer reads each one+ first.+ - `mergeOperator`, a new field of `Config`, gives a column family its+ operator, or the default one when it is in the `Config` of the+ database. `UInt64Add` is RocksDB's own, for counters. `MergeFunctions`+ makes one of a name and the addresses of two C functions, a full merge+ and a partial merge, with the types `FullMerge` and `PartialMerge`.+ They are C functions because RocksDB calls them from its own threads;+ `cbits/rocksdb_haskell.h` says what they must do, and a third function+ of `cbits/rocksdb_haskell.c` makes the operator around them.+ - `Merge` and `MergeCF` are new constructors of `BatchOp`, written by+ `write` and packed by `packOps` as `PackedMerge` records. `merge` and+ `mergeCF` write one record. Code that matches on every `BatchOp` has+ two more cases.+ - `get`, `multiGet`, iterators and chunks return merged values, and+ need no change.+ - A column family with merge records must be given its operator by+ every program that opens the database. RocksDB 11.1 opens it without:+ it then drops the merge records that are still in the write-ahead+ log, with no error, and a read of a key with a value under merge+ records crashes the process. The README has the whole table.+ * `flush` writes the memtable of a column family to a table file and waits+ for it, and `disableAutoCompactions`, a new field of `Config`, stops+ RocksDB from compacting a column family in the background. They are for+ tests and benchmarks that need data in table files, a file for each+ flush, and for bulk loads that end with `compactRange`.+ * The `Config` record has new fields, so code that builds one without `def`+ has to set them; code that starts from `def` needs no change. This is why+ the version is 2.8.0 and not 2.7.2.+ ## 2.7.1 * Documentation and tests only; the library behaves as in 2.7.0.
README.md view
@@ -68,7 +68,10 @@ - **Stay close to the C API.** Functions map nearly one to one to RocksDB's C functions, so RocksDB's own documentation and tuning guides apply directly. Conveniences such as typed keys live in separate libraries- ([rocksdb-query](https://github.com/jprupp/rocksdb-query)).+ ([rocksdb-query](https://github.com/jprupp/rocksdb-query)). The two+ exceptions are `iterChunk`, which walks an iterator, and `writePacked`,+ which fills a batch, each in a C function of this package: the cost they+ remove is that of calling C from Haskell. - **Scope every resource.** Databases, snapshots and iterators are C objects that must be freed once and never touched again. `withDB`, `withSnapshot` and `withIter` free them when their block ends, even on an exception.@@ -88,20 +91,25 @@ | -------------- | ------------------------------------------------------------------------------ | | Basic access | `get`, `put`, `delete` on strict `ByteString` keys and values | | Batches | Atomic `write` of many puts and deletes, across column families |+| Packed batches | A batch of any size handed to RocksDB in a few foreign calls, from your buffers |+| Merges | Change a value without reading it: merge records, and a merge operator in C | | Multi-get | `multiGet` reads many keys in one call to RocksDB's `MultiGet` | | Column families| Open a database with several independent key spaces; add new ones on open | | Snapshots | Consistent point-in-time reads across many queries and iterators | | Iterators | Ordered scans, forwards and backwards, optionally pinned to a snapshot | | Bounds | Iterators that stop at a key or at the end of a prefix, without reading past it |+| Chunks | Hundreds of entries from one foreign call, in either direction, within a prefix | | Tuning | Bloom filters, fixed-length prefix extractor, shared LRU block cache | | Limits | Memtable and WAL size limits, open file limit | | Administration | Properties (memory use, statistics, SST files), approximate sizes, destroy, repair | ### What is not included -- No transactions (`TransactionDB`, `OptimisticTransactionDB`), merge- operators, TTL databases, backups or checkpoints.-- No manual compaction, `DeleteRange` or `SingleDelete`.+- No transactions (`TransactionDB`, `OptimisticTransactionDB`), TTL+ databases, backups or checkpoints.+- No merge operators written in Haskell: RocksDB calls a merge operator+ from its own threads, so it is a C function.+- No `DeleteRange` or `SingleDelete`. - No custom comparators: keys always sort by their bytes. Pull requests that add bindings for these are welcome.@@ -185,7 +193,7 @@ ```yaml extra-deps:- - rocksdb-haskell-jprupp-2.7.1+ - rocksdb-haskell-jprupp-2.8.0 ``` With `stack --nix`, add RocksDB to the Nix packages:@@ -288,6 +296,9 @@ go [] ``` +A list of thousands of events is read faster in chunks: see+[Reading many entries in one call](#reading-many-entries-in-one-call).+ ### 4. Read consistently with a snapshot If another thread logs events while you read the counter and then the events,@@ -464,6 +475,257 @@ iterator together with the action that destroys it, which also frees the bounds: RocksDB keeps pointers to them for as long as the iterator lives. +### Reading many entries in one call++`iterNext` may wait for the disk, so it is a foreign call of the kind that+lets other Haskell threads run meanwhile. For that, the thread gives up its+capability and has to get one back when the call returns. A scan that moves+one step at a time pays this for every entry, and under load, when more+threads want to run than there are capabilities, it waits each time.++`iterChunk` walks the iterator in C and returns up to some hundreds of+entries from one call, with a flag that tells whether more follow:++```haskell+-- every key that starts with "user:", in chunks+withIter db $ \it -> do+ iterSeek it "user:"+ let go acc = do+ (entries, more) <- iterChunk it def {chunkPrefix = "user:"}+ let acc' = entries : acc+ if more then go acc' else return (concat (reverse acc'))+ go []+```++The `ChunkConfig` says what one call reads:++| Field | Default | What it does |+| ---------------- | ------- | ------------------------------------------------------------------- |+| `chunkEntries` | 256 | The most entries to read |+| `chunkBytes` | 1 MiB | Stop before keys and values take more; one entry is always read |+| `chunkPrefix` | empty | Stop at the first key that does not start with it |+| `chunkBackwards` | `False` | Go backwards, as `iterPrev` does |++The iterator is left on the first entry that was not returned, so the next+call continues there. A bound of the iterator, from `withIterConfig`, ends a+chunk as it ends a scan by single steps. And `iterChunk` throws if the+iterator stopped at an error, so there is no `iterGetError` to remember.++For a known number of entries there are short forms, which return just the+list: `iterEntries it n`, `iterEntriesBack it n`, `iterEntriesPrefix it+prefix n` and `iterEntriesPrefixBack it prefix n`. They have no limit of+bytes, so a list shorter than `n` means that the scan is over.++```haskell+-- a page of twenty users, starting after the last one seen+iterSeek it (lastSeen <> "\0")+page <- iterEntriesPrefix it "user:" 20+```++What it saves, in this package's benchmark (`stack bench`): seconds for+every thread to read 200,000 entries of 52 bytes, on four capabilities and+four cores.++| Scan | 1 thread | 16 threads |+| -------------------------------------- | -------- | ---------- |+| Single steps, loop not tail recursive | 0.78 | 3.9 |+| Single steps, loop tail recursive | 0.090 | 0.50 |+| Chunks of 256 entries | 0.048 | 0.25 |++The first row is a loop that builds its list with `(entry :) <$> go`. The+run-time system walks the stack of a thread at every foreign call that may+block, and such a loop leaves a frame there for every entry read so far.+Whatever you read with, keep the result in an argument of the loop.++With more cores than capabilities the chunks gain more, because a thread+that is inside the call holds no capability: with twenty more cores free,+the sixteen threads took 0.12 seconds in chunks and 0.83 in single steps.++Keep to single steps where each move depends on the entry just read, or+where one entry is all you want: `iterEntry` after a seek reads it without+moving the iterator.++### Writing a large batch in a few calls++`write` hands a batch to RocksDB one operation at a time, and each is a+foreign call of the kind that lets other Haskell threads run meanwhile. For+a batch of a few operations that is nothing. For tens of thousands it adds+up, and so do the `ByteString`s, one for each key and each value.++A packed batch is the same operations written one after another into a few+large buffers, the chunks. `writePacked` gives each chunk to a C function of+this package, which adds its operations to the batch, and then writes the+batch as `write` does: atomically, with one write.++```haskell+-- the same as: write db ops+writePacked db [packOps ops]+```++`packOps` packs a list of `BatchOp` into one chunk. It is the simple way,+and it copies every key and value once more. A program that serialises its+keys and values itself can skip the list, the `ByteString`s and the copy,+and write records straight into a buffer of its own:++```haskell+-- one record at a pointer: room for the header, the key, the value, and+-- then the header, now that the lengths are known+packRecord :: Ptr Word8 -> ColumnFamily -> IO (Ptr Word8)+packRecord ptr cf = do+ let body = ptr `plusPtr` packedHeaderSize+ keyLen <- serialiseKey body+ valueLen <- serialiseValue (body `plusPtr` keyLen)+ pokePackedHeader ptr (Just cf) PackedPut keyLen valueLen+ return (body `plusPtr` (keyLen + valueLen))+```++Here `serialiseKey` and `serialiseValue` stand for whatever writes your+bytes at a pointer and says how many. The next record goes where this one+ended: records need no alignment and nothing goes between them. A chunk is+then a `ByteString` over the part of the buffer that holds whole records.++What to know:++- **All the chunks are one batch.** `writePacked db chunks` applies the+ records of every chunk in order, atomically, with one write. Chunks only+ decide how much RocksDB is handed in one call: a few hundred kilobytes is+ plenty. A program that packs on several cores gives the chunks of each+ core, one core after another.+- **A chunk holds whole records.** A record cannot start in one chunk and+ end in the next.+- **A chunk is checked before it is used.** One that is cut short, or whose+ lengths say more than it holds, is refused with an exception, and nothing+ of the batch is written. The check cannot cover the column family handle+ of a record.+- **A chunk is only good here and now.** It holds column family handles,+ which are pointers of this process and of the open database.+- **A record can be a merge.** `PackedMerge` adds a merge record, as+ `Merge` does in a `BatchOp`, for a column family that has a merge+ operator: see [Merge records](#merge-records-change-a-value-without-reading-it).++The layout of a record, for a program that writes them by other means, is+in `cbits/rocksdb_haskell.h`. It is this package's own format, and what a+chunk becomes is a series of calls to RocksDB's C API. It is not RocksDB's+internal encoding of a batch, which a binding should not depend on.++What it saves, in this package's benchmark+(`stack bench rocksdb-haskell-jprupp:bench:batch`): milliseconds to write a+batch of 65,000 operations and 6 MB over four column families, on four+capabilities and four cores, alone and next to eight threads that compute.++| Way to write the batch | Alone | Next to busy threads |+| ----------------------------------------- | ----- | -------------------- |+| `write`: a call for each operation | 48 | 420 |+| `writePacked`, with `packOps` in the time | 28 | 115 |+| `writePacked`, chunks packed beforehand | 22 | 28 |++All three end with the same write, which takes about 20 ms here: the rest+is the handing over. Alone, a thread gets its capability back at once+after each call, and still pays 400 ns an operation. Next to threads that+want the capabilities too, as in a server under load, it waits for one+after every call. `packOps` is work in Haskell, which has to share the+capabilities with those threads like any other.++With all 24 hardware threads of the machine free, the batch took `write`+1,160 ms next to the busy threads, and 31 ms from packed chunks.++Keep to `write` for small batches: it is as fast there, and simpler.++### Merge records: change a value without reading it++A put replaces the value of a key. A program that wants to change a value,+to add to a counter or to add an entry to a list, has to read it first.+For a batch of thousands of such changes those reads are most of the work,+and each may wait for the disk.++A **merge record** is a third kind of write, next to a put and a delete. It+says how the value changes, not what it becomes: "add 5", "add this+entry". RocksDB stores the record as it is and reads nothing.++The work is done later by the **merge operator** of the column family: a+function that takes the value that the key had and the records written+since, oldest first, and gives the value that the key has now. RocksDB+calls it when the key is read, and when a compaction brings the records+and the value together. After that the key holds a plain value again.++```haskell+-- a column family of counters, with RocksDB's own operator for them+let counters = def {mergeOperator = Just UInt64Add}+withDBCF "/tmp/example-merge" def {createIfMissing = True} [("hits", counters)] $ \db -> do+ let [hits] = columnFamilies db+ five = BS.pack [5, 0, 0, 0, 0, 0, 0, 0] -- eight bytes, least significant first+ mergeCF db hits "home" five -- no read+ mergeCF db hits "home" five+ getCF db hits "home" >>= print -- Just "\n\NUL\NUL\NUL\NUL\NUL\NUL\NUL": ten+ write db [MergeCF hits "home" five, MergeCF hits "about" five]+```++What to know:++- **Readers see values.** `get`, `multiGet` and iterators return merged+ values, in a snapshot too. For a key with records pending they pay for+ the merge, and for a look into each table file that holds one of the+ records.+- **A put or a delete hides the records before it**, like any older value.+ A record written after a delete, or to a key that never had a value, is+ merged with no value: the operator decides what that means.+- **The operator is set when the database is opened**, with the+ `mergeOperator` field of the `Config` of the column family. The one in+ the `Config` of the database is for the default column family.+- **`UInt64Add`** is RocksDB's own: values and records are unsigned 64-bit+ numbers in eight bytes, least significant first, added modulo 2^64.+- **`MergeFunctions`** makes an operator of a name and two C functions: one+ that works out a value (the full merge), and one that makes a single+ record of several (the partial merge), which RocksDB uses to shorten a+ list of records before it has the value. They are C because RocksDB+ calls them from its own threads, during reads, flushes and compactions.++```c+/* my_merge.c, listed under c-sources of your package */+char *my_full_merge(void *state, const char *key, size_t key_length,+ const char *existing_value, size_t existing_value_length,+ const char *const *operands_list,+ const size_t *operands_list_length, int num_operands,+ unsigned char *success, size_t *new_value_length);+char *my_partial_merge(void *state, const char *key, size_t key_length,+ const char *const *operands_list,+ const size_t *operands_list_length, int num_operands,+ unsigned char *success, size_t *new_value_length);+```++```haskell+foreign import ccall "&my_full_merge" myFullMerge :: FunPtr FullMerge+foreign import ccall "&my_partial_merge" myPartialMerge :: FunPtr PartialMerge++mine :: Config+mine = def {mergeOperator = Just (MergeFunctions "my-merge" myFullMerge myPartialMerge)}+```++A function returns the new bytes in memory from `malloc`, which RocksDB+frees, with their length in `*new_value_length` and 1 in `*success`.+`existing_value` is `NULL` when the key has no value. To fail, it stores 0+in both and returns `NULL`: the read of that key then throws. A partial+merge may always fail, and RocksDB then keeps the records as they are.+`cbits/rocksdb_haskell.h` has the two types and the rules in full, and+`test/cbits/merge_test.c` is a small operator to start from.++**Every program that opens the database must set the same operator.**+RocksDB opens a column family that has merge records without one, and+this is what version 11.1 was seen to do then:++| Where the merge records are | What happens without the operator |+| --------------------------- | -------------------------------------------------------------- |+| In the write-ahead log | Dropped when the database is opened, without an error: lost |+| In table files | Kept, also through a compaction, but they cannot be read |+| Read: records and no value | `get` throws; an iterator stops there with an error |+| Read: records over a value | `get` and `multiGet` crash the process |+| Writing a merge record | Refused with an error |++So when a program starts to write merge records to a database that older+versions of it can open, make sure that they no longer can. A new column+family is enough: RocksDB refuses to open a database without all its+column families, and does so before it reads the write-ahead log.+ ### Properties and maintenance ```haskell@@ -491,12 +753,18 @@ | `dbWriteBufferSize` | database | Bytes of all memtables together (default unlimited) | | `blockCacheSize` | database | Bytes of an LRU block cache shared by all column families | | `maxTotalWalSize` | database | Bytes of write-ahead log that trigger flushes |+| `maxBackgroundJobs` | database | Threads for flushes and compactions together (RocksDB default 2) | | `writeBufferSize` | column family | Bytes per memtable (RocksDB default 64 MiB) | | `bloomFilter` | column family | Full Bloom filter with 10 bits per key | | `prefixLength` | column family | Length of a fixed key prefix for the prefix extractor | | `cacheIndexAndFilterBlocks` | column family | Keep index and filter blocks in the block cache |+| `partitionIndexAndFilters` | column family | Write index and filter in 4 KiB pieces, loaded as needed |+| `mergeOperator` | column family | How merge records are combined with a value (default: none) |+| `disableAutoCompactions` | column family | No background compaction: for bulk loads, tests and benchmarks | Database-wide fields are ignored in the configuration of a column family.+The column family fields of the database's own `Config` are for its default+column family. ### Choosing a configuration @@ -534,17 +802,25 @@ | Part | Contents | | ------------------------ | -------------------------------------------------------------- | | Types | `DB`, `Config`, `BatchOp`, `ColumnFamily`, `Snapshot`, `Iterator`, `Property`, `Range` |+| Merge operators | `MergeOperator`, `FullMerge`, `PartialMerge` | | Opening | `withDB`, `withDBCF` | | Reading and writing | `get`, `getCF`, `multiGet`, `multiGetCF`, `put`, `putCF`, `delete`, `deleteCF`, `write` |+| Merging | `merge`, `mergeCF`, and `Merge`, `MergeCF` in a `BatchOp` |+| Packed batches | `writePacked`, `packOps`, `PackedOp`, `packedHeaderSize`, `pokePackedHeader` | | Snapshots | `withSnapshot`, `createSnapshot`, `releaseSnapshot` | | Creating iterators | `withIter`, `withIterCF`, `withIterSnap`, `withIterSnapCF` | | Bounded iterators | `IterConfig`, `withIterConfig`, `createIteratorConfig`, `prefixUpperBound` | | Moving iterators | `iterSeek`, `iterFirst`, `iterLast`, `iterNext`, `iterPrev` | | Reading from iterators | `iterKey`, `iterValue`, `iterEntry`, `iterValid`, `iterGetError` |-| Administration | `getProperty`, `getPropertyCF`, `approximateSize`, `destroy`, `repair` |+| Reading in chunks | `iterChunk`, `ChunkConfig` |+| Reading a number of entries | `iterEntries`, `iterEntriesBack`, `iterEntriesPrefix`, `iterEntriesPrefixBack` |+| Administration | `getProperty`, `getPropertyCF`, `approximateSize`, `compactRange`, `destroy`, `repair` |+| Flushing | `flush` | The FFI bindings (`Database.RocksDB.C`) and helpers (`Database.RocksDB.Internal`)-are hidden modules.+are hidden modules. `cbits/rocksdb_haskell.c` is the one piece of C in the+package: the functions behind `iterChunk` and `writePacked`, and the one+that makes a merge operator of two functions. ## Things to watch out for @@ -576,7 +852,29 @@ with it, not with `destroyIterator`, which would leave its bounds and read options allocated. - Calls into RocksDB are `safe` foreign calls, so they do not block other- Haskell threads, but each one costs a little more than an `unsafe` call.+ Haskell threads, but each one costs more than an `unsafe` call, and much+ more when threads compete for capabilities. A scan by single steps makes+ one for every entry: read ranges with `iterChunk`. A `write` makes one+ for every operation: write batches of thousands with `writePacked`.+- A chunk for `writePacked` is checked for its lengths, not for its column+ family handles. A handle that is not of the open database crashes the+ process, as it does in `PutCF`.+- A loop over an iterator that is not tail recursive gets slower with every+ entry, because the run-time system walks the stack at each foreign call.+ Keep what has been read in an argument of the loop.+- `iterChunk` and the `iterEntries` functions leave the iterator on the+ first entry that they did not return, not on the last one that they did.+- An `iterChunk` with a `chunkPrefix` still moves the iterator to the first+ key outside the prefix, as a loop written by hand does. Where deleted keys+ follow the range, the iterator needs a bound as well.+- A column family with merge records must get its `mergeOperator` from+ every program that opens the database. Without it RocksDB drops the+ records that are still in its write-ahead log, and a read of a key that+ has a value under merge records crashes the process: see+ [Merge records](#merge-records-change-a-value-without-reading-it).+- A merge operator is called from RocksDB's threads, several at a time. It+ must be a C function that keeps no state between calls, and must return+ memory from `malloc`. - Releases before 2.6.2 built no Bloom filters on RocksDB 6.24 or later, whatever `bloomFilter` said: they passed the bits per key as an integer to a function that had come to take a `double`. A database written by one of them@@ -598,11 +896,19 @@ stack build # build the library stack test # run the tests+stack bench # time scans and large batches stack haddock --no-haddock-deps # build the API documentation ``` With `stack --nix`, the `nix` section of `stack.yaml` provides RocksDB. +`bench/Scan.hs` times a scan of 200,000 entries by single steps and in+chunks, from one thread and from sixteen on four capabilities.+`bench/Batch.hs` times a batch of 65,000 operations written with `write`+and with `writePacked`, alone and next to threads that compute. To see what+a machine with four cores does, confine them: `taskset -c 0-3 stack bench`.+To run one of the two: `stack bench rocksdb-haskell-jprupp:bench:batch`.+ ### Nix ```sh@@ -629,7 +935,8 @@ - **Properties** apply random writes to the database and to a `Map`, and require that every way of reading the database agrees with the map: point reads, `multiGet`, iteration in both directions, seeks, snapshots and- column families. The keys are short and drawn from the bytes a binding is+ column families. Chunks are held to what single steps read, with random+ limits, prefixes and bounds. The keys are short and drawn from the bytes a binding is most likely to mishandle, zero and 255, so that operations often meet on the same key.
Setup.hs view
@@ -1,2 +1,3 @@ import Distribution.Simple+ main = defaultMain
+ bench/Batch.hs view
@@ -0,0 +1,187 @@+{-# LANGUAGE ImportQualifiedPost #-}+{-# LANGUAGE OverloadedStrings #-}++-- |+-- What it costs to hand a large batch to RocksDB with one foreign call for+-- each operation, and with one for each chunk of packed operations.+--+-- The question it answers: 'write' gives RocksDB the operations of a batch+-- one call at a time, and a Haskell program pays for every foreign call+-- that may block, since the thread gives up its capability for the call and+-- has to get one back afterwards. How much of the time of a large batch is+-- that, and how much is left when 'writePacked' makes the call once for a+-- few thousand operations?+--+-- A batch is 'operationCount' operations over four column families, about+-- what the import of a Bitcoin block writes. It is written three ways:+--+-- * with 'write', from a list of t'BatchOp';+--+-- * with 'writePacked', from the same list packed by 'packOps' into chunks+-- of 'chunkOperations', the packing included in the time;+--+-- * with 'writePacked', from chunks that were packed beforehand: what is+-- left for a program that serialises its keys and values straight into+-- chunks, and so has no list to pack.+--+-- All three end with the same @rocksdb_write@, which adds the operations to+-- the memtables and takes most of the time. The difference between the+-- rows is what the hand-over costs.+--+-- The answer depends on whether other threads want the capabilities. A+-- thread alone gets its own back at once after a call. In a server under+-- load it finds it taken, and waits. So every way is timed alone and next+-- to 'busyThreads' threads that compute, on four capabilities whatever the+-- machine has: the suite is built with @-N4@.+--+-- Run it with @stack bench rocksdb-haskell-jprupp:bench:batch@, and on+-- four cores with @taskset -c 0-3@ before it. The figures of the CHANGELOG+-- and of the README come from it.+module Main (main) where++import Control.Concurrent (getNumCapabilities)+import Control.Concurrent.Async (withAsync)+import Control.Exception (evaluate)+import Control.Monad (forM, forM_, unless)+import Data.ByteString (ByteString)+import Data.ByteString qualified as B+import Data.ByteString.Char8 qualified as C+import Data.Default (def)+import Data.IORef (atomicModifyIORef', newIORef)+import Data.List (sort)+import Database.RocksDB+import GHC.Clock (getMonotonicTime)+import GHC.Conc (getNumProcessors)+import System.IO.Temp (withSystemTempDirectory)+import System.Mem (performGC)+import Text.Printf (printf)++-- | Operations in a batch.+operationCount :: Int+operationCount = 65000++-- | Operations in a chunk of a packed batch: with keys of 34 bytes, values+-- of 66 and the header of a record, a chunk of about 256 KiB.+chunkOperations :: Int+chunkOperations = 2400++-- | Times each measurement is taken. The median is reported: other programs+-- on the machine make single runs slow, never fast.+runs :: Int+runs = 15++-- | Threads that compute while a batch is written, in the second+-- measurement of each way.+busyThreads :: Int+busyThreads = 8++-- | A key of 34 bytes that is new in every batch: the number of the batch+-- and that of the operation.+key :: Int -> Int -> ByteString+key batch i = C.pack (printf "key:%010d:%019d" batch i)++-- | A value of 66 bytes.+value :: Int -> ByteString+value = C.pack . printf "value:%060d"++-- | The operations of a batch: one in eight deletes a key, the others set+-- one, in the four column families in turn.+operations :: [ColumnFamily] -> Int -> [BatchOp]+operations families batch = zipWith op (cycle families) [1 .. operationCount]+ where+ op cf i+ | i `mod` 8 == 0 = DelCF cf (key batch i)+ | otherwise = PutCF cf (key batch i) (value i)++-- | A list in pieces of a length, the last one shorter.+groupsOf :: Int -> [a] -> [[a]]+groupsOf _ [] = []+groupsOf n xs = let (a, b) = splitAt n xs in a : groupsOf n b++-- | A way to write the operations of a batch: what has to be ready before+-- the clock starts, and the write that is timed.+data Way a = Way+ { name :: String,+ prepare :: [BatchOp] -> IO a,+ timed :: DB -> a -> IO ()+ }++-- | Every operation with its key and value in memory.+ready :: [BatchOp] -> IO [BatchOp]+ready ops = do+ forM_ ops $ \op -> evaluate $ case op of+ Put k v -> B.length k + B.length v+ PutCF _ k v -> B.length k + B.length v+ Del k -> B.length k+ DelCF _ k -> B.length k+ return ops++-- | The seconds that a way takes to write a batch of new keys.+timeWrite :: DB -> IO Int -> Way a -> IO Double+timeWrite db next way = do+ batch <- next+ input <- prepare way =<< ready (operations (columnFamilies db) batch)+ performGC+ t0 <- getMonotonicTime+ timed way db input+ t1 <- getMonotonicTime+ return (t1 - t0)++-- | Run an action next to so many threads that compute and allocate, as+-- the threads of a server do, and stop them afterwards.+besideBusy :: Int -> IO a -> IO a+besideBusy 0 action = action+besideBusy n action = withAsync (spin 0) (const (besideBusy (n - 1) action))+ where+ -- it allocates, so that it can be stopped, and depends on its+ -- argument, so that it is not worked out once and for all+ spin :: Int -> IO ()+ spin i = do+ _ <- evaluate (sum (map fromEnum (show [i .. i + 2000])))+ spin (i + 1)++-- | The median of 'runs' measurements.+median :: IO Double -> IO Double+median measure = do+ times <- forM [1 .. runs] (const measure)+ return (sort times !! (runs `div` 2))++main :: IO ()+main = withSystemTempDirectory "rocksdb-bench" $ \dir -> do+ let config = def {createIfMissing = True, createMissingColumnFamilies = True}+ families = [(n, config) | n <- ["one", "two", "three", "four"]]+ withDBCF dir config families $ \db -> do+ caps <- getNumCapabilities+ cores <- getNumProcessors+ counter <- newIORef 0+ let next = atomicModifyIORef' counter (\n -> (n + 1, n))+ row :: Way a -> IO ()+ row way = do+ alone <- median (timeWrite db next way)+ busy <- median (besideBusy busyThreads (timeWrite db next way))+ printf "%-44s %10.1f %10.1f\n" (name way) (1000 * alone) (1000 * busy)+ printf "batches of %d operations, about %d bytes of keys and values\n" operationCount size+ printf "%d capabilities, %d cores, median of %d runs, milliseconds a batch\n" caps cores runs+ printf "%-44s %10s %10s\n" (s "") (s "alone") (s (show busyThreads <> " busy"))+ -- once unmeasured, so that the first row does not pay for the start+ _ <- timeWrite db next oneCall+ unless (caps == 4) $ putStrLn "note: the figures of the documentation are for four capabilities"+ row oneCall+ row packedList+ row packedAhead+ where+ s :: String -> String+ s = id+ -- one operation in eight is a delete, with a key of 34 bytes and no+ -- value of 66+ size = operationCount * 100 - operationCount `div` 8 * 66+ oneCall = Way "write: a call for each operation" return write+ packedList =+ Way "writePacked, packOps included" return $ \db ops ->+ writePacked db (map packOps (groupsOf chunkOperations ops))+ packedAhead =+ Way "writePacked, chunks packed beforehand" pack writePacked+ pack ops = do+ let chunks = map packOps (groupsOf chunkOperations ops)+ forM_ chunks (evaluate . B.length)+ return chunks
+ bench/Scan.hs view
@@ -0,0 +1,164 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE ImportQualifiedPost #-}+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE OverloadedStrings #-}++-- |+-- What a scan costs with one foreign call for each entry, and with one for+-- each chunk of entries.+--+-- The question it answers: a Haskell program pays for every foreign call+-- that may block, since the thread gives up its capability for the call and+-- has to get one back afterwards. How much of the time of a scan is that,+-- and how much is left when 'iterChunk' makes the call once for a few+-- hundred entries?+--+-- The answer depends on how many threads want a capability. A thread alone+-- gets its own back at once. Sixteen threads on four capabilities, as in a+-- server under load, each find theirs taken when the call returns, and wait.+-- So every scan is timed with one thread and with sixteen, on four+-- capabilities, whatever the machine has: the suite is built with @-N4@.+--+-- It also times the loop that is not tail recursive, @(entry :) \<$\> go@,+-- because the run-time system looks at the stack of a thread at every such+-- call, and that loop leaves a frame on it for every entry read so far.+--+-- Each scan reads every entry of a database of 'entryCount' small entries+-- from a table file, and adds up their sizes. Run it with @stack bench@; to+-- run some of the scans only, name them: @stack bench --ba chunks@. The figures of+-- the CHANGELOG and of the README come from it.+module Main (main) where++import Control.Concurrent (getNumCapabilities)+import Control.Concurrent.Async (forConcurrently)+import Control.Monad (forM, forM_, unless)+import Data.ByteString (ByteString)+import Data.ByteString qualified as B+import Data.ByteString.Char8 qualified as C+import Data.Default (def)+import Data.List (isInfixOf, sort)+import Database.RocksDB+import GHC.Clock (getMonotonicTime)+import GHC.Conc (getNumProcessors)+import System.Environment (getArgs)+import System.IO.Temp (withSystemTempDirectory)+import System.Mem (performGC)+import Text.Printf (printf)++-- | Entries in the database, all of which every scan reads.+entryCount :: Int+entryCount = 200000++-- | Times each measurement is taken. The median is reported: other programs+-- on the machine make single runs slow, never fast.+runs :: Int+runs = 5++-- | A key of 20 bytes that sorts by its number.+key :: Int -> ByteString+key = C.pack . printf "key:%016d"++-- | A value of 32 bytes: about what an index entry holds.+value :: Int -> ByteString+value = C.pack . printf "value:%026d"++-- | A way to read everything from where the iterator is, giving the number+-- of entries and the bytes of their keys and values. No scan keeps the+-- entries: what a program does with them, and what the garbage collector+-- then has to do, is not what is measured here.+type Scan = Iterator -> IO Total++-- | Entries read, and the bytes of their keys and values.+data Total = Total !Int !Int+ deriving (Eq, Show)++-- | Count an entry.+add :: Total -> (ByteString, ByteString) -> Total+add (Total n bytes) (k, v) = Total (n + 1) (bytes + B.length k + B.length v)++-- | One step for each entry, in a loop that keeps its total in an argument:+-- the stack stays as it is.+steps :: Scan+steps it = go (Total 0 0)+ where+ go !total =+ iterEntry it >>= \case+ Just entry -> iterNext it >> go (add total entry)+ Nothing -> return total++-- | One step for each entry, in a loop that is not tail recursive, as one+-- that returns a list with @(entry :) \<$\> go@ is: each entry waits on the+-- stack for those that follow.+stepsOnStack :: Scan+stepsOnStack it = go+ where+ go =+ iterEntry it >>= \case+ Just entry -> do+ iterNext it+ !total <- go+ return (add total entry)+ Nothing -> return (Total 0 0)++-- | Chunks of so many entries.+chunks :: Int -> Scan+chunks n it = go (Total 0 0)+ where+ go !total = do+ (entries, more) <- iterChunk it def {chunkEntries = n}+ let total' = foldl' add total entries+ if more then go total' else return total'++-- | The seconds that so many threads take to scan the whole database at the+-- same time, each with its own iterator.+timeScan :: DB -> Scan -> Int -> IO Double+timeScan db scan threads = do+ performGC+ t0 <- getMonotonicTime+ totals <- forConcurrently [1 .. threads] $ \_ ->+ withIter db $ \it -> iterFirst it >> scan it+ t1 <- getMonotonicTime+ -- every key is of 20 bytes and every value of 32+ unless (all (== Total entryCount (entryCount * 52)) totals) $+ fail ("a scan read " <> show totals <> ", and not every entry")+ return (t1 - t0)++-- | The median of 'runs' measurements.+median :: IO Double -> IO Double+median measure = do+ times <- forM [1 .. runs] (const measure)+ return (sort times !! (runs `div` 2))++main :: IO ()+main = withSystemTempDirectory "rocksdb-bench" $ \dir ->+ withDB dir def {createIfMissing = True} $ \db -> do+ forM_ [0, 1000 .. entryCount - 1] $ \i ->+ write db [Put (key j) (value j) | j <- [i .. min entryCount (i + 1000) - 1]]+ -- Compacted, the entries are in a table file, where those of a database+ -- in production are, and not in the memtable.+ compactRange db Nothing Nothing Nothing+ caps <- getNumCapabilities+ cores <- getNumProcessors+ printf "%d entries, %d capabilities, %d cores, median of %d runs\n" entryCount caps cores runs+ printf "%-34s %8s %10s %18s\n" (s "scan") (s "threads") (s "seconds") (s "entries a second")+ -- once unmeasured, to fill the block cache+ _ <- timeScan db (chunks 256) 1+ wanted <- getArgs+ let chosen = [x | x@(name, _) <- scans, null wanted || any (`isInfixOf` name) wanted]+ forM_ chosen $ \(name, scan) ->+ forM_ [1, 16] $ \threads -> do+ seconds <- median (timeScan db scan threads)+ let rate = fromIntegral (threads * entryCount) / seconds :: Double+ printf "%-34s %8d %10.3f %18.0f\n" name threads seconds rate+ where+ s :: String -> String+ s = id+ scans :: [(String, Scan)]+ scans =+ [ ("single steps, not tail recursive", stepsOnStack),+ ("single steps", steps),+ ("chunks of 16", chunks 16),+ ("chunks of 64", chunks 64),+ ("chunks of 256", chunks 256),+ ("chunks of 1024", chunks 1024)+ ]
+ cbits/rocksdb_haskell.c view
@@ -0,0 +1,379 @@+/*+ * C helpers of rocksdb-haskell-jprupp. See rocksdb_haskell.h for what each+ * function is for and what it promises.+ */++#include "rocksdb_haskell.h"++#include <stdint.h>+#include <stdlib.h>+#include <string.h>++/* The two lengths that come before the bytes of an entry. */+#define HEADER (2 * sizeof(size_t))++/* The first size of the buffer, which doubles when it is full. */+#define FIRST_CAPACITY 1024++/*+ * Whether the iterator is on an entry whose key starts with the prefix. If+ * it is, the key is in "*key" and "*key_len", and belongs to the iterator+ * until it moves.+ */+static int at_match(+ const rocksdb_iterator_t *iter,+ const char *prefix,+ size_t prefix_len,+ const char **key,+ size_t *key_len)+{+ if (!rocksdb_iter_valid(iter)) {+ return 0;+ }+ *key_len = 0;+ *key = rocksdb_iter_key(iter, key_len);+ if (*key == NULL) {+ /* no bytes to read, whatever the length says */+ *key_len = 0;+ }+ if (*key_len < prefix_len) {+ return 0;+ }+ /* memcmp must not be given a null pointer, even to compare no bytes */+ return prefix_len == 0 || memcmp(*key, prefix, prefix_len) == 0;+}++/*+ * The size that an entry takes in the buffer: header, key, value and+ * padding. Zero if it does not fit in a size_t, which no real entry does.+ */+static size_t entry_size(size_t key_len, size_t value_len)+{+ const size_t align = sizeof(size_t);+ size_t size = HEADER;++ if (key_len > SIZE_MAX - size) {+ return 0;+ }+ size += key_len;+ if (value_len > SIZE_MAX - size) {+ return 0;+ }+ size += value_len;+ if (size > SIZE_MAX - (align - 1)) {+ return 0;+ }+ return (size + (align - 1)) / align * align;+}++/*+ * Make room for "size" more bytes after the "used" ones of the buffer.+ * Returns zero, with the buffer as it was, if there is no memory for it.+ */+static int reserve(char **buffer, size_t *capacity, size_t used, size_t size)+{+ size_t needed;+ size_t grown;+ char *moved;++ if (size > SIZE_MAX - used) {+ return 0;+ }+ needed = used + size;+ if (needed <= *capacity) {+ return 1;+ }+ grown = *capacity < FIRST_CAPACITY ? FIRST_CAPACITY : *capacity;+ while (grown < needed) {+ if (grown > SIZE_MAX / 2) {+ grown = needed;+ } else {+ grown *= 2;+ }+ }+ moved = realloc(*buffer, grown);+ if (moved == NULL) {+ return 0;+ }+ *buffer = moved;+ *capacity = grown;+ return 1;+}++int rocksdb_haskell_iter_chunk(+ rocksdb_iterator_t *iter,+ int backwards,+ size_t max_entries,+ size_t max_bytes,+ const char *prefix,+ size_t prefix_len,+ char **buffer,+ size_t *count,+ unsigned char *more,+ char **errptr)+{+ char *entries = NULL; /* the buffer */+ size_t capacity = 0; /* its size */+ size_t used = 0; /* how much of it holds entries */+ size_t read = 0; /* how many entries */+ size_t bytes = 0; /* their keys and values, without the rest */+ int no_memory = 0;++ *buffer = NULL;+ *count = 0;+ *more = 0;++ while (read < max_entries) {+ const char *key;+ const char *value;+ size_t key_len;+ size_t value_len = 0;+ size_t size;+ char *entry;++ if (!at_match(iter, prefix, prefix_len, &key, &key_len)) {+ break;+ }+ value = rocksdb_iter_value(iter, &value_len);+ if (value == NULL) {+ value_len = 0;+ }++ size = entry_size(key_len, value_len);+ if (size == 0) {+ no_memory = 1;+ break;+ }+ /*+ * The limit on bytes, which the first entry is exempt from. Since+ * entry_size did not overflow, neither does the sum of the lengths.+ * "bytes" is above "max_bytes" only after a first entry that was.+ * At the limit exactly, an entry of no bytes is still read: it+ * does not take the bytes read to more.+ */+ if (read > 0 && (bytes > max_bytes || key_len + value_len > max_bytes - bytes)) {+ break;+ }+ if (!reserve(&entries, &capacity, used, size)) {+ no_memory = 1;+ break;+ }++ entry = entries + used;+ memcpy(entry, &key_len, sizeof(size_t));+ memcpy(entry + sizeof(size_t), &value_len, sizeof(size_t));+ if (key_len > 0) {+ memcpy(entry + HEADER, key, key_len);+ }+ if (value_len > 0) {+ memcpy(entry + HEADER + key_len, value, value_len);+ }+ used += size;+ bytes += key_len + value_len;+ read += 1;++ /* only now is the entry safe, and the iterator may leave it */+ if (backwards) {+ rocksdb_iter_prev(iter);+ } else {+ rocksdb_iter_next(iter);+ }+ }++ if (no_memory) {+ /* the iterator is still on the entry that did not fit */+ if (read == 0) {+ free(entries);+ return ROCKSDB_HASKELL_NO_MEMORY;+ }+ *buffer = entries;+ *count = read;+ *more = 1;+ return ROCKSDB_HASKELL_OK;+ }++ /*+ * An iterator that met an error is not valid, like one that reached the+ * end. Ask it, so that the caller does not take one for the other.+ */+ rocksdb_iter_get_error(iter, errptr);+ if (*errptr != NULL) {+ free(entries);+ return ROCKSDB_HASKELL_OK;+ }++ {+ const char *key;+ size_t key_len;++ *more = at_match(iter, prefix, prefix_len, &key, &key_len) ? 1 : 0;+ }+ *buffer = entries;+ *count = read;+ return ROCKSDB_HASKELL_OK;+}++/* A record of a packed batch, read out of its buffer. */+struct record {+ rocksdb_column_family_handle_t *family; /* NULL: the default one */+ uint32_t key_len;+ uint32_t value_len;+ unsigned char operation;+ const char *key; /* in the buffer */+ const char *value; /* in the buffer, after the key */+ size_t size; /* of the whole record, header included */+};++/*+ * Read the record at the start of the "left" bytes at "bytes". Returns zero+ * if there is no whole and well formed record there, and reads nothing past+ * those bytes either way.+ *+ * The header is copied out with memcpy because it is not aligned: a record+ * starts wherever the one before it ended.+ */+static int record_at(const char *bytes, size_t left, struct record *record)+{+ const size_t header = ROCKSDB_HASKELL_RECORD_HEADER;+ const char *field = bytes;++ if (left < header) {+ return 0;+ }+ memcpy(&record->family, field, sizeof(record->family));+ field += sizeof(record->family);+ memcpy(&record->key_len, field, sizeof(uint32_t));+ field += sizeof(uint32_t);+ memcpy(&record->value_len, field, sizeof(uint32_t));+ field += sizeof(uint32_t);+ record->operation = (unsigned char)*field;++ if (record->operation > ROCKSDB_HASKELL_MERGE) {+ return 0;+ }+ if (record->operation == ROCKSDB_HASKELL_DELETE && record->value_len != 0) {+ return 0;+ }+ /* one length at a time, so that their sum cannot overflow */+ left -= header;+ if (record->key_len > left) {+ return 0;+ }+ left -= record->key_len;+ if (record->value_len > left) {+ return 0;+ }++ record->key = bytes + header;+ record->value = record->key + record->key_len;+ record->size = header + record->key_len + record->value_len;+ return 1;+}++/* Add a record to a batch, with the function of RocksDB that it names. */+static void add_record(rocksdb_writebatch_t *batch, const struct record *record)+{+ rocksdb_column_family_handle_t *family = record->family;+ const char *key = record->key;+ const char *value = record->value;+ size_t key_len = record->key_len;+ size_t value_len = record->value_len;++ switch (record->operation) {+ case ROCKSDB_HASKELL_PUT:+ if (family == NULL) {+ rocksdb_writebatch_put(batch, key, key_len, value, value_len);+ } else {+ rocksdb_writebatch_put_cf(batch, family, key, key_len, value, value_len);+ }+ break;+ case ROCKSDB_HASKELL_DELETE:+ if (family == NULL) {+ rocksdb_writebatch_delete(batch, key, key_len);+ } else {+ rocksdb_writebatch_delete_cf(batch, family, key, key_len);+ }+ break;+ case ROCKSDB_HASKELL_MERGE:+ if (family == NULL) {+ rocksdb_writebatch_merge(batch, key, key_len, value, value_len);+ } else {+ rocksdb_writebatch_merge_cf(batch, family, key, key_len, value, value_len);+ }+ break;+ }+}++int rocksdb_haskell_writebatch_add_packed(+ rocksdb_writebatch_t *batch,+ const char *records,+ size_t len,+ size_t *count)+{+ struct record record;+ size_t at;+ size_t whole = 0;++ /*+ * Check every record first, so that a bad one leaves the batch as it+ * was. It costs a second look at headers that are in the cache.+ */+ for (at = 0; at < len; at += record.size) {+ if (!record_at(records + at, len - at, &record)) {+ *count = whole;+ return ROCKSDB_HASKELL_MALFORMED;+ }+ whole += 1;+ }++ for (at = 0; at < len; at += record.size) {+ /* cannot fail: the same bytes passed above */+ if (!record_at(records + at, len - at, &record)) {+ break;+ }+ add_record(batch, &record);+ }+ *count = whole;+ return ROCKSDB_HASKELL_OK;+}++/* The state of a merge operator is its name, which it owns. */+static void merge_operator_destroy(void *state)+{+ free(state);+}++static const char *merge_operator_name(void *state)+{+ return state;+}++int rocksdb_haskell_options_set_merge_operator(+ rocksdb_options_t *options,+ const char *name,+ rocksdb_haskell_full_merge_t full_merge,+ rocksdb_haskell_partial_merge_t partial_merge)+{+ size_t size = strlen(name) + 1;+ char *copy = malloc(size);+ rocksdb_mergeoperator_t *merge_operator;++ if (copy == NULL) {+ return ROCKSDB_HASKELL_NO_MEMORY;+ }+ memcpy(copy, name, size);+ /*+ * No function to delete a value: RocksDB then frees with free() what+ * the two functions return.+ */+ merge_operator = rocksdb_mergeoperator_create(+ copy,+ merge_operator_destroy,+ full_merge,+ partial_merge,+ NULL,+ merge_operator_name);+ /* the options take the operator, and destroy it when nothing uses it */+ rocksdb_options_set_merge_operator(options, merge_operator);+ return ROCKSDB_HASKELL_OK;+}
+ cbits/rocksdb_haskell.h view
@@ -0,0 +1,228 @@+/*+ * C helpers of rocksdb-haskell-jprupp: what the binding needs and the C API+ * of RocksDB does not have.+ *+ * The Haskell imports in Database.RocksDB.C name this header with the capi+ * calling convention, so the C compiler checks each of them against the+ * prototypes below.+ */++#ifndef ROCKSDB_HASKELL_H+#define ROCKSDB_HASKELL_H++#include <stddef.h>+#include <stdint.h>++#include <rocksdb/c.h>++/* Results of the functions below. */+#define ROCKSDB_HASKELL_OK 0+/* of rocksdb_haskell_iter_chunk and rocksdb_haskell_options_set_merge_operator */+#define ROCKSDB_HASKELL_NO_MEMORY 1+/* of rocksdb_haskell_writebatch_add_packed */+#define ROCKSDB_HASKELL_MALFORMED 2++/*+ * Read entries of an iterator, from the one it is on, in one call.+ *+ * Why: a Haskell program pays for every foreign call that may block, so a+ * scan that calls rocksdb_iter_next once for each entry pays once for each+ * entry. This function walks the iterator in C and copies the entries out,+ * so that the scan pays once for a few hundred.+ *+ * It goes forwards with rocksdb_iter_next, or backwards with+ * rocksdb_iter_prev if "backwards" is not zero, and stops at the first of:+ *+ * - an iterator that is not valid: the end of the data or of its bounds, or+ * an error;+ * - an entry whose key does not start with the "prefix_len" bytes at+ * "prefix" (no bytes: every key does);+ * - "max_entries" entries read;+ * - an entry whose key and value would take the bytes of keys and values+ * read to more than "max_bytes". The first entry is read whatever its+ * size, so that a scan always moves.+ *+ * The iterator is left on the first entry that was not read. "*more" tells+ * whether that is an entry whose key starts with the prefix: whether there is+ * anything left for another call.+ *+ * "*count" is the number of entries read, and "*buffer" holds them, one+ * after another, each laid out as+ *+ * size_t key_len; size_t value_len; key bytes; value bytes; padding+ *+ * with padding up to a multiple of sizeof(size_t), so that the lengths are+ * always aligned. The caller frees "*buffer" with free(); it is NULL when no+ * entry was read.+ *+ * Errors:+ *+ * - An error of the iterator is stored in "*errptr" as by+ * rocksdb_iter_get_error: a message that the caller frees. Nothing is+ * returned then: "*buffer" is NULL and "*count" is zero. "*errptr" must be+ * NULL on entry.+ * - If memory runs out with some entries read, they are returned as a short+ * chunk, and the iterator is on the entry that did not fit. If it runs out+ * on the first entry, the result is ROCKSDB_HASKELL_NO_MEMORY, with+ * nothing returned and the iterator where it was.+ *+ * The result is ROCKSDB_HASKELL_OK otherwise, also when "*errptr" is set.+ */+int rocksdb_haskell_iter_chunk(+ rocksdb_iterator_t *iter,+ int backwards,+ size_t max_entries,+ size_t max_bytes,+ const char *prefix,+ size_t prefix_len,+ char **buffer,+ size_t *count,+ unsigned char *more,+ char **errptr);++/* The operation of a record of rocksdb_haskell_writebatch_add_packed. */+#define ROCKSDB_HASKELL_PUT 0+#define ROCKSDB_HASKELL_DELETE 1+#define ROCKSDB_HASKELL_MERGE 2++/*+ * The bytes that come before the key of such a record: a column family+ * handle, two lengths and an operation.+ */+#define ROCKSDB_HASKELL_RECORD_HEADER \+ (sizeof(rocksdb_column_family_handle_t *) + 2 * sizeof(uint32_t) + 1)++/*+ * Add many operations to a write batch in one call.+ *+ * Why: a Haskell program pays for every foreign call that may block, so a+ * batch that is built with rocksdb_writebatch_put_cf pays once for each+ * key. For the 65,000 keys of a Bitcoin block that was more than half of+ * the time that the batch took to write, and nearly all of it when other+ * threads of the program were busy: bench/Batch.hs. This function takes+ * the operations packed one after another in a buffer and makes those+ * calls from C, so that the program pays once for a few thousand.+ *+ * "records" holds "len" bytes: any number of records, with nothing between+ * them and nothing after the last. A record is+ *+ * rocksdb_column_family_handle_t *family; NULL: the default one+ * uint32_t key_len;+ * uint32_t value_len; zero for a delete+ * unsigned char operation; ROCKSDB_HASKELL_PUT, ...+ * key bytes; value bytes+ *+ * with no padding anywhere: the header is ROCKSDB_HASKELL_RECORD_HEADER+ * bytes, 17 where a pointer has eight, and nothing in it is aligned. The+ * handle and the lengths are as this machine holds them in memory, written+ * with memcpy or its like. Being unaligned is what lets a program write a+ * record where the last one ended, in a buffer of its own, and fill in the+ * lengths once it has serialised the key and the value after the header.+ *+ * The operations are added in the order of the records, each as the+ * function of RocksDB that it names would add it: rocksdb_writebatch_put,+ * rocksdb_writebatch_delete or rocksdb_writebatch_merge for the default+ * column family, and the ones ending in _cf for a handle. RocksDB copies+ * the keys and values, so the buffer is not needed after the call.+ *+ * A merge is only added here. Whether the column family has a merge+ * operator to apply it with is for RocksDB to say, when the batch is+ * written or when the key is read.+ *+ * The buffer is checked before anything is added, and the call never reads+ * outside it. If a record is cut short by the end of the buffer, names an+ * operation that does not exist, or is a delete with a value, the result is+ * ROCKSDB_HASKELL_MALFORMED, the batch is as it was, and "*count" is the+ * number of whole records before the bad one. A handle cannot be checked:+ * one that is not NULL has to be a column family of the database that the+ * batch is written to.+ *+ * The result is ROCKSDB_HASKELL_OK otherwise, and "*count" is the number of+ * operations added.+ */+int rocksdb_haskell_writebatch_add_packed(+ rocksdb_writebatch_t *batch,+ const char *records,+ size_t len,+ size_t *count);++/*+ * The two functions of a merge operator, with the types that+ * rocksdb_mergeoperator_create of rocksdb/c.h asks for.+ *+ * The first argument of both is the state of the operator, which belongs+ * to rocksdb_haskell_options_set_merge_operator: a function must not use+ * it. "key" is the key that is merged, for an operator that needs it.+ *+ * A full merge works out the value of a key: from the value that the key+ * has ("existing_value", which is NULL when it has none) and the+ * "num_operands" merge records written since, oldest first, at+ * "operands_list" with their lengths at "operands_list_length".+ *+ * A partial merge is given merge records only, oldest first, and makes one+ * record of them that does to any value what they did one after another.+ * RocksDB uses it to shorten the list before the value is known, when it+ * compacts files that do not have it.+ *+ * Both return the new bytes in memory from malloc(), which RocksDB frees+ * with free(), their length in "*new_value_length", and 1 in "*success".+ * To fail, a function stores 0 in "*success" and in "*new_value_length",+ * and returns NULL: RocksDB copies that many bytes from the result before+ * it looks at "*success". A read of the key then fails, and so does a+ * compaction that meets it. A partial merge that fails is not an error:+ * RocksDB keeps the records as they are.+ *+ * They are called from the threads of RocksDB, during reads, flushes and+ * compactions, several at a time. They must be plain C: a function that+ * enters the Haskell run-time system from such a thread is not what this+ * is for.+ */+typedef char *(*rocksdb_haskell_full_merge_t)(+ void *state,+ const char *key,+ size_t key_length,+ const char *existing_value,+ size_t existing_value_length,+ const char *const *operands_list,+ const size_t *operands_list_length,+ int num_operands,+ unsigned char *success,+ size_t *new_value_length);++typedef char *(*rocksdb_haskell_partial_merge_t)(+ void *state,+ const char *key,+ size_t key_length,+ const char *const *operands_list,+ const size_t *operands_list_length,+ int num_operands,+ unsigned char *success,+ size_t *new_value_length);++/*+ * Give the options of a database or of a column family a merge operator+ * made of a name and two functions.+ *+ * Why: rocksdb_mergeoperator_create wants a state with a destructor and a+ * function that returns the name, besides the two that merge. A Haskell+ * program has no C function to give for those, and should not make one+ * that calls back into Haskell, since RocksDB calls them from its own+ * threads. This function makes the operator around a copy of the name,+ * which is all the state it has and is freed with the operator.+ *+ * The options own the operator from then on, and the column families+ * opened with them keep it for as long as they need it: there is nothing+ * for the caller to destroy. RocksDB frees what the two functions return+ * with free(), since the operator is made without a function of its own+ * for that.+ *+ * The result is ROCKSDB_HASKELL_OK, or ROCKSDB_HASKELL_NO_MEMORY with the+ * options as they were.+ */+int rocksdb_haskell_options_set_merge_operator(+ rocksdb_options_t *options,+ const char *name,+ rocksdb_haskell_full_merge_t full_merge,+ rocksdb_haskell_partial_merge_t partial_merge);++#endif
rocksdb-haskell-jprupp.cabal view
@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: rocksdb-haskell-jprupp-version: 2.7.1+version: 2.8.0 synopsis: Haskell bindings for RocksDB description: See README at <https://github.com/jprupp/rocksdb-haskell#readme> category: Database, FFI@@ -19,6 +19,7 @@ README.md CHANGELOG.md AUTHORS+ cbits/rocksdb_haskell.h source-repository head type: git@@ -36,6 +37,11 @@ hs-source-dirs: src ghc-options: -Wall+ cc-options: -Wall+ include-dirs:+ cbits+ c-sources:+ cbits/rocksdb_haskell.c extra-libraries: rocksdb build-depends:@@ -53,6 +59,9 @@ hs-source-dirs: test ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N+ cc-options: -Wall+ c-sources:+ test/cbits/merge_test.c build-depends: QuickCheck , async@@ -61,7 +70,40 @@ , containers , data-default , directory+ , filepath , hspec+ , rocksdb-haskell-jprupp+ , temporary+ default-language: Haskell2010++benchmark batch+ type: exitcode-stdio-1.0+ main-is: Batch.hs+ hs-source-dirs:+ bench+ ghc-options: -Wall -O2 -threaded -rtsopts -with-rtsopts=-N4+ build-depends:+ async+ , base >=4.9 && <5+ , bytestring+ , data-default+ , directory+ , rocksdb-haskell-jprupp+ , temporary+ default-language: Haskell2010++benchmark scan+ type: exitcode-stdio-1.0+ main-is: Scan.hs+ hs-source-dirs:+ bench+ ghc-options: -Wall -O2 -threaded -rtsopts -with-rtsopts=-N4+ build-depends:+ async+ , base >=4.9 && <5+ , bytestring+ , data-default+ , directory , rocksdb-haskell-jprupp , temporary default-language: Haskell2010
src/Database/RocksDB.hs view
@@ -79,6 +79,14 @@ -- to 'withDBCF'. Functions ending in @CF@ act on a given column family; the -- others act on the default column family. --+-- = Merges+--+-- A value can be changed without being read: a merge record ('merge', or+-- 'Merge' in a batch) says how it changes, and the t'MergeOperator' that+-- the 'mergeOperator' of the t'Config' gives the column family works out+-- the value when the key is read or compacted. See t'MergeOperator' for+-- what that is and what it asks of every program that opens the database.+-- -- = Errors -- -- Errors reported by RocksDB are thrown as 'IOError' exceptions created with
src/Database/RocksDB/Base.hs view
@@ -1,552 +1,968 @@ {-# LANGUAGE LambdaCase #-}--- |--- Module : Database.RocksDB.Base--- Copyright : (c) 2012-2013 The leveldb-haskell Authors--- (c) 2014-2020 The rocksdb-haskell Authors--- License : BSD3--- Maintainer : jprupp@protonmail.ch--- Stability : experimental--- Portability : non-portable------ RocksDB Haskell binding.------ The API closely follows the C-API of RocksDB. All resources (databases,--- snapshots, iterators) are managed with bracket-style functions such as--- 'withDB', 'withSnapshot' and 'withIter', which release them when the--- supplied action returns or throws. Manual variants such as 'createSnapshot'--- and 'createIterator' exist for cases where the lifetime of a resource cannot--- be expressed as a scope.------ A t'DB' handle is safe to use from many threads at once.--module Database.RocksDB.Base- ( -- * Exported Types- DB (..)- , BatchOp (..)- , Range- , ColumnFamily- , Snapshot- , ReadOpts-- -- * Options- , Config (..)-- -- * Basic Database Manipulations- , withDB- , withDBCF- , put- , putCF- , delete- , deleteCF- , write- , get- , getCF- , multiGet- , multiGetCF- , withSnapshot- , createSnapshot- , releaseSnapshot-- -- * Administrative Functions- , Property (..), getProperty, getPropertyCF- , destroy- , repair- , approximateSize-- -- * Iteration- , module Database.RocksDB.Iterator- ) where--import Control.Exception-import Control.Monad (forM, when)-import Data.ByteString (ByteString)-import qualified Data.ByteString as BS-import Data.ByteString.Internal (ByteString (..))-import qualified Data.ByteString.Unsafe as BU-import Data.Int-import Database.RocksDB.C-import Database.RocksDB.Internal-import Database.RocksDB.Iterator-import Foreign.C.String-import Foreign.ForeignPtr-import Foreign.Marshal.Alloc-import Foreign.Marshal.Array-import Foreign.Marshal.Utils (withMany)-import Foreign.Ptr-import Foreign.Storable-import System.Directory---- | Properties exposed by RocksDB. Read them with 'getProperty' or--- 'getPropertyCF'. Every value is returned by RocksDB as text.-data Property- -- | @rocksdb.num-files-at-level\<N\>@: number of SST files at level @N@- = NumFilesAtLevel Int- -- | @rocksdb.stats@: multi-line human-readable statistics- | Stats- -- | @rocksdb.sstables@: description of the SST files that make up the- -- database- | SSTables- -- | @rocksdb.estimate-table-readers-mem@: memory for index and filter- -- blocks of open table files, outside the block cache- | EstimateTableReadersMem- -- | @rocksdb.cur-size-all-mem-tables@: memory of all memtables- | CurSizeAllMemTables- -- | @rocksdb.block-cache-usage@: memory used by the block cache- | BlockCacheUsage- -- | any other property, by its full name- | PropertyName String- deriving (Eq, Show)---- | A single operation in an atomic batch written with 'write'.------ Batches are how a program keeps several keys consistent with each other: an--- index and the data it points to, for example, must change together or not at--- all. RocksDB applies a batch atomically and in one write to its log, so a batch--- is also much faster than the same writes made one by one.-data BatchOp = Put !ByteString !ByteString- -- ^ set a key to a value in the default column family- | Del !ByteString- -- ^ delete a key from the default column family- | PutCF !ColumnFamily !ByteString !ByteString- -- ^ set a key to a value in a column family- | DelCF !ColumnFamily !ByteString- -- ^ delete a key from a column family- deriving (Eq, Show)---- | Open a database at the given path and pass its handle to a function.------ The handle is closed automatically when the function returns or throws, and--- must not be used afterwards. If 'createIfMissing' is set, the directory is--- created first if needed. Throws an 'IOError' if the database cannot be--- opened, for example if it does not exist and 'createIfMissing' is not set,--- or if it is locked by another process.------ Only the default column family is opened. A database that has other column--- families must be opened with 'withDBCF'.------ Opening is expensive and a database can only be opened by one process at a--- time, so open it once at program start and share the t'DB' between threads for--- the life of the program.-withDB :: FilePath -> Config -> (DB -> IO a) -> IO a-withDB path config f =- withBlockCache config $ \cache ->- withOptionsCache cache config $ \opts_ptr ->- withReadOpts Nothing $ \read_opts ->- withWriteOpts $ \write_opts ->- bracket (create_db opts_ptr read_opts write_opts) destroy_db f- where- destroy_db db = c_rocksdb_close $ rocksDB db- create_db opts_ptr read_opts write_opts = do- when (createIfMissing config) $- createDirectoryIfMissing True path- withCString path $ \path_ptr -> do- db_ptr <- throwIfErr "open" $- c_rocksdb_open opts_ptr path_ptr- return DB { rocksDB = db_ptr- , columnFamilies = []- , readOpts = read_opts- , snapshot = Nothing- , writeOpts = write_opts- }---- | Open a database with column families and pass its handle to a function.------ The second argument configures the database and its default column family.--- The third argument lists the other column families by name, each with its own--- t'Config'. Their handles are stored in 'columnFamilies', in the same order.------ If the database directory is empty, a new database is created along with all--- the listed column families. Otherwise the existing database is opened with--- those column families; if some of them do not exist yet, the open fails unless--- 'createMissingColumnFamilies' is set in the database t'Config'. RocksDB also--- requires every column family that exists in the database to be listed.------ The handle and all column family handles are released when the function--- returns or throws.------ Column families are separate key spaces in one database. They share the--- write-ahead log, so a batch can change several of them atomically, but each--- has its own files, memtables and tuning. Use them to keep data of different--- shapes apart, such as an index with short keys and a table of large values.-withDBCF :: FilePath- -> Config- -> [(String, Config)]- -> (DB -> IO a)- -> IO a-withDBCF path config cf_cfgs f =- withBlockCache config $ \cache ->- withOptionsCache cache config $ \opts_ptr ->- withOptionsCFCache cache (map snd cf_cfgs) $ \cf_opts ->- withReadOpts Nothing $ \read_opts ->- withWriteOpts $ \write_opts ->- withStrings (map fst cf_cfgs) $ \cf_names ->- allocaArray (length cf_cfgs + 1) $ \cf_names_array ->- allocaArray (length cf_cfgs + 1) $ \cf_opts_array ->- allocaArray (length cf_cfgs + 1) $ \cf_ptrs_array ->- bracket ( create_db opts_ptr- cf_names- cf_names_array- cf_opts- cf_opts_array- read_opts- cf_ptrs_array- write_opts- ) destroy_db f- where- create_new cf_names cf_opts opts_ptr read_opts write_opts = do- createDirectoryIfMissing True path- withCString path $ \path_ptr -> do- db_ptr <- throwIfErr "open" $- c_rocksdb_open opts_ptr path_ptr- cfs <- forM (zip cf_names cf_opts) $ \(n, o) ->- throwIfErr "create_column_family" $- c_rocksdb_create_column_family- db_ptr o n- return DB { rocksDB = db_ptr- , columnFamilies = cfs- , readOpts = read_opts- , snapshot = Nothing- , writeOpts = write_opts- }- destroy_db db = do- mapM_ c_rocksdb_column_family_handle_destroy (columnFamilies db)- c_rocksdb_close $ rocksDB db- create_db opts_ptr- cf_names- cf_names_array- cf_opts- cf_opts_array- read_opts- cf_ptrs_array- write_opts = do- when (createIfMissing config) $- createDirectoryIfMissing True path- listDirectory path >>= \case- [] -> create_new cf_names cf_opts opts_ptr read_opts write_opts- _ -> withCString path $ \path_ptr ->- withCString "default" $ \cf_deflt_name -> do- pokeArray cf_names_array (cf_deflt_name : cf_names)- pokeArray cf_opts_array (opts_ptr : cf_opts)- db_ptr <- throwIfErr "open" $- c_rocksdb_open_column_families- opts_ptr- path_ptr- (intToCInt (length cf_cfgs + 1))- cf_names_array- cf_opts_array- cf_ptrs_array- (deflt, cfs) <- peekArray (length cf_cfgs + 1) cf_ptrs_array >>= \case- d : cs -> return (d, cs)- [] -> error "rocksdb_open_column_families returned no handles"- -- Every handle must be destroyed before the database is- -- closed. The default column family is not kept, so- -- release its handle now; the database keeps its own.- c_rocksdb_column_family_handle_destroy deflt- return DB { rocksDB = db_ptr- , columnFamilies = cfs- , readOpts = read_opts- , snapshot = Nothing- , writeOpts = write_opts- }---- | Run an action with a snapshot of the database.--- The t'DB' object is not valid after the action ends.------ The t'DB' passed to the action reads from a consistent point-in-time view of--- the database: 'get', 'multiGet' and iterators created from it do not see--- writes made after the snapshot was taken. Writes through it still go to the--- live database.------ Without a snapshot, two reads made one after the other can see different--- states of the database if another thread writes in between. Anything that--- reads several related keys and expects them to agree, such as an index and the--- records it points to, should read through a snapshot.-withSnapshot :: DB -> (DB -> IO a) -> IO a-withSnapshot db f =- bracket (createSnapshot db) releaseSnapshot (f . fst)---- | Manually create an unmanaged snapshot.--- The returned t'DB' has 'readOpts' configured for the snapshot.--- Use 'releaseSnapshot' to release both the snapshot and its read options.-createSnapshot :: DB -> IO (DB, Snapshot)-createSnapshot db@DB{rocksDB = db_ptr} = do- snap_ptr <- c_rocksdb_create_snapshot db_ptr- read_opts <- createReadOpts (Just snap_ptr)- return (db{readOpts = read_opts, snapshot = Just snap_ptr}, snap_ptr)---- | Function to release an unmanaged snapshot.--- Also releases the read options associated with the snapshot DB.-releaseSnapshot :: (DB, Snapshot) -> IO ()-releaseSnapshot (DB{rocksDB = db_ptr, readOpts = read_opts}, snap_ptr) = do- destroyReadOpts read_opts- c_rocksdb_release_snapshot db_ptr snap_ptr---- | Get a property of the default column family, or of the database as a--- whole. Returns 'Nothing' if RocksDB does not know the property.-getProperty :: DB -> Property -> IO (Maybe ByteString)-getProperty DB{rocksDB = db_ptr} p =- withCString (propertyName p) $ \name ->- c_rocksdb_property_value db_ptr name >>= propertyResult---- | Get a property of a column family. Many properties, such as the memory--- of memtables and table readers, are kept per column family.-getPropertyCF :: DB -> ColumnFamily -> Property -> IO (Maybe ByteString)-getPropertyCF DB{rocksDB = db_ptr} cf p =- withCString (propertyName p) $ \name ->- c_rocksdb_property_value_cf db_ptr cf name >>= propertyResult---- | Turn a property value returned by RocksDB into a 'ByteString', freeing the--- C string.-propertyResult :: CString -> IO (Maybe ByteString)-propertyResult val_ptr- | val_ptr == nullPtr = return Nothing- | otherwise = do- res <- BS.packCString val_ptr- freeCString val_ptr- return (Just res)---- | The name RocksDB uses for a property.-propertyName :: Property -> String-propertyName = \case- NumFilesAtLevel i -> "rocksdb.num-files-at-level" ++ show i- Stats -> "rocksdb.stats"- SSTables -> "rocksdb.sstables"- EstimateTableReadersMem -> "rocksdb.estimate-table-readers-mem"- CurSizeAllMemTables -> "rocksdb.cur-size-all-mem-tables"- BlockCacheUsage -> "rocksdb.block-cache-usage"- PropertyName n -> n---- | Destroy the given RocksDB database, deleting all of its files. The--- database must not be open. Throws an 'IOError' on failure.-destroy :: FilePath -> Options -> IO ()-destroy path opts_ptr =- withCString path $ \path_ptr ->- throwIfErr "destroy" $ c_rocksdb_destroy_db opts_ptr path_ptr---- | Try to recover as much data as possible from a corrupted database. Some--- data may be lost. The database must not be open. Throws an 'IOError' on--- failure.-repair :: FilePath -> Options -> IO ()-repair path opts_ptr =- withCString path $ \path_ptr ->- throwIfErr "repair" $ c_rocksdb_repair_db opts_ptr path_ptr----- | A key range given as @(start, limit)@. The start key is included and the--- limit key is excluded.------ TODO: support [Range], like C API does-type Range = (ByteString, ByteString)---- | Approximate number of bytes of file system space used by the keys in a--- 'Range' of the default column family. Data still in memtables may not be--- counted. Throws an 'IOError' on failure.-approximateSize :: DB -> Range -> IO Int64-approximateSize DB{rocksDB = db_ptr} (from, to) =- BU.unsafeUseAsCStringLen from $ \(from_ptr, flen) ->- BU.unsafeUseAsCStringLen to $ \(to_ptr, tlen) ->- withArray [from_ptr] $ \from_ptrs ->- withArray [intToCSize flen] $ \flen_ptrs ->- withArray [to_ptr] $ \to_ptrs ->- withArray [intToCSize tlen] $ \tlen_ptrs ->- alloca $ \size_ptr -> do- throwIfErr "approximate_sizes" $- c_rocksdb_approximate_sizes db_ptr 1- from_ptrs flen_ptrs- to_ptrs tlen_ptrs- size_ptr- fromIntegral <$> peek size_ptr---- | Write a key/value pair to the default column family, replacing any--- previous value. Throws an 'IOError' on failure.-put :: DB -> ByteString -> ByteString -> IO ()-put db = putCommon db Nothing---- | Like 'put', but write to the given column family.-putCF :: DB -> ColumnFamily -> ByteString -> ByteString -> IO ()-putCF db cf = putCommon db (Just cf)---- | Write a key/value pair to the default column family, or to a column family--- if one is given.-putCommon :: DB -> Maybe ColumnFamily -> ByteString -> ByteString -> IO ()-putCommon DB{rocksDB = db_ptr, writeOpts = write_opts} mcf key value =- BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->- BU.unsafeUseAsCStringLen value $ \(val_ptr, vlen) ->- throwIfErr "put" $ case mcf of- Just cf -> c_rocksdb_put_cf- db_ptr write_opts cf- key_ptr (intToCSize klen)- val_ptr (intToCSize vlen)- Nothing -> c_rocksdb_put- db_ptr write_opts- key_ptr (intToCSize klen)- val_ptr (intToCSize vlen)---- | Read a value by key from the default column family. Returns 'Nothing' if--- the key is not found. Throws an 'IOError' on failure.-get :: DB -> ByteString -> IO (Maybe ByteString)-get db = getCommon db Nothing---- | Like 'get', but read from the given column family.-getCF :: DB -> ColumnFamily -> ByteString -> IO (Maybe ByteString)-getCF db cf = getCommon db (Just cf)---- | Read a value from the default column family, or from a column family if one--- is given.-getCommon :: DB -> Maybe ColumnFamily -> ByteString -> IO (Maybe ByteString)-getCommon DB{rocksDB = db_ptr, readOpts = read_opts} mcf key =- BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->- alloca $ \vlen_ptr -> do- val_ptr <- throwIfErr "get" $- case mcf of- Just cf -> c_rocksdb_get_cf- db_ptr read_opts cf- key_ptr (intToCSize klen) vlen_ptr- Nothing -> c_rocksdb_get- db_ptr read_opts- key_ptr (intToCSize klen) vlen_ptr- if val_ptr == nullPtr- then return Nothing- else do- vlen <- peek vlen_ptr- res <- BU.unsafePackMallocCStringLen (val_ptr, cSizeToInt vlen)- return $ Just res---- | Read the values of many keys in one call. RocksDB looks the keys up--- together, sharing the work of finding the files and blocks that hold--- them, which is faster than reading them one by one. The values are in the--- order of the keys, with 'Nothing' for keys that are not found. Duplicate keys--- are allowed. Throws an 'IOError' if any of the lookups fails.------ For a query that needs hundreds of keys at once, such as the balances of many--- addresses, this cuts the time spent finding blocks and crossing into C to a--- fraction of separate 'get' calls.-multiGet :: DB -> [ByteString] -> IO [Maybe ByteString]-multiGet db = multiGetCommon db Nothing---- | 'multiGet' on a column family.-multiGetCF :: DB -> ColumnFamily -> [ByteString] -> IO [Maybe ByteString]-multiGetCF db cf = multiGetCommon db (Just cf)---- | Read many values from the default column family, or from a column family if--- one is given.-multiGetCommon :: DB -> Maybe ColumnFamily -> [ByteString] -> IO [Maybe ByteString]-multiGetCommon _ _ [] = return []-multiGetCommon DB{rocksDB = db_ptr, readOpts = read_opts} mcf keys =- withMany BU.unsafeUseAsCStringLen keys $ \kls ->- withArrayLen (map fst kls) $ \n key_ptrs ->- withArray (map (intToCSize . snd) kls) $ \key_lens ->- allocaArray n $ \val_ptrs ->- allocaArray n $ \val_lens ->- allocaArray n $ \err_ptrs -> do- case mcf of- Just cf ->- withArray (replicate n cf) $ \cfs ->- c_rocksdb_multi_get_cf- db_ptr read_opts cfs (intToCSize n)- key_ptrs key_lens val_ptrs val_lens err_ptrs- Nothing ->- c_rocksdb_multi_get- db_ptr read_opts (intToCSize n)- key_ptrs key_lens val_ptrs val_lens err_ptrs- vals <- peekArray n val_ptrs- lens <- peekArray n val_lens- errs <- peekArray n err_ptrs- -- Own every value and free every error before throwing, so that- -- none of them leaks.- res <- forM (zip vals lens) $ \(v, l) ->- if v == nullPtr- then return Nothing- else Just <$> BU.unsafePackMallocCStringLen (v, cSizeToInt l)- msgs <- forM errs $ \e ->- if e == nullPtr- then return Nothing- else do- m <- peekCString e- free e- return (Just m)- case [m | Just m <- msgs] of- m : _ -> throwIO $ userError $ "multi_get: " ++ m- [] -> return res---- | Delete a key from the default column family. Deleting a key that does not--- exist is not an error. Throws an 'IOError' on failure.-delete :: DB -> ByteString -> IO ()-delete db = deleteCommon db Nothing---- | Like 'delete', but delete from the given column family.-deleteCF :: DB -> ColumnFamily -> ByteString -> IO ()-deleteCF db cf = deleteCommon db (Just cf)---- | Delete a key/value pair from the default column family, or from a column--- family if one is given.-deleteCommon :: DB -> Maybe ColumnFamily -> ByteString -> IO ()-deleteCommon DB{rocksDB = db_ptr, writeOpts = write_opts} mcf key =- BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->- throwIfErr "delete" $ case mcf of- Just cf -> c_rocksdb_delete_cf db_ptr write_opts cf key_ptr (intToCSize klen)- Nothing -> c_rocksdb_delete db_ptr write_opts key_ptr (intToCSize klen)---- | Perform a batch mutation. All operations in the list are applied atomically:--- either all of them are written or none are. They are applied in order, so a--- later operation on a key overrides an earlier one. Throws an 'IOError' on--- failure.------ The keys and values are kept alive until RocksDB has copied them, because--- they are handed to C as pointers into Haskell memory.-write :: DB -> [BatchOp] -> IO ()-write DB{rocksDB = db_ptr, writeOpts = write_opts} batch =- bracket- c_rocksdb_writebatch_create- c_rocksdb_writebatch_destroy $ \batch_ptr -> do- mapM_ (batchAdd batch_ptr) batch- throwIfErr "write" $ c_rocksdb_write db_ptr write_opts batch_ptr- -- ensure @ByteString@s (and respective shared @CStringLen@s) aren't- -- GC'ed until here- mapM_ touch batch- where- batchAdd batch_ptr (Put key val) =- BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->- BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->- c_rocksdb_writebatch_put- batch_ptr- key_ptr (intToCSize klen)- val_ptr (intToCSize vlen)- batchAdd batch_ptr (PutCF cf_ptr key val) =- BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->- BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->- c_rocksdb_writebatch_put_cf- batch_ptr- cf_ptr- key_ptr (intToCSize klen)- val_ptr (intToCSize vlen)- batchAdd batch_ptr (Del key) =- BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->- c_rocksdb_writebatch_delete- batch_ptr- key_ptr (intToCSize klen)- batchAdd batch_ptr (DelCF cf_ptr key) =- BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->- c_rocksdb_writebatch_delete_cf- batch_ptr- cf_ptr- key_ptr (intToCSize klen)- touch (Put (PS p _ _) (PS p' _ _)) = do- touchForeignPtr p- touchForeignPtr p'- touch (PutCF _ (PS p _ _) (PS p' _ _)) = do- touchForeignPtr p- touchForeignPtr p'- touch (Del (PS p _ _)) =- touchForeignPtr p- touch (DelCF _ (PS p _ _)) =- touchForeignPtr p---- | Marshal a list of strings into C strings that are valid while the function--- runs.-withStrings :: [String] -> ([CString] -> IO a) -> IO a-withStrings ss f =- go [] ss- where- go acc [] = f (reverse acc)- go acc (x:xs) = withCString x $ \p -> go (p:acc) xs++-- |+-- Module : Database.RocksDB.Base+-- Copyright : (c) 2012-2013 The leveldb-haskell Authors+-- (c) 2014-2020 The rocksdb-haskell Authors+-- License : BSD3+-- Maintainer : jprupp@protonmail.ch+-- Stability : experimental+-- Portability : non-portable+--+-- RocksDB Haskell binding.+--+-- The API closely follows the C-API of RocksDB. All resources (databases,+-- snapshots, iterators) are managed with bracket-style functions such as+-- 'withDB', 'withSnapshot' and 'withIter', which release them when the+-- supplied action returns or throws. Manual variants such as 'createSnapshot'+-- and 'createIterator' exist for cases where the lifetime of a resource cannot+-- be expressed as a scope.+--+-- A t'DB' handle is safe to use from many threads at once.+module Database.RocksDB.Base+ ( -- * Exported Types+ DB (..),+ BatchOp (..),+ Range,+ ColumnFamily,+ Snapshot,+ ReadOpts,++ -- * Options+ Config (..),+ MergeOperator (..),+ FullMerge,+ PartialMerge,++ -- * Basic Database Manipulations+ withDB,+ withDBCF,+ put,+ putCF,+ delete,+ deleteCF,+ merge,+ mergeCF,+ write,+ get,+ getCF,+ multiGet,+ multiGetCF,+ withSnapshot,+ createSnapshot,+ releaseSnapshot,++ -- * Packed batches+ -- $packed+ writePacked,+ packOps,+ PackedOp (..),+ packedHeaderSize,+ pokePackedHeader,++ -- * Administrative Functions+ Property (..),+ getProperty,+ getPropertyCF,+ destroy,+ repair,+ approximateSize,+ compactRange,+ flush,++ -- * Iteration+ module Database.RocksDB.Iterator,+ )+where++import Control.Exception+import Control.Monad (foldM_, forM, forM_, when)+import Data.ByteString (ByteString)+import qualified Data.ByteString as BS+import Data.ByteString.Internal (ByteString (..))+import qualified Data.ByteString.Internal as BI+import qualified Data.ByteString.Unsafe as BU+import Data.Int+import Data.Maybe (fromMaybe)+import Data.Word (Word32, Word8)+import Database.RocksDB.C+import Database.RocksDB.Internal+import Database.RocksDB.Iterator+import Foreign.C.String+import Foreign.ForeignPtr+import Foreign.Marshal.Alloc+import Foreign.Marshal.Array+import Foreign.Marshal.Utils (copyBytes, withMany)+import Foreign.Ptr+import Foreign.Storable+import System.Directory++-- | Properties exposed by RocksDB. Read them with 'getProperty' or+-- 'getPropertyCF'. Every value is returned by RocksDB as text.+data Property+ = -- | @rocksdb.num-files-at-level\<N\>@: number of SST files at level @N@+ NumFilesAtLevel Int+ | -- | @rocksdb.stats@: multi-line human-readable statistics+ Stats+ | -- | @rocksdb.sstables@: description of the SST files that make up the+ -- database+ SSTables+ | -- | @rocksdb.estimate-table-readers-mem@: memory for index and filter+ -- blocks of open table files, outside the block cache+ EstimateTableReadersMem+ | -- | @rocksdb.cur-size-all-mem-tables@: memory of all memtables+ CurSizeAllMemTables+ | -- | @rocksdb.block-cache-usage@: memory used by the block cache+ BlockCacheUsage+ | -- | any other property, by its full name+ PropertyName String+ deriving (Eq, Show)++-- | A single operation in an atomic batch written with 'write'.+--+-- Batches are how a program keeps several keys consistent with each other: an+-- index and the data it points to, for example, must change together or not at+-- all. RocksDB applies a batch atomically and in one write to its log, so a batch+-- is also much faster than the same writes made one by one.+data BatchOp+ = -- | set a key to a value in the default column family+ Put !ByteString !ByteString+ | -- | delete a key from the default column family+ Del !ByteString+ | -- | set a key to a value in a column family+ PutCF !ColumnFamily !ByteString !ByteString+ | -- | delete a key from a column family+ DelCF !ColumnFamily !ByteString+ | -- | add a merge record to a key of the default column family: the+ -- second byte string is for the t'MergeOperator' of the column family+ -- to combine with the value that the key has. Nothing is read when+ -- the batch is written. RocksDB refuses the batch if the column family+ -- has no merge operator, and 'write' throws. What is left of such a+ -- batch depends on the version: RocksDB 11 keeps none of it, RocksDB+ -- 8.9 keeps the operations that came before the merge. Give every+ -- column family that gets merge records its operator.+ Merge !ByteString !ByteString+ | -- | add a merge record to a key of a column family+ MergeCF !ColumnFamily !ByteString !ByteString+ deriving (Eq, Show)++-- | Open a database at the given path and pass its handle to a function.+--+-- The handle is closed automatically when the function returns or throws, and+-- must not be used afterwards. If 'createIfMissing' is set, the directory is+-- created first if needed. Throws an 'IOError' if the database cannot be+-- opened, for example if it does not exist and 'createIfMissing' is not set,+-- or if it is locked by another process.+--+-- Only the default column family is opened. A database that has other column+-- families must be opened with 'withDBCF'.+--+-- Opening is expensive and a database can only be opened by one process at a+-- time, so open it once at program start and share the t'DB' between threads for+-- the life of the program.+withDB :: FilePath -> Config -> (DB -> IO a) -> IO a+withDB path config f =+ withBlockCache config $ \cache ->+ withOptionsCache cache config $ \opts_ptr ->+ withReadOpts Nothing $ \read_opts ->+ withWriteOpts $ \write_opts ->+ bracket (create_db opts_ptr read_opts write_opts) destroy_db f+ where+ destroy_db db = c_rocksdb_close $ rocksDB db+ create_db opts_ptr read_opts write_opts = do+ when (createIfMissing config) $+ createDirectoryIfMissing True path+ withCString path $ \path_ptr -> do+ db_ptr <-+ throwIfErr "open" $+ c_rocksdb_open opts_ptr path_ptr+ return+ DB+ { rocksDB = db_ptr,+ columnFamilies = [],+ readOpts = read_opts,+ snapshot = Nothing,+ writeOpts = write_opts+ }++-- | Open a database with column families and pass its handle to a function.+--+-- The second argument configures the database and its default column family.+-- The third argument lists the other column families by name, each with its own+-- t'Config'. Their handles are stored in 'columnFamilies', in the same order.+--+-- If the database directory is empty, a new database is created along with all+-- the listed column families. Otherwise the existing database is opened with+-- those column families; if some of them do not exist yet, the open fails unless+-- 'createMissingColumnFamilies' is set in the database t'Config'. RocksDB also+-- requires every column family that exists in the database to be listed.+--+-- The handle and all column family handles are released when the function+-- returns or throws.+--+-- Column families are separate key spaces in one database. They share the+-- write-ahead log, so a batch can change several of them atomically, but each+-- has its own files, memtables and tuning. Use them to keep data of different+-- shapes apart, such as an index with short keys and a table of large values.+withDBCF ::+ FilePath ->+ Config ->+ [(String, Config)] ->+ (DB -> IO a) ->+ IO a+withDBCF path config cf_cfgs f =+ withBlockCache config $ \cache ->+ withOptionsCache cache config $ \opts_ptr ->+ withOptionsCFCache cache (map snd cf_cfgs) $ \cf_opts ->+ withReadOpts Nothing $ \read_opts ->+ withWriteOpts $ \write_opts ->+ withStrings (map fst cf_cfgs) $ \cf_names ->+ allocaArray (length cf_cfgs + 1) $ \cf_names_array ->+ allocaArray (length cf_cfgs + 1) $ \cf_opts_array ->+ allocaArray (length cf_cfgs + 1) $ \cf_ptrs_array ->+ bracket+ ( create_db+ opts_ptr+ cf_names+ cf_names_array+ cf_opts+ cf_opts_array+ read_opts+ cf_ptrs_array+ write_opts+ )+ destroy_db+ f+ where+ create_new cf_names cf_opts opts_ptr read_opts write_opts = do+ createDirectoryIfMissing True path+ withCString path $ \path_ptr -> do+ db_ptr <-+ throwIfErr "open" $+ c_rocksdb_open opts_ptr path_ptr+ cfs <- forM (zip cf_names cf_opts) $ \(n, o) ->+ throwIfErr "create_column_family" $+ c_rocksdb_create_column_family+ db_ptr+ o+ n+ return+ DB+ { rocksDB = db_ptr,+ columnFamilies = cfs,+ readOpts = read_opts,+ snapshot = Nothing,+ writeOpts = write_opts+ }+ destroy_db db = do+ mapM_ c_rocksdb_column_family_handle_destroy (columnFamilies db)+ c_rocksdb_close $ rocksDB db+ create_db+ opts_ptr+ cf_names+ cf_names_array+ cf_opts+ cf_opts_array+ read_opts+ cf_ptrs_array+ write_opts = do+ when (createIfMissing config) $+ createDirectoryIfMissing True path+ listDirectory path >>= \case+ [] -> create_new cf_names cf_opts opts_ptr read_opts write_opts+ _ -> withCString path $ \path_ptr ->+ withCString "default" $ \cf_deflt_name -> do+ pokeArray cf_names_array (cf_deflt_name : cf_names)+ pokeArray cf_opts_array (opts_ptr : cf_opts)+ db_ptr <-+ throwIfErr "open" $+ c_rocksdb_open_column_families+ opts_ptr+ path_ptr+ (intToCInt (length cf_cfgs + 1))+ cf_names_array+ cf_opts_array+ cf_ptrs_array+ (deflt, cfs) <-+ peekArray (length cf_cfgs + 1) cf_ptrs_array >>= \case+ d : cs -> return (d, cs)+ [] -> error "rocksdb_open_column_families returned no handles"+ -- Every handle must be destroyed before the database is+ -- closed. The default column family is not kept, so+ -- release its handle now; the database keeps its own.+ c_rocksdb_column_family_handle_destroy deflt+ return+ DB+ { rocksDB = db_ptr,+ columnFamilies = cfs,+ readOpts = read_opts,+ snapshot = Nothing,+ writeOpts = write_opts+ }++-- | Run an action with a snapshot of the database.+-- The t'DB' object is not valid after the action ends.+--+-- The t'DB' passed to the action reads from a consistent point-in-time view of+-- the database: 'get', 'multiGet' and iterators created from it do not see+-- writes made after the snapshot was taken. Writes through it still go to the+-- live database.+--+-- Without a snapshot, two reads made one after the other can see different+-- states of the database if another thread writes in between. Anything that+-- reads several related keys and expects them to agree, such as an index and the+-- records it points to, should read through a snapshot.+withSnapshot :: DB -> (DB -> IO a) -> IO a+withSnapshot db f =+ bracket (createSnapshot db) releaseSnapshot (f . fst)++-- | Manually create an unmanaged snapshot.+-- The returned t'DB' has 'readOpts' configured for the snapshot.+-- Use 'releaseSnapshot' to release both the snapshot and its read options.+createSnapshot :: DB -> IO (DB, Snapshot)+createSnapshot db@DB {rocksDB = db_ptr} = do+ snap_ptr <- c_rocksdb_create_snapshot db_ptr+ read_opts <- createReadOpts (Just snap_ptr)+ return (db {readOpts = read_opts, snapshot = Just snap_ptr}, snap_ptr)++-- | Function to release an unmanaged snapshot.+-- Also releases the read options associated with the snapshot DB.+releaseSnapshot :: (DB, Snapshot) -> IO ()+releaseSnapshot (DB {rocksDB = db_ptr, readOpts = read_opts}, snap_ptr) = do+ destroyReadOpts read_opts+ c_rocksdb_release_snapshot db_ptr snap_ptr++-- | Get a property of the default column family, or of the database as a+-- whole. Returns 'Nothing' if RocksDB does not know the property.+getProperty :: DB -> Property -> IO (Maybe ByteString)+getProperty DB {rocksDB = db_ptr} p =+ withCString (propertyName p) $ \name ->+ c_rocksdb_property_value db_ptr name >>= propertyResult++-- | Get a property of a column family. Many properties, such as the memory+-- of memtables and table readers, are kept per column family.+getPropertyCF :: DB -> ColumnFamily -> Property -> IO (Maybe ByteString)+getPropertyCF DB {rocksDB = db_ptr} cf p =+ withCString (propertyName p) $ \name ->+ c_rocksdb_property_value_cf db_ptr cf name >>= propertyResult++-- | Turn a property value returned by RocksDB into a 'ByteString', freeing the+-- C string.+propertyResult :: CString -> IO (Maybe ByteString)+propertyResult val_ptr+ | val_ptr == nullPtr = return Nothing+ | otherwise = do+ res <- BS.packCString val_ptr+ freeCString val_ptr+ return (Just res)++-- | The name RocksDB uses for a property.+propertyName :: Property -> String+propertyName = \case+ NumFilesAtLevel i -> "rocksdb.num-files-at-level" ++ show i+ Stats -> "rocksdb.stats"+ SSTables -> "rocksdb.sstables"+ EstimateTableReadersMem -> "rocksdb.estimate-table-readers-mem"+ CurSizeAllMemTables -> "rocksdb.cur-size-all-mem-tables"+ BlockCacheUsage -> "rocksdb.block-cache-usage"+ PropertyName n -> n++-- | Destroy the given RocksDB database, deleting all of its files. The+-- database must not be open. Throws an 'IOError' on failure.+destroy :: FilePath -> Options -> IO ()+destroy path opts_ptr =+ withCString path $ \path_ptr ->+ throwIfErr "destroy" $ c_rocksdb_destroy_db opts_ptr path_ptr++-- | Try to recover as much data as possible from a corrupted database. Some+-- data may be lost. The database must not be open. Throws an 'IOError' on+-- failure.+repair :: FilePath -> Options -> IO ()+repair path opts_ptr =+ withCString path $ \path_ptr ->+ throwIfErr "repair" $ c_rocksdb_repair_db opts_ptr path_ptr++-- | A key range given as @(start, limit)@. The start key is included and the+-- limit key is excluded.+--+-- TODO: support [Range], like C API does+type Range = (ByteString, ByteString)++-- | Approximate number of bytes of file system space used by the keys in a+-- 'Range' of the default column family. Data still in memtables may not be+-- counted. Throws an 'IOError' on failure.+approximateSize :: DB -> Range -> IO Int64+approximateSize DB {rocksDB = db_ptr} (from, to) =+ BU.unsafeUseAsCStringLen from $ \(from_ptr, flen) ->+ BU.unsafeUseAsCStringLen to $ \(to_ptr, tlen) ->+ withArray [from_ptr] $ \from_ptrs ->+ withArray [intToCSize flen] $ \flen_ptrs ->+ withArray [to_ptr] $ \to_ptrs ->+ withArray [intToCSize tlen] $ \tlen_ptrs ->+ alloca $ \size_ptr -> do+ throwIfErr "approximate_sizes" $+ c_rocksdb_approximate_sizes+ db_ptr+ 1+ from_ptrs+ flen_ptrs+ to_ptrs+ tlen_ptrs+ size_ptr+ fromIntegral <$> peek size_ptr++-- | Write again every table file that has keys from the first to the second+-- of the two given, in the default column family or in the one given, and+-- return when it is done. 'Nothing' leaves that end of the range open, so+-- two of them are the whole column family.+--+-- RocksDB compacts by itself, and a program does not need this to keep a+-- database in order. It is for the one thing that only a rewrite does:+-- giving the files that a database already has the layout of its present+-- t'Config', such as a Bloom filter or 'partitionIndexAndFilters', which+-- otherwise reach a file only when RocksDB happens to compact it, and the+-- files of the last level, which hold most of the data, rarely are. Every+-- file in the range is written, those of the last level included.+--+-- It reads and writes all the data in the range, with the background+-- threads of the database ('maxBackgroundJobs'): hours on a large one, and+-- room on the disk for the files being written until the old ones are+-- deleted. The database can be read and written meanwhile. RocksDB reports+-- no error from it; what it could not do stays as it was.+compactRange :: DB -> Maybe ColumnFamily -> Maybe ByteString -> Maybe ByteString -> IO ()+compactRange DB {rocksDB = db_ptr} mcf from to =+ bracket c_rocksdb_compactoptions_create c_rocksdb_compactoptions_destroy $ \opts -> do+ c_rocksdb_compactoptions_set_bottommost_level_compaction opts forceOptimized+ bound from $ \(from_ptr, flen) ->+ bound to $ \(to_ptr, tlen) ->+ case mcf of+ Just cf -> c_rocksdb_compact_range_cf_opt db_ptr cf opts from_ptr flen to_ptr tlen+ Nothing -> c_rocksdb_compact_range_opt db_ptr opts from_ptr flen to_ptr tlen+ where+ -- @BottommostLevelCompaction::kForceOptimized@: the files of the last+ -- level too, but not those that this compaction has just written+ forceOptimized = 3+ bound Nothing f = f (nullPtr, 0)+ bound (Just k) f = BU.unsafeUseAsCStringLen k $ \(p, n) -> f (p, intToCSize n)++-- | Write a key/value pair to the default column family, replacing any+-- previous value. Throws an 'IOError' on failure.+put :: DB -> ByteString -> ByteString -> IO ()+put db = putCommon db Nothing++-- | Like 'put', but write to the given column family.+putCF :: DB -> ColumnFamily -> ByteString -> ByteString -> IO ()+putCF db cf = putCommon db (Just cf)++-- | Write a key/value pair to the default column family, or to a column family+-- if one is given.+putCommon :: DB -> Maybe ColumnFamily -> ByteString -> ByteString -> IO ()+putCommon DB {rocksDB = db_ptr, writeOpts = write_opts} mcf key value =+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ BU.unsafeUseAsCStringLen value $ \(val_ptr, vlen) ->+ throwIfErr "put" $ case mcf of+ Just cf ->+ c_rocksdb_put_cf+ db_ptr+ write_opts+ cf+ key_ptr+ (intToCSize klen)+ val_ptr+ (intToCSize vlen)+ Nothing ->+ c_rocksdb_put+ db_ptr+ write_opts+ key_ptr+ (intToCSize klen)+ val_ptr+ (intToCSize vlen)++-- | Read a value by key from the default column family. Returns 'Nothing' if+-- the key is not found. Throws an 'IOError' on failure.+get :: DB -> ByteString -> IO (Maybe ByteString)+get db = getCommon db Nothing++-- | Like 'get', but read from the given column family.+getCF :: DB -> ColumnFamily -> ByteString -> IO (Maybe ByteString)+getCF db cf = getCommon db (Just cf)++-- | Read a value from the default column family, or from a column family if one+-- is given.+getCommon :: DB -> Maybe ColumnFamily -> ByteString -> IO (Maybe ByteString)+getCommon DB {rocksDB = db_ptr, readOpts = read_opts} mcf key =+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ alloca $ \vlen_ptr -> do+ val_ptr <- throwIfErr "get" $+ case mcf of+ Just cf ->+ c_rocksdb_get_cf+ db_ptr+ read_opts+ cf+ key_ptr+ (intToCSize klen)+ vlen_ptr+ Nothing ->+ c_rocksdb_get+ db_ptr+ read_opts+ key_ptr+ (intToCSize klen)+ vlen_ptr+ if val_ptr == nullPtr+ then return Nothing+ else do+ vlen <- peek vlen_ptr+ res <- BU.unsafePackMallocCStringLen (val_ptr, cSizeToInt vlen)+ return $ Just res++-- | Read the values of many keys in one call. RocksDB looks the keys up+-- together, sharing the work of finding the files and blocks that hold+-- them, which is faster than reading them one by one. The values are in the+-- order of the keys, with 'Nothing' for keys that are not found. Duplicate keys+-- are allowed. Throws an 'IOError' if any of the lookups fails.+--+-- For a query that needs hundreds of keys at once, such as the balances of many+-- addresses, this cuts the time spent finding blocks and crossing into C to a+-- fraction of separate 'get' calls.+multiGet :: DB -> [ByteString] -> IO [Maybe ByteString]+multiGet db = multiGetCommon db Nothing++-- | 'multiGet' on a column family.+multiGetCF :: DB -> ColumnFamily -> [ByteString] -> IO [Maybe ByteString]+multiGetCF db cf = multiGetCommon db (Just cf)++-- | Read many values from the default column family, or from a column family if+-- one is given.+multiGetCommon :: DB -> Maybe ColumnFamily -> [ByteString] -> IO [Maybe ByteString]+multiGetCommon _ _ [] = return []+multiGetCommon DB {rocksDB = db_ptr, readOpts = read_opts} mcf keys =+ withMany BU.unsafeUseAsCStringLen keys $ \kls ->+ withArrayLen (map fst kls) $ \n key_ptrs ->+ withArray (map (intToCSize . snd) kls) $ \key_lens ->+ allocaArray n $ \val_ptrs ->+ allocaArray n $ \val_lens ->+ allocaArray n $ \err_ptrs -> do+ case mcf of+ Just cf ->+ withArray (replicate n cf) $ \cfs ->+ c_rocksdb_multi_get_cf+ db_ptr+ read_opts+ cfs+ (intToCSize n)+ key_ptrs+ key_lens+ val_ptrs+ val_lens+ err_ptrs+ Nothing ->+ c_rocksdb_multi_get+ db_ptr+ read_opts+ (intToCSize n)+ key_ptrs+ key_lens+ val_ptrs+ val_lens+ err_ptrs+ vals <- peekArray n val_ptrs+ lens <- peekArray n val_lens+ errs <- peekArray n err_ptrs+ -- Own every value and free every error before throwing, so that+ -- none of them leaks.+ res <- forM (zip vals lens) $ \(v, l) ->+ if v == nullPtr+ then return Nothing+ else Just <$> BU.unsafePackMallocCStringLen (v, cSizeToInt l)+ msgs <- forM errs $ \e ->+ if e == nullPtr+ then return Nothing+ else do+ m <- peekCString e+ free e+ return (Just m)+ case [m | Just m <- msgs] of+ m : _ -> throwIO $ userError $ "multi_get: " ++ m+ [] -> return res++-- | Delete a key from the default column family. Deleting a key that does not+-- exist is not an error. Throws an 'IOError' on failure.+delete :: DB -> ByteString -> IO ()+delete db = deleteCommon db Nothing++-- | Like 'delete', but delete from the given column family.+deleteCF :: DB -> ColumnFamily -> ByteString -> IO ()+deleteCF db cf = deleteCommon db (Just cf)++-- | Delete a key/value pair from the default column family, or from a column+-- family if one is given.+deleteCommon :: DB -> Maybe ColumnFamily -> ByteString -> IO ()+deleteCommon DB {rocksDB = db_ptr, writeOpts = write_opts} mcf key =+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ throwIfErr "delete" $ case mcf of+ Just cf -> c_rocksdb_delete_cf db_ptr write_opts cf key_ptr (intToCSize klen)+ Nothing -> c_rocksdb_delete db_ptr write_opts key_ptr (intToCSize klen)++-- | Add a merge record to a key of the default column family, for the+-- t'MergeOperator' of the column family to combine with the value that the+-- key has. Nothing is read: the merge is done when the key is read, and+-- when RocksDB compacts the files that hold the record and the value.+--+-- Throws an 'IOError' if the column family has no merge operator, and on+-- any other failure. For many keys at once use a 'Merge' in a batch.+merge :: DB -> ByteString -> ByteString -> IO ()+merge db = mergeCommon db Nothing++-- | Like 'merge', but for a key of the given column family.+mergeCF :: DB -> ColumnFamily -> ByteString -> ByteString -> IO ()+mergeCF db cf = mergeCommon db (Just cf)++-- | Add a merge record to a key of the default column family, or of a+-- column family if one is given.+mergeCommon :: DB -> Maybe ColumnFamily -> ByteString -> ByteString -> IO ()+mergeCommon DB {rocksDB = db_ptr, writeOpts = write_opts} mcf key value =+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ BU.unsafeUseAsCStringLen value $ \(val_ptr, vlen) ->+ throwIfErr "merge" $ case mcf of+ Just cf ->+ c_rocksdb_merge_cf+ db_ptr+ write_opts+ cf+ key_ptr+ (intToCSize klen)+ val_ptr+ (intToCSize vlen)+ Nothing ->+ c_rocksdb_merge+ db_ptr+ write_opts+ key_ptr+ (intToCSize klen)+ val_ptr+ (intToCSize vlen)++-- | Write the memtable of the default column family, or of the one given,+-- to a table file, and return when it is done.+--+-- RocksDB flushes by itself when a memtable is full, and a program does+-- not need this to keep its data: what is in a memtable is in the+-- write-ahead log too. It is for tests and benchmarks that want data in+-- table files, where the data of a database in production is, and with+-- 'disableAutoCompactions' in a file of its own for each flush. Throws an+-- 'IOError' on failure.+flush :: DB -> Maybe ColumnFamily -> IO ()+flush DB {rocksDB = db_ptr} mcf =+ bracket c_rocksdb_flushoptions_create c_rocksdb_flushoptions_destroy $ \opts -> do+ c_rocksdb_flushoptions_set_wait opts 1+ throwIfErr "flush" $ case mcf of+ Just cf -> c_rocksdb_flush_cf db_ptr opts cf+ Nothing -> c_rocksdb_flush db_ptr opts++-- | Perform a batch mutation. All operations in the list are applied atomically:+-- either all of them are written or none are. They are applied in order, so a+-- later operation on a key overrides an earlier one. Throws an 'IOError' on+-- failure.+--+-- The keys and values are kept alive until RocksDB has copied them, because+-- they are handed to C as pointers into Haskell memory.+--+-- Each operation is handed to RocksDB with a foreign call of its own. For a+-- batch of tens of thousands of operations, 'writePacked' hands them over a+-- few thousand at a call.+write :: DB -> [BatchOp] -> IO ()+write DB {rocksDB = db_ptr, writeOpts = write_opts} batch =+ bracket+ c_rocksdb_writebatch_create+ c_rocksdb_writebatch_destroy+ $ \batch_ptr -> do+ mapM_ (batchAdd batch_ptr) batch+ throwIfErr "write" $ c_rocksdb_write db_ptr write_opts batch_ptr+ -- ensure @ByteString@s (and respective shared @CStringLen@s) aren't+ -- GC'ed until here+ mapM_ touch batch+ where+ batchAdd batch_ptr (Put key val) =+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->+ c_rocksdb_writebatch_put+ batch_ptr+ key_ptr+ (intToCSize klen)+ val_ptr+ (intToCSize vlen)+ batchAdd batch_ptr (PutCF cf_ptr key val) =+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->+ c_rocksdb_writebatch_put_cf+ batch_ptr+ cf_ptr+ key_ptr+ (intToCSize klen)+ val_ptr+ (intToCSize vlen)+ batchAdd batch_ptr (Del key) =+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ c_rocksdb_writebatch_delete+ batch_ptr+ key_ptr+ (intToCSize klen)+ batchAdd batch_ptr (DelCF cf_ptr key) =+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ c_rocksdb_writebatch_delete_cf+ batch_ptr+ cf_ptr+ key_ptr+ (intToCSize klen)+ batchAdd batch_ptr (Merge key val) =+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->+ c_rocksdb_writebatch_merge+ batch_ptr+ key_ptr+ (intToCSize klen)+ val_ptr+ (intToCSize vlen)+ batchAdd batch_ptr (MergeCF cf_ptr key val) =+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->+ c_rocksdb_writebatch_merge_cf+ batch_ptr+ cf_ptr+ key_ptr+ (intToCSize klen)+ val_ptr+ (intToCSize vlen)+ touch (Put (PS p _ _) (PS p' _ _)) = do+ touchForeignPtr p+ touchForeignPtr p'+ touch (PutCF _ (PS p _ _) (PS p' _ _)) = do+ touchForeignPtr p+ touchForeignPtr p'+ touch (Del (PS p _ _)) =+ touchForeignPtr p+ touch (DelCF _ (PS p _ _)) =+ touchForeignPtr p+ touch (Merge (PS p _ _) (PS p' _ _)) = do+ touchForeignPtr p+ touchForeignPtr p'+ touch (MergeCF _ (PS p _ _) (PS p' _ _)) = do+ touchForeignPtr p+ touchForeignPtr p'++-- $packed+--+-- 'write' hands a batch to RocksDB one operation at a time, and each is a+-- foreign call of the kind that lets other Haskell threads run meanwhile:+-- the thread gives up its capability for it, and has to get one back. For+-- a batch of a few operations that is nothing. For the 65,000 keys of a+-- Bitcoin block it was more than half of the time of the batch in the+-- benchmark of this package, 26 ms of 48, and 400 ms or more when eight+-- other threads wanted the four capabilities. It comes on top of a+-- 'ByteString' for each key and each value, which is memory that the+-- collector cannot move.+--+-- A packed batch is the same operations written one after another into a+-- few large buffers: /chunks/. 'writePacked' gives each chunk to a small C+-- function of this package, which adds its operations to the batch, and+-- then writes the batch as 'write' does: atomically, with one write.+--+-- There are two ways to make chunks:+--+-- * 'packOps' packs a list of t'BatchOp' into one chunk. It is the simple+-- way, for small batches and for tests.+--+-- * A program that serialises its keys and values itself can do so+-- straight into a buffer of its own, with no 'ByteString' for each: leave+-- 'packedHeaderSize' bytes, serialise the key and then the value after+-- them, and fill in the header with 'pokePackedHeader' once the two+-- lengths are known. The next record starts where the value ended.+--+-- A record is a header and then the bytes of the key and of the value, with+-- no padding. The header is the column family handle (a null pointer for+-- the default one), the length of the key and that of the value as 32-bit+-- numbers, and one byte for the operation, all as the machine holds them+-- in memory and none of them aligned. @cbits\/rocksdb_haskell.h@ has the+-- layout for a program that writes records by other means.+--+-- The format is this package's own, and what a chunk becomes is a series of+-- calls to the C API of RocksDB. The alternative was to write RocksDB's own+-- encoding of a batch in Haskell and hand it over whole+-- (@rocksdb_writebatch_create_from@), which saves the copy in C. That+-- encoding is internal to RocksDB, with column family numbers and record+-- types that a binding has no business knowing, and the copy costs a few+-- milliseconds for a batch that takes hundreds to write.+--+-- A chunk holds column family handles, which are pointers: it means+-- something only in the process that made it, and only while the database+-- that the handles came from is open.++-- | What a record of a packed batch does to its key.+data PackedOp+ = -- | set the key to the value, as 'Put' and 'PutCF' do+ PackedPut+ | -- | delete the key, as 'Del' and 'DelCF' do; the record has no value+ PackedDel+ | -- | add a merge record to the key, as 'Merge' and 'MergeCF' do: the+ -- value of the record is for the t'MergeOperator' of the column family+ -- to combine with the value that the key has.+ --+ -- On a column family without a merge operator, RocksDB refuses the+ -- batch when it is written, and 'writePacked' throws. RocksDB 11 then+ -- keeps none of the batch; RocksDB 8.9 keeps the operations that came+ -- before the merge. Do not count on either: a batch that is not all+ -- or nothing is what a batch is there to prevent.+ PackedMerge+ deriving (Eq, Show, Enum, Bounded)++-- | Bytes of the header that comes before the key in a record of a packed+-- batch: 17 where a pointer has eight.+packedHeaderSize :: Int+packedHeaderSize = sizeOf (nullPtr :: ColumnFamily) + 2 * sizeOf (0 :: Word32) + 1++-- | Write the header of a record of a packed batch at a pointer, which+-- needs no alignment: the column family, or 'Nothing' for the default one,+-- the operation, the length of the key and the length of the value. It+-- writes 'packedHeaderSize' bytes. The bytes of the key come right after+-- them, and those of the value right after the key.+--+-- It is for a program that serialises keys and values into its own buffer:+-- see the top of this section. The lengths are only known once that is+-- done, so the header is written last, into the room left for it.+--+-- Throws an 'IOError', and writes nothing, if a length is negative or does+-- not fit in 32 bits, or if a 'PackedDel' is given a value.+pokePackedHeader :: Ptr a -> Maybe ColumnFamily -> PackedOp -> Int -> Int -> IO ()+pokePackedHeader ptr mcf op klen vlen = do+ when (outside klen || outside vlen) $+ throwIO (userError "pokePackedHeader: a length that does not fit in 32 bits")+ when (op == PackedDel && vlen /= 0) $+ throwIO (userError "pokePackedHeader: a delete with a value")+ -- Unaligned writes, in the byte order of the machine, which is how+ -- the C side reads them back with memcpy.+ poke (castPtr ptr) (fromMaybe nullPtr mcf)+ pokeByteOff ptr word (fromIntegral klen :: Word32)+ pokeByteOff ptr (word + 4) (fromIntegral vlen :: Word32)+ pokeByteOff ptr (word + 8) code+ where+ word = sizeOf (nullPtr :: ColumnFamily)+ -- compared as words, which have 32 bits at least, so that the limit is+ -- not a negative number where an 'Int' has 32+ outside n = n < 0 || (fromIntegral n :: Word) > fromIntegral (maxBound :: Word32)+ -- ROCKSDB_HASKELL_PUT, _DELETE and _MERGE of rocksdb_haskell.h+ code :: Word8+ code = case op of+ PackedPut -> 0+ PackedDel -> 1+ PackedMerge -> 2+{-# INLINE pokePackedHeader #-}++-- | Pack the operations of a batch into one chunk for 'writePacked', in+-- their order. @'writePacked' db ['packOps' ops]@ does what @'write' db ops@+-- does.+--+-- This is the simple way to a packed batch: it copies every key and value+-- once more, into a buffer as large as all of them together. Use it for+-- small batches, for the odd operation that a program does not serialise+-- itself, and for tests. A program with a large batch gains more by+-- serialising straight into its own chunks with 'pokePackedHeader'.+--+-- The chunk holds the column family handles of the operations, so it is+-- only good while their database is open. Evaluating it throws an 'IOError'+-- if a key or a value is of four gigabytes or more.+packOps :: [BatchOp] -> ByteString+packOps ops = BI.unsafeCreate (sum (map size ops)) $ \start -> foldM_ pack start ops+ where+ parts (Put k v) = (Nothing, PackedPut, k, v)+ parts (Del k) = (Nothing, PackedDel, k, BS.empty)+ parts (PutCF cf k v) = (Just cf, PackedPut, k, v)+ parts (DelCF cf k) = (Just cf, PackedDel, k, BS.empty)+ parts (Merge k v) = (Nothing, PackedMerge, k, v)+ parts (MergeCF cf k v) = (Just cf, PackedMerge, k, v)+ size op = case parts op of+ (_, _, k, v) -> packedHeaderSize + BS.length k + BS.length v+ -- write the record of an operation, and give where the next one goes+ pack :: Ptr Word8 -> BatchOp -> IO (Ptr Word8)+ pack ptr op = do+ let (mcf, kind, k, v) = parts op+ key_ptr = ptr `plusPtr` packedHeaderSize+ val_ptr = key_ptr `plusPtr` BS.length k+ pokePackedHeader ptr mcf kind (BS.length k) (BS.length v)+ copy k key_ptr+ copy v val_ptr+ return (val_ptr `plusPtr` BS.length v)+ -- memcpy must not be given the pointer of an empty string+ copy bytes to =+ BU.unsafeUseAsCStringLen bytes $ \(from, len) ->+ when (len > 0) $ copyBytes to (castPtr from) len++-- | Write a batch that is given as packed chunks: see the top of this+-- section for what they are and how to make them. All the operations of all+-- the chunks are applied atomically, as one batch and with one write:+-- either all of them are written or none are. They are applied in the order+-- of the chunks, and within a chunk in the order of its records, so a later+-- operation on a key overrides an earlier one.+--+-- Every chunk must hold whole records: a record cannot start in one chunk+-- and end in the next. Where a chunk is cut otherwise does not matter, and+-- an empty chunk or no chunk at all is fine. So a program that packs on+-- several cores gives the chunks of each core, one core after another.+--+-- Use it for batches of thousands of operations, where it saves the+-- foreign call that 'write' makes for each operation, and the 'ByteString'+-- for each key and value if the program packs its own chunks. It makes one+-- foreign call for each chunk, so chunks of a few hundred kilobytes are+-- large enough. For a few operations, 'write' is as fast and simpler.+--+-- Throws an 'IOError', with nothing written, if a chunk is not made of+-- whole records: one that is cut short, an operation that does not exist,+-- a delete with a value. The message names the chunk and the record, both+-- counted from zero. Each chunk is checked in C before any of it is added+-- to the batch, and nothing is read outside it. What cannot be checked is+-- the column family handle of a record: a wrong one crashes the process,+-- as it would in 'PutCF'.+--+-- Also throws an 'IOError' if RocksDB refuses the batch, as 'write' does.+-- That is what happens to a batch with a 'PackedMerge' for a column family+-- that has no 'mergeOperator': RocksDB 11 refuses it when it is written,+-- with nothing written.+writePacked :: DB -> [ByteString] -> IO ()+writePacked DB {rocksDB = db_ptr, writeOpts = write_opts} chunks =+ bracket+ c_rocksdb_writebatch_create+ c_rocksdb_writebatch_destroy+ $ \batch_ptr -> do+ forM_ (zip [0 :: Int ..] chunks) $ \(n, chunk) ->+ -- the chunk cannot move or be freed during the call, and RocksDB+ -- has copied what it needs when the call returns+ BU.unsafeUseAsCStringLen chunk $ \(chunk_ptr, len) ->+ alloca $ \count_ptr -> do+ status <-+ c_rocksdb_haskell_writebatch_add_packed+ batch_ptr+ chunk_ptr+ (intToCSize len)+ count_ptr+ when (status /= 0) $ do+ count <- peek count_ptr+ -- both are counted from zero+ throwIO . userError . unwords $+ [ "writePacked: chunk",+ show n,+ "is not made of whole records: record",+ show (cSizeToInt count),+ "is malformed or cut short"+ ]+ throwIfErr "write" $ c_rocksdb_write db_ptr write_opts batch_ptr++-- | Marshal a list of strings into C strings that are valid while the function+-- runs.+withStrings :: [String] -> ([CString] -> IO a) -> IO a+withStrings ss f =+ go [] ss+ where+ go acc [] = f (reverse acc)+ go acc (x : xs) = withCString x $ \p -> go (p : acc) xs
src/Database/RocksDB/C.hs view
@@ -1,7 +1,8 @@-{-# LANGUAGE CApiFFI #-}-{-# LANGUAGE CPP #-}-{-# LANGUAGE EmptyDataDecls #-}+{-# LANGUAGE CApiFFI #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE EmptyDataDecls #-} {-# LANGUAGE ForeignFunctionInterface #-}+ -- | -- Module : Database.RocksDB.C -- Copyright : (c) 2012-2020 The rocksdb-haskell Authors@@ -20,8 +21,13 @@ -- -- The bindings are @safe@ foreign calls. RocksDB calls can take a long time, -- for example when they wait for disk or for a compaction to make room, and a--- @safe@ call lets other Haskell threads keep running meanwhile. The small--- extra cost per call is worth that in a concurrent program.+-- @safe@ call lets other Haskell threads keep running meanwhile. The extra+-- cost per call is worth that in a concurrent program. Where it adds up, in+-- a scan that moves an iterator once for every entry or a batch that is+-- given its keys one by one, the answer is fewer calls and not unsafe ones:+-- 'c_rocksdb_haskell_iter_chunk' reads many entries in one, and+-- 'c_rocksdb_haskell_writebatch_add_packed' adds many operations to a batch+-- in one. -- -- = Keeping the types in step with the header --@@ -37,166 +43,226 @@ -- function with @capi@ instead of @ccall@, as -- 'c_rocksdb_filterpolicy_create_bloom_full' is: the C compiler then reads -- the header that is installed, and converts the argument.- module Database.RocksDB.C- ( RocksDB- , Options- , ReadOpts- , WriteOpts- , ColumnFamily- , SliceTransform- , Snapshot- , Iterator- , WriteBatch- , FilterPolicy- , Cache- , ErrPtr- , DBName- , CFName- , Key- , Val- , c_rocksdb_open- , c_rocksdb_open_column_families- , c_rocksdb_close- , c_rocksdb_create_column_family- , c_rocksdb_drop_column_family- , c_rocksdb_column_family_handle_destroy- , c_rocksdb_put- , c_rocksdb_put_cf- , c_rocksdb_delete- , c_rocksdb_delete_cf- , c_rocksdb_write- , c_rocksdb_get- , c_rocksdb_get_cf- , c_rocksdb_multi_get- , c_rocksdb_multi_get_cf- , c_rocksdb_create_snapshot- , c_rocksdb_release_snapshot- , c_rocksdb_property_value- , c_rocksdb_property_value_cf- , c_rocksdb_approximate_sizes- , c_rocksdb_destroy_db- , c_rocksdb_repair_db- , c_rocksdb_create_iterator- , c_rocksdb_create_iterator_cf- , c_rocksdb_iter_destroy- , c_rocksdb_iter_valid- , c_rocksdb_iter_seek_to_first- , c_rocksdb_iter_seek_to_last- , c_rocksdb_iter_seek- , c_rocksdb_iter_next- , c_rocksdb_iter_prev- , c_rocksdb_iter_key- , c_rocksdb_iter_value- , c_rocksdb_iter_get_error- , c_rocksdb_writebatch_create- , c_rocksdb_writebatch_destroy- , c_rocksdb_writebatch_clear- , c_rocksdb_writebatch_put- , c_rocksdb_writebatch_put_cf- , c_rocksdb_writebatch_delete- , c_rocksdb_writebatch_delete_cf- , c_rocksdb_options_create- , c_rocksdb_options_destroy- , c_rocksdb_options_set_create_if_missing- , c_rocksdb_options_set_error_if_exists- , c_rocksdb_options_set_paranoid_checks- , c_rocksdb_options_set_max_open_files- , c_rocksdb_options_set_write_buffer_size- , c_rocksdb_options_set_db_write_buffer_size- , c_rocksdb_options_set_max_total_wal_size- , c_rocksdb_options_set_create_missing_column_families- , c_rocksdb_options_set_prefix_extractor- , c_rocksdb_options_set_block_based_table_factory- , c_rocksdb_block_based_options_create- , c_rocksdb_block_based_options_set_filter_policy- , c_rocksdb_block_based_options_destroy- , c_rocksdb_block_based_options_set_block_cache- , c_rocksdb_block_based_options_set_cache_index_and_filter_blocks- , c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority- , c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache- , c_rocksdb_cache_create_lru- , c_rocksdb_cache_destroy- , c_rocksdb_slicetransform_create_fixed_prefix- , c_rocksdb_slicetransform_destroy- , c_rocksdb_filterpolicy_destroy- , c_rocksdb_filterpolicy_create_bloom_full- , c_rocksdb_readoptions_create- , c_rocksdb_readoptions_destroy- , c_rocksdb_readoptions_set_snapshot- , c_rocksdb_readoptions_set_iterate_upper_bound- , c_rocksdb_readoptions_set_iterate_lower_bound- , c_rocksdb_readoptions_set_prefix_same_as_start- , c_rocksdb_readoptions_set_total_order_seek- , c_rocksdb_writeoptions_create- , c_rocksdb_writeoptions_destroy- , c_rocksdb_free- ) where+ ( RocksDB,+ Options,+ ReadOpts,+ WriteOpts,+ ColumnFamily,+ SliceTransform,+ Snapshot,+ Iterator,+ WriteBatch,+ FilterPolicy,+ Cache,+ ErrPtr,+ DBName,+ CFName,+ Key,+ Val,+ c_rocksdb_open,+ c_rocksdb_open_column_families,+ c_rocksdb_close,+ c_rocksdb_create_column_family,+ c_rocksdb_drop_column_family,+ c_rocksdb_column_family_handle_destroy,+ c_rocksdb_put,+ c_rocksdb_put_cf,+ c_rocksdb_delete,+ c_rocksdb_delete_cf,+ c_rocksdb_merge,+ c_rocksdb_merge_cf,+ c_rocksdb_write,+ c_rocksdb_get,+ c_rocksdb_get_cf,+ c_rocksdb_multi_get,+ c_rocksdb_multi_get_cf,+ c_rocksdb_create_snapshot,+ c_rocksdb_release_snapshot,+ c_rocksdb_property_value,+ c_rocksdb_property_value_cf,+ c_rocksdb_approximate_sizes,+ CompactOptions,+ c_rocksdb_compactoptions_create,+ c_rocksdb_compactoptions_destroy,+ c_rocksdb_compactoptions_set_bottommost_level_compaction,+ c_rocksdb_compact_range_opt,+ c_rocksdb_compact_range_cf_opt,+ c_rocksdb_destroy_db,+ c_rocksdb_repair_db,+ c_rocksdb_create_iterator,+ c_rocksdb_create_iterator_cf,+ c_rocksdb_iter_destroy,+ c_rocksdb_iter_valid,+ c_rocksdb_iter_seek_to_first,+ c_rocksdb_iter_seek_to_last,+ c_rocksdb_iter_seek,+ c_rocksdb_iter_next,+ c_rocksdb_iter_prev,+ c_rocksdb_iter_key,+ c_rocksdb_iter_value,+ c_rocksdb_iter_get_error,+ c_rocksdb_haskell_iter_chunk,+ c_rocksdb_writebatch_create,+ c_rocksdb_writebatch_destroy,+ c_rocksdb_writebatch_clear,+ c_rocksdb_writebatch_put,+ c_rocksdb_writebatch_put_cf,+ c_rocksdb_writebatch_delete,+ c_rocksdb_writebatch_delete_cf,+ c_rocksdb_writebatch_merge,+ c_rocksdb_writebatch_merge_cf,+ c_rocksdb_haskell_writebatch_add_packed,+ FullMerge,+ PartialMerge,+ c_rocksdb_haskell_options_set_merge_operator,+ c_rocksdb_options_set_uint64add_merge_operator,+ c_rocksdb_options_set_disable_auto_compactions,+ FlushOptions,+ c_rocksdb_flushoptions_create,+ c_rocksdb_flushoptions_destroy,+ c_rocksdb_flushoptions_set_wait,+ c_rocksdb_flush,+ c_rocksdb_flush_cf,+ c_rocksdb_options_create,+ c_rocksdb_options_destroy,+ c_rocksdb_options_set_create_if_missing,+ c_rocksdb_options_set_error_if_exists,+ c_rocksdb_options_set_paranoid_checks,+ c_rocksdb_options_set_max_open_files,+ c_rocksdb_options_set_write_buffer_size,+ c_rocksdb_options_set_db_write_buffer_size,+ c_rocksdb_options_set_max_total_wal_size,+ c_rocksdb_options_set_max_background_jobs,+ c_rocksdb_options_set_create_missing_column_families,+ c_rocksdb_options_set_prefix_extractor,+ c_rocksdb_options_set_block_based_table_factory,+ c_rocksdb_block_based_options_create,+ c_rocksdb_block_based_options_set_filter_policy,+ c_rocksdb_block_based_options_destroy,+ c_rocksdb_block_based_options_set_block_cache,+ c_rocksdb_block_based_options_set_cache_index_and_filter_blocks,+ c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority,+ c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache,+ c_rocksdb_block_based_options_set_index_type,+ c_rocksdb_block_based_options_set_partition_filters,+ c_rocksdb_block_based_options_set_metadata_block_size,+ c_rocksdb_block_based_options_set_pin_top_level_index_and_filter,+ c_rocksdb_cache_create_lru,+ c_rocksdb_cache_destroy,+ c_rocksdb_slicetransform_create_fixed_prefix,+ c_rocksdb_slicetransform_destroy,+ c_rocksdb_filterpolicy_destroy,+ c_rocksdb_filterpolicy_create_bloom_full,+ c_rocksdb_readoptions_create,+ c_rocksdb_readoptions_destroy,+ c_rocksdb_readoptions_set_snapshot,+ c_rocksdb_readoptions_set_iterate_upper_bound,+ c_rocksdb_readoptions_set_iterate_lower_bound,+ c_rocksdb_readoptions_set_prefix_same_as_start,+ c_rocksdb_readoptions_set_total_order_seek,+ c_rocksdb_writeoptions_create,+ c_rocksdb_writeoptions_destroy,+ c_rocksdb_free,+ )+where -import Foreign-import Foreign.C.String-import Foreign.C.Types+import Foreign+import Foreign.C.String+import Foreign.C.Types -- | Opaque C database object. data LRocksDB+ -- | Opaque C column family handle. data LColumnFamily+ -- | Opaque C iterator. data LIterator+ -- | Opaque C database or column family options. data LOptions+ -- | Opaque C block-based table options. data LBlockBasedOptions+ -- | Opaque C read options. data LReadOpts+ -- | Opaque C snapshot. data LSnapshot+ -- | Opaque C write batch. data LWriteBatch+ -- | Opaque C write options. data LWriteOpts+ -- | Opaque C filter policy. data LFilterPolicy+ -- | Opaque C slice transform (prefix extractor). data LSliceTransform+ -- | Opaque C block cache. data LCache -- | Pointer to an open database.-type RocksDB = Ptr LRocksDB+type RocksDB = Ptr LRocksDB+ -- | Pointer to a column family handle.-type ColumnFamily = Ptr LColumnFamily+type ColumnFamily = Ptr LColumnFamily+ -- | Pointer to database or column family options.-type Options = Ptr LOptions+type Options = Ptr LOptions+ -- | Pointer to a batch of writes applied atomically.-type WriteBatch = Ptr LWriteBatch+type WriteBatch = Ptr LWriteBatch+ -- | Pointer to a key transformation, used to extract prefixes.-type SliceTransform = Ptr LSliceTransform+type SliceTransform = Ptr LSliceTransform+ -- | Pointer to options for reads and iterators.-type ReadOpts = Ptr LReadOpts+type ReadOpts = Ptr LReadOpts+ -- | Pointer to options for writes.-type WriteOpts = Ptr LWriteOpts+type WriteOpts = Ptr LWriteOpts++-- | Phantom type of the options of a manual compaction.+data LCompactOptions++-- | Pointer to the options of a manual compaction.+type CompactOptions = Ptr LCompactOptions+ -- | Pointer to a consistent point-in-time view of a database.-type Snapshot = Ptr LSnapshot+type Snapshot = Ptr LSnapshot+ -- | Pointer to an iterator over keys.-type Iterator = Ptr LIterator+type Iterator = Ptr LIterator+ -- | Pointer to a filter policy, such as a bloom filter.-type FilterPolicy = Ptr LFilterPolicy+type FilterPolicy = Ptr LFilterPolicy+ -- | Pointer to block-based table options.-type BlockBasedOptions = Ptr LBlockBasedOptions+type BlockBasedOptions = Ptr LBlockBasedOptions+ -- | Pointer to a block cache.-type Cache = Ptr LCache+type Cache = Ptr LCache -- | Pointer to a location where a C function stores an error message, or leaves @NULL@ on success.-type ErrPtr = Ptr CString+type ErrPtr = Ptr CString+ -- | Path of a database, as a C string.-type DBName = CString+type DBName = CString+ -- | Name of a column family, as a C string.-type CFName = CString+type CFName = CString+ -- | Pointer to the bytes of a key. Its length is passed separately.-type Key = CString+type Key = CString+ -- | Pointer to the bytes of a value. Its length is passed separately.-type Val = CString+type Val = CString -- | Binding to @rocksdb_open@. foreign import ccall safe "rocksdb/c.h rocksdb_open"@@ -204,14 +270,15 @@ -- | Binding to @rocksdb_open_column_families@. foreign import ccall safe "rocksdb/c.h rocksdb_open_column_families"- c_rocksdb_open_column_families :: Options- -> DBName- -> CInt- -> Ptr CFName- -> Ptr Options- -> Ptr ColumnFamily- -> ErrPtr- -> IO RocksDB+ c_rocksdb_open_column_families ::+ Options ->+ DBName ->+ CInt ->+ Ptr CFName ->+ Ptr Options ->+ Ptr ColumnFamily ->+ ErrPtr ->+ IO RocksDB -- | Binding to @rocksdb_close@. foreign import ccall safe "rocksdb/c.h rocksdb_close"@@ -219,18 +286,20 @@ -- | Binding to @rocksdb_create_column_family@. foreign import ccall safe "rocksdb/c.h rocksdb_create_column_family"- c_rocksdb_create_column_family :: RocksDB- -> Options- -> CFName- -> ErrPtr- -> IO ColumnFamily+ c_rocksdb_create_column_family ::+ RocksDB ->+ Options ->+ CFName ->+ ErrPtr ->+ IO ColumnFamily -- | Binding to @rocksdb_drop_column_family@. foreign import ccall safe "rocksdb/c.h rocksdb_drop_column_family"- c_rocksdb_drop_column_family :: RocksDB- -> ColumnFamily- -> ErrPtr- -> IO ()+ c_rocksdb_drop_column_family ::+ RocksDB ->+ ColumnFamily ->+ ErrPtr ->+ IO () -- | Binding to @rocksdb_column_family_handle_destroy@. foreign import ccall safe "rocksdb/c.h rocksdb_column_family_handle_destroy"@@ -238,94 +307,149 @@ -- | Binding to @rocksdb_put@. foreign import ccall safe "rocksdb/c.h rocksdb_put"- c_rocksdb_put :: RocksDB- -> WriteOpts- -> Key -> CSize- -> Val -> CSize- -> ErrPtr- -> IO ()+ c_rocksdb_put ::+ RocksDB ->+ WriteOpts ->+ Key ->+ CSize ->+ Val ->+ CSize ->+ ErrPtr ->+ IO () -- | Binding to @rocksdb_put_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_put_cf"- c_rocksdb_put_cf :: RocksDB- -> WriteOpts- -> ColumnFamily- -> Key -> CSize- -> Val -> CSize- -> ErrPtr- -> IO ()+ c_rocksdb_put_cf ::+ RocksDB ->+ WriteOpts ->+ ColumnFamily ->+ Key ->+ CSize ->+ Val ->+ CSize ->+ ErrPtr ->+ IO () -- | Binding to @rocksdb_delete@. foreign import ccall safe "rocksdb/c.h rocksdb_delete"- c_rocksdb_delete :: RocksDB- -> WriteOpts- -> Key -> CSize- -> ErrPtr- -> IO ()+ c_rocksdb_delete ::+ RocksDB ->+ WriteOpts ->+ Key ->+ CSize ->+ ErrPtr ->+ IO () -- | Binding to @rocksdb_delete_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_delete_cf"- c_rocksdb_delete_cf :: RocksDB- -> WriteOpts- -> ColumnFamily- -> Key -> CSize- -> ErrPtr- -> IO ()+ c_rocksdb_delete_cf ::+ RocksDB ->+ WriteOpts ->+ ColumnFamily ->+ Key ->+ CSize ->+ ErrPtr ->+ IO () +-- | Binding to @rocksdb_merge@.+foreign import ccall safe "rocksdb/c.h rocksdb_merge"+ c_rocksdb_merge ::+ RocksDB ->+ WriteOpts ->+ Key ->+ CSize ->+ Val ->+ CSize ->+ ErrPtr ->+ IO ()++-- | Binding to @rocksdb_merge_cf@.+foreign import ccall safe "rocksdb/c.h rocksdb_merge_cf"+ c_rocksdb_merge_cf ::+ RocksDB ->+ WriteOpts ->+ ColumnFamily ->+ Key ->+ CSize ->+ Val ->+ CSize ->+ ErrPtr ->+ IO ()+ -- | Binding to @rocksdb_write@. foreign import ccall safe "rocksdb/c.h rocksdb_write"- c_rocksdb_write :: RocksDB- -> WriteOpts- -> WriteBatch- -> ErrPtr- -> IO ()+ c_rocksdb_write ::+ RocksDB ->+ WriteOpts ->+ WriteBatch ->+ ErrPtr ->+ IO () -- | Returns NULL if not found. A malloc()ed array otherwise. -- Stores the length of the array in *vallen. -- -- Binding to @rocksdb_get@. foreign import ccall safe "rocksdb/c.h rocksdb_get"- c_rocksdb_get :: RocksDB- -> ReadOpts- -> Key -> CSize- -> Ptr CSize -- ^ value length- -> ErrPtr- -> IO CString+ c_rocksdb_get ::+ RocksDB ->+ ReadOpts ->+ Key ->+ CSize ->+ -- | value length+ Ptr CSize ->+ ErrPtr ->+ IO CString -- | Binding to @rocksdb_get_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_get_cf"- c_rocksdb_get_cf :: RocksDB- -> ReadOpts- -> ColumnFamily- -> Key -> CSize- -> Ptr CSize -- ^ value length- -> ErrPtr- -> IO CString+ c_rocksdb_get_cf ::+ RocksDB ->+ ReadOpts ->+ ColumnFamily ->+ Key ->+ CSize ->+ -- | value length+ Ptr CSize ->+ ErrPtr ->+ IO CString -- | Binding to @rocksdb_multi_get@. foreign import ccall safe "rocksdb/c.h rocksdb_multi_get"- c_rocksdb_multi_get :: RocksDB- -> ReadOpts- -> CSize -- ^ number of keys- -> Ptr CString -- ^ keys- -> Ptr CSize -- ^ key sizes- -> Ptr CString -- ^ values- -> Ptr CSize -- ^ values sizes- -> ErrPtr- -> IO ()+ c_rocksdb_multi_get ::+ RocksDB ->+ ReadOpts ->+ -- | number of keys+ CSize ->+ -- | keys+ Ptr CString ->+ -- | key sizes+ Ptr CSize ->+ -- | values+ Ptr CString ->+ -- | values sizes+ Ptr CSize ->+ ErrPtr ->+ IO () -- | Binding to @rocksdb_multi_get_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_multi_get_cf"- c_rocksdb_multi_get_cf :: RocksDB- -> ReadOpts- -> Ptr ColumnFamily -- ^ column families- -> CSize -- ^ number of keys- -> Ptr CString -- ^ keys- -> Ptr CSize -- ^ key sizes- -> Ptr CString -- ^ values- -> Ptr CSize -- ^ values sizes- -> ErrPtr- -> IO ()+ c_rocksdb_multi_get_cf ::+ RocksDB ->+ ReadOpts ->+ -- | column families+ Ptr ColumnFamily ->+ -- | number of keys+ CSize ->+ -- | keys+ Ptr CString ->+ -- | key sizes+ Ptr CSize ->+ -- | values+ Ptr CString ->+ -- | values sizes+ Ptr CSize ->+ ErrPtr ->+ IO () -- | Binding to @rocksdb_create_snapshot@. foreign import ccall safe "rocksdb/c.h rocksdb_create_snapshot"@@ -353,17 +477,20 @@ -- read a pointer that was never passed, and to write through it on failure. -- Older versions do not take the argument, and ignore it. foreign import ccall safe "rocksdb/c.h rocksdb_approximate_sizes"- c_rocksdb_approximate_sizes :: RocksDB- -> CInt- -- ^ num ranges- -> Ptr CString -> Ptr CSize- -- ^ range start keys (array)- -> Ptr CString -> Ptr CSize- -- ^ range limit keys (array)- -> Ptr Word64- -- ^ array of approx. sizes of ranges- -> ErrPtr- -> IO ()+ c_rocksdb_approximate_sizes ::+ RocksDB ->+ -- | num ranges+ CInt ->+ Ptr CString ->+ -- | range start keys (array)+ Ptr CSize ->+ Ptr CString ->+ -- | range limit keys (array)+ Ptr CSize ->+ -- | array of approx. sizes of ranges+ Ptr Word64 ->+ ErrPtr ->+ IO () -- | Binding to @rocksdb_destroy_db@. foreign import ccall safe "rocksdb/c.h rocksdb_destroy_db"@@ -373,23 +500,24 @@ foreign import ccall safe "rocksdb/c.h rocksdb_repair_db" c_rocksdb_repair_db :: Options -> DBName -> ErrPtr -> IO () - -- -- Iterator -- -- | Binding to @rocksdb_create_iterator@. foreign import ccall safe "rocksdb/c.h rocksdb_create_iterator"- c_rocksdb_create_iterator :: RocksDB- -> ReadOpts- -> IO Iterator+ c_rocksdb_create_iterator ::+ RocksDB ->+ ReadOpts ->+ IO Iterator -- | Binding to @rocksdb_create_iterator_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_create_iterator_cf"- c_rocksdb_create_iterator_cf :: RocksDB- -> ReadOpts- -> ColumnFamily- -> IO Iterator+ c_rocksdb_create_iterator_cf ::+ RocksDB ->+ ReadOpts ->+ ColumnFamily ->+ IO Iterator -- | Binding to @rocksdb_iter_destroy@. foreign import ccall safe "rocksdb/c.h rocksdb_iter_destroy"@@ -439,6 +567,35 @@ foreign import ccall safe "rocksdb/c.h rocksdb_iter_get_error" c_rocksdb_iter_get_error :: Iterator -> ErrPtr -> IO () +-- | Binding to @rocksdb_haskell_iter_chunk@, which is not part of RocksDB: it+-- is this package's own, in @cbits\/rocksdb_haskell.c@, and its header says+-- what it promises. It reads many entries of an iterator in one call.+--+-- The arguments are the iterator, whether to go backwards, the most entries+-- and the most bytes to read, the prefix that keys have to start with and its+-- length, and where to store the buffer with the entries, their number,+-- whether more follow, and an error of the iterator. The result is not zero+-- if there was no memory for the first entry. The caller frees the buffer.+--+-- A safe call, since it moves the iterator and so may wait for the disk. That+-- is the point of it: a scan that moves the iterator from Haskell makes a+-- safe call for every entry, and this makes one for hundreds.+--+-- It is imported with @capi@ so that the C compiler checks the types below+-- against the prototype in the header.+foreign import capi safe "rocksdb_haskell.h rocksdb_haskell_iter_chunk"+ c_rocksdb_haskell_iter_chunk ::+ Iterator ->+ CInt ->+ CSize ->+ CSize ->+ CString ->+ CSize ->+ Ptr CString ->+ Ptr CSize ->+ Ptr CUChar ->+ ErrPtr ->+ IO CInt -- -- Write batch@@ -458,33 +615,181 @@ -- | Binding to @rocksdb_writebatch_put@. foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_put"- c_rocksdb_writebatch_put :: WriteBatch- -> Key -> CSize- -> Val -> CSize- -> IO ()+ c_rocksdb_writebatch_put ::+ WriteBatch ->+ Key ->+ CSize ->+ Val ->+ CSize ->+ IO () -- | Binding to @rocksdb_writebatch_put_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_put_cf"- c_rocksdb_writebatch_put_cf :: WriteBatch- -> ColumnFamily- -> Key -> CSize- -> Val -> CSize- -> IO ()+ c_rocksdb_writebatch_put_cf ::+ WriteBatch ->+ ColumnFamily ->+ Key ->+ CSize ->+ Val ->+ CSize ->+ IO () -- | Binding to @rocksdb_writebatch_delete@. foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_delete"- c_rocksdb_writebatch_delete :: WriteBatch- -> Key -> CSize -> IO ()+ c_rocksdb_writebatch_delete ::+ WriteBatch ->+ Key ->+ CSize ->+ IO () -- | Binding to @rocksdb_writebatch_delete_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_delete_cf"- c_rocksdb_writebatch_delete_cf :: WriteBatch- -> ColumnFamily- -> Key- -> CSize- -> IO ()+ c_rocksdb_writebatch_delete_cf ::+ WriteBatch ->+ ColumnFamily ->+ Key ->+ CSize ->+ IO () +-- | Binding to @rocksdb_writebatch_merge@.+foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_merge"+ c_rocksdb_writebatch_merge ::+ WriteBatch ->+ Key ->+ CSize ->+ Val ->+ CSize ->+ IO ()++-- | Binding to @rocksdb_writebatch_merge_cf@.+foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_merge_cf"+ c_rocksdb_writebatch_merge_cf ::+ WriteBatch ->+ ColumnFamily ->+ Key ->+ CSize ->+ Val ->+ CSize ->+ IO ()++-- | Binding to @rocksdb_haskell_writebatch_add_packed@, which is not part of+-- RocksDB: it is this package's own, in @cbits\/rocksdb_haskell.c@, and its+-- header says what it promises and how a record is laid out. It adds many+-- operations to a write batch in one call. --+-- The arguments are the batch, the buffer of packed records and its length,+-- and where to store the number of records. The result is not zero if the+-- buffer does not hold whole and well formed records: the batch is then as+-- it was, and the number is that of the records before the bad one.+--+-- A safe call, like the calls to @rocksdb_writebatch_put_cf@ that it stands+-- for, so that a buffer of megabytes does not hold up the other Haskell+-- threads while RocksDB copies it. A program makes one for a few thousand+-- operations, and no longer one for each.+--+-- It is imported with @capi@ so that the C compiler checks the types below+-- against the prototype in the header.+foreign import capi safe "rocksdb_haskell.h rocksdb_haskell_writebatch_add_packed"+ c_rocksdb_haskell_writebatch_add_packed ::+ WriteBatch ->+ CString ->+ CSize ->+ Ptr CSize ->+ IO CInt++--+-- Merge operators+--++-- | The type of the function of a merge operator that works out the value+-- of a key: the full merge of @rocksdb_mergeoperator_create@.+--+-- Its arguments are a state that it must not use, the key and its length,+-- the value that the key has and its length (a null pointer if it has+-- none), the merge records, oldest first, as an array of pointers and one+-- of lengths, their number, where to store whether it succeeded, and where+-- to store the length of the result. It returns the new value in memory+-- from @malloc@. @cbits\/rocksdb_haskell.h@ says the rest.+type FullMerge =+ Ptr () ->+ CString ->+ CSize ->+ CString ->+ CSize ->+ Ptr CString ->+ Ptr CSize ->+ CInt ->+ Ptr CUChar ->+ Ptr CSize ->+ IO CString++-- | The type of the function of a merge operator that makes one merge+-- record of several: the partial merge of @rocksdb_mergeoperator_create@.+-- Its arguments are those of a t'FullMerge' without the value that the key+-- has.+type PartialMerge =+ Ptr () ->+ CString ->+ CSize ->+ Ptr CString ->+ Ptr CSize ->+ CInt ->+ Ptr CUChar ->+ Ptr CSize ->+ IO CString++-- | Binding to @rocksdb_haskell_options_set_merge_operator@, which is not+-- part of RocksDB: it is this package's own, in+-- @cbits\/rocksdb_haskell.c@. It gives options a merge operator made of a+-- name and two C functions, and keeps a copy of the name for as long as+-- the operator lives. The options own the operator. The result is not+-- zero if there was no memory for it.+--+-- It is imported with @capi@ so that the C compiler checks the types below+-- against the prototype in the header.+foreign import capi safe "rocksdb_haskell.h rocksdb_haskell_options_set_merge_operator"+ c_rocksdb_haskell_options_set_merge_operator ::+ Options ->+ CString ->+ FunPtr FullMerge ->+ FunPtr PartialMerge ->+ IO CInt++-- | Binding to @rocksdb_options_set_uint64add_merge_operator@.+foreign import ccall safe "rocksdb/c.h rocksdb_options_set_uint64add_merge_operator"+ c_rocksdb_options_set_uint64add_merge_operator :: Options -> IO ()++--+-- Flush+--++-- | Phantom type of the options of a flush.+data LFlushOptions++-- | Pointer to the options of a flush.+type FlushOptions = Ptr LFlushOptions++-- | Binding to @rocksdb_flushoptions_create@.+foreign import ccall safe "rocksdb/c.h rocksdb_flushoptions_create"+ c_rocksdb_flushoptions_create :: IO FlushOptions++-- | Binding to @rocksdb_flushoptions_destroy@.+foreign import ccall safe "rocksdb/c.h rocksdb_flushoptions_destroy"+ c_rocksdb_flushoptions_destroy :: FlushOptions -> IO ()++-- | Binding to @rocksdb_flushoptions_set_wait@.+foreign import ccall safe "rocksdb/c.h rocksdb_flushoptions_set_wait"+ c_rocksdb_flushoptions_set_wait :: FlushOptions -> CUChar -> IO ()++-- | Binding to @rocksdb_flush@.+foreign import ccall safe "rocksdb/c.h rocksdb_flush"+ c_rocksdb_flush :: RocksDB -> FlushOptions -> ErrPtr -> IO ()++-- | Binding to @rocksdb_flush_cf@.+foreign import ccall safe "rocksdb/c.h rocksdb_flush_cf"+ c_rocksdb_flush_cf :: RocksDB -> FlushOptions -> ColumnFamily -> ErrPtr -> IO ()++-- -- Options -- @@ -524,15 +829,24 @@ foreign import ccall safe "rocksdb/c.h rocksdb_options_set_max_total_wal_size" c_rocksdb_options_set_max_total_wal_size :: Options -> Word64 -> IO () +-- | Binding to @rocksdb_options_set_disable_auto_compactions@.+foreign import ccall safe "rocksdb/c.h rocksdb_options_set_disable_auto_compactions"+ c_rocksdb_options_set_disable_auto_compactions :: Options -> CInt -> IO ()++-- | Binding to @rocksdb_options_set_max_background_jobs@.+foreign import ccall safe "rocksdb/c.h rocksdb_options_set_max_background_jobs"+ c_rocksdb_options_set_max_background_jobs :: Options -> CInt -> IO ()+ -- | Binding to @rocksdb_options_set_create_missing_column_families@. foreign import ccall safe "rocksdb/c.h rocksdb_options_set_create_missing_column_families" c_rocksdb_options_set_create_missing_column_families :: Options -> CBool -> IO () -- | Binding to @rocksdb_options_set_prefix_extractor@. foreign import ccall safe "rocksdb/c.h rocksdb_options_set_prefix_extractor"- c_rocksdb_options_set_prefix_extractor :: Options- -> SliceTransform- -> IO ()+ c_rocksdb_options_set_prefix_extractor ::+ Options ->+ SliceTransform ->+ IO () -- -- Block-Based Options@@ -544,45 +858,91 @@ -- | Binding to @rocksdb_block_based_options_destroy@. foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_destroy"- c_rocksdb_block_based_options_destroy :: BlockBasedOptions- -> IO ()+ c_rocksdb_block_based_options_destroy ::+ BlockBasedOptions ->+ IO () -- | Binding to @rocksdb_block_based_options_set_filter_policy@. foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_filter_policy"- c_rocksdb_block_based_options_set_filter_policy :: BlockBasedOptions- -> FilterPolicy- -> IO ()+ c_rocksdb_block_based_options_set_filter_policy ::+ BlockBasedOptions ->+ FilterPolicy ->+ IO () -- | Binding to @rocksdb_block_based_options_set_block_cache@. foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_block_cache"- c_rocksdb_block_based_options_set_block_cache :: BlockBasedOptions- -> Cache- -> IO ()+ c_rocksdb_block_based_options_set_block_cache ::+ BlockBasedOptions ->+ Cache ->+ IO () -- | Binding to @rocksdb_block_based_options_set_cache_index_and_filter_blocks@. foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_cache_index_and_filter_blocks"- c_rocksdb_block_based_options_set_cache_index_and_filter_blocks :: BlockBasedOptions- -> CUChar- -> IO ()+ c_rocksdb_block_based_options_set_cache_index_and_filter_blocks ::+ BlockBasedOptions ->+ CUChar ->+ IO () -- | Binding to @rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority@.-foreign import ccall safe- "rocksdb/c.h rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority"- c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority :: BlockBasedOptions- -> CUChar- -> IO ()+foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority"+ c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority ::+ BlockBasedOptions ->+ CUChar ->+ IO () +-- | Binding to @rocksdb_compactoptions_create@.+foreign import ccall safe "rocksdb/c.h rocksdb_compactoptions_create"+ c_rocksdb_compactoptions_create :: IO CompactOptions++-- | Binding to @rocksdb_compactoptions_destroy@.+foreign import ccall safe "rocksdb/c.h rocksdb_compactoptions_destroy"+ c_rocksdb_compactoptions_destroy :: CompactOptions -> IO ()++-- | Binding to @rocksdb_compactoptions_set_bottommost_level_compaction@.+foreign import ccall safe "rocksdb/c.h rocksdb_compactoptions_set_bottommost_level_compaction"+ c_rocksdb_compactoptions_set_bottommost_level_compaction :: CompactOptions -> CUChar -> IO ()++-- | Binding to @rocksdb_compact_range_opt@. A null key leaves that end of+-- the range open.+foreign import ccall safe "rocksdb/c.h rocksdb_compact_range_opt"+ c_rocksdb_compact_range_opt ::+ RocksDB -> CompactOptions -> CString -> CSize -> CString -> CSize -> IO ()++-- | Binding to @rocksdb_compact_range_cf_opt@. A null key leaves that end+-- of the range open.+foreign import ccall safe "rocksdb/c.h rocksdb_compact_range_cf_opt"+ c_rocksdb_compact_range_cf_opt ::+ RocksDB -> ColumnFamily -> CompactOptions -> CString -> CSize -> CString -> CSize -> IO ()++-- | Binding to @rocksdb_block_based_options_set_index_type@.+foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_index_type"+ c_rocksdb_block_based_options_set_index_type :: BlockBasedOptions -> CInt -> IO ()++-- | Binding to @rocksdb_block_based_options_set_partition_filters@.+foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_partition_filters"+ c_rocksdb_block_based_options_set_partition_filters :: BlockBasedOptions -> CUChar -> IO ()++-- | Binding to @rocksdb_block_based_options_set_metadata_block_size@.+foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_metadata_block_size"+ c_rocksdb_block_based_options_set_metadata_block_size :: BlockBasedOptions -> Word64 -> IO ()++-- | Binding to @rocksdb_block_based_options_set_pin_top_level_index_and_filter@.+foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_pin_top_level_index_and_filter"+ c_rocksdb_block_based_options_set_pin_top_level_index_and_filter :: BlockBasedOptions -> CUChar -> IO ()+ -- | Binding to @rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache@. foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache"- c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache :: BlockBasedOptions- -> CUChar- -> IO ()+ c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache ::+ BlockBasedOptions ->+ CUChar ->+ IO () -- | Binding to @rocksdb_options_set_block_based_table_factory@. foreign import ccall safe "rocksdb/c.h rocksdb_options_set_block_based_table_factory"- c_rocksdb_options_set_block_based_table_factory :: Options- -> BlockBasedOptions- -> IO ()+ c_rocksdb_options_set_block_based_table_factory ::+ Options ->+ BlockBasedOptions ->+ IO () -- -- Block Cache@@ -667,7 +1027,6 @@ -- | Binding to @rocksdb_readoptions_set_total_order_seek@. foreign import ccall safe "rocksdb/c.h rocksdb_readoptions_set_total_order_seek" c_rocksdb_readoptions_set_total_order_seek :: ReadOpts -> CUChar -> IO ()- -- -- Write options
src/Database/RocksDB/Internal.hs view
@@ -1,4 +1,6 @@+{-# LANGUAGE LambdaCase #-} {-# LANGUAGE RecordWildCards #-}+ -- | -- Module : Database.RocksDB.Internal -- Copyright : (c) 2012-2013 The leveldb-haskell Authors@@ -11,61 +13,63 @@ -- Internal types and helpers shared by the other modules. The t'DB' and t'Config' -- types are re-exported by "Database.RocksDB"; everything else is an -- implementation detail.- module Database.RocksDB.Internal- ( Config (..)- , DB (..)+ ( Config (..),+ MergeOperator (..),+ DB (..), -- * Smart constructors & extractors- , withOptions- , withOptionsCache- , withOptionsCF- , withOptionsCFCache- , withBlockCache- , withReadOpts- , createReadOpts- , destroyReadOpts- , withWriteOpts+ withOptions,+ withOptionsCache,+ withOptionsCF,+ withOptionsCFCache,+ withBlockCache,+ withReadOpts,+ createReadOpts,+ destroyReadOpts,+ withWriteOpts, -- * Utilities- , freeCString- , throwIfErr- , cSizeToInt- , intToCSize- , intToCInt- , cIntToInt- , boolToNum- ) where+ freeCString,+ throwIfErr,+ cSizeToInt,+ intToCSize,+ intToCInt,+ cIntToInt,+ boolToNum,+ )+where -import Control.Exception-import Control.Monad-import Data.Default-import Data.Maybe (isJust)-import Database.RocksDB.C-import Foreign-import Foreign.C.String-import Foreign.C.Types+import Control.Exception+import Control.Monad+import Data.Default+import Data.Maybe (isJust)+import Database.RocksDB.C+import Foreign+import Foreign.C.String+import Foreign.C.Types -- | Handle to an open database. Obtain one with 'Database.RocksDB.withDB' or -- 'Database.RocksDB.withDBCF'. It is only valid inside the function passed to -- those, and can be shared between threads.-data DB = DB { rocksDB :: !RocksDB- -- ^ pointer to the C database object- , columnFamilies :: ![ColumnFamily]- -- ^ handles of the column families opened with- -- 'Database.RocksDB.withDBCF', in the order they were given;- -- empty for 'Database.RocksDB.withDB'- , readOpts :: !ReadOpts- -- ^ read options used by reads and iterators; set to read from- -- a snapshot by 'Database.RocksDB.withSnapshot'- , snapshot :: !(Maybe Snapshot)- -- ^ the snapshot that 'readOpts' read from, if any. RocksDB- -- cannot be asked what snapshot its read options hold, and- -- 'Database.RocksDB.withIterConfig' has to make read options- -- of its own that read from the same one.- , writeOpts :: !WriteOpts- -- ^ write options used by all writes- }+data DB = DB+ { -- | pointer to the C database object+ rocksDB :: !RocksDB,+ -- | handles of the column families opened with+ -- 'Database.RocksDB.withDBCF', in the order they were given;+ -- empty for 'Database.RocksDB.withDB'+ columnFamilies :: ![ColumnFamily],+ -- | read options used by reads and iterators; set to read from+ -- a snapshot by 'Database.RocksDB.withSnapshot'+ readOpts :: !ReadOpts,+ -- | the snapshot that 'readOpts' read from, if any. RocksDB+ -- cannot be asked what snapshot its read options hold, and+ -- 'Database.RocksDB.withIterConfig' has to make read options+ -- of its own that read from the same one.+ snapshot :: !(Maybe Snapshot),+ -- | write options used by all writes+ writeOpts :: !WriteOpts+ } -- | Configuration of a database or column family. Use 'def' for the defaults -- and override the fields you need:@@ -83,69 +87,224 @@ -- database these grow without bound. Setting 'maxFiles', 'blockCacheSize', -- 'cacheIndexAndFilterBlocks' and 'dbWriteBufferSize' turns that into a fixed -- budget.-data Config = Config { createIfMissing :: !Bool- -- ^ create the database if it does not exist- , errorIfExists :: !Bool- -- ^ fail to open if the database already exists- , paranoidChecks :: !Bool- -- ^ check data aggressively and stop early on any- -- corruption- , createMissingColumnFamilies :: !Bool- -- ^ create the column families passed to- -- 'Database.RocksDB.withDBCF' that an existing- -- database does not have yet- , maxFiles :: !(Maybe Int)- -- ^ maximum number of files RocksDB keeps open- -- ('Nothing': RocksDB default, unlimited)- , prefixLength :: !(Maybe Int)- -- ^ length in bytes of a fixed key prefix, used by- -- bloom filters and prefix seeks- , bloomFilter :: !Bool- -- ^ use a full bloom filter with 10 bits per key to- -- skip files that cannot contain a key; the filters- -- take about 1.25 bytes of memory per key- , writeBufferSize :: !(Maybe Int)- -- ^ bytes per memtable (RocksDB default 64 MiB);- -- per column family- , dbWriteBufferSize :: !(Maybe Int)- -- ^ bytes for all memtables across all column- -- families (RocksDB default unlimited); database-wide,- -- ignored in column family configs- , blockCacheSize :: !(Maybe Int)- -- ^ bytes of a least-recently-used block cache shared- -- by the database and all its column families- -- ('Nothing': RocksDB gives each column family its- -- own default cache, of 32 MiB in RocksDB 8);- -- database-wide, ignored in column family configs- , cacheIndexAndFilterBlocks :: !Bool- -- ^ keep index and filter blocks in the block cache,- -- at high priority, with those of level 0 pinned,- -- instead of in memory outside it for every open- -- file; per column family- , maxTotalWalSize :: !(Maybe Int)- -- ^ bytes of write-ahead log after which the column- -- families holding the oldest log are flushed, so it- -- can be deleted (RocksDB default: four times the- -- memtables); database-wide, ignored in column family- -- configs- } deriving (Eq, Show)+data Config = Config+ { -- | create the database if it does not exist+ createIfMissing :: !Bool,+ -- | fail to open if the database already exists+ errorIfExists :: !Bool,+ -- | check data aggressively and stop early on any+ -- corruption+ paranoidChecks :: !Bool,+ -- | create the column families passed to+ -- 'Database.RocksDB.withDBCF' that an existing+ -- database does not have yet+ createMissingColumnFamilies :: !Bool,+ -- | maximum number of files RocksDB keeps open+ -- ('Nothing': RocksDB default, unlimited)+ maxFiles :: !(Maybe Int),+ -- | length in bytes of a fixed key prefix, used by+ -- bloom filters and prefix seeks+ prefixLength :: !(Maybe Int),+ -- | use a full bloom filter with 10 bits per key to+ -- skip files that cannot contain a key; the filters+ -- take about 1.25 bytes of memory per key+ bloomFilter :: !Bool,+ -- | bytes per memtable (RocksDB default 64 MiB);+ -- per column family+ writeBufferSize :: !(Maybe Int),+ -- | bytes for all memtables across all column+ -- families (RocksDB default unlimited); database-wide,+ -- ignored in column family configs+ dbWriteBufferSize :: !(Maybe Int),+ -- | bytes of a least-recently-used block cache shared+ -- by the database and all its column families+ -- ('Nothing': RocksDB gives each column family its+ -- own default cache, of 32 MiB in RocksDB 8);+ -- database-wide, ignored in column family configs+ blockCacheSize :: !(Maybe Int),+ -- | keep index and filter blocks in the block cache,+ -- at high priority, with those of level 0 pinned,+ -- instead of in memory outside it for every open+ -- file; per column family+ cacheIndexAndFilterBlocks :: !Bool,+ -- | write the index and the filter of each table file in+ -- pieces of 4 KiB under a small index of their own, where+ -- each is otherwise one block that grows with the file: on+ -- files of 64 MiB the filter is one to three mebibytes and+ -- the index a few hundred kibibytes. A lookup then loads+ -- the two pieces that it needs and not megabytes, and a+ -- block cache far smaller than all the indices and filters+ -- of the database still serves most lookups from memory.+ -- Without it, a database with more of them than its cache+ -- holds loads and drops them at nearly every lookup: a+ -- millisecond of copying, checksums and decompression+ -- each, on a disk that is idle. The small top-level index+ -- of each open file is kept in the cache. It applies to+ -- the files written from then on; the files that a+ -- database already has keep their layout until they are+ -- compacted, and both kinds are read. It costs one more+ -- lookup in memory for each read. Use it with+ -- 'cacheIndexAndFilterBlocks'; per column family+ partitionIndexAndFilters :: !Bool,+ -- | bytes of write-ahead log after which the column+ -- families holding the oldest log are flushed, so it+ -- can be deleted (RocksDB default: four times the+ -- memtables); database-wide, ignored in column family+ -- configs+ maxTotalWalSize :: !(Maybe Int),+ -- | threads that RocksDB may use at once to flush+ -- memtables and to compact files (RocksDB default 2:+ -- one flush and one compaction). Of @n@ jobs, a+ -- quarter are for flushes and the rest for+ -- compactions, at least one of each. More of them+ -- shorten a backlog of compactions on a disk and+ -- cores that have room for it, and take both from+ -- reads when they have not: on a slow disk a single+ -- compaction already waits for the disk, and a+ -- second one only makes every read wait longer.+ -- Database-wide, ignored in column family configs+ maxBackgroundJobs :: !(Maybe Int),+ -- | how RocksDB works out the value of a key that has+ -- merge records ('Database.RocksDB.Merge'): see+ -- t'MergeOperator'. 'Nothing', the default, for a+ -- column family that is never written with merges; one+ -- that is must be given the same operator every time+ -- the database is opened, by every program that opens+ -- it. Per column family: the one in the configuration+ -- of the database is for the default column family+ mergeOperator :: !(Maybe MergeOperator),+ -- | do not compact the files of the column family in+ -- the background. Files then pile up as memtables are+ -- flushed, and reads get slower with each: it is for+ -- a bulk load that ends with+ -- 'Database.RocksDB.compactRange', and for tests and+ -- benchmarks that need to know which file a key is+ -- in. Per column family+ disableAutoCompactions :: !Bool+ }+ deriving (Eq, Show) +-- | How RocksDB combines the merge records of a key with its value.+--+-- = What a merge is+--+-- A put replaces the value of a key, and a program that wants to change+-- a value has to read it first: add to a counter, add an entry to a list.+-- For a batch of thousands of such changes, those reads are most of the+-- work, and each may wait for the disk.+--+-- A /merge record/ is a third kind of write, next to a put and a delete+-- ('Database.RocksDB.Merge', 'Database.RocksDB.merge'). It says how the+-- value changes and not what it becomes: \"add 5\", \"add this entry\".+-- RocksDB stores the record as it is, without reading anything. The work+-- is done later by the /merge operator/ of the column family: a function+-- that takes the value that the key had and the records written since,+-- oldest first, and gives the value that the key now has. RocksDB calls+-- it when the key is read, and when a compaction brings the records and+-- the value together, after which the key holds a plain value again.+--+-- Readers see none of this: 'Database.RocksDB.get',+-- 'Database.RocksDB.multiGet' and iterators return merged values. What+-- they pay is the merge, for a key that has records pending, and a look+-- into each file that holds one of them.+--+-- A put or a delete after merge records hides them, like any older+-- value. A merge record after a delete, or on a key that was never+-- written, is merged with no value: the operator decides what that means.+--+-- = Which operator+--+-- The operator is part of the meaning of the data, like the format of+-- the values. Every program that opens the database must give the column+-- family the same operator, because RocksDB does not refuse to open one+-- that has merge records without it. This is what RocksDB 11.1 was seen to+-- do then:+--+-- * Merge records that are still in the write-ahead log, not yet flushed+-- to a table file, are __dropped when the database is opened__, without+-- an error. They are lost for good.+--+-- * Merge records in table files are kept, also through a compaction, and+-- are there again for a program that opens the database with the+-- operator. Meanwhile they cannot be read: 'Database.RocksDB.get' throws+-- for a key that has records and no value under them, an iterator stops+-- at such a key with an error, and a 'Database.RocksDB.get' or+-- 'Database.RocksDB.multiGet' of a key that has a value under its+-- records __crashes the process__.+--+-- * Writing a merge record is refused.+--+-- A column family opened with another operator gives whatever that+-- operator makes of the records. So a program that starts to write merge+-- records to a database that older versions of it can open should make+-- sure that they no longer can: a column family that they do not know of+-- is enough, as RocksDB refuses to open a database without all its column+-- families, before it reads the write-ahead log.+data MergeOperator+ = -- | RocksDB's own operator for counters. A value and a merge record+ -- are both an unsigned 64-bit number in eight bytes, least significant+ -- byte first, and merging adds them, modulo 2^64. A key without a+ -- value counts from zero, and so does one whose value is not eight+ -- bytes long.+ UInt64Add+ | -- | An operator made of two C functions.+ --+ -- They are C functions and not Haskell ones because RocksDB calls+ -- them from its own threads, during reads, flushes and compactions,+ -- several at once. Import their addresses:+ --+ -- @+ -- foreign import ccall "&my_full_merge" myFullMerge :: FunPtr FullMerge+ -- foreign import ccall "&my_partial_merge" myPartialMerge :: FunPtr PartialMerge+ -- @+ --+ -- @cbits\/rocksdb_haskell.h@ in this package has their C types, as+ -- @rocksdb_haskell_full_merge_t@ and @rocksdb_haskell_partial_merge_t@,+ -- and what they must do: return the result in memory from @malloc@,+ -- which RocksDB frees, and how to report a failure.+ MergeFunctions+ { -- | the name of the operator, which RocksDB writes in its+ -- options file and in every table file. Change it when what+ -- the functions do changes.+ mergeName :: !String,+ -- | the value of a key from the value it had, if any, and its+ -- merge records, oldest first+ fullMerge :: !(FunPtr FullMerge),+ -- | one merge record from several, oldest first, that does what+ -- they did. RocksDB uses it to shorten the list where it has+ -- records and not the value under them. A function that always+ -- fails is allowed: the records are then kept as they are.+ partialMerge :: !(FunPtr PartialMerge)+ }+ deriving (Eq, Show)+ -- | All flags off and all sizes left to RocksDB defaults. instance Default Config where- def = Config { createIfMissing = False- , errorIfExists = False- , paranoidChecks = False- , createMissingColumnFamilies = False- , maxFiles = Nothing- , prefixLength = Nothing- , bloomFilter = False- , writeBufferSize = Nothing- , dbWriteBufferSize = Nothing- , blockCacheSize = Nothing- , cacheIndexAndFilterBlocks = False- , maxTotalWalSize = Nothing- }+ def =+ Config+ { createIfMissing = False,+ errorIfExists = False,+ paranoidChecks = False,+ createMissingColumnFamilies = False,+ maxFiles = Nothing,+ prefixLength = Nothing,+ bloomFilter = False,+ writeBufferSize = Nothing,+ dbWriteBufferSize = Nothing,+ blockCacheSize = Nothing,+ cacheIndexAndFilterBlocks = False,+ partitionIndexAndFilters = False,+ maxTotalWalSize = Nothing,+ maxBackgroundJobs = Nothing,+ mergeOperator = Nothing,+ disableAutoCompactions = False+ } +-- | @rocksdb_block_based_table_index_type_two_level_index_search@: an index+-- of the pieces of the index.+twoLevelIndexSearch :: CInt+twoLevelIndexSearch = 2+ -- | Create RocksDB options from a t'Config' and pass them to a function. The -- options are destroyed when it returns. withOptions :: Config -> (Options -> IO a) -> IO a@@ -155,46 +314,67 @@ -- any. The options are destroyed when the function returns. withOptionsCache :: Maybe Cache -> Config -> (Options -> IO a) -> IO a withOptionsCache cache Config {..} f = with_opts $ \opts -> do- when (bloomFilter || isJust cache || cacheIndexAndFilterBlocks) $- bracket c_rocksdb_block_based_options_create- c_rocksdb_block_based_options_destroy $ \bo -> do- when bloomFilter $ do- fp <- c_rocksdb_filterpolicy_create_bloom_full 10- c_rocksdb_block_based_options_set_filter_policy bo fp- forM_ cache $ c_rocksdb_block_based_options_set_block_cache bo- when cacheIndexAndFilterBlocks $ do- c_rocksdb_block_based_options_set_cache_index_and_filter_blocks bo 1- c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority bo 1- c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache bo 1- -- the table factory copies the options, filter policy and cache- c_rocksdb_options_set_block_based_table_factory opts bo- forM_ prefixLength $ \l -> do- t <- c_rocksdb_slicetransform_create_fixed_prefix (intToCSize l)- c_rocksdb_options_set_prefix_extractor opts t- forM_ maxFiles $- c_rocksdb_options_set_max_open_files opts . intToCInt- forM_ writeBufferSize $- c_rocksdb_options_set_write_buffer_size opts . intToCSize- forM_ dbWriteBufferSize $- c_rocksdb_options_set_db_write_buffer_size opts . intToCSize- forM_ maxTotalWalSize $- c_rocksdb_options_set_max_total_wal_size opts . fromIntegral- c_rocksdb_options_set_create_if_missing- opts (boolToCBool createIfMissing)- c_rocksdb_options_set_error_if_exists- opts (boolToCBool errorIfExists)- c_rocksdb_options_set_paranoid_checks- opts (boolToCBool paranoidChecks)- c_rocksdb_options_set_create_missing_column_families- opts (boolToCBool createMissingColumnFamilies)- f opts+ when (bloomFilter || isJust cache || cacheIndexAndFilterBlocks || partitionIndexAndFilters)+ $ bracket+ c_rocksdb_block_based_options_create+ c_rocksdb_block_based_options_destroy+ $ \bo -> do+ when bloomFilter $ do+ fp <- c_rocksdb_filterpolicy_create_bloom_full 10+ c_rocksdb_block_based_options_set_filter_policy bo fp+ forM_ cache $ c_rocksdb_block_based_options_set_block_cache bo+ when cacheIndexAndFilterBlocks $ do+ c_rocksdb_block_based_options_set_cache_index_and_filter_blocks bo 1+ c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority bo 1+ c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache bo 1+ when partitionIndexAndFilters $ do+ c_rocksdb_block_based_options_set_index_type bo twoLevelIndexSearch+ c_rocksdb_block_based_options_set_partition_filters bo 1+ c_rocksdb_block_based_options_set_metadata_block_size bo 4096+ c_rocksdb_block_based_options_set_pin_top_level_index_and_filter bo 1+ -- the table factory copies the options, filter policy and cache+ c_rocksdb_options_set_block_based_table_factory opts bo+ forM_ prefixLength $ \l -> do+ t <- c_rocksdb_slicetransform_create_fixed_prefix (intToCSize l)+ c_rocksdb_options_set_prefix_extractor opts t+ forM_ maxFiles $+ c_rocksdb_options_set_max_open_files opts . intToCInt+ forM_ writeBufferSize $+ c_rocksdb_options_set_write_buffer_size opts . intToCSize+ forM_ dbWriteBufferSize $+ c_rocksdb_options_set_db_write_buffer_size opts . intToCSize+ forM_ maxTotalWalSize $+ c_rocksdb_options_set_max_total_wal_size opts . fromIntegral+ forM_ maxBackgroundJobs $+ c_rocksdb_options_set_max_background_jobs opts . intToCInt+ forM_ mergeOperator $ \case+ UInt64Add -> c_rocksdb_options_set_uint64add_merge_operator opts+ MergeFunctions {..} ->+ withCString mergeName $ \name -> do+ status <- c_rocksdb_haskell_options_set_merge_operator opts name fullMerge partialMerge+ when (status /= 0) $+ throwIO (userError "mergeOperator: no memory for the merge operator")+ when disableAutoCompactions $+ c_rocksdb_options_set_disable_auto_compactions opts 1+ c_rocksdb_options_set_create_if_missing+ opts+ (boolToCBool createIfMissing)+ c_rocksdb_options_set_error_if_exists+ opts+ (boolToCBool errorIfExists)+ c_rocksdb_options_set_paranoid_checks+ opts+ (boolToCBool paranoidChecks)+ c_rocksdb_options_set_create_missing_column_families+ opts+ (boolToCBool createMissingColumnFamilies)+ f opts where with_opts =- bracket+ bracket c_rocksdb_options_create c_rocksdb_options_destroy - -- | Create options for several column families at once, in the same order as -- the configurations. withOptionsCF :: [Config] -> ([Options] -> IO a) -> IO a@@ -203,10 +383,10 @@ -- | Options of column families that use this block cache, if any. withOptionsCFCache :: Maybe Cache -> [Config] -> ([Options] -> IO a) -> IO a withOptionsCFCache cache cfgs f =- go [] cfgs+ go [] cfgs where go acc [] = f (reverse acc)- go acc (c:cs) = withOptionsCache cache c $ \o -> go (o:acc) cs+ go acc (c : cs) = withOptionsCache cache c $ \o -> go (o : acc) cs -- | The block cache of 'blockCacheSize', if it is set. The database and -- column families that use it keep it alive after this returns.@@ -216,15 +396,16 @@ withBlockCache :: Config -> (Maybe Cache -> IO a) -> IO a withBlockCache Config {blockCacheSize = Nothing} f = f Nothing withBlockCache Config {blockCacheSize = Just n} f =- bracket (c_rocksdb_cache_create_lru (intToCSize n))- c_rocksdb_cache_destroy- (f . Just)+ bracket+ (c_rocksdb_cache_create_lru (intToCSize n))+ c_rocksdb_cache_destroy+ (f . Just) -- | Create read options, reading from a snapshot if one is given, and pass them -- to a function. They are destroyed when it returns. withReadOpts :: Maybe Snapshot -> (ReadOpts -> IO a) -> IO a withReadOpts maybe_snap_ptr =- bracket+ bracket (createReadOpts maybe_snap_ptr) c_rocksdb_readoptions_destroy @@ -232,9 +413,9 @@ -- Caller is responsible for calling 'destroyReadOpts'. createReadOpts :: Maybe Snapshot -> IO ReadOpts createReadOpts maybe_snap_ptr = do- read_opts_ptr <- c_rocksdb_readoptions_create- forM_ maybe_snap_ptr $ c_rocksdb_readoptions_set_snapshot read_opts_ptr- return read_opts_ptr+ read_opts_ptr <- c_rocksdb_readoptions_create+ forM_ maybe_snap_ptr $ c_rocksdb_readoptions_set_snapshot read_opts_ptr+ return read_opts_ptr -- | Destroy read options created with 'createReadOpts'. destroyReadOpts :: ReadOpts -> IO ()@@ -244,7 +425,7 @@ -- destroyed when it returns. withWriteOpts :: (WriteOpts -> IO a) -> IO a withWriteOpts =- bracket+ bracket c_rocksdb_writeoptions_create c_rocksdb_writeoptions_destroy @@ -261,18 +442,18 @@ -- freed. throwIfErr :: String -> (ErrPtr -> IO a) -> IO a throwIfErr s f = alloca $ \err_ptr -> do- poke err_ptr nullPtr- res <- f err_ptr- err_cstr <- peek err_ptr- when (err_cstr /= nullPtr) $ do- err <- peekCString err_cstr- free err_cstr- throwIO $ userError $ s ++ ": " ++ err- return res+ poke err_ptr nullPtr+ res <- f err_ptr+ err_cstr <- peek err_ptr+ when (err_cstr /= nullPtr) $ do+ err <- peekCString err_cstr+ free err_cstr+ throwIO $ userError $ s ++ ": " ++ err+ return res -- | Convert a 'Bool' to a C boolean. boolToCBool :: Bool -> CBool-boolToCBool True = 1+boolToCBool True = 1 boolToCBool False = 0 {-# INLINE boolToCBool #-} @@ -297,7 +478,7 @@ {-# INLINE cIntToInt #-} -- | Convert a 'Bool' to @1@ or @0@.-boolToNum :: Num b => Bool -> b-boolToNum True = fromIntegral (1 :: Int)+boolToNum :: (Num b) => Bool -> b+boolToNum True = fromIntegral (1 :: Int) boolToNum False = fromIntegral (0 :: Int) {-# INLINE boolToNum #-}
src/Database/RocksDB/Iterator.hs view
@@ -73,6 +73,57 @@ -- go [] -- @ --+-- = Reading many entries in one call+--+-- 'iterNext' and 'iterPrev' may wait for the disk, so they are foreign calls+-- of the kind that lets other Haskell threads run meanwhile. That has a+-- price: the thread gives up its capability for the call and has to get one+-- back afterwards, and when more threads want to run than there are+-- capabilities, it waits for it. A scan that moves one step at a time pays+-- that for every entry.+--+-- The run-time system also walks the stack of the thread at every such call.+-- A loop that is not tail recursive, such as one that ends in+-- @(entry :) \<$\> go@, leaves a frame there for every entry read so far, so+-- each step costs more the deeper the scan is.+--+-- 'iterChunk' walks the iterator in C instead, and returns up to some+-- hundreds of entries from one call. 'iterEntries', 'iterEntriesBack',+-- 'iterEntriesPrefix' and 'iterEntriesPrefixBack' are short forms of it for+-- a known number of entries.+--+-- @+-- -- the same scan again, in chunks+-- usersFrom db = 'withIter' db $ \it -> do+-- 'iterSeek' it "user:"+-- let go acc = do+-- (entries, more) <- 'iterChunk' it def{'chunkPrefix' = "user:"}+-- let acc' = entries : acc+-- if more then go acc' else return (concat (reverse acc'))+-- go []+-- @+--+-- Seconds for every thread to read 200,000 entries of 52 bytes, on four+-- capabilities and four cores, from the benchmark of this package:+--+-- > scan 1 thread 16 threads+-- > single steps, not tail recursive 0.78 3.9+-- > single steps, tail recursive 0.090 0.50+-- > chunks of 256 entries 0.048 0.25+--+-- So a scan in chunks takes half the time of the best loop of single steps,+-- and a sixteenth of the time of a loop that is not tail recursive. On a+-- machine with more cores than capabilities it gains more, since the threads+-- that are inside the call do not hold a capability: with twenty more cores+-- free, the sixteen threads took 0.12 seconds in chunks and 0.83 in steps.+--+-- Use chunks to read a range. Keep to single steps where each move depends+-- on the entry just read, or where one entry is all that is wanted:+-- 'iterEntry' after a seek reads it with no call that moves the iterator.+--+-- A chunk also reports errors by itself: 'iterChunk' throws if the iterator+-- stopped at one, where single steps need 'iterGetError' to be asked.+-- -- = Safety -- -- The C API crashes the process if an invalid iterator is moved, and the bytes@@ -81,21 +132,25 @@ -- and values are copied into Haskell memory before they are returned. module Database.RocksDB.Iterator ( Iterator,+ -- * Bracket-style iteration withIter, withIterCF, withIterSnap, withIterSnapCF,+ -- * Bounded iteration IterConfig (..), withIterConfig, prefixUpperBound,+ -- * Manual iterator management createIterator, createIteratorSnap, createIteratorConfig, destroyIterator, destroyReadOpts,+ -- * Iterator operations iterEntry, iterFirst,@@ -107,10 +162,18 @@ iterSeek, iterValid, iterValue,++ -- * Reading many entries in one call+ iterEntries,+ iterEntriesBack,+ iterEntriesPrefix,+ iterEntriesPrefixBack,+ ChunkConfig (..),+ iterChunk, ) where -import Control.Exception (bracket, mask_, onException)+import Control.Exception (bracket, finally, mask, mask_, onException, throwIO) import Control.Monad (forM, forM_, when) import Data.ByteString (ByteString) import Data.ByteString qualified as B@@ -121,7 +184,7 @@ import Foreign import Foreign.C.Error (throwErrnoIfNull) import Foreign.C.String (CString)-import Foreign.C.Types (CSize)+import Foreign.C.Types (CChar, CSize) -- | Create 'Iterator' and use it. --@@ -145,54 +208,54 @@ -- enabling consistent reads across multiple iterators and point queries. -- -- Iterator should not be used after computation ends.-withIterSnap- :: DB- -> Maybe Snapshot- -> (Iterator -> IO a)- -> IO a+withIterSnap ::+ DB ->+ Maybe Snapshot ->+ (Iterator -> IO a) ->+ IO a withIterSnap db msnap = withIterCommon db msnap Nothing -- | Create 'Iterator' on a column family with a specific snapshot. -- -- If 'Nothing' is passed for snapshot, the iterator creates its own implicit snapshot. -- If 'Just snapshot' is passed, the iterator uses the provided snapshot.-withIterSnapCF- :: DB- -> Maybe Snapshot- -> ColumnFamily- -> (Iterator -> IO a)- -> IO a+withIterSnapCF ::+ DB ->+ Maybe Snapshot ->+ ColumnFamily ->+ (Iterator -> IO a) ->+ IO a withIterSnapCF db msnap cf = withIterCommon db msnap (Just cf) -- | Create an iterator over the default column family, or over a column family -- if one is given, reading from a snapshot if one is given.-withIterCommon- :: DB- -> Maybe Snapshot- -> Maybe ColumnFamily- -> (Iterator -> IO a)- -> IO a-withIterCommon DB{rocksDB = rocks_db, readOpts = read_opts} msnap mcf f =- case msnap of- Nothing ->- -- Use DB's readOpts (iterator creates implicit snapshot)- bracket (create_iterator read_opts) destroy_iterator f- Just snap ->- -- Create temporary ReadOpts with the provided snapshot- bracket create_read_opts destroy_read_opts $ \ro ->- bracket (create_iterator ro) destroy_iterator f- where- create_read_opts = do- ro <- c_rocksdb_readoptions_create- c_rocksdb_readoptions_set_snapshot ro snap- return ro- destroy_read_opts = c_rocksdb_readoptions_destroy+withIterCommon ::+ DB ->+ Maybe Snapshot ->+ Maybe ColumnFamily ->+ (Iterator -> IO a) ->+ IO a+withIterCommon DB {rocksDB = rocks_db, readOpts = read_opts} msnap mcf f =+ case msnap of+ Nothing ->+ -- Use DB's readOpts (iterator creates implicit snapshot)+ bracket (create_iterator read_opts) destroy_iterator f+ Just snap ->+ -- Create temporary ReadOpts with the provided snapshot+ bracket create_read_opts destroy_read_opts $ \ro ->+ bracket (create_iterator ro) destroy_iterator f+ where+ create_read_opts = do+ ro <- c_rocksdb_readoptions_create+ c_rocksdb_readoptions_set_snapshot ro snap+ return ro+ destroy_read_opts = c_rocksdb_readoptions_destroy where destroy_iterator = c_rocksdb_iter_destroy create_iterator ro =- throwErrnoIfNull "create_iterator" $ case mcf of- Just cf -> c_rocksdb_create_iterator_cf rocks_db ro cf- Nothing -> c_rocksdb_create_iterator rocks_db ro+ throwErrnoIfNull "create_iterator" $ case mcf of+ Just cf -> c_rocksdb_create_iterator_cf rocks_db ro cf+ Nothing -> c_rocksdb_create_iterator rocks_db ro -- | Where an iterator stops, and what a seek promises. 'def' asks for -- nothing, which gives the iterator that 'withIter' creates; override the@@ -217,52 +280,52 @@ -- * 'totalOrderSeek' when a database with a 'prefixLength' has to be walked -- from end to end. data IterConfig = IterConfig- { lowerBound :: !(Maybe ByteString)- -- ^ the smallest key that the iterator can be at. Going backwards, it- -- becomes invalid instead of moving to a smaller key; a seek to a- -- smaller key lands on the first key at or after the bound.- , upperBound :: !(Maybe ByteString)- -- ^ a key that the iterator stays below: the bound itself is outside.- -- Going forwards, the iterator becomes invalid instead of reaching the- -- bound or a larger key, and 'iterLast' goes to the last key below it.- -- 'prefixUpperBound' gives the bound that ends a prefix.- , prefixSameAsStart :: !Bool- -- ^ stay within the prefix of the key that the iterator was last sent- -- to: it becomes invalid instead of moving to a key with another- -- prefix. The prefix is the first 'Database.RocksDB.prefixLength'- -- bytes of a key. After 'iterSeek' it is the prefix of the key- -- sought; after 'iterFirst' or 'iterLast', that of the key found, so- -- that those two give the first or the last prefix of the database- -- and no more. It has no effect in a database or column family- -- without a prefix length, nor together with 'totalOrderSeek'.- --- -- __Every key has to be at least as long as the prefix__: the keys- -- sought, and the keys stored. RocksDB takes the prefix of a key- -- without looking at its length, so with a shorter key it reads the- -- bytes that follow it in memory, and where the iterator stops is- -- then anyone's guess. This binding cannot check it for you, because- -- an 'Iterator' does not know how it was made. If keys can be- -- shorter, use an 'upperBound', which has no such condition.- , totalOrderSeek :: !Bool- -- ^ see every key in order, whatever its prefix, in a database or- -- column family that has a 'Database.RocksDB.prefixLength'. There, a- -- plain iterator is only sure to return the keys that share the prefix- -- of the key it was sent to; what it does once it moves past them, or- -- after 'iterFirst', is not defined. That is the price of the prefix- -- filters, which let a seek skip the files that have nothing for a- -- prefix. This option gives up the filters and gets the order back.- }- deriving (Eq, Show)+ { -- | the smallest key that the iterator can be at. Going backwards, it+ -- becomes invalid instead of moving to a smaller key; a seek to a+ -- smaller key lands on the first key at or after the bound.+ lowerBound :: !(Maybe ByteString),+ -- | a key that the iterator stays below: the bound itself is outside.+ -- Going forwards, the iterator becomes invalid instead of reaching the+ -- bound or a larger key, and 'iterLast' goes to the last key below it.+ -- 'prefixUpperBound' gives the bound that ends a prefix.+ upperBound :: !(Maybe ByteString),+ -- | stay within the prefix of the key that the iterator was last sent+ -- to: it becomes invalid instead of moving to a key with another+ -- prefix. The prefix is the first 'Database.RocksDB.prefixLength'+ -- bytes of a key. After 'iterSeek' it is the prefix of the key+ -- sought; after 'iterFirst' or 'iterLast', that of the key found, so+ -- that those two give the first or the last prefix of the database+ -- and no more. It has no effect in a database or column family+ -- without a prefix length, nor together with 'totalOrderSeek'.+ --+ -- __Every key has to be at least as long as the prefix__: the keys+ -- sought, and the keys stored. RocksDB takes the prefix of a key+ -- without looking at its length, so with a shorter key it reads the+ -- bytes that follow it in memory, and where the iterator stops is+ -- then anyone's guess. This binding cannot check it for you, because+ -- an 'Iterator' does not know how it was made. If keys can be+ -- shorter, use an 'upperBound', which has no such condition.+ prefixSameAsStart :: !Bool,+ -- | see every key in order, whatever its prefix, in a database or+ -- column family that has a 'Database.RocksDB.prefixLength'. There, a+ -- plain iterator is only sure to return the keys that share the prefix+ -- of the key it was sent to; what it does once it moves past them, or+ -- after 'iterFirst', is not defined. That is the price of the prefix+ -- filters, which let a seek skip the files that have nothing for a+ -- prefix. This option gives up the filters and gets the order back.+ totalOrderSeek :: !Bool+ }+ deriving (Eq, Show) -- | No bounds, and seeks as RocksDB does them by default. instance Default IterConfig where- def =- IterConfig- { lowerBound = Nothing- , upperBound = Nothing- , prefixSameAsStart = False- , totalOrderSeek = False- }+ def =+ IterConfig+ { lowerBound = Nothing,+ upperBound = Nothing,+ prefixSameAsStart = False,+ totalOrderSeek = False+ } -- | The smallest key that is greater than every key that starts with the -- given prefix: the 'upperBound' of a scan of the prefix.@@ -276,9 +339,9 @@ -- Just "user;" prefixUpperBound :: ByteString -> Maybe ByteString prefixUpperBound prefix =- case B.unsnoc (B.dropWhileEnd (== 0xff) prefix) of- Nothing -> Nothing- Just (start, byte) -> Just (B.snoc start (byte + 1))+ case B.unsnoc (B.dropWhileEnd (== 0xff) prefix) of+ Nothing -> Nothing+ Just (start, byte) -> Just (B.snoc start (byte + 1)) -- | Create an 'Iterator' that stops where an t'IterConfig' says, over the -- default column family or over a column family if one is given, and use it.@@ -297,7 +360,7 @@ -- Iterator should not be used after computation ends. withIterConfig :: DB -> Maybe ColumnFamily -> IterConfig -> (Iterator -> IO a) -> IO a withIterConfig db mcf cfg f =- bracket (createIteratorConfig db mcf cfg) snd (f . fst)+ bracket (createIteratorConfig db mcf cfg) snd (f . fst) -- | Manually create an unmanaged iterator like that of 'withIterConfig'. --@@ -310,34 +373,34 @@ -- For automatic resource management, prefer 'withIterConfig'. createIteratorConfig :: DB -> Maybe ColumnFamily -> IterConfig -> IO (Iterator, IO ()) createIteratorConfig db mcf cfg = mask_ $ do- ro <- c_rocksdb_readoptions_create- lower <- bound ro c_rocksdb_readoptions_set_iterate_lower_bound (lowerBound cfg)- upper <- bound ro c_rocksdb_readoptions_set_iterate_upper_bound (upperBound cfg)- let release = do- c_rocksdb_readoptions_destroy ro- mapM_ free lower- mapM_ free upper- it <- flip onException release $ do- forM_ (snapshot db) (c_rocksdb_readoptions_set_snapshot ro)- when (prefixSameAsStart cfg) $ c_rocksdb_readoptions_set_prefix_same_as_start ro 1- when (totalOrderSeek cfg) $ c_rocksdb_readoptions_set_total_order_seek ro 1- createIteratorWithOpts db ro mcf- return (it, c_rocksdb_iter_destroy it >> release)+ ro <- c_rocksdb_readoptions_create+ lower <- bound ro c_rocksdb_readoptions_set_iterate_lower_bound (lowerBound cfg)+ upper <- bound ro c_rocksdb_readoptions_set_iterate_upper_bound (upperBound cfg)+ let release = do+ c_rocksdb_readoptions_destroy ro+ mapM_ free lower+ mapM_ free upper+ it <- flip onException release $ do+ forM_ (snapshot db) (c_rocksdb_readoptions_set_snapshot ro)+ when (prefixSameAsStart cfg) $ c_rocksdb_readoptions_set_prefix_same_as_start ro 1+ when (totalOrderSeek cfg) $ c_rocksdb_readoptions_set_total_order_seek ro 1+ createIteratorWithOpts db ro mcf+ return (it, c_rocksdb_iter_destroy it >> release) where -- A copy of the bound outside the Haskell heap, where it does not move -- and lives until it is freed. One byte is asked for at least, so that -- an empty bound has a pointer too. bound ::- ReadOpts ->- (ReadOpts -> CString -> CSize -> IO ()) ->- Maybe ByteString ->- IO (Maybe CString)+ ReadOpts ->+ (ReadOpts -> CString -> CSize -> IO ()) ->+ Maybe ByteString ->+ IO (Maybe CString) bound ro set mkey = forM mkey $ \key -> do- let len = B.length key- ptr <- mallocBytes (max 1 len)- BU.unsafeUseAsCString key $ \src -> copyBytes ptr src len- set ro ptr (intToCSize len)- return ptr+ let len = B.length key+ ptr <- mallocBytes (max 1 len)+ BU.unsafeUseAsCString key $ \src -> copyBytes ptr src len+ set ro ptr (intToCSize len)+ return ptr -- | Manually create unmanaged iterator, over the default column family or over -- a column family if one is given. It uses the 'readOpts' of the t'DB', so if@@ -345,10 +408,10 @@ -- snapshot. It must be released with 'destroyIterator', and must not outlive -- the t'DB'. createIterator :: DB -> Maybe ColumnFamily -> IO Iterator-createIterator DB{rocksDB = rocks_db, readOpts = read_opts} mcf =- throwErrnoIfNull "create_iterator" $ case mcf of- Just cf -> c_rocksdb_create_iterator_cf rocks_db read_opts cf- Nothing -> c_rocksdb_create_iterator rocks_db read_opts+createIterator DB {rocksDB = rocks_db, readOpts = read_opts} mcf =+ throwErrnoIfNull "create_iterator" $ case mcf of+ Just cf -> c_rocksdb_create_iterator_cf rocks_db read_opts cf+ Nothing -> c_rocksdb_create_iterator rocks_db read_opts -- | Manually create unmanaged iterator on a specific snapshot. --@@ -360,29 +423,29 @@ -- and 'destroyReadOpts' for the read options. -- -- For automatic resource management, prefer 'withIterSnap'.-createIteratorSnap- :: DB- -> Maybe Snapshot- -> Maybe ColumnFamily- -> IO (Iterator, Maybe ReadOpts)+createIteratorSnap ::+ DB ->+ Maybe Snapshot ->+ Maybe ColumnFamily ->+ IO (Iterator, Maybe ReadOpts) createIteratorSnap db msnap mcf = case msnap of- Nothing -> (, Nothing) <$> createIterator db mcf- Just snap -> do- ro <- c_rocksdb_readoptions_create- c_rocksdb_readoptions_set_snapshot ro snap- it <- createIteratorWithOpts db ro mcf- return (it, Just ro)+ Nothing -> (,Nothing) <$> createIterator db mcf+ Just snap -> do+ ro <- c_rocksdb_readoptions_create+ c_rocksdb_readoptions_set_snapshot ro snap+ it <- createIteratorWithOpts db ro mcf+ return (it, Just ro) -- | Internal: create iterator with explicit ReadOpts.-createIteratorWithOpts- :: DB- -> ReadOpts- -> Maybe ColumnFamily- -> IO Iterator-createIteratorWithOpts DB{rocksDB = rocks_db} ro mcf =- throwErrnoIfNull "create_iterator" $ case mcf of- Just cf -> c_rocksdb_create_iterator_cf rocks_db ro cf- Nothing -> c_rocksdb_create_iterator rocks_db ro+createIteratorWithOpts ::+ DB ->+ ReadOpts ->+ Maybe ColumnFamily ->+ IO Iterator+createIteratorWithOpts DB {rocksDB = rocks_db} ro mcf =+ throwErrnoIfNull "create_iterator" $ case mcf of+ Just cf -> c_rocksdb_create_iterator_cf rocks_db ro cf+ Nothing -> c_rocksdb_create_iterator rocks_db ro -- | Destroy unmanaged iterator created with 'createIterator' or -- 'createIteratorSnap'. It must not be used afterwards.@@ -393,16 +456,16 @@ -- function returns /true/ iff the iterator is valid. iterValid :: Iterator -> IO Bool iterValid iter_ptr = do- x <- c_rocksdb_iter_valid iter_ptr- return (x /= 0)+ x <- c_rocksdb_iter_valid iter_ptr+ return (x /= 0) -- | Position at the first key in the source that is at or past target. The -- iterator is /valid/ after this call iff the source contains an entry that -- comes at or past target. iterSeek :: Iterator -> ByteString -> IO () iterSeek iter_ptr key =- BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->- c_rocksdb_iter_seek iter_ptr key_ptr (intToCSize klen)+ BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+ c_rocksdb_iter_seek iter_ptr key_ptr (intToCSize klen) -- | Position at the first key in the source. The iterator is /valid/ after this -- call iff the source is not empty.@@ -425,8 +488,8 @@ -- which is the right trade for a Haskell binding. iterNext :: Iterator -> IO () iterNext it = do- valid <- iterValid it- when valid $ c_rocksdb_iter_next it+ valid <- iterValid it+ when valid $ c_rocksdb_iter_next it -- | Moves to the previous entry in the source. After this call, 'iterValid' is -- /true/ iff the iterator was not positioned at the first entry in the source.@@ -436,8 +499,8 @@ -- 'iterFirst' or 'iterLast' instead. iterPrev :: Iterator -> IO () iterPrev it = do- valid <- iterValid it- when valid $ c_rocksdb_iter_prev it+ valid <- iterValid it+ when valid $ c_rocksdb_iter_prev it -- | Return the key for the current entry if the iterator is currently -- positioned at an entry, ie. 'iterValid'.@@ -456,9 +519,9 @@ -- at an entry, ie. 'iterValid'. iterEntry :: Iterator -> IO (Maybe (ByteString, ByteString)) iterEntry it = do- mkey <- iterKey it- mval <- iterValue it- return $ (,) <$> mkey <*> mval+ mkey <- iterKey it+ mval <- iterValue it+ return $ (,) <$> mkey <*> mval -- | Check for errors --@@ -468,31 +531,230 @@ -- apart. Returns 'Nothing' if there was no error. iterGetError :: Iterator -> IO (Maybe ByteString) iterGetError iter_ptr = alloca $ \err_ptr -> do- poke err_ptr nullPtr- c_rocksdb_iter_get_error iter_ptr err_ptr- err_str <- peek err_ptr- if err_str == nullPtr- then return Nothing- else Just <$> BU.unsafePackMallocCString err_str+ poke err_ptr nullPtr+ c_rocksdb_iter_get_error iter_ptr err_ptr+ err_str <- peek err_ptr+ if err_str == nullPtr+ then return Nothing+ else Just <$> BU.unsafePackMallocCString err_str +-- | The next entries, at most as many as asked for, from the one the+-- iterator is on and going forwards. The iterator is left on the entry after+-- the last one returned, so that calling again continues the scan. --+-- Fewer entries than asked for means that the iterator reached the end, of+-- the data or of its bounds, and is no longer valid. A count of zero or less+-- returns nothing and does not move the iterator.+--+-- It returns what that many rounds of 'iterEntry' and 'iterNext' would, from+-- one foreign call for all of them; see the module header for what that+-- saves. Every entry is held in memory at once, twice for a moment, so ask+-- for no more than fit: to read a range of unknown size, call it in a loop,+-- or use 'iterChunk', which also takes a limit in bytes.+--+-- Throws an 'IOError' if the iterator stopped because of an error.+iterEntries :: Iterator -> Int -> IO [(ByteString, ByteString)]+iterEntries it n = fst <$> iterChunk it whole {chunkEntries = n}++-- | Like 'iterEntries', going backwards: the entry that the iterator is on+-- comes first and those before it follow, in descending order. The iterator+-- is left on the entry before the last one returned.+iterEntriesBack :: Iterator -> Int -> IO [(ByteString, ByteString)]+iterEntriesBack it n = fst <$> iterChunk it whole {chunkEntries = n, chunkBackwards = True}++-- | Like 'iterEntries', for the keys that start with a prefix: it also stops+-- at the first entry whose key does not, and leaves the iterator on that+-- entry, which is not returned. Fewer entries than asked for means that the+-- keys with the prefix have all been read.+--+-- It does not seek: send the iterator to the prefix with 'iterSeek' first.+--+-- @+-- -- the first hundred users+-- 'iterSeek' it "user:"+-- users <- 'iterEntriesPrefix' it "user:" 100+-- @+iterEntriesPrefix :: Iterator -> ByteString -> Int -> IO [(ByteString, ByteString)]+iterEntriesPrefix it prefix n =+ fst <$> iterChunk it whole {chunkEntries = n, chunkPrefix = prefix}++-- | Like 'iterEntriesPrefix', going backwards as 'iterEntriesBack' does. The+-- iterator has to be on the last key of the prefix to begin with: 'iterLast'+-- puts it there if its 'upperBound' is the 'prefixUpperBound' of the prefix.+iterEntriesPrefixBack :: Iterator -> ByteString -> Int -> IO [(ByteString, ByteString)]+iterEntriesPrefixBack it prefix n =+ fst <$> iterChunk it whole {chunkEntries = n, chunkPrefix = prefix, chunkBackwards = True}++-- | A chunk with no limit on its bytes and no prefix, for the functions that+-- promise a number of entries.+whole :: ChunkConfig+whole = def {chunkBytes = maxBound}++-- | What one call of 'iterChunk' reads: how much, which keys, and in which+-- direction. 'def' reads forwards, any key, up to 256 entries or 1 MiB;+-- override the fields you need:+--+-- @+-- def { 'chunkPrefix' = "user:" }+-- @+--+-- The two limits are there for different reasons. 'chunkEntries' spreads+-- the cost of the foreign call over many entries. In the benchmark of this+-- package, chunks of 16 take up to a fifth longer than chunks of 256, and chunks+-- of 1024 are no faster; a larger chunk only makes a reader that stops early+-- pay for more entries that it does not want. 'chunkBytes' bounds the+-- memory: a chunk is held twice for a moment, in the buffer that C fills and+-- in the 'ByteString's copied from it, and 256 values can be of any size.+data ChunkConfig = ChunkConfig+ { -- | the most entries to read. Zero or less reads nothing.+ chunkEntries :: !Int,+ -- | stop before the keys and values read take more bytes than this. The+ -- first entry is read whatever its size, so a chunk is only empty where+ -- there is nothing to read, and a scan gets past an entry that is+ -- larger than the limit.+ chunkBytes :: !Int,+ -- | read only entries whose key starts with these bytes, and stop at the+ -- first one that does not. Empty, the default, for any key.+ --+ -- The iterator still has to move to that first key to see that it is+ -- outside, as in a loop written by hand, and that can be slow where+ -- deleted keys follow the range: see the module header. An 'upperBound'+ -- or 'prefixSameAsStart' stops the iterator before it, and this prefix+ -- is then not needed, though it does no harm.+ chunkPrefix :: !ByteString,+ -- | go backwards, with 'iterPrev', and return the entries in descending+ -- order.+ chunkBackwards :: !Bool+ }+ deriving (Eq, Show)++-- | Forwards, any key, up to 256 entries or 1 MiB of keys and values.+instance Default ChunkConfig where+ def =+ ChunkConfig+ { chunkEntries = 256,+ chunkBytes = 1024 * 1024,+ chunkPrefix = B.empty,+ chunkBackwards = False+ }++-- | Read the next entries, as a t'ChunkConfig' says, from the one the+-- iterator is on, and tell whether more follow.+--+-- This is the function to build a scan on. The iterator is walked in C, in+-- one foreign call for the whole chunk, where a loop of 'iterEntry' and+-- 'iterNext' makes several for every entry; the module header has the+-- figures.+--+-- @+-- -- every entry whose key starts with "user:"+-- 'iterSeek' it "user:"+-- let go acc = do+-- (entries, more) <- 'iterChunk' it def{'chunkPrefix' = "user:"}+-- let acc' = entries : acc+-- if more then go acc' else return (concat (reverse acc'))+-- go []+-- @+--+-- The chunk ends at the first of: the end of the data; a bound of the+-- iterator, from 'upperBound', 'lowerBound' or 'prefixSameAsStart', which+-- all work as they do with 'iterNext' and 'iterPrev'; a key that does not+-- start with 'chunkPrefix'; 'chunkEntries' entries; 'chunkBytes' bytes.+--+-- __Where the iterator is left:__ on the first entry that was not returned,+-- or not valid if there is none. So the next call continues where this one+-- stopped, 'iterEntry' shows the entry that comes next, and after a chunk+-- that a prefix ended, the iterator is on the first key outside the prefix.+--+-- __The flag__ is 'True' if the iterator is left on an entry whose key+-- starts with 'chunkPrefix': the next call would return something. It is+-- 'False' when the scan is over, which saves the call that would find that+-- out by returning nothing.+--+-- The entries are ordinary 'ByteString's, each a copy of its own that the+-- garbage collector frees on its own.+--+-- Unlike the functions that move one step, this one throws an 'IOError' if+-- the iterator stopped because of an error, such as a corrupt block or a+-- failed read: there is no need to ask 'iterGetError' after it, and a scan+-- cannot take an error for the end of the data. It also throws if there is+-- no memory for a single entry.+iterChunk :: Iterator -> ChunkConfig -> IO ([(ByteString, ByteString)], Bool)+iterChunk it cfg =+ BU.unsafeUseAsCStringLen (chunkPrefix cfg) $ \(prefix_ptr, prefix_len) ->+ alloca $ \buffer_ptr ->+ alloca $ \count_ptr ->+ alloca $ \more_ptr ->+ -- Nothing may interrupt between the call that allocates the+ -- buffer and the handler that frees it.+ mask $ \restore -> do+ poke buffer_ptr nullPtr+ status <-+ throwIfErr "iterChunk" $+ c_rocksdb_haskell_iter_chunk+ it+ (if chunkBackwards cfg then 1 else 0)+ (limit (chunkEntries cfg))+ (limit (chunkBytes cfg))+ prefix_ptr+ (intToCSize prefix_len)+ buffer_ptr+ count_ptr+ more_ptr+ buffer <- peek buffer_ptr+ flip finally (free buffer) $ do+ when (status /= 0) $+ throwIO (userError "iterChunk: no memory for an entry")+ count <- peek count_ptr+ more <- peek more_ptr+ entries <- restore (unpackChunk buffer (cSizeToInt count))+ return (entries, more /= 0)+ where+ -- a limit below zero is a limit of zero, and not a huge unsigned number+ limit = intToCSize . max 0++-- -- Internal -- +-- | Copy so many entries out of a buffer filled by+-- 'c_rocksdb_haskell_iter_chunk', in the order they are in. Each entry is+-- two lengths, the key, the value, and padding up to the size of a length;+-- see @cbits\/rocksdb_haskell.h@.+--+-- Every key and value becomes a 'ByteString' of its own. Slices of the one+-- buffer would save the copy, and keep the whole chunk alive for as long as+-- any one key of it is.+unpackChunk :: Ptr CChar -> Int -> IO [(ByteString, ByteString)]+unpackChunk buffer = go [] 0+ where+ word = sizeOf (0 :: CSize)+ go acc _ 0 = return (reverse acc)+ go acc offset n = do+ key_len <- cSizeToInt <$> peekByteOff buffer offset+ value_len <- cSizeToInt <$> peekByteOff buffer (offset + word)+ let key_ptr = buffer `plusPtr` (offset + 2 * word)+ value_ptr = key_ptr `plusPtr` key_len+ size = 2 * word + key_len + value_len+ padded = (size + word - 1) `div` word * word+ key <- B.packCStringLen (key_ptr, key_len)+ value <- B.packCStringLen (value_ptr, value_len)+ go ((key, value) : acc) (offset + padded) (n - 1 :: Int)+ -- | Read a C string from the iterator using the given accessor, if the -- iterator is valid. The string is copied, since it belongs to the iterator. iterString ::- Iterator ->- (Iterator -> Ptr CSize -> IO CString) ->- IO (Maybe ByteString)+ Iterator ->+ (Iterator -> Ptr CSize -> IO CString) ->+ IO (Maybe ByteString) iterString it f = do- valid <- iterValid it- if valid- then alloca $ \len_ptr -> do- str_ptr <- f it len_ptr- if str_ptr == nullPtr- then return Nothing- else do- len <- peek len_ptr- Just <$> B.packCStringLen (str_ptr, cSizeToInt len)- else return Nothing+ valid <- iterValid it+ if valid+ then alloca $ \len_ptr -> do+ str_ptr <- f it len_ptr+ if str_ptr == nullPtr+ then return Nothing+ else do+ len <- peek len_ptr+ Just <$> B.packCStringLen (str_ptr, cSizeToInt len)+ else return Nothing
test/Spec.hs view
@@ -1,4 +1,6 @@+{-# LANGUAGE ForeignFunctionInterface #-} {-# LANGUAGE ImportQualifiedPost #-}+{-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TupleSections #-} @@ -26,17 +28,27 @@ import Control.Concurrent import Control.Concurrent.Async-import Control.Exception (bracket)+import Control.Exception (bracket, evaluate, try) import Control.Monad+import Data.Bits (shiftR) import Data.ByteString (ByteString) import Data.ByteString qualified as BS import Data.ByteString.Char8 qualified as C+import Data.ByteString.Internal qualified as BI import Data.Default (def)+import Data.List (isInfixOf, isPrefixOf, isSuffixOf) import Data.Map.Strict (Map) import Data.Map.Strict qualified as Map import Data.Maybe+import Data.Word (Word64, Word8) import Database.RocksDB+import Foreign.Marshal.Utils (copyBytes)+import Foreign.Ptr (FunPtr, Ptr, castPtr, plusPtr) import GHC.Clock (getMonotonicTime)+import System.Directory (listDirectory)+import System.FilePath ((</>))+import System.IO (IOMode (ReadWriteMode), SeekMode (AbsoluteSeek), hSeek, withBinaryFile)+import System.IO.Error (ioeGetErrorString) import System.IO.Temp import Test.Hspec import Test.Hspec.QuickCheck (modifyMaxSuccess, prop)@@ -48,6 +60,10 @@ main = hspec $ do around (withTunedDB ["one", "two", "tree"]) tunedSpec around withPlainDB plainSpec+ around withPlainDB chunkSpec+ chunkErrorSpec+ packedSpec+ mergeSpec boundSpec openingSpec tuningSpec@@ -260,6 +276,14 @@ withIterConfig db Nothing def {prefixSameAsStart = True} $ \itr -> do iterSeek itr "bbb2" map fst <$> scan iterPrev itr `shouldReturn` ["bbb2", "bbb1"]+ it "stays within the prefix when read in chunks" $ \db -> do+ fill db+ withIterConfig db Nothing def {prefixSameAsStart = True} $ \itr -> do+ iterSeek itr "bbb"+ iterChunk itr def `shouldReturn` ([("bbb1", "bbb1"), ("bbb2", "bbb2")], False)+ iterValid itr `shouldReturn` False+ iterSeek itr "ccc"+ iterEntriesBack itr 10 `shouldReturn` [("ccc1", "ccc1")] it "stays within the prefix when the data is in table files" $ \db -> do fill db -- enough writes to flush the keys out of memory, where the prefix@@ -597,6 +621,719 @@ forwards itr `shouldReturn` [("m", "changed"), ("n", "20")] -------------------------------------------------------------------------------+-- Reading many entries in one call+-------------------------------------------------------------------------------++-- | The bytes that the keys and values of some entries take, which is what+-- 'chunkBytes' limits.+sizeOf :: [(ByteString, ByteString)] -> Int+sizeOf entries = sum [BS.length k + BS.length v | (k, v) <- entries]++-- | Read chunk after chunk until one says that no more follow, and give the+-- chunks. Fails if a chunk is empty when the one before promised more: the+-- loop would never end.+chunksOf :: Iterator -> ChunkConfig -> IO [[(ByteString, ByteString)]]+chunksOf itr cfg = go []+ where+ go acc = do+ (entries, more) <- iterChunk itr cfg+ if more+ then do+ entries `shouldNotBe` []+ go (entries : acc)+ else return (reverse (entries : acc))++chunkSpec :: SpecWith DB+chunkSpec = describe "Reading many entries in one call" $ do+ let keys = ["a1", "a2", "b1", "b2", "c1"] :: [ByteString]+ fill db = forM_ keys $ \k -> put db k k+ same = map (\k -> (k, k))+ -- ten entries of 108 bytes each: a key of 8 and a value of 100+ wide = [(key i, BS.replicate 100 (fromIntegral i)) | i <- [0 .. 9]]+ it "reads the next entries and leaves the iterator after them" $ \db -> do+ fill db+ withIter db $ \itr -> do+ iterFirst itr+ iterEntries itr 2 `shouldReturn` same ["a1", "a2"]+ iterKey itr `shouldReturn` Just "b1"+ iterEntries itr 100 `shouldReturn` same ["b1", "b2", "c1"]+ iterValid itr `shouldReturn` False+ iterEntries itr 100 `shouldReturn` []+ it "reads backwards, in descending order" $ \db -> do+ fill db+ withIter db $ \itr -> do+ iterLast itr+ iterEntriesBack itr 2 `shouldReturn` same ["c1", "b2"]+ iterKey itr `shouldReturn` Just "b1"+ iterEntriesBack itr 100 `shouldReturn` same ["b1", "a2", "a1"]+ iterValid itr `shouldReturn` False+ it "reads nothing, and stays, when asked for no entries" $ \db -> do+ fill db+ withIter db $ \itr -> do+ iterSeek itr "b1"+ iterEntries itr 0 `shouldReturn` []+ iterEntries itr (-3) `shouldReturn` []+ iterEntriesBack itr minBound `shouldReturn` []+ iterChunk itr def {chunkEntries = 0} `shouldReturn` ([], True)+ iterKey itr `shouldReturn` Just "b1"+ it "stops at the first key outside the prefix, and stays on it" $ \db -> do+ fill db+ withIter db $ \itr -> do+ iterSeek itr "b"+ iterEntriesPrefix itr "b" 100 `shouldReturn` same ["b1", "b2"]+ iterKey itr `shouldReturn` Just "c1"+ iterEntriesPrefix itr "b" 100 `shouldReturn` []+ iterKey itr `shouldReturn` Just "c1"+ iterSeek itr "b2"+ iterEntriesPrefixBack itr "b" 100 `shouldReturn` same ["b2", "b1"]+ iterKey itr `shouldReturn` Just "a2"+ it "takes a key equal to the prefix, and not one shorter than it" $ \db -> do+ forM_ ["a", "ab", "abc", "b"] $ \k -> put db k ""+ withIter db $ \itr -> do+ iterFirst itr+ iterEntriesPrefix itr "ab" 100 `shouldReturn` []+ iterSeek itr "ab"+ map fst <$> iterEntriesPrefix itr "ab" 100 `shouldReturn` ["ab", "abc"]+ it "tells whether more entries follow" $ \db -> do+ fill db+ withIter db $ \itr -> do+ let cfg = def {chunkPrefix = "a"}+ iterFirst itr+ iterChunk itr cfg {chunkEntries = 1} `shouldReturn` (same ["a1"], True)+ -- the limit is reached where the prefix ends: nothing follows+ iterFirst itr+ iterChunk itr cfg {chunkEntries = 2} `shouldReturn` (same ["a1", "a2"], False)+ iterChunk itr cfg `shouldReturn` ([], False)+ -- and where the data ends+ iterSeek itr "c1"+ iterChunk itr def {chunkEntries = 1} `shouldReturn` (same ["c1"], False)+ it "stops before the limit of bytes" $ \db -> do+ forM_ wide $ uncurry (put db)+ withIter db $ \itr -> do+ let sizes bytes = do+ iterFirst itr+ map length <$> chunksOf itr def {chunkBytes = bytes}+ -- two entries are 216 bytes+ sizes 250 `shouldReturn` [2, 2, 2, 2, 2]+ sizes 216 `shouldReturn` [2, 2, 2, 2, 2]+ sizes 215 `shouldReturn` [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]+ sizes 500 `shouldReturn` [4, 4, 2]+ sizes maxBound `shouldReturn` [10]+ it "reads one entry at least, whatever the limit of bytes" $ \db -> do+ forM_ wide $ uncurry (put db)+ withIter db $ \itr -> forM_ [0, 1, 107, -1, minBound] $ \bytes -> do+ iterFirst itr+ chunks <- chunksOf itr def {chunkBytes = bytes}+ chunks `shouldBe` map (: []) wide+ it "gets past an entry larger than the limit of bytes" $ \db -> do+ -- three megabytes between small entries, with a limit of one+ let big = BS.replicate (3 * 1024 * 1024) 7+ entries = [("a", "small"), ("b", big), ("c", "small"), ("d", "small")]+ forM_ entries $ uncurry (put db)+ withIter db $ \itr -> do+ iterFirst itr+ chunks <- chunksOf itr def+ map (map fst) chunks `shouldBe` [["a"], ["b"], ["c", "d"]]+ concat chunks `shouldBe` entries+ iterLast itr+ back <- chunksOf itr def {chunkBackwards = True}+ map (map fst) back `shouldBe` [["d", "c"], ["b"], ["a"]]+ it "reads across many chunks what single steps read" $ \db -> do+ let entries = [(key i, C.pack (show i)) | i <- [0 .. 9999]]+ write db (map (uncurry Put) entries)+ withIter db $ \itr -> forM_ [1, 7, 256, 10000, 20000] $ \n -> do+ iterFirst itr+ chunks <- chunksOf itr def {chunkEntries = n}+ concat chunks `shouldBe` entries+ map length (init chunks) `shouldSatisfy` all (== n)+ iterLast itr+ back <- chunksOf itr def {chunkEntries = n, chunkBackwards = True}+ concat back `shouldBe` reverse entries+ it "stops at the bounds of the iterator" $ \db -> do+ fill db+ withIterConfig db Nothing def {lowerBound = Just "a2", upperBound = Just "b2"} $ \itr -> do+ iterFirst itr+ iterChunk itr def `shouldReturn` (same ["a2", "b1"], False)+ iterValid itr `shouldReturn` False+ iterLast itr+ iterChunk itr def {chunkBackwards = True} `shouldReturn` (same ["b1", "a2"], False)+ iterValid itr `shouldReturn` False+ it "reads a prefix backwards from the bound that ends it" $ \db -> do+ fill db+ withIterConfig db Nothing def {upperBound = prefixUpperBound "a"} $ \itr -> do+ iterLast itr+ iterEntriesPrefixBack itr "a" 100 `shouldReturn` same ["a2", "a1"]+ it "returns nothing where there is nothing" $ \db -> do+ withIter db $ \itr -> do+ iterFirst itr+ iterChunk itr def `shouldReturn` ([], False)+ iterEntries itr 10 `shouldReturn` []+ fill db+ withIter db $ \itr -> do+ -- not sent anywhere yet+ iterChunk itr def `shouldReturn` ([], False)+ -- a prefix that no key has+ iterSeek itr "b3"+ iterChunk itr def {chunkPrefix = "b3"} `shouldReturn` ([], False)+ iterKey itr `shouldReturn` Just "c1"+ -- past the last key+ iterSeek itr "d"+ iterChunk itr def `shouldReturn` ([], False)+ iterChunk itr def {chunkBackwards = True} `shouldReturn` ([], False)+ it "keeps bytes that C takes for the end of a string, and empty keys and values" $ \db -> do+ let entries = [("", "empty key"), ("\0", ""), ("\0\0a", "\0b\0"), ("\0\255", "\255\0")]+ forM_ entries $ uncurry (put db)+ withIter db $ \itr -> do+ iterFirst itr+ iterEntries itr 100 `shouldReturn` entries+ iterSeek itr "\0"+ iterEntriesPrefix itr "\0\0" 100 `shouldReturn` []+ iterEntriesPrefix itr "\0" 100 `shouldReturn` drop 1 entries+ it "stays within its column family" $ \db -> do+ other <- firstFamily db+ fill db+ putCF db other "b9" "other"+ withIterCF db other $ \itr -> do+ iterFirst itr+ iterEntries itr 100 `shouldReturn` [("b9", "other")]+ it "returns entries that outlive the iterator" $ \db -> do+ let entries = [(key i, BS.replicate 50 (fromIntegral i)) | i <- [0 .. 999]]+ write db (map (uncurry Put) entries)+ got <- withIter db $ \itr -> iterFirst itr >> iterEntries itr 1000+ -- the heap is churned before the entries are looked at+ sum (map BS.length [key j | j <- [1 .. 20000]]) `shouldBe` 160000+ got `shouldBe` entries+ it "reads the same from many threads at once" $ \db -> do+ let entries = [(key i, C.pack (show i)) | i <- [0 .. 19999]]+ write db (map (uncurry Put) entries)+ results <- forConcurrently [1 .. 16 :: Int] $ \n ->+ withIter db $ \itr -> do+ iterFirst itr+ concat <$> chunksOf itr def {chunkEntries = n * 10}+ forM_ results (`shouldBe` entries)++-- | An iterator that meets a block it cannot read is no longer valid, which+-- is also how it looks at the end of the data. A scan in chunks has to tell+-- the two apart by itself, since its caller gets a list and nothing else.+chunkErrorSpec :: Spec+chunkErrorSpec = describe "Reading many entries from a damaged file" $ do+ it "throws, where single steps look as if the data had ended" $+ withTempPath $ \path -> do+ -- compacted, so that the entries are in a table file+ withDB path plain $ \db -> do+ forM_ [0 .. 999] $ \i -> put db (key i) (BS.replicate 100 (fromIntegral i))+ compactRange db Nothing Nothing Nothing+ tables <- filter (".sst" `isSuffixOf`) <$> listDirectory path+ tables `shouldNotBe` []+ -- garbage over the start of each file, where the first block of+ -- entries is: its checksum no longer matches+ forM_ tables $ \table ->+ withBinaryFile (path </> table) ReadWriteMode $ \h -> do+ hSeek h AbsoluteSeek 100+ BS.hPut h (BS.replicate 100 0xa5)+ withDB path plain $ \db -> withIter db $ \itr -> do+ iterFirst itr+ iterValid itr `shouldReturn` False+ iterGetError itr >>= (`shouldSatisfy` isJust)+ iterFirst itr+ iterChunk itr def `shouldThrow` anyIOException+ iterFirst itr+ iterEntries itr 10 `shouldThrow` anyIOException++-------------------------------------------------------------------------------+-- Packed batches+-------------------------------------------------------------------------------++-- | A write to a column family that is named by its number, so that the same+-- operation can be given to two databases: zero is the default family, and+-- the others are those of 'columnFamilies' from one. 'Nothing' deletes.+data FamilyOp = FamilyOp Int ByteString (Maybe ByteString)++-- | The operation of a batch for a database.+batchOp :: DB -> FamilyOp -> BatchOp+batchOp _ (FamilyOp 0 k (Just v)) = Put k v+batchOp _ (FamilyOp 0 k Nothing) = Del k+batchOp db (FamilyOp n k mv) = case drop (n - 1) (columnFamilies db) of+ cf : _ -> maybe (DelCF cf k) (PutCF cf k) mv+ [] -> error ("no column family " <> show n)++-- | Every entry of every column family of a database, the default one+-- first.+contents :: DB -> IO [[(ByteString, ByteString)]]+contents db = (:) <$> withIter db forwards <*> forM (columnFamilies db) (\cf -> withIterCF db cf forwards)++-- | Two new databases with the same three column families besides the+-- default: one to write with 'write', and one with 'writePacked'.+withTwoDBs :: (DB -> DB -> IO a) -> IO a+withTwoDBs go = new (new . go)+ where+ families = map (,plain) ["one", "two", "three"]+ new f = withTempPath $ \path -> withDBCF path plain {createMissingColumnFamilies = True} families f++-- | Require that the operations, cut into chunks of so many, leave what+-- they leave when they are written with 'write': in every column family.+shouldWriteAs :: Int -> [FamilyOp] -> IO ()+shouldWriteAs size ops = withTwoDBs $ \expected packed -> do+ write expected (map (batchOp expected) ops)+ writePacked packed (map packOps (groupsOf size (map (batchOp packed) ops)))+ written <- contents packed+ wanted <- contents expected+ map length written `shouldBe` map length wanted+ written `shouldBe` wanted++-- | A list in pieces of a length, the last one shorter.+groupsOf :: Int -> [a] -> [[a]]+groupsOf _ [] = []+groupsOf n xs = let (a, b) = splitAt n xs in a : groupsOf n b++-- | A chunk with one record, written as a program that packs its own+-- chunks does: room for the header, the key and the value after it, and+-- then the header. The record starts at an odd address, since records are+-- not aligned.+record :: Maybe ColumnFamily -> PackedOp -> ByteString -> ByteString -> ByteString+record mcf op k v =+ BS.drop 1 . BI.unsafeCreate (1 + size) $ \buffer -> do+ let start = buffer `plusPtr` 1 :: Ptr Word8+ key_ptr = start `plusPtr` packedHeaderSize+ copy k key_ptr+ copy v (key_ptr `plusPtr` BS.length k)+ pokePackedHeader start mcf op (BS.length k) (BS.length v)+ where+ size = packedHeaderSize + BS.length k + BS.length v+ copy bytes to =+ BS.useAsCStringLen bytes $ \(from, len) -> copyBytes to (castPtr from) len++-- | Replace the bytes at an offset.+patch :: Int -> [Word8] -> ByteString -> ByteString+patch at bytes whole = BS.take at whole <> BS.pack bytes <> BS.drop (at + length bytes) whole++-- | A batch that a small C function of the package hands to RocksDB, a few+-- thousand operations at a call. It has to do what 'write' does with the+-- same operations, and it reads a buffer that the caller made: so it must+-- not believe what the buffer says about its own lengths.+packedSpec :: Spec+packedSpec = describe "Packed batches" $ do+ let several =+ [ FamilyOp 0 "a" (Just "default"),+ FamilyOp 1 "a" (Just "one"),+ FamilyOp 2 "a" (Just "two"),+ FamilyOp 3 "a" (Just "three"),+ FamilyOp 0 "empty value" (Just ""),+ FamilyOp 2 "" (Just "empty key"),+ FamilyOp 3 "zero\0inside" (Just "\0\0"),+ FamilyOp 1 "gone" (Just "v"),+ FamilyOp 1 "gone" Nothing,+ FamilyOp 2 "never there" Nothing+ ]+ -- twenty thousand operations over four families: every key is set,+ -- every third one is set again and every seventh deleted+ many =+ concat+ [ [FamilyOp (i `mod` 4) (key i) (Just (C.pack (show i)))]+ <> [FamilyOp (i `mod` 4) (key i) (Just "again") | i `mod` 3 == 0]+ <> [FamilyOp (i `mod` 4) (key i) Nothing | i `mod` 7 == 0]+ | i <- [1 .. 14000]+ ]+ it "writes what the same operations written with write leave, in every column family" $ do+ shouldWriteAs (length several) several+ withTwoDBs $ \_ db -> do+ writePacked db [packOps (map (batchOp db) several)]+ contents db+ `shouldReturn` [ [("a", "default"), ("empty value", "")],+ [("a", "one")],+ [("", "empty key"), ("a", "two")],+ [("a", "three"), ("zero\0inside", "\0\0")]+ ]+ it "applies operations in order, within a chunk and from one chunk to the next" $ do+ let ops =+ [ FamilyOp 0 "k" (Just "first"),+ FamilyOp 0 "k" (Just "second"),+ FamilyOp 1 "gone" (Just "v"),+ FamilyOp 1 "gone" Nothing,+ FamilyOp 1 "back" Nothing,+ FamilyOp 1 "back" (Just "v")+ ]+ forM_ [1, 2, 3, 6] $ \size -> do+ shouldWriteAs size ops+ withTwoDBs $ \_ db -> do+ writePacked db (map packOps (groupsOf size (map (batchOp db) ops)))+ contents db `shouldReturn` [[("k", "second")], [("back", "v")], [], []]+ it "accepts no chunk, and empty chunks among the others" $+ withTwoDBs $ \_ db -> do+ writePacked db []+ writePacked db [BS.empty]+ contents db `shouldReturn` [[], [], [], []]+ writePacked db [BS.empty, packOps [Put "a" "1"], BS.empty, packOps [], packOps [Put "b" "2"]]+ contents db `shouldReturn` [[("a", "1"), ("b", "2")], [], [], []]+ it "writes a large batch, in one chunk and over many" $ do+ length many `shouldSatisfy` (> 20000)+ shouldWriteAs (length many) many+ shouldWriteAs 97 many+ shouldWriteAs 1 (take 3000 many)+ it "takes records that a program wrote into a buffer of its own" $+ withTwoDBs $ \_ db -> do+ other <- firstFamily db+ packedHeaderSize `shouldBe` BS.length (packOps [Put "" ""])+ record Nothing PackedPut "key" "value" `shouldBe` packOps [Put "key" "value"]+ record (Just other) PackedPut "key" "" `shouldBe` packOps [PutCF other "key" ""]+ record (Just other) PackedDel "key" "" `shouldBe` packOps [DelCF other "key"]+ put db "old" "v"+ writePacked+ db+ [ record Nothing PackedPut "a" "1" <> record (Just other) PackedPut "b" "2",+ record Nothing PackedDel "old" ""+ ]+ contents db `shouldReturn` [[("a", "1")], [("b", "2")], [], []]+ it "refuses a header that cannot be: a length that is negative, a delete with a value" $ do+ evaluate (record Nothing PackedPut "k" "v") `shouldReturn` packOps [Put "k" "v"]+ evaluate (record Nothing PackedDel "k" "v") `shouldThrow` anyIOException+ BI.create 64 (\p -> pokePackedHeader p Nothing PackedPut (-1) 0) `shouldThrow` anyIOException+ BI.create 64 (\p -> pokePackedHeader p Nothing PackedPut 0 (-1)) `shouldThrow` anyIOException+ describe "A chunk that is not made of whole records" $ do+ let good = packOps [Put "before" "v"]+ -- two records of 17 bytes of header, a key of 3 and a value of 5+ chunk = packOps [Put "aaa" "11111", Put "bbb" "22222"]+ size = BS.length chunk `div` 2+ operation = packedHeaderSize - 1+ key_len = packedHeaderSize - 9+ value_len = packedHeaderSize - 5+ -- the chunk is refused, by this package and not by RocksDB, with+ -- the number of its first bad record, and nothing of the batch is+ -- written: not the good chunks around it either+ refused n bad = withTwoDBs $ \_ db -> do+ writePacked db [good, bad, good] `shouldThrow` \e ->+ ("writePacked: chunk 1 " `isPrefixOf` ioeGetErrorString e)+ && ((": record " <> show (n :: Int) <> " is") `isInfixOf` ioeGetErrorString e)+ writePacked db [bad] `shouldThrow` anyIOException+ contents db `shouldReturn` [[], [], [], []]+ -- and the database is still good for a batch+ writePacked db [good]+ contents db `shouldReturn` [[("before", "v")], [], [], []]+ it "is written when nothing is wrong with it" $+ withTwoDBs $ \_ db -> do+ writePacked db [good, chunk, good]+ contents db `shouldReturn` [[("aaa", "11111"), ("bbb", "22222"), ("before", "v")], [], [], []]+ it "is refused when it ends within a value" $+ refused 1 (BS.init chunk)+ it "is refused when it ends within a key" $+ refused 1 (BS.take (size + packedHeaderSize + 1) chunk)+ it "is refused when it ends within a header, or right after one" $ do+ refused 1 (BS.take (size + 1) chunk)+ refused 1 (BS.take (size + packedHeaderSize - 1) chunk)+ refused 1 (BS.take (size + packedHeaderSize) chunk)+ refused 0 (BS.take 1 chunk)+ it "is refused when bytes follow its last record" $+ refused 2 (chunk <> "\0")+ it "is refused when a length says more than there is" $ do+ refused 1 (patch (size + key_len) [9, 0, 0, 0] chunk)+ refused 1 (patch (size + value_len) [6, 0, 0, 0] chunk)+ refused 0 (patch key_len [255, 255, 255, 255] chunk)+ refused 0 (patch value_len [255, 255, 255, 255] chunk)+ refused 0 (patch key_len [255, 255, 255, 255] (patch value_len [255, 255, 255, 255] chunk))+ it "is refused when it names an operation that does not exist" $ do+ refused 1 (patch (size + operation) [3] chunk)+ refused 0 (patch operation [255] chunk)+ it "is refused when a delete has a value" $+ refused 1 (patch (size + operation) [1] chunk)+ describe "A merge" $ do+ -- What a merge does to a value is for the tests of merge records. Here:+ -- that a packed record is taken for a merge and handed to RocksDB as+ -- one. RocksDB 11 says so itself, by refusing the batch for want of a+ -- merge operator. A RocksDB that took such a batch would have a merge+ -- record under the key, and no way to read it.+ it "is handed to RocksDB as a merge, in the default column family and in another" $+ withTwoDBs $ \_ db -> do+ other <- firstFamily db+ let asMerge chunks reading = do+ written <- try (writePacked db chunks)+ case written of+ Left e -> do+ ioeGetErrorString e `shouldSatisfy` (not . ("writePacked" `isPrefixOf`))+ ioeGetErrorString e `shouldSatisfy` ("erge" `isInfixOf`)+ -- the merge was not written; the put that came before+ -- it in the batch was not either with RocksDB 11, and+ -- was with RocksDB 8.9+ contents db >>= (`shouldSatisfy` (`elem` [[[], [], [], []], [[("a", "1")], [], [], []]]))+ Right () -> reading `shouldThrow` anyIOException+ asMerge [packOps [Put "a" "1"], record Nothing PackedMerge "k" "v"] (get db "k")+ asMerge [record (Just other) PackedMerge "k" "v", packOps [Put "a" "1"]] (getCF db other "k")+ record Nothing PackedMerge "k" "v" `shouldBe` packOps [Merge "k" "v"]+ record (Just other) PackedMerge "k" "v" `shouldBe` packOps [MergeCF other "k" "v"]++-------------------------------------------------------------------------------+-- Merge records and merge operators+-------------------------------------------------------------------------------++-- The merge operator of the tests, in @test\/cbits\/merge_test.c@: the+-- value of a key is the value it had and its merge records, oldest first,+-- joined with commas. The value shows which records were merged and in+-- which order, which a sum would not.+foreign import ccall "&rocksdb_haskell_test_join_full" joinFull :: FunPtr FullMerge++foreign import ccall "&rocksdb_haskell_test_join_partial" joinPartial :: FunPtr PartialMerge++foreign import ccall "&rocksdb_haskell_test_no_partial" noPartial :: FunPtr PartialMerge++-- | Join with commas.+joining :: MergeOperator+joining = MergeFunctions {mergeName = "test-join", fullMerge = joinFull, partialMerge = joinPartial}++-- | Join with commas, with a partial merge that always fails: RocksDB has+-- to keep every record until it has the value under them.+joiningWhole :: MergeOperator+joiningWhole = MergeFunctions {mergeName = "test-join-whole", fullMerge = joinFull, partialMerge = noPartial}++-- | A number as RocksDB's own operator for counters reads and writes it:+-- eight bytes, the least significant first.+counter :: Word64 -> ByteString+counter n = BS.pack [fromIntegral (n `shiftR` (8 * i)) | i <- [0 .. 7]]++-- | The configuration of a column family, or of the database, with a merge+-- operator. Nothing is compacted behind the back of a test, so that each+-- flush leaves a table file of its own and merge records stay where they+-- were written until the test compacts.+merging :: Maybe MergeOperator -> Config+merging op = plain {createMissingColumnFamilies = True, mergeOperator = op, disableAutoCompactions = True}++-- | The families of 'withMergeDB' besides the default.+mergeFamilies :: [(String, Config)]+mergeFamilies =+ [ ("count", merging (Just UInt64Add)),+ ("whole", merging (Just joiningWhole)),+ ("none", merging Nothing)+ ]++-- | A database whose default column family joins with commas, with three+-- more: one that counts, one that joins and cannot merge records without+-- the value, and one without a merge operator.+withMergeDB :: (DB -> ColumnFamily -> ColumnFamily -> ColumnFamily -> IO a) -> IO a+withMergeDB go = withTempPath $ \path -> openMergeDB path go++openMergeDB :: FilePath -> (DB -> ColumnFamily -> ColumnFamily -> ColumnFamily -> IO a) -> IO a+openMergeDB path go =+ withDBCF path (merging (Just joining)) mergeFamilies $ \db -> case columnFamilies db of+ [count, whole, none] -> go db count whole none+ cfs -> fail ("expected three column families, got " <> show (length cfs))++-- | Flush every column family of a database.+flushAll :: DB -> IO ()+flushAll db = flush db Nothing >> mapM_ (flush db . Just) (columnFamilies db)++-- | Compact every column family of a database, to its last level.+compactAll :: DB -> IO ()+compactAll db = do+ compactRange db Nothing Nothing Nothing+ forM_ (columnFamilies db) $ \cf -> compactRange db (Just cf) Nothing Nothing++-- | A merge record says how a value changes, and the merge operator of the+-- column family works out the value when it is read or compacted. What can+-- go wrong is the order of the records, what a put or a delete among them+-- hides, and a value that depends on where its parts happen to be: in the+-- memtable, in several table files, or compacted into one.+mergeSpec :: Spec+mergeSpec = describe "Merge records" $ do+ it "adds to a counter, from zero where the key has no value" $+ withMergeDB $ \db count _ _ -> do+ mergeCF db count "k" (counter 5)+ getCF db count "k" `shouldReturn` Just (counter 5)+ mergeCF db count "k" (counter 7)+ getCF db count "k" `shouldReturn` Just (counter 12)+ putCF db count "set" (counter 100)+ mergeCF db count "set" (counter 1)+ getCF db count "set" `shouldReturn` Just (counter 101)+ -- modulo 2^64, so a record can take away as well+ mergeCF db count "set" (counter (negate 101))+ getCF db count "set" `shouldReturn` Just (counter 0)+ getCF db count "absent" `shouldReturn` Nothing+ it "gives a custom operator the value and the records, oldest first" $+ withMergeDB $ \db _ _ _ -> do+ merge db "new" "a"+ get db "new" `shouldReturn` Just "a"+ merge db "new" "b"+ merge db "new" "c"+ get db "new" `shouldReturn` Just "a,b,c"+ put db "old" "x"+ merge db "old" "y"+ get db "old" `shouldReturn` Just "x,y"+ -- a value of no bytes is a value, and a record of no bytes a record+ put db "empty" ""+ merge db "empty" ""+ get db "empty" `shouldReturn` Just ","+ it "hides the records before a put or a delete, and applies those after" $+ withMergeDB $ \db _ _ _ -> do+ merge db "put" "a" >> put db "put" "x" >> merge db "put" "b"+ get db "put" `shouldReturn` Just "x,b"+ merge db "del" "a" >> delete db "del"+ get db "del" `shouldReturn` Nothing+ merge db "del" "b"+ get db "del" `shouldReturn` Just "b"+ put db "last" "x" >> merge db "last" "a" >> delete db "last"+ get db "last" `shouldReturn` Nothing+ -- and the same when each write is in a table file of its own+ forM_ [merge db "files" "a", put db "files" "x", merge db "files" "b", delete db "files", merge db "files" "c"] $+ \step -> step >> flush db Nothing+ get db "files" `shouldReturn` Just "c"+ merge db "files" "d"+ get db "files" `shouldReturn` Just "c,d"+ it "takes merges in a batch, among puts and deletes, written either way" $ do+ let ops count whole =+ [ Put "a" "1",+ Merge "a" "2",+ MergeCF count "n" (counter 3),+ Merge "b" "x",+ MergeCF whole "a" "w",+ Merge "a" "3",+ MergeCF count "n" (counter 4),+ Put "gone" "v",+ Merge "gone" "w",+ Del "gone",+ MergeCF whole "a" "v"+ ]+ wanted = [[("a", "1,2,3"), ("b", "x")], [("n", counter 7)], [("a", "w,v")], []]+ withMergeDB $ \db count whole _ -> do+ write db (ops count whole)+ contents db `shouldReturn` wanted+ withMergeDB $ \db count whole _ -> do+ writePacked db [packOps (ops count whole)]+ contents db `shouldReturn` wanted+ withMergeDB $ \db count whole _ -> do+ writePacked db (map packOps (groupsOf 2 (ops count whole)))+ contents db `shouldReturn` wanted+ it "gives the same values from the memtable, from table files and after a compaction" $+ withMergeDB $ \db count whole _ -> do+ let keys = ["both", "records", "value"] :: [ByteString]+ wanted =+ [ [("both", "x,a,b,c"), ("records", "a,b,c"), ("value", "x")],+ [("both", counter 16), ("records", counter 6), ("value", counter 10)],+ [("both", "x,a,b,c"), ("records", "a,b,c"), ("value", "x")],+ []+ ]+ -- every way of reading, in every family+ agree = do+ contents db `shouldReturn` wanted+ forM_ (zip (Nothing : map Just (columnFamilies db)) wanted) $ \(mcf, entries) -> do+ let values = map (`lookup` entries) keys+ case mcf of+ Nothing -> do+ mapM (get db) keys `shouldReturn` values+ multiGet db keys `shouldReturn` values+ withIter db backwards `shouldReturn` reverse entries+ withIter db (\itr -> iterFirst itr >> iterEntries itr 10) `shouldReturn` entries+ Just cf -> do+ mapM (getCF db cf) keys `shouldReturn` values+ multiGetCF db cf keys `shouldReturn` values+ withIterCF db cf backwards `shouldReturn` reverse entries+ withIterCF db cf (\itr -> iterFirst itr >> iterEntries itr 10) `shouldReturn` entries+ each k v n = do+ merge db k v+ mergeCF db whole k v+ mergeCF db count k (counter n)+ forM_ ["both", "value"] $ \k -> do+ put db k "x"+ putCF db whole k "x"+ putCF db count k (counter 10)+ flushAll db+ forM_ [("a", 1), ("b", 2)] $ \(v, n) -> do+ each "both" v n+ each "records" v n+ flushAll db+ -- the last record of each key is in the memtable, the others in a+ -- table file each, and the value in another+ each "both" "c" 3+ each "records" "c" 3+ agree+ withSnapshot db $ \snap -> do+ merge db "both" "later"+ get snap "both" `shouldReturn` Just "x,a,b,c"+ delete db "both" >> put db "both" "x,a,b,c"+ flushAll db+ agree+ compactAll db+ agree+ -- and what is merged after the compaction is merged with its result+ each "both" "d" 4+ get db "both" `shouldReturn` Just "x,a,b,c,d"+ getCF db whole "both" `shouldReturn` Just "x,a,b,c,d"+ getCF db count "both" `shouldReturn` Just (counter 20)+ it "keeps its values when the database is closed and opened again" $+ withTempPath $ \path -> do+ openMergeDB path $ \db count whole _ -> do+ put db "k" "x" >> merge db "k" "a"+ flushAll db+ -- these are in the write-ahead log only+ merge db "k" "b"+ mergeCF db count "n" (counter 2)+ mergeCF db whole "k" "w"+ openMergeDB path $ \db _ _ _ ->+ contents db `shouldReturn` [[("k", "x,a,b")], [("n", counter 2)], [("k", "w")], []]+ it "throws at the read of a key whose merge fails, and at no other" $+ withMergeDB $ \db _ _ _ -> do+ merge db "bad" "a" >> merge db "bad" "!" >> merge db "bad" "b"+ put db "good" "x" >> merge db "good" "y"+ get db "bad" `shouldThrow` anyIOException+ get db "good" `shouldReturn` Just "x,y"+ withIter db (\itr -> iterFirst itr >> iterEntries itr 10) `shouldThrow` anyIOException+ -- a put hides the record that cannot be merged+ put db "bad" "mended"+ get db "bad" `shouldReturn` Just "mended"+ withIter db forwards `shouldReturn` [("bad", "mended"), ("good", "x,y")]+ it "refuses a merge in a column family that has no merge operator" $+ withMergeDB $ \db _ _ none -> do+ mergeCF db none "k" "v" `shouldThrow` anyIOException+ write db [Put "a" "1", MergeCF none "k" "v"] `shouldThrow` anyIOException+ writePacked db [packOps [Put "a" "1", MergeCF none "k" "v"]] `shouldThrow` anyIOException+ -- the merge is nowhere; RocksDB 8.9 keeps the put that came before+ -- it in the refused batch, RocksDB 11 does not+ contents db >>= (`shouldSatisfy` (`elem` [[[], [], [], []], [[("a", "1")], [], [], []]]))+ -- A database is opened without the merge operator that its records need+ -- by a program that does not know of them. RocksDB opens it all the+ -- same. This is what RocksDB 11 does with records that are in table+ -- files, as far as it can be tried: a read of a key that has a value+ -- under its records, with 'get' or 'multiGet', crashes the process.+ it "cannot read merge records in table files without their operator, and keeps them" $+ withTempPath $ \path -> do+ let without = [(name, cfg {mergeOperator = Nothing}) | (name, cfg) <- mergeFamilies]+ openMergeDB path $ \db _ _ _ -> do+ put db "plain" "x"+ merge db "records" "a" >> merge db "records" "b"+ flushAll db+ withDBCF path (merging Nothing) without $ \db -> do+ get db "plain" `shouldReturn` Just "x"+ get db "records" `shouldThrow` anyIOException+ withIter db (\itr -> iterFirst itr >> iterEntries itr 10) `shouldThrow` anyIOException+ merge db "records" "c" `shouldThrow` anyIOException+ -- it can still be written, flushed and compacted+ put db "more" "y"+ flushAll db+ compactAll db+ openMergeDB path $ \db _ _ _ ->+ withIter db forwards `shouldReturn` [("more", "y"), ("plain", "x"), ("records", "a,b")]+ -- Records that are still in the write-ahead log fare worse. RocksDB 11+ -- drops them when it opens the database without the operator, and says+ -- nothing: they are lost for the program that has the operator too. It+ -- refuses to open a database without all its column families, and does+ -- so before it reads the log: a column family that the other program+ -- does not know of keeps the records safe.+ it "loses the merge records of the log to an open without their operator, unless the open is refused" $ do+ let without = [(name, cfg {mergeOperator = Nothing}) | (name, cfg) <- mergeFamilies]+ -- a value in a table file, and merge records that are in the log+ -- and in no table file when the database is closed+ records path = openMergeDB path $ \db _ _ _ -> do+ put db "value" "x"+ flush db Nothing+ merge db "value" "a" >> merge db "new" "b"+ withTempPath $ \path -> do+ records path+ withDBCF path (merging Nothing) without $ \_ -> return ()+ openMergeDB path $ \db _ _ _ ->+ withIter db forwards `shouldReturn` [("value", "x")]+ withTempPath $ \path -> do+ records path+ withDBCF path (merging Nothing) (take 2 without) (\_ -> return ()) `shouldThrow` anyIOException+ openMergeDB path $ \db _ _ _ ->+ withIter db forwards `shouldReturn` [("new", "b"), ("value", "x,a")]++------------------------------------------------------------------------------- -- Opening and closing ------------------------------------------------------------------------------- @@ -742,11 +1479,56 @@ multiGetCF db one (map key [1, 10000, 20000]) `shouldReturn` replicate 3 (Just value) getProperty db BlockCacheUsage >>= (`shouldSatisfy` positive)+ it "loads a piece of the index and of the filter for a lookup, not all of them" $ do+ -- One table file of 200,000 keys has a filter of some 250 kB and an+ -- index of about as much. A lookup in a file that the cache does not+ -- know loads both whole, or two pieces of 4 KiB.+ let size = 64 * 1024 * 1024 :: Int+ whole =+ tuned+ { errorIfExists = False,+ bloomFilter = True,+ blockCacheSize = Just size,+ cacheIndexAndFilterBlocks = True+ }+ pieces = whole {partitionIndexAndFilters = True}+ value = C.replicate 20 'v'+ afterOneLookup cfg =+ withTempPath $ \path -> do+ withDBCF path cfg [("one", cfg)] $ \db -> do+ one <- firstFamily db+ forM_ [1 .. 200000] $ \i -> putCF db one (key i) value+ -- opening it writes what was in memory to a file of the first+ -- level, all of whose index and filter the cache keeps: the+ -- compaction moves it down, where they are loaded as needed+ withDBCF path cfg [("one", cfg)] $ \db -> do+ one <- firstFamily db+ compactRange db (Just one) Nothing Nothing+ withDBCF path cfg [("one", cfg)] $ \db -> do+ one <- firstFamily db+ getCF db one (key 100000) `shouldReturn` Just value+ usage <- getProperty db BlockCacheUsage+ return (maybe 0 (read . C.unpack) usage :: Int)+ held <- afterOneLookup whole+ heldInPieces <- afterOneLookup pieces+ putStrLn $ "block cache after one lookup: " <> show held <> " bytes whole, " <> show heldInPieces <> " in pieces"+ held `shouldSatisfy` (> 200 * 1024)+ heldInPieces `shouldSatisfy` (< held `div` 4) it "opens with a bound on the write-ahead log" $ withTempPath $ \path -> withDBCF path tuned {maxTotalWalSize = Just (1024 * 1024)} [("one", tuned)] $ \db -> do put db "key" "value" get db "key" `shouldReturn` Just "value"+ it "opens with the number of background jobs that it is given" $+ withTempPath $ \path -> do+ let setting = "max_background_jobs="+ -- RocksDB writes the options that it runs with to a file+ jobs = do+ files <- filter ("OPTIONS-" `isPrefixOf`) <$> listDirectory path+ concat <$> forM files (fmap (settings . lines) . readFile . (path </>))+ settings = filter (setting `isPrefixOf`) . map (dropWhile (== ' '))+ withDB path plain {maxBackgroundJobs = Just 5} $ \db -> put db "key" "value"+ jobs >>= (`shouldSatisfy` elem (setting <> "5")) it "works with few open files and paranoid checks" $ withTempPath $ \path -> do let strict =@@ -784,6 +1566,39 @@ n <- frequency [(1, return 0), (4, choose (1, 24))] BS.pack <$> vectorOf n arbitrary +-- | A prefix that keys of 'genKey' often start with, and often the empty+-- one, which every key starts with.+genPrefix :: Gen ByteString+genPrefix = do+ n <- frequency [(2, return 0), (2, return 1), (1, return 2)]+ BS.pack <$> vectorOf n (elements [0, 1, 97, 255])++-- | Chunks small enough that a scan of a few entries takes several, with a+-- limit of bytes that is often below the size of one entry: 'genKey' and+-- 'genValue' give entries of up to 27 bytes.+genChunk :: Gen ChunkConfig+genChunk = do+ entries <- choose (1, 5)+ bytes <- frequency [(1, return 0), (3, choose (1, 60)), (1, return maxBound)]+ prefix <- genPrefix+ back <- arbitrary+ return+ ChunkConfig+ { chunkEntries = entries,+ chunkBytes = bytes,+ chunkPrefix = prefix,+ chunkBackwards = back+ }++-- | An iterator with no bound, one, or two that leave a range between them.+genBounds :: Gen IterConfig+genBounds = do+ a <- genKey+ b <- genKey+ lower <- elements [Nothing, Just (min a b)]+ upper <- elements [Nothing, Just (max a b)]+ return def {lowerBound = lower, upperBound = upper}+ genOps :: Gen [Op] genOps = listOf $ frequency [(3, Set <$> genKey <*> genValue), (1, Unset <$> genKey)] @@ -795,6 +1610,43 @@ opKey (Set k _) = k opKey (Unset k) = k +-- | A step of a history with merge records: a write, a merge record for the+-- operator that joins with commas, or a flush of the memtable to a table+-- file of its own.+data Step+ = Write Op+ | Join ByteString ByteString+ | Flush+ deriving (Show)++stepKey :: Step -> Maybe ByteString+stepKey (Write op) = Just (opKey op)+stepKey (Join k _) = Just k+stepKey Flush = Nothing++-- | Histories in which keys often have several merge records, with puts,+-- deletes and flushes among them. The records are letters: the one byte+-- that makes the operator of the tests fail is not among them.+genMergeOps :: Gen [Step]+genMergeOps =+ listOf . frequency $+ [ (2, Write <$> (Set <$> genKey <*> genValue)),+ (1, Write . Unset <$> genKey),+ (5, Join <$> genKey <*> (BS.pack <$> listOf (elements [97 .. 122]))),+ (2, return Flush)+ ]++-- | What the database should hold after a history, with the operator that+-- joins with commas: a record on a key without a value is the value, and+-- on a key with one it follows a comma.+joinModel :: [Step] -> Map ByteString ByteString+joinModel = foldl apply Map.empty+ where+ apply m (Write (Set k v)) = Map.insert k v m+ apply m (Write (Unset k)) = Map.delete k m+ apply m (Join k v) = Map.insertWith (\new old -> old <> "," <> new) k v m+ apply m Flush = m+ -- | What the database should hold after the operations, in order. model :: [Op] -> Map ByteString ByteString model = foldl apply Map.empty@@ -840,6 +1692,39 @@ forAllOps $ \ops -> inNewDB $ \db -> do write db (batchOf ops) shouldHold db (map opKey ops) (model ops)+ prop "after one packed batch, however it is cut into chunks" $+ forAllOps $ \ops -> forAll (choose (1, 5)) $ \size -> inNewDB $ \db -> do+ writePacked db (map packOps (groupsOf size (batchOf ops)))+ shouldHold db (map opKey ops) (model ops)+ prop "with merge records among the writes, in memtables, in table files and compacted" $+ forAll genMergeOps $ \steps -> ioProperty $ withTempPath $ \path ->+ withDB path (merging (Just joining)) $ \db -> do+ let keys = [k | Just k <- map stepKey steps]+ wanted = joinModel steps+ forM_ steps $ \case+ Write (Set k v) -> put db k v+ Write (Unset k) -> delete db k+ Join k v -> merge db k v+ Flush -> flush db Nothing+ shouldHold db keys wanted+ flush db Nothing+ shouldHold db keys wanted+ compactRange db Nothing Nothing Nothing+ shouldHold db keys wanted+ prop "with merge records in one batch, plain or packed" $+ forAll genMergeOps $ \steps -> forAll arbitrary $ \packed -> ioProperty $ withTempPath $ \path ->+ withDB path (merging (Just joining)) $ \db -> do+ let batch =+ [ op+ | step <- steps,+ op <- case step of+ Write (Set k v) -> [Put k v]+ Write (Unset k) -> [Del k]+ Join k v -> [Merge k v]+ Flush -> []+ ]+ if packed then writePacked db (map packOps (groupsOf 3 batch)) else write db batch+ shouldHold db [k | Just k <- map stepKey steps] (joinModel steps) prop "after the database is closed and opened again" $ forAllOps $ \ops -> ioProperty $ withTempPath $ \path -> do withDB path plain $ \db -> runOps db ops@@ -861,6 +1746,58 @@ withIter db $ \itr -> forM_ targets $ \target -> do iterSeek itr target iterEntry itr `shouldReturn` Map.lookupGE target (model ops)+ prop "in chunks, as one step at a time" $+ forAllOps $ \ops -> forAll genChunk $ \cfg -> forAll genBounds $ \bounds ->+ forAll genKey $ \start -> inNewDB $ \db -> do+ runOps db ops+ withIterConfig db Nothing bounds $ \itr -> do+ let step = if chunkBackwards cfg then iterPrev else iterNext+ place = do+ iterSeek itr start+ valid <- iterValid itr+ when (chunkBackwards cfg && not valid) (iterLast itr)+ -- what single steps read from there to the end, and the part of+ -- it that the chunks are to return+ place+ rest <- scan step itr+ let wanted = takeWhile ((chunkPrefix cfg `BS.isPrefixOf`) . fst) rest+ place+ let go done = do+ (entries, more) <- iterChunk itr cfg+ let done' = done + length entries+ left = drop done' wanted+ -- the entries that come next, within both limits+ entries `shouldBe` take (length entries) (drop done wanted)+ length entries `shouldSatisfy` (<= chunkEntries cfg)+ when (length entries > 1) $+ sizeOf entries `shouldSatisfy` (<= chunkBytes cfg)+ -- and not fewer than the limits allow+ when (length entries < chunkEntries cfg) $+ sizeOf (entries <> take 1 left) `shouldSatisfy` (\n -> null left || n > chunkBytes cfg)+ -- the flag says whether anything is left+ more `shouldBe` not (null left)+ -- the iterator is on the first entry that was not returned+ iterEntry itr `shouldReturn` listToMaybe (drop done' rest)+ when more $ do+ entries `shouldNotBe` []+ go done'+ go 0+ iterGetError itr `shouldReturn` Nothing+ prop "a number of entries at once, as one step at a time" $+ forAllOps $ \ops -> forAll (choose (0, 8)) $ \n -> forAll arbitrary $ \back ->+ forAll genPrefix $ \prefix -> forAll genKey $ \start -> inNewDB $ \db -> do+ runOps db ops+ withIter db $ \itr -> do+ let step = if back then iterPrev else iterNext+ iterSeek itr start+ rest <- scan step itr+ iterSeek itr start+ (if back then iterEntriesBack else iterEntries) itr n `shouldReturn` take n rest+ iterEntry itr `shouldReturn` listToMaybe (drop n rest)+ let wanted = take n (takeWhile ((prefix `BS.isPrefixOf`) . fst) rest)+ iterSeek itr start+ (if back then iterEntriesPrefixBack else iterEntriesPrefix) itr prefix n `shouldReturn` wanted+ iterEntry itr `shouldReturn` listToMaybe (drop (length wanted) rest) prop "through a snapshot, whatever is written afterwards" $ forAllOps $ \earlier -> forAllOps $ \later -> inNewDB $ \db -> do let keys = map opKey (earlier <> later)
+ test/cbits/merge_test.c view
@@ -0,0 +1,134 @@+/*+ * A merge operator for the tests of rocksdb-haskell-jprupp: it joins byte+ * strings with commas.+ *+ * The value of a key is the value it had, if any, then its merge records+ * from the oldest to the newest, with a comma between each two. So the+ * value shows which records were merged and in which order, which a sum+ * would not. A record of the single byte '!' makes a merge fail, to see+ * what RocksDB does then.+ *+ * The functions have the types of rocksdb_haskell_full_merge_t and+ * rocksdb_haskell_partial_merge_t in cbits/rocksdb_haskell.h.+ */++#include <stddef.h>+#include <stdlib.h>+#include <string.h>++/* Report a failure the way the header says to. */+static char *fail(unsigned char *success, size_t *new_value_length)+{+ *success = 0;+ *new_value_length = 0;+ return NULL;+}++/*+ * Join a first byte string, if there is one, and some more, with commas.+ */+static char *join(+ const char *first,+ size_t first_length,+ const char *const *list,+ const size_t *lengths,+ int count,+ unsigned char *success,+ size_t *new_value_length)+{+ size_t size = first == NULL ? 0 : first_length;+ size_t used = 0;+ char *joined;+ int i;++ for (i = 0; i < count; i++) {+ if (lengths[i] == 1 && list[i][0] == '!') {+ return fail(success, new_value_length);+ }+ /* a comma before every part but the first */+ size += lengths[i] + (first != NULL || i > 0 ? 1 : 0);+ }+ /* malloc may return NULL for no bytes, which is not a failure here */+ joined = malloc(size > 0 ? size : 1);+ if (joined == NULL) {+ return fail(success, new_value_length);+ }+ if (first != NULL && first_length > 0) {+ memcpy(joined, first, first_length);+ used = first_length;+ }+ for (i = 0; i < count; i++) {+ if (first != NULL || i > 0) {+ joined[used] = ',';+ used += 1;+ }+ if (lengths[i] > 0) {+ memcpy(joined + used, list[i], lengths[i]);+ used += lengths[i];+ }+ }+ *success = 1;+ *new_value_length = used;+ return joined;+}++char *rocksdb_haskell_test_join_full(+ void *state,+ const char *key,+ size_t key_length,+ const char *existing_value,+ size_t existing_value_length,+ const char *const *operands_list,+ const size_t *operands_list_length,+ int num_operands,+ unsigned char *success,+ size_t *new_value_length)+{+ (void)state;+ (void)key;+ (void)key_length;+ return join(+ existing_value,+ existing_value_length,+ operands_list,+ operands_list_length,+ num_operands,+ success,+ new_value_length);+}++char *rocksdb_haskell_test_join_partial(+ void *state,+ const char *key,+ size_t key_length,+ const char *const *operands_list,+ const size_t *operands_list_length,+ int num_operands,+ unsigned char *success,+ size_t *new_value_length)+{+ (void)state;+ (void)key;+ (void)key_length;+ return join(NULL, 0, operands_list, operands_list_length, num_operands, success, new_value_length);+}++/* A partial merge that always fails: RocksDB then keeps the records. */+char *rocksdb_haskell_test_no_partial(+ void *state,+ const char *key,+ size_t key_length,+ const char *const *operands_list,+ const size_t *operands_list_length,+ int num_operands,+ unsigned char *success,+ size_t *new_value_length)+{+ (void)state;+ (void)key;+ (void)key_length;+ (void)operands_list;+ (void)operands_list_length;+ (void)num_operands;+ return fail(success, new_value_length);+}