packages feed

rocksdb-haskell-jprupp 2.7.1 → 2.8.0

raw patch · 15 files changed

+4880/−1140 lines, 15 filesdep +filepathsetup-changedPVP ok

version bump matches the API change (PVP)

Dependencies added: filepath

API changes (from Hackage documentation)

+ Database.RocksDB: ChunkConfig :: !Int -> !Int -> !ByteString -> !Bool -> ChunkConfig
+ Database.RocksDB: Merge :: !ByteString -> !ByteString -> BatchOp
+ Database.RocksDB: MergeCF :: !ColumnFamily -> !ByteString -> !ByteString -> BatchOp
+ Database.RocksDB: MergeFunctions :: !String -> !FunPtr FullMerge -> !FunPtr PartialMerge -> MergeOperator
+ Database.RocksDB: PackedDel :: PackedOp
+ Database.RocksDB: PackedMerge :: PackedOp
+ Database.RocksDB: PackedPut :: PackedOp
+ Database.RocksDB: UInt64Add :: MergeOperator
+ Database.RocksDB: [chunkBackwards] :: ChunkConfig -> !Bool
+ Database.RocksDB: [chunkBytes] :: ChunkConfig -> !Int
+ Database.RocksDB: [chunkEntries] :: ChunkConfig -> !Int
+ Database.RocksDB: [chunkPrefix] :: ChunkConfig -> !ByteString
+ Database.RocksDB: [disableAutoCompactions] :: Config -> !Bool
+ Database.RocksDB: [fullMerge] :: MergeOperator -> !FunPtr FullMerge
+ Database.RocksDB: [maxBackgroundJobs] :: Config -> !Maybe Int
+ Database.RocksDB: [mergeName] :: MergeOperator -> !String
+ Database.RocksDB: [mergeOperator] :: Config -> !Maybe MergeOperator
+ Database.RocksDB: [partialMerge] :: MergeOperator -> !FunPtr PartialMerge
+ Database.RocksDB: [partitionIndexAndFilters] :: Config -> !Bool
+ Database.RocksDB: compactRange :: DB -> Maybe ColumnFamily -> Maybe ByteString -> Maybe ByteString -> IO ()
+ Database.RocksDB: data ChunkConfig
+ Database.RocksDB: data MergeOperator
+ Database.RocksDB: data PackedOp
+ Database.RocksDB: flush :: DB -> Maybe ColumnFamily -> IO ()
+ Database.RocksDB: iterChunk :: Iterator -> ChunkConfig -> IO ([(ByteString, ByteString)], Bool)
+ Database.RocksDB: iterEntries :: Iterator -> Int -> IO [(ByteString, ByteString)]
+ Database.RocksDB: iterEntriesBack :: Iterator -> Int -> IO [(ByteString, ByteString)]
+ Database.RocksDB: iterEntriesPrefix :: Iterator -> ByteString -> Int -> IO [(ByteString, ByteString)]
+ Database.RocksDB: iterEntriesPrefixBack :: Iterator -> ByteString -> Int -> IO [(ByteString, ByteString)]
+ Database.RocksDB: merge :: DB -> ByteString -> ByteString -> IO ()
+ Database.RocksDB: mergeCF :: DB -> ColumnFamily -> ByteString -> ByteString -> IO ()
+ Database.RocksDB: packOps :: [BatchOp] -> ByteString
+ Database.RocksDB: packedHeaderSize :: Int
+ Database.RocksDB: pokePackedHeader :: Ptr a -> Maybe ColumnFamily -> PackedOp -> Int -> Int -> IO ()
+ Database.RocksDB: type FullMerge = Ptr () -> CString -> CSize -> CString -> CSize -> Ptr CString -> Ptr CSize -> CInt -> Ptr CUChar -> Ptr CSize -> IO CString
+ Database.RocksDB: type PartialMerge = Ptr () -> CString -> CSize -> Ptr CString -> Ptr CSize -> CInt -> Ptr CUChar -> Ptr CSize -> IO CString
+ Database.RocksDB: writePacked :: DB -> [ByteString] -> IO ()
- Database.RocksDB: Config :: !Bool -> !Bool -> !Bool -> !Bool -> !Maybe Int -> !Maybe Int -> !Bool -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Bool -> !Maybe Int -> Config
+ Database.RocksDB: Config :: !Bool -> !Bool -> !Bool -> !Bool -> !Maybe Int -> !Maybe Int -> !Bool -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Bool -> !Bool -> !Maybe Int -> !Maybe Int -> !Maybe MergeOperator -> !Bool -> Config

Files

CHANGELOG.md view
@@ -1,3 +1,138 @@+## 2.8.0++ * `maxBackgroundJobs`, a new field of `Config`: how many threads RocksDB may+   use at once to flush memtables and to compact files, where its default is+   two, one of each. A database that has a backlog of compactions, after a+   bulk load for instance, works through it with one thread unless it is+   given more. More threads only help where the disk and the cores have room+   for them: a compaction that already waits for a slow disk is not made+   faster by a second one, and both take from the reads.+ * `partitionIndexAndFilters`, another new field of `Config`: the index and+   the filter of each table file are written in pieces of 4 KiB under a small+   index of their own. Each is otherwise one block that grows with the file,+   megabytes for a file of 64 MiB with a filter, and a database that has more+   of them than its block cache holds loads and drops them at nearly every+   lookup. Measured on a database of 1.6 TB with a cache of 1 GiB: the cache+   held the index and filter of 400 of 32,000 files and three megabytes of+   data, and a lookup took 0.8 ms of processor time on an idle disk. With+   pieces, a lookup loads the two that it needs. It applies to files written+   from then on; existing files keep their layout until compacted.+ * `compactRange` writes again every table file of a range of keys, those of+   the last level included, so that the files a database already has get the+   layout of its present `Config`: a Bloom filter, or+   `partitionIndexAndFilters`. RocksDB's own compactions rarely reach the+   last level, where most of the data is.+ * Many entries of an iterator in one foreign call. `iterNext` and `iterPrev`+   can wait for the disk, so each is a call for which the Haskell thread+   gives up its capability and has to get one back, and a scan made one for+   every entry. `iterChunk` walks the iterator in C, in a small function+   that now comes with the package (`cbits/rocksdb_haskell.c`), and returns+   up to 256 entries or 1 MiB from one call, forwards or backwards, within a+   prefix if one is given, with a flag that tells whether more follow. A+   `ChunkConfig` sets those limits. `iterEntries`, `iterEntriesBack`,+   `iterEntriesPrefix` and `iterEntriesPrefixBack` are short forms for a+   known number of entries. All of them leave the iterator on the first+   entry that they did not return, respect the bounds of an `IterConfig`,+   and throw if the iterator stopped at an error. The single steps are+   unchanged.++   Seconds for every thread to read 200,000 entries of 52 bytes, on four+   capabilities and four cores (`stack bench`):++   | Scan                                  | 1 thread | 16 threads |+   | ------------------------------------- | -------- | ---------- |+   | Single steps, loop not tail recursive | 0.78     | 3.9        |+   | Single steps, loop tail recursive     | 0.090    | 0.50       |+   | Chunks of 256 entries                 | 0.048    | 0.25       |++   With all 24 hardware threads of the machine free, the sixteen threads+   took 4.4, 0.83 and 0.12 seconds: threads inside the call hold no+   capability, so chunks gain more where there are more cores than+   capabilities. The first row is a loop that returns its list with+   `(entry :) <$> go`; the run-time system walks the stack at every such+   call, and that loop leaves a frame on it for every entry.+ * A large batch in a few foreign calls. `write` hands RocksDB the+   operations of a batch one call at a time, each a call for which the+   Haskell thread gives up its capability and has to get one back. For the+   65,000 keys of a Bitcoin block that was more than half of the time of+   the batch, and nearly all of it in a program whose other threads were+   busy. `writePacked` takes the batch as chunks of packed records, and a+   second function of `cbits/rocksdb_haskell.c` adds the operations of a+   chunk to the batch in one call. The batch is then written as before:+   atomically, with one write, however many chunks it came in.++   `packOps` packs a list of `BatchOp` into a chunk. A program that+   serialises its keys and values itself can write records straight into+   its own buffers instead, with no `ByteString` for each: it leaves+   `packedHeaderSize` bytes, serialises the key and the value after them,+   and fills in the header with `pokePackedHeader` once the lengths are+   known. Records are not aligned, so the next one starts where the last+   one ended.++   A chunk is checked in C before anything of it is used. One that is cut+   short, that has a length reaching past its end, an operation that does+   not exist or a delete with a value, makes `writePacked` throw, with+   nothing written.++   A record can also ask for a merge (`PackedMerge`), which is handed to+   `rocksdb_writebatch_merge`. The binding still has no way to give a+   column family a merge operator, so RocksDB refuses such a batch; the+   format has it so that it need not change later.++   The format is this package's own. It is not RocksDB's internal encoding+   of a batch (`rocksdb_writebatch_create_from`), which would save a copy+   and tie the binding to the insides of RocksDB.++   Milliseconds to write a batch of 65,000 operations and 6 MB over four+   column families, on four capabilities and four cores+   (`stack bench rocksdb-haskell-jprupp:bench:batch`):++   | Way to write the batch                    | Alone | Next to 8 busy threads |+   | ----------------------------------------- | ----- | ---------------------- |+   | `write`: a call for each operation        | 48    | 420                    |+   | `writePacked`, with `packOps` in the time | 28    | 115                    |+   | `writePacked`, chunks packed beforehand   | 22    | 28                     |++   All three end with the same write, of about 20 ms. Alone, the calls of+   `write` cost 400 ns an operation. Next to threads that compute, the+   thread waits for a capability after each of its 65,000 calls. With all+   24 hardware threads of the machine free, that took `write` 1,160 ms,+   and packed chunks 31.+ * Merge records and merge operators. A merge record is a third kind of+   write, next to a put and a delete: it says how a value changes, and+   RocksDB stores it without reading the value. The merge operator of the+   column family works out the value when the key is read, and when a+   compaction brings the records and the value together. A program that+   adds to counters or to lists by the thousand no longer reads each one+   first.+   - `mergeOperator`, a new field of `Config`, gives a column family its+     operator, or the default one when it is in the `Config` of the+     database. `UInt64Add` is RocksDB's own, for counters. `MergeFunctions`+     makes one of a name and the addresses of two C functions, a full merge+     and a partial merge, with the types `FullMerge` and `PartialMerge`.+     They are C functions because RocksDB calls them from its own threads;+     `cbits/rocksdb_haskell.h` says what they must do, and a third function+     of `cbits/rocksdb_haskell.c` makes the operator around them.+   - `Merge` and `MergeCF` are new constructors of `BatchOp`, written by+     `write` and packed by `packOps` as `PackedMerge` records. `merge` and+     `mergeCF` write one record. Code that matches on every `BatchOp` has+     two more cases.+   - `get`, `multiGet`, iterators and chunks return merged values, and+     need no change.+   - A column family with merge records must be given its operator by+     every program that opens the database. RocksDB 11.1 opens it without:+     it then drops the merge records that are still in the write-ahead+     log, with no error, and a read of a key with a value under merge+     records crashes the process. The README has the whole table.+ * `flush` writes the memtable of a column family to a table file and waits+   for it, and `disableAutoCompactions`, a new field of `Config`, stops+   RocksDB from compacting a column family in the background. They are for+   tests and benchmarks that need data in table files, a file for each+   flush, and for bulk loads that end with `compactRange`.+ * The `Config` record has new fields, so code that builds one without `def`+   has to set them; code that starts from `def` needs no change. This is why+   the version is 2.8.0 and not 2.7.2.+ ## 2.7.1   * Documentation and tests only; the library behaves as in 2.7.0.
README.md view
@@ -68,7 +68,10 @@ - **Stay close to the C API.** Functions map nearly one to one to RocksDB's C   functions, so RocksDB's own documentation and tuning guides apply directly.   Conveniences such as typed keys live in separate libraries-  ([rocksdb-query](https://github.com/jprupp/rocksdb-query)).+  ([rocksdb-query](https://github.com/jprupp/rocksdb-query)). The two+  exceptions are `iterChunk`, which walks an iterator, and `writePacked`,+  which fills a batch, each in a C function of this package: the cost they+  remove is that of calling C from Haskell. - **Scope every resource.** Databases, snapshots and iterators are C objects   that must be freed once and never touched again. `withDB`, `withSnapshot` and   `withIter` free them when their block ends, even on an exception.@@ -88,20 +91,25 @@ | -------------- | ------------------------------------------------------------------------------ | | Basic access   | `get`, `put`, `delete` on strict `ByteString` keys and values                  | | Batches        | Atomic `write` of many puts and deletes, across column families                |+| Packed batches | A batch of any size handed to RocksDB in a few foreign calls, from your buffers |+| Merges         | Change a value without reading it: merge records, and a merge operator in C    | | Multi-get      | `multiGet` reads many keys in one call to RocksDB's `MultiGet`                 | | Column families| Open a database with several independent key spaces; add new ones on open     | | Snapshots      | Consistent point-in-time reads across many queries and iterators               | | Iterators      | Ordered scans, forwards and backwards, optionally pinned to a snapshot         | | Bounds         | Iterators that stop at a key or at the end of a prefix, without reading past it |+| Chunks         | Hundreds of entries from one foreign call, in either direction, within a prefix | | Tuning         | Bloom filters, fixed-length prefix extractor, shared LRU block cache           | | Limits         | Memtable and WAL size limits, open file limit                                  | | Administration | Properties (memory use, statistics, SST files), approximate sizes, destroy, repair |  ### What is not included -- No transactions (`TransactionDB`, `OptimisticTransactionDB`), merge-  operators, TTL databases, backups or checkpoints.-- No manual compaction, `DeleteRange` or `SingleDelete`.+- No transactions (`TransactionDB`, `OptimisticTransactionDB`), TTL+  databases, backups or checkpoints.+- No merge operators written in Haskell: RocksDB calls a merge operator+  from its own threads, so it is a C function.+- No `DeleteRange` or `SingleDelete`. - No custom comparators: keys always sort by their bytes.  Pull requests that add bindings for these are welcome.@@ -185,7 +193,7 @@  ```yaml extra-deps:-  - rocksdb-haskell-jprupp-2.7.1+  - rocksdb-haskell-jprupp-2.8.0 ```  With `stack --nix`, add RocksDB to the Nix packages:@@ -288,6 +296,9 @@   go [] ``` +A list of thousands of events is read faster in chunks: see+[Reading many entries in one call](#reading-many-entries-in-one-call).+ ### 4. Read consistently with a snapshot  If another thread logs events while you read the counter and then the events,@@ -464,6 +475,257 @@ iterator together with the action that destroys it, which also frees the bounds: RocksDB keeps pointers to them for as long as the iterator lives. +### Reading many entries in one call++`iterNext` may wait for the disk, so it is a foreign call of the kind that+lets other Haskell threads run meanwhile. For that, the thread gives up its+capability and has to get one back when the call returns. A scan that moves+one step at a time pays this for every entry, and under load, when more+threads want to run than there are capabilities, it waits each time.++`iterChunk` walks the iterator in C and returns up to some hundreds of+entries from one call, with a flag that tells whether more follow:++```haskell+-- every key that starts with "user:", in chunks+withIter db $ \it -> do+  iterSeek it "user:"+  let go acc = do+        (entries, more) <- iterChunk it def {chunkPrefix = "user:"}+        let acc' = entries : acc+        if more then go acc' else return (concat (reverse acc'))+  go []+```++The `ChunkConfig` says what one call reads:++| Field            | Default | What it does                                                        |+| ---------------- | ------- | ------------------------------------------------------------------- |+| `chunkEntries`   | 256     | The most entries to read                                            |+| `chunkBytes`     | 1 MiB   | Stop before keys and values take more; one entry is always read     |+| `chunkPrefix`    | empty   | Stop at the first key that does not start with it                   |+| `chunkBackwards` | `False` | Go backwards, as `iterPrev` does                                    |++The iterator is left on the first entry that was not returned, so the next+call continues there. A bound of the iterator, from `withIterConfig`, ends a+chunk as it ends a scan by single steps. And `iterChunk` throws if the+iterator stopped at an error, so there is no `iterGetError` to remember.++For a known number of entries there are short forms, which return just the+list: `iterEntries it n`, `iterEntriesBack it n`, `iterEntriesPrefix it+prefix n` and `iterEntriesPrefixBack it prefix n`. They have no limit of+bytes, so a list shorter than `n` means that the scan is over.++```haskell+-- a page of twenty users, starting after the last one seen+iterSeek it (lastSeen <> "\0")+page <- iterEntriesPrefix it "user:" 20+```++What it saves, in this package's benchmark (`stack bench`): seconds for+every thread to read 200,000 entries of 52 bytes, on four capabilities and+four cores.++| Scan                                   | 1 thread | 16 threads |+| -------------------------------------- | -------- | ---------- |+| Single steps, loop not tail recursive  | 0.78     | 3.9        |+| Single steps, loop tail recursive      | 0.090    | 0.50       |+| Chunks of 256 entries                  | 0.048    | 0.25       |++The first row is a loop that builds its list with `(entry :) <$> go`. The+run-time system walks the stack of a thread at every foreign call that may+block, and such a loop leaves a frame there for every entry read so far.+Whatever you read with, keep the result in an argument of the loop.++With more cores than capabilities the chunks gain more, because a thread+that is inside the call holds no capability: with twenty more cores free,+the sixteen threads took 0.12 seconds in chunks and 0.83 in single steps.++Keep to single steps where each move depends on the entry just read, or+where one entry is all you want: `iterEntry` after a seek reads it without+moving the iterator.++### Writing a large batch in a few calls++`write` hands a batch to RocksDB one operation at a time, and each is a+foreign call of the kind that lets other Haskell threads run meanwhile. For+a batch of a few operations that is nothing. For tens of thousands it adds+up, and so do the `ByteString`s, one for each key and each value.++A packed batch is the same operations written one after another into a few+large buffers, the chunks. `writePacked` gives each chunk to a C function of+this package, which adds its operations to the batch, and then writes the+batch as `write` does: atomically, with one write.++```haskell+-- the same as: write db ops+writePacked db [packOps ops]+```++`packOps` packs a list of `BatchOp` into one chunk. It is the simple way,+and it copies every key and value once more. A program that serialises its+keys and values itself can skip the list, the `ByteString`s and the copy,+and write records straight into a buffer of its own:++```haskell+-- one record at a pointer: room for the header, the key, the value, and+-- then the header, now that the lengths are known+packRecord :: Ptr Word8 -> ColumnFamily -> IO (Ptr Word8)+packRecord ptr cf = do+  let body = ptr `plusPtr` packedHeaderSize+  keyLen <- serialiseKey body+  valueLen <- serialiseValue (body `plusPtr` keyLen)+  pokePackedHeader ptr (Just cf) PackedPut keyLen valueLen+  return (body `plusPtr` (keyLen + valueLen))+```++Here `serialiseKey` and `serialiseValue` stand for whatever writes your+bytes at a pointer and says how many. The next record goes where this one+ended: records need no alignment and nothing goes between them. A chunk is+then a `ByteString` over the part of the buffer that holds whole records.++What to know:++- **All the chunks are one batch.** `writePacked db chunks` applies the+  records of every chunk in order, atomically, with one write. Chunks only+  decide how much RocksDB is handed in one call: a few hundred kilobytes is+  plenty. A program that packs on several cores gives the chunks of each+  core, one core after another.+- **A chunk holds whole records.** A record cannot start in one chunk and+  end in the next.+- **A chunk is checked before it is used.** One that is cut short, or whose+  lengths say more than it holds, is refused with an exception, and nothing+  of the batch is written. The check cannot cover the column family handle+  of a record.+- **A chunk is only good here and now.** It holds column family handles,+  which are pointers of this process and of the open database.+- **A record can be a merge.** `PackedMerge` adds a merge record, as+  `Merge` does in a `BatchOp`, for a column family that has a merge+  operator: see [Merge records](#merge-records-change-a-value-without-reading-it).++The layout of a record, for a program that writes them by other means, is+in `cbits/rocksdb_haskell.h`. It is this package's own format, and what a+chunk becomes is a series of calls to RocksDB's C API. It is not RocksDB's+internal encoding of a batch, which a binding should not depend on.++What it saves, in this package's benchmark+(`stack bench rocksdb-haskell-jprupp:bench:batch`): milliseconds to write a+batch of 65,000 operations and 6 MB over four column families, on four+capabilities and four cores, alone and next to eight threads that compute.++| Way to write the batch                    | Alone | Next to busy threads |+| ----------------------------------------- | ----- | -------------------- |+| `write`: a call for each operation        | 48    | 420                  |+| `writePacked`, with `packOps` in the time | 28    | 115                  |+| `writePacked`, chunks packed beforehand   | 22    | 28                   |++All three end with the same write, which takes about 20 ms here: the rest+is the handing over. Alone, a thread gets its capability back at once+after each call, and still pays 400 ns an operation. Next to threads that+want the capabilities too, as in a server under load, it waits for one+after every call. `packOps` is work in Haskell, which has to share the+capabilities with those threads like any other.++With all 24 hardware threads of the machine free, the batch took `write`+1,160 ms next to the busy threads, and 31 ms from packed chunks.++Keep to `write` for small batches: it is as fast there, and simpler.++### Merge records: change a value without reading it++A put replaces the value of a key. A program that wants to change a value,+to add to a counter or to add an entry to a list, has to read it first.+For a batch of thousands of such changes those reads are most of the work,+and each may wait for the disk.++A **merge record** is a third kind of write, next to a put and a delete. It+says how the value changes, not what it becomes: "add 5", "add this+entry". RocksDB stores the record as it is and reads nothing.++The work is done later by the **merge operator** of the column family: a+function that takes the value that the key had and the records written+since, oldest first, and gives the value that the key has now. RocksDB+calls it when the key is read, and when a compaction brings the records+and the value together. After that the key holds a plain value again.++```haskell+-- a column family of counters, with RocksDB's own operator for them+let counters = def {mergeOperator = Just UInt64Add}+withDBCF "/tmp/example-merge" def {createIfMissing = True} [("hits", counters)] $ \db -> do+  let [hits] = columnFamilies db+      five = BS.pack [5, 0, 0, 0, 0, 0, 0, 0] -- eight bytes, least significant first+  mergeCF db hits "home" five -- no read+  mergeCF db hits "home" five+  getCF db hits "home" >>= print -- Just "\n\NUL\NUL\NUL\NUL\NUL\NUL\NUL": ten+  write db [MergeCF hits "home" five, MergeCF hits "about" five]+```++What to know:++- **Readers see values.** `get`, `multiGet` and iterators return merged+  values, in a snapshot too. For a key with records pending they pay for+  the merge, and for a look into each table file that holds one of the+  records.+- **A put or a delete hides the records before it**, like any older value.+  A record written after a delete, or to a key that never had a value, is+  merged with no value: the operator decides what that means.+- **The operator is set when the database is opened**, with the+  `mergeOperator` field of the `Config` of the column family. The one in+  the `Config` of the database is for the default column family.+- **`UInt64Add`** is RocksDB's own: values and records are unsigned 64-bit+  numbers in eight bytes, least significant first, added modulo 2^64.+- **`MergeFunctions`** makes an operator of a name and two C functions: one+  that works out a value (the full merge), and one that makes a single+  record of several (the partial merge), which RocksDB uses to shorten a+  list of records before it has the value. They are C because RocksDB+  calls them from its own threads, during reads, flushes and compactions.++```c+/* my_merge.c, listed under c-sources of your package */+char *my_full_merge(void *state, const char *key, size_t key_length,+                    const char *existing_value, size_t existing_value_length,+                    const char *const *operands_list,+                    const size_t *operands_list_length, int num_operands,+                    unsigned char *success, size_t *new_value_length);+char *my_partial_merge(void *state, const char *key, size_t key_length,+                       const char *const *operands_list,+                       const size_t *operands_list_length, int num_operands,+                       unsigned char *success, size_t *new_value_length);+```++```haskell+foreign import ccall "&my_full_merge" myFullMerge :: FunPtr FullMerge+foreign import ccall "&my_partial_merge" myPartialMerge :: FunPtr PartialMerge++mine :: Config+mine = def {mergeOperator = Just (MergeFunctions "my-merge" myFullMerge myPartialMerge)}+```++A function returns the new bytes in memory from `malloc`, which RocksDB+frees, with their length in `*new_value_length` and 1 in `*success`.+`existing_value` is `NULL` when the key has no value. To fail, it stores 0+in both and returns `NULL`: the read of that key then throws. A partial+merge may always fail, and RocksDB then keeps the records as they are.+`cbits/rocksdb_haskell.h` has the two types and the rules in full, and+`test/cbits/merge_test.c` is a small operator to start from.++**Every program that opens the database must set the same operator.**+RocksDB opens a column family that has merge records without one, and+this is what version 11.1 was seen to do then:++| Where the merge records are | What happens without the operator                              |+| --------------------------- | -------------------------------------------------------------- |+| In the write-ahead log      | Dropped when the database is opened, without an error: lost    |+| In table files              | Kept, also through a compaction, but they cannot be read       |+| Read: records and no value  | `get` throws; an iterator stops there with an error            |+| Read: records over a value  | `get` and `multiGet` crash the process                         |+| Writing a merge record      | Refused with an error                                          |++So when a program starts to write merge records to a database that older+versions of it can open, make sure that they no longer can. A new column+family is enough: RocksDB refuses to open a database without all its+column families, and does so before it reads the write-ahead log.+ ### Properties and maintenance  ```haskell@@ -491,12 +753,18 @@ | `dbWriteBufferSize`           | database        | Bytes of all memtables together (default unlimited)             | | `blockCacheSize`              | database        | Bytes of an LRU block cache shared by all column families       | | `maxTotalWalSize`             | database        | Bytes of write-ahead log that trigger flushes                   |+| `maxBackgroundJobs`           | database        | Threads for flushes and compactions together (RocksDB default 2) | | `writeBufferSize`             | column family   | Bytes per memtable (RocksDB default 64 MiB)                     | | `bloomFilter`                 | column family   | Full Bloom filter with 10 bits per key                          | | `prefixLength`                | column family   | Length of a fixed key prefix for the prefix extractor           | | `cacheIndexAndFilterBlocks`   | column family   | Keep index and filter blocks in the block cache                 |+| `partitionIndexAndFilters`    | column family   | Write index and filter in 4 KiB pieces, loaded as needed        |+| `mergeOperator`               | column family   | How merge records are combined with a value (default: none)     |+| `disableAutoCompactions`      | column family   | No background compaction: for bulk loads, tests and benchmarks  |  Database-wide fields are ignored in the configuration of a column family.+The column family fields of the database's own `Config` are for its default+column family.  ### Choosing a configuration @@ -534,17 +802,25 @@ | Part                     | Contents                                                       | | ------------------------ | -------------------------------------------------------------- | | Types                    | `DB`, `Config`, `BatchOp`, `ColumnFamily`, `Snapshot`, `Iterator`, `Property`, `Range` |+| Merge operators          | `MergeOperator`, `FullMerge`, `PartialMerge`                   | | Opening                  | `withDB`, `withDBCF`                                           | | Reading and writing      | `get`, `getCF`, `multiGet`, `multiGetCF`, `put`, `putCF`, `delete`, `deleteCF`, `write` |+| Merging                  | `merge`, `mergeCF`, and `Merge`, `MergeCF` in a `BatchOp`      |+| Packed batches           | `writePacked`, `packOps`, `PackedOp`, `packedHeaderSize`, `pokePackedHeader` | | Snapshots                | `withSnapshot`, `createSnapshot`, `releaseSnapshot`            | | Creating iterators       | `withIter`, `withIterCF`, `withIterSnap`, `withIterSnapCF`     | | Bounded iterators        | `IterConfig`, `withIterConfig`, `createIteratorConfig`, `prefixUpperBound` | | Moving iterators         | `iterSeek`, `iterFirst`, `iterLast`, `iterNext`, `iterPrev`    | | Reading from iterators   | `iterKey`, `iterValue`, `iterEntry`, `iterValid`, `iterGetError` |-| Administration           | `getProperty`, `getPropertyCF`, `approximateSize`, `destroy`, `repair` |+| Reading in chunks        | `iterChunk`, `ChunkConfig`                                     |+| Reading a number of entries | `iterEntries`, `iterEntriesBack`, `iterEntriesPrefix`, `iterEntriesPrefixBack` |+| Administration           | `getProperty`, `getPropertyCF`, `approximateSize`, `compactRange`, `destroy`, `repair` |+| Flushing                 | `flush`                                                        |  The FFI bindings (`Database.RocksDB.C`) and helpers (`Database.RocksDB.Internal`)-are hidden modules.+are hidden modules. `cbits/rocksdb_haskell.c` is the one piece of C in the+package: the functions behind `iterChunk` and `writePacked`, and the one+that makes a merge operator of two functions.  ## Things to watch out for @@ -576,7 +852,29 @@   with it, not with `destroyIterator`, which would leave its bounds and read   options allocated. - Calls into RocksDB are `safe` foreign calls, so they do not block other-  Haskell threads, but each one costs a little more than an `unsafe` call.+  Haskell threads, but each one costs more than an `unsafe` call, and much+  more when threads compete for capabilities. A scan by single steps makes+  one for every entry: read ranges with `iterChunk`. A `write` makes one+  for every operation: write batches of thousands with `writePacked`.+- A chunk for `writePacked` is checked for its lengths, not for its column+  family handles. A handle that is not of the open database crashes the+  process, as it does in `PutCF`.+- A loop over an iterator that is not tail recursive gets slower with every+  entry, because the run-time system walks the stack at each foreign call.+  Keep what has been read in an argument of the loop.+- `iterChunk` and the `iterEntries` functions leave the iterator on the+  first entry that they did not return, not on the last one that they did.+- An `iterChunk` with a `chunkPrefix` still moves the iterator to the first+  key outside the prefix, as a loop written by hand does. Where deleted keys+  follow the range, the iterator needs a bound as well.+- A column family with merge records must get its `mergeOperator` from+  every program that opens the database. Without it RocksDB drops the+  records that are still in its write-ahead log, and a read of a key that+  has a value under merge records crashes the process: see+  [Merge records](#merge-records-change-a-value-without-reading-it).+- A merge operator is called from RocksDB's threads, several at a time. It+  must be a C function that keeps no state between calls, and must return+  memory from `malloc`. - Releases before 2.6.2 built no Bloom filters on RocksDB 6.24 or later,   whatever `bloomFilter` said: they passed the bits per key as an integer to a   function that had come to take a `double`. A database written by one of them@@ -598,11 +896,19 @@  stack build                        # build the library stack test                         # run the tests+stack bench                        # time scans and large batches stack haddock --no-haddock-deps    # build the API documentation ```  With `stack --nix`, the `nix` section of `stack.yaml` provides RocksDB. +`bench/Scan.hs` times a scan of 200,000 entries by single steps and in+chunks, from one thread and from sixteen on four capabilities.+`bench/Batch.hs` times a batch of 65,000 operations written with `write`+and with `writePacked`, alone and next to threads that compute. To see what+a machine with four cores does, confine them: `taskset -c 0-3 stack bench`.+To run one of the two: `stack bench rocksdb-haskell-jprupp:bench:batch`.+ ### Nix  ```sh@@ -629,7 +935,8 @@ - **Properties** apply random writes to the database and to a `Map`, and   require that every way of reading the database agrees with the map: point   reads, `multiGet`, iteration in both directions, seeks, snapshots and-  column families. The keys are short and drawn from the bytes a binding is+  column families. Chunks are held to what single steps read, with random+  limits, prefixes and bounds. The keys are short and drawn from the bytes a binding is   most likely to mishandle, zero and 255, so that operations often meet on   the same key. 
Setup.hs view
@@ -1,2 +1,3 @@ import Distribution.Simple+ main = defaultMain
+ bench/Batch.hs view
@@ -0,0 +1,187 @@+{-# LANGUAGE ImportQualifiedPost #-}+{-# LANGUAGE OverloadedStrings #-}++-- |+-- What it costs to hand a large batch to RocksDB with one foreign call for+-- each operation, and with one for each chunk of packed operations.+--+-- The question it answers: 'write' gives RocksDB the operations of a batch+-- one call at a time, and a Haskell program pays for every foreign call+-- that may block, since the thread gives up its capability for the call and+-- has to get one back afterwards. How much of the time of a large batch is+-- that, and how much is left when 'writePacked' makes the call once for a+-- few thousand operations?+--+-- A batch is 'operationCount' operations over four column families, about+-- what the import of a Bitcoin block writes. It is written three ways:+--+-- * with 'write', from a list of t'BatchOp';+--+-- * with 'writePacked', from the same list packed by 'packOps' into chunks+--   of 'chunkOperations', the packing included in the time;+--+-- * with 'writePacked', from chunks that were packed beforehand: what is+--   left for a program that serialises its keys and values straight into+--   chunks, and so has no list to pack.+--+-- All three end with the same @rocksdb_write@, which adds the operations to+-- the memtables and takes most of the time. The difference between the+-- rows is what the hand-over costs.+--+-- The answer depends on whether other threads want the capabilities. A+-- thread alone gets its own back at once after a call. In a server under+-- load it finds it taken, and waits. So every way is timed alone and next+-- to 'busyThreads' threads that compute, on four capabilities whatever the+-- machine has: the suite is built with @-N4@.+--+-- Run it with @stack bench rocksdb-haskell-jprupp:bench:batch@, and on+-- four cores with @taskset -c 0-3@ before it. The figures of the CHANGELOG+-- and of the README come from it.+module Main (main) where++import Control.Concurrent (getNumCapabilities)+import Control.Concurrent.Async (withAsync)+import Control.Exception (evaluate)+import Control.Monad (forM, forM_, unless)+import Data.ByteString (ByteString)+import Data.ByteString qualified as B+import Data.ByteString.Char8 qualified as C+import Data.Default (def)+import Data.IORef (atomicModifyIORef', newIORef)+import Data.List (sort)+import Database.RocksDB+import GHC.Clock (getMonotonicTime)+import GHC.Conc (getNumProcessors)+import System.IO.Temp (withSystemTempDirectory)+import System.Mem (performGC)+import Text.Printf (printf)++-- | Operations in a batch.+operationCount :: Int+operationCount = 65000++-- | Operations in a chunk of a packed batch: with keys of 34 bytes, values+-- of 66 and the header of a record, a chunk of about 256 KiB.+chunkOperations :: Int+chunkOperations = 2400++-- | Times each measurement is taken. The median is reported: other programs+-- on the machine make single runs slow, never fast.+runs :: Int+runs = 15++-- | Threads that compute while a batch is written, in the second+-- measurement of each way.+busyThreads :: Int+busyThreads = 8++-- | A key of 34 bytes that is new in every batch: the number of the batch+-- and that of the operation.+key :: Int -> Int -> ByteString+key batch i = C.pack (printf "key:%010d:%019d" batch i)++-- | A value of 66 bytes.+value :: Int -> ByteString+value = C.pack . printf "value:%060d"++-- | The operations of a batch: one in eight deletes a key, the others set+-- one, in the four column families in turn.+operations :: [ColumnFamily] -> Int -> [BatchOp]+operations families batch = zipWith op (cycle families) [1 .. operationCount]+  where+    op cf i+      | i `mod` 8 == 0 = DelCF cf (key batch i)+      | otherwise = PutCF cf (key batch i) (value i)++-- | A list in pieces of a length, the last one shorter.+groupsOf :: Int -> [a] -> [[a]]+groupsOf _ [] = []+groupsOf n xs = let (a, b) = splitAt n xs in a : groupsOf n b++-- | A way to write the operations of a batch: what has to be ready before+-- the clock starts, and the write that is timed.+data Way a = Way+  { name :: String,+    prepare :: [BatchOp] -> IO a,+    timed :: DB -> a -> IO ()+  }++-- | Every operation with its key and value in memory.+ready :: [BatchOp] -> IO [BatchOp]+ready ops = do+  forM_ ops $ \op -> evaluate $ case op of+    Put k v -> B.length k + B.length v+    PutCF _ k v -> B.length k + B.length v+    Del k -> B.length k+    DelCF _ k -> B.length k+  return ops++-- | The seconds that a way takes to write a batch of new keys.+timeWrite :: DB -> IO Int -> Way a -> IO Double+timeWrite db next way = do+  batch <- next+  input <- prepare way =<< ready (operations (columnFamilies db) batch)+  performGC+  t0 <- getMonotonicTime+  timed way db input+  t1 <- getMonotonicTime+  return (t1 - t0)++-- | Run an action next to so many threads that compute and allocate, as+-- the threads of a server do, and stop them afterwards.+besideBusy :: Int -> IO a -> IO a+besideBusy 0 action = action+besideBusy n action = withAsync (spin 0) (const (besideBusy (n - 1) action))+  where+    -- it allocates, so that it can be stopped, and depends on its+    -- argument, so that it is not worked out once and for all+    spin :: Int -> IO ()+    spin i = do+      _ <- evaluate (sum (map fromEnum (show [i .. i + 2000])))+      spin (i + 1)++-- | The median of 'runs' measurements.+median :: IO Double -> IO Double+median measure = do+  times <- forM [1 .. runs] (const measure)+  return (sort times !! (runs `div` 2))++main :: IO ()+main = withSystemTempDirectory "rocksdb-bench" $ \dir -> do+  let config = def {createIfMissing = True, createMissingColumnFamilies = True}+      families = [(n, config) | n <- ["one", "two", "three", "four"]]+  withDBCF dir config families $ \db -> do+    caps <- getNumCapabilities+    cores <- getNumProcessors+    counter <- newIORef 0+    let next = atomicModifyIORef' counter (\n -> (n + 1, n))+        row :: Way a -> IO ()+        row way = do+          alone <- median (timeWrite db next way)+          busy <- median (besideBusy busyThreads (timeWrite db next way))+          printf "%-44s %10.1f %10.1f\n" (name way) (1000 * alone) (1000 * busy)+    printf "batches of %d operations, about %d bytes of keys and values\n" operationCount size+    printf "%d capabilities, %d cores, median of %d runs, milliseconds a batch\n" caps cores runs+    printf "%-44s %10s %10s\n" (s "") (s "alone") (s (show busyThreads <> " busy"))+    -- once unmeasured, so that the first row does not pay for the start+    _ <- timeWrite db next oneCall+    unless (caps == 4) $ putStrLn "note: the figures of the documentation are for four capabilities"+    row oneCall+    row packedList+    row packedAhead+  where+    s :: String -> String+    s = id+    -- one operation in eight is a delete, with a key of 34 bytes and no+    -- value of 66+    size = operationCount * 100 - operationCount `div` 8 * 66+    oneCall = Way "write: a call for each operation" return write+    packedList =+      Way "writePacked, packOps included" return $ \db ops ->+        writePacked db (map packOps (groupsOf chunkOperations ops))+    packedAhead =+      Way "writePacked, chunks packed beforehand" pack writePacked+    pack ops = do+      let chunks = map packOps (groupsOf chunkOperations ops)+      forM_ chunks (evaluate . B.length)+      return chunks
+ bench/Scan.hs view
@@ -0,0 +1,164 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE ImportQualifiedPost #-}+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE OverloadedStrings #-}++-- |+-- What a scan costs with one foreign call for each entry, and with one for+-- each chunk of entries.+--+-- The question it answers: a Haskell program pays for every foreign call+-- that may block, since the thread gives up its capability for the call and+-- has to get one back afterwards. How much of the time of a scan is that,+-- and how much is left when 'iterChunk' makes the call once for a few+-- hundred entries?+--+-- The answer depends on how many threads want a capability. A thread alone+-- gets its own back at once. Sixteen threads on four capabilities, as in a+-- server under load, each find theirs taken when the call returns, and wait.+-- So every scan is timed with one thread and with sixteen, on four+-- capabilities, whatever the machine has: the suite is built with @-N4@.+--+-- It also times the loop that is not tail recursive, @(entry :) \<$\> go@,+-- because the run-time system looks at the stack of a thread at every such+-- call, and that loop leaves a frame on it for every entry read so far.+--+-- Each scan reads every entry of a database of 'entryCount' small entries+-- from a table file, and adds up their sizes. Run it with @stack bench@; to+-- run some of the scans only, name them: @stack bench --ba chunks@. The figures of+-- the CHANGELOG and of the README come from it.+module Main (main) where++import Control.Concurrent (getNumCapabilities)+import Control.Concurrent.Async (forConcurrently)+import Control.Monad (forM, forM_, unless)+import Data.ByteString (ByteString)+import Data.ByteString qualified as B+import Data.ByteString.Char8 qualified as C+import Data.Default (def)+import Data.List (isInfixOf, sort)+import Database.RocksDB+import GHC.Clock (getMonotonicTime)+import GHC.Conc (getNumProcessors)+import System.Environment (getArgs)+import System.IO.Temp (withSystemTempDirectory)+import System.Mem (performGC)+import Text.Printf (printf)++-- | Entries in the database, all of which every scan reads.+entryCount :: Int+entryCount = 200000++-- | Times each measurement is taken. The median is reported: other programs+-- on the machine make single runs slow, never fast.+runs :: Int+runs = 5++-- | A key of 20 bytes that sorts by its number.+key :: Int -> ByteString+key = C.pack . printf "key:%016d"++-- | A value of 32 bytes: about what an index entry holds.+value :: Int -> ByteString+value = C.pack . printf "value:%026d"++-- | A way to read everything from where the iterator is, giving the number+-- of entries and the bytes of their keys and values. No scan keeps the+-- entries: what a program does with them, and what the garbage collector+-- then has to do, is not what is measured here.+type Scan = Iterator -> IO Total++-- | Entries read, and the bytes of their keys and values.+data Total = Total !Int !Int+  deriving (Eq, Show)++-- | Count an entry.+add :: Total -> (ByteString, ByteString) -> Total+add (Total n bytes) (k, v) = Total (n + 1) (bytes + B.length k + B.length v)++-- | One step for each entry, in a loop that keeps its total in an argument:+-- the stack stays as it is.+steps :: Scan+steps it = go (Total 0 0)+  where+    go !total =+      iterEntry it >>= \case+        Just entry -> iterNext it >> go (add total entry)+        Nothing -> return total++-- | One step for each entry, in a loop that is not tail recursive, as one+-- that returns a list with @(entry :) \<$\> go@ is: each entry waits on the+-- stack for those that follow.+stepsOnStack :: Scan+stepsOnStack it = go+  where+    go =+      iterEntry it >>= \case+        Just entry -> do+          iterNext it+          !total <- go+          return (add total entry)+        Nothing -> return (Total 0 0)++-- | Chunks of so many entries.+chunks :: Int -> Scan+chunks n it = go (Total 0 0)+  where+    go !total = do+      (entries, more) <- iterChunk it def {chunkEntries = n}+      let total' = foldl' add total entries+      if more then go total' else return total'++-- | The seconds that so many threads take to scan the whole database at the+-- same time, each with its own iterator.+timeScan :: DB -> Scan -> Int -> IO Double+timeScan db scan threads = do+  performGC+  t0 <- getMonotonicTime+  totals <- forConcurrently [1 .. threads] $ \_ ->+    withIter db $ \it -> iterFirst it >> scan it+  t1 <- getMonotonicTime+  -- every key is of 20 bytes and every value of 32+  unless (all (== Total entryCount (entryCount * 52)) totals) $+    fail ("a scan read " <> show totals <> ", and not every entry")+  return (t1 - t0)++-- | The median of 'runs' measurements.+median :: IO Double -> IO Double+median measure = do+  times <- forM [1 .. runs] (const measure)+  return (sort times !! (runs `div` 2))++main :: IO ()+main = withSystemTempDirectory "rocksdb-bench" $ \dir ->+  withDB dir def {createIfMissing = True} $ \db -> do+    forM_ [0, 1000 .. entryCount - 1] $ \i ->+      write db [Put (key j) (value j) | j <- [i .. min entryCount (i + 1000) - 1]]+    -- Compacted, the entries are in a table file, where those of a database+    -- in production are, and not in the memtable.+    compactRange db Nothing Nothing Nothing+    caps <- getNumCapabilities+    cores <- getNumProcessors+    printf "%d entries, %d capabilities, %d cores, median of %d runs\n" entryCount caps cores runs+    printf "%-34s %8s %10s %18s\n" (s "scan") (s "threads") (s "seconds") (s "entries a second")+    -- once unmeasured, to fill the block cache+    _ <- timeScan db (chunks 256) 1+    wanted <- getArgs+    let chosen = [x | x@(name, _) <- scans, null wanted || any (`isInfixOf` name) wanted]+    forM_ chosen $ \(name, scan) ->+      forM_ [1, 16] $ \threads -> do+        seconds <- median (timeScan db scan threads)+        let rate = fromIntegral (threads * entryCount) / seconds :: Double+        printf "%-34s %8d %10.3f %18.0f\n" name threads seconds rate+  where+    s :: String -> String+    s = id+    scans :: [(String, Scan)]+    scans =+      [ ("single steps, not tail recursive", stepsOnStack),+        ("single steps", steps),+        ("chunks of 16", chunks 16),+        ("chunks of 64", chunks 64),+        ("chunks of 256", chunks 256),+        ("chunks of 1024", chunks 1024)+      ]
+ cbits/rocksdb_haskell.c view
@@ -0,0 +1,379 @@+/*+ * C helpers of rocksdb-haskell-jprupp. See rocksdb_haskell.h for what each+ * function is for and what it promises.+ */++#include "rocksdb_haskell.h"++#include <stdint.h>+#include <stdlib.h>+#include <string.h>++/* The two lengths that come before the bytes of an entry. */+#define HEADER (2 * sizeof(size_t))++/* The first size of the buffer, which doubles when it is full. */+#define FIRST_CAPACITY 1024++/*+ * Whether the iterator is on an entry whose key starts with the prefix. If+ * it is, the key is in "*key" and "*key_len", and belongs to the iterator+ * until it moves.+ */+static int at_match(+    const rocksdb_iterator_t *iter,+    const char *prefix,+    size_t prefix_len,+    const char **key,+    size_t *key_len)+{+    if (!rocksdb_iter_valid(iter)) {+        return 0;+    }+    *key_len = 0;+    *key = rocksdb_iter_key(iter, key_len);+    if (*key == NULL) {+        /* no bytes to read, whatever the length says */+        *key_len = 0;+    }+    if (*key_len < prefix_len) {+        return 0;+    }+    /* memcmp must not be given a null pointer, even to compare no bytes */+    return prefix_len == 0 || memcmp(*key, prefix, prefix_len) == 0;+}++/*+ * The size that an entry takes in the buffer: header, key, value and+ * padding. Zero if it does not fit in a size_t, which no real entry does.+ */+static size_t entry_size(size_t key_len, size_t value_len)+{+    const size_t align = sizeof(size_t);+    size_t size = HEADER;++    if (key_len > SIZE_MAX - size) {+        return 0;+    }+    size += key_len;+    if (value_len > SIZE_MAX - size) {+        return 0;+    }+    size += value_len;+    if (size > SIZE_MAX - (align - 1)) {+        return 0;+    }+    return (size + (align - 1)) / align * align;+}++/*+ * Make room for "size" more bytes after the "used" ones of the buffer.+ * Returns zero, with the buffer as it was, if there is no memory for it.+ */+static int reserve(char **buffer, size_t *capacity, size_t used, size_t size)+{+    size_t needed;+    size_t grown;+    char *moved;++    if (size > SIZE_MAX - used) {+        return 0;+    }+    needed = used + size;+    if (needed <= *capacity) {+        return 1;+    }+    grown = *capacity < FIRST_CAPACITY ? FIRST_CAPACITY : *capacity;+    while (grown < needed) {+        if (grown > SIZE_MAX / 2) {+            grown = needed;+        } else {+            grown *= 2;+        }+    }+    moved = realloc(*buffer, grown);+    if (moved == NULL) {+        return 0;+    }+    *buffer = moved;+    *capacity = grown;+    return 1;+}++int rocksdb_haskell_iter_chunk(+    rocksdb_iterator_t *iter,+    int backwards,+    size_t max_entries,+    size_t max_bytes,+    const char *prefix,+    size_t prefix_len,+    char **buffer,+    size_t *count,+    unsigned char *more,+    char **errptr)+{+    char *entries = NULL; /* the buffer */+    size_t capacity = 0;  /* its size */+    size_t used = 0;      /* how much of it holds entries */+    size_t read = 0;      /* how many entries */+    size_t bytes = 0;     /* their keys and values, without the rest */+    int no_memory = 0;++    *buffer = NULL;+    *count = 0;+    *more = 0;++    while (read < max_entries) {+        const char *key;+        const char *value;+        size_t key_len;+        size_t value_len = 0;+        size_t size;+        char *entry;++        if (!at_match(iter, prefix, prefix_len, &key, &key_len)) {+            break;+        }+        value = rocksdb_iter_value(iter, &value_len);+        if (value == NULL) {+            value_len = 0;+        }++        size = entry_size(key_len, value_len);+        if (size == 0) {+            no_memory = 1;+            break;+        }+        /*+         * The limit on bytes, which the first entry is exempt from. Since+         * entry_size did not overflow, neither does the sum of the lengths.+         * "bytes" is above "max_bytes" only after a first entry that was.+         * At the limit exactly, an entry of no bytes is still read: it+         * does not take the bytes read to more.+         */+        if (read > 0 && (bytes > max_bytes || key_len + value_len > max_bytes - bytes)) {+            break;+        }+        if (!reserve(&entries, &capacity, used, size)) {+            no_memory = 1;+            break;+        }++        entry = entries + used;+        memcpy(entry, &key_len, sizeof(size_t));+        memcpy(entry + sizeof(size_t), &value_len, sizeof(size_t));+        if (key_len > 0) {+            memcpy(entry + HEADER, key, key_len);+        }+        if (value_len > 0) {+            memcpy(entry + HEADER + key_len, value, value_len);+        }+        used += size;+        bytes += key_len + value_len;+        read += 1;++        /* only now is the entry safe, and the iterator may leave it */+        if (backwards) {+            rocksdb_iter_prev(iter);+        } else {+            rocksdb_iter_next(iter);+        }+    }++    if (no_memory) {+        /* the iterator is still on the entry that did not fit */+        if (read == 0) {+            free(entries);+            return ROCKSDB_HASKELL_NO_MEMORY;+        }+        *buffer = entries;+        *count = read;+        *more = 1;+        return ROCKSDB_HASKELL_OK;+    }++    /*+     * An iterator that met an error is not valid, like one that reached the+     * end. Ask it, so that the caller does not take one for the other.+     */+    rocksdb_iter_get_error(iter, errptr);+    if (*errptr != NULL) {+        free(entries);+        return ROCKSDB_HASKELL_OK;+    }++    {+        const char *key;+        size_t key_len;++        *more = at_match(iter, prefix, prefix_len, &key, &key_len) ? 1 : 0;+    }+    *buffer = entries;+    *count = read;+    return ROCKSDB_HASKELL_OK;+}++/* A record of a packed batch, read out of its buffer. */+struct record {+    rocksdb_column_family_handle_t *family; /* NULL: the default one */+    uint32_t key_len;+    uint32_t value_len;+    unsigned char operation;+    const char *key;   /* in the buffer */+    const char *value; /* in the buffer, after the key */+    size_t size;       /* of the whole record, header included */+};++/*+ * Read the record at the start of the "left" bytes at "bytes". Returns zero+ * if there is no whole and well formed record there, and reads nothing past+ * those bytes either way.+ *+ * The header is copied out with memcpy because it is not aligned: a record+ * starts wherever the one before it ended.+ */+static int record_at(const char *bytes, size_t left, struct record *record)+{+    const size_t header = ROCKSDB_HASKELL_RECORD_HEADER;+    const char *field = bytes;++    if (left < header) {+        return 0;+    }+    memcpy(&record->family, field, sizeof(record->family));+    field += sizeof(record->family);+    memcpy(&record->key_len, field, sizeof(uint32_t));+    field += sizeof(uint32_t);+    memcpy(&record->value_len, field, sizeof(uint32_t));+    field += sizeof(uint32_t);+    record->operation = (unsigned char)*field;++    if (record->operation > ROCKSDB_HASKELL_MERGE) {+        return 0;+    }+    if (record->operation == ROCKSDB_HASKELL_DELETE && record->value_len != 0) {+        return 0;+    }+    /* one length at a time, so that their sum cannot overflow */+    left -= header;+    if (record->key_len > left) {+        return 0;+    }+    left -= record->key_len;+    if (record->value_len > left) {+        return 0;+    }++    record->key = bytes + header;+    record->value = record->key + record->key_len;+    record->size = header + record->key_len + record->value_len;+    return 1;+}++/* Add a record to a batch, with the function of RocksDB that it names. */+static void add_record(rocksdb_writebatch_t *batch, const struct record *record)+{+    rocksdb_column_family_handle_t *family = record->family;+    const char *key = record->key;+    const char *value = record->value;+    size_t key_len = record->key_len;+    size_t value_len = record->value_len;++    switch (record->operation) {+    case ROCKSDB_HASKELL_PUT:+        if (family == NULL) {+            rocksdb_writebatch_put(batch, key, key_len, value, value_len);+        } else {+            rocksdb_writebatch_put_cf(batch, family, key, key_len, value, value_len);+        }+        break;+    case ROCKSDB_HASKELL_DELETE:+        if (family == NULL) {+            rocksdb_writebatch_delete(batch, key, key_len);+        } else {+            rocksdb_writebatch_delete_cf(batch, family, key, key_len);+        }+        break;+    case ROCKSDB_HASKELL_MERGE:+        if (family == NULL) {+            rocksdb_writebatch_merge(batch, key, key_len, value, value_len);+        } else {+            rocksdb_writebatch_merge_cf(batch, family, key, key_len, value, value_len);+        }+        break;+    }+}++int rocksdb_haskell_writebatch_add_packed(+    rocksdb_writebatch_t *batch,+    const char *records,+    size_t len,+    size_t *count)+{+    struct record record;+    size_t at;+    size_t whole = 0;++    /*+     * Check every record first, so that a bad one leaves the batch as it+     * was. It costs a second look at headers that are in the cache.+     */+    for (at = 0; at < len; at += record.size) {+        if (!record_at(records + at, len - at, &record)) {+            *count = whole;+            return ROCKSDB_HASKELL_MALFORMED;+        }+        whole += 1;+    }++    for (at = 0; at < len; at += record.size) {+        /* cannot fail: the same bytes passed above */+        if (!record_at(records + at, len - at, &record)) {+            break;+        }+        add_record(batch, &record);+    }+    *count = whole;+    return ROCKSDB_HASKELL_OK;+}++/* The state of a merge operator is its name, which it owns. */+static void merge_operator_destroy(void *state)+{+    free(state);+}++static const char *merge_operator_name(void *state)+{+    return state;+}++int rocksdb_haskell_options_set_merge_operator(+    rocksdb_options_t *options,+    const char *name,+    rocksdb_haskell_full_merge_t full_merge,+    rocksdb_haskell_partial_merge_t partial_merge)+{+    size_t size = strlen(name) + 1;+    char *copy = malloc(size);+    rocksdb_mergeoperator_t *merge_operator;++    if (copy == NULL) {+        return ROCKSDB_HASKELL_NO_MEMORY;+    }+    memcpy(copy, name, size);+    /*+     * No function to delete a value: RocksDB then frees with free() what+     * the two functions return.+     */+    merge_operator = rocksdb_mergeoperator_create(+        copy,+        merge_operator_destroy,+        full_merge,+        partial_merge,+        NULL,+        merge_operator_name);+    /* the options take the operator, and destroy it when nothing uses it */+    rocksdb_options_set_merge_operator(options, merge_operator);+    return ROCKSDB_HASKELL_OK;+}
+ cbits/rocksdb_haskell.h view
@@ -0,0 +1,228 @@+/*+ * C helpers of rocksdb-haskell-jprupp: what the binding needs and the C API+ * of RocksDB does not have.+ *+ * The Haskell imports in Database.RocksDB.C name this header with the capi+ * calling convention, so the C compiler checks each of them against the+ * prototypes below.+ */++#ifndef ROCKSDB_HASKELL_H+#define ROCKSDB_HASKELL_H++#include <stddef.h>+#include <stdint.h>++#include <rocksdb/c.h>++/* Results of the functions below. */+#define ROCKSDB_HASKELL_OK 0+/* of rocksdb_haskell_iter_chunk and rocksdb_haskell_options_set_merge_operator */+#define ROCKSDB_HASKELL_NO_MEMORY 1+/* of rocksdb_haskell_writebatch_add_packed */+#define ROCKSDB_HASKELL_MALFORMED 2++/*+ * Read entries of an iterator, from the one it is on, in one call.+ *+ * Why: a Haskell program pays for every foreign call that may block, so a+ * scan that calls rocksdb_iter_next once for each entry pays once for each+ * entry. This function walks the iterator in C and copies the entries out,+ * so that the scan pays once for a few hundred.+ *+ * It goes forwards with rocksdb_iter_next, or backwards with+ * rocksdb_iter_prev if "backwards" is not zero, and stops at the first of:+ *+ *  - an iterator that is not valid: the end of the data or of its bounds, or+ *    an error;+ *  - an entry whose key does not start with the "prefix_len" bytes at+ *    "prefix" (no bytes: every key does);+ *  - "max_entries" entries read;+ *  - an entry whose key and value would take the bytes of keys and values+ *    read to more than "max_bytes". The first entry is read whatever its+ *    size, so that a scan always moves.+ *+ * The iterator is left on the first entry that was not read. "*more" tells+ * whether that is an entry whose key starts with the prefix: whether there is+ * anything left for another call.+ *+ * "*count" is the number of entries read, and "*buffer" holds them, one+ * after another, each laid out as+ *+ *     size_t key_len; size_t value_len; key bytes; value bytes; padding+ *+ * with padding up to a multiple of sizeof(size_t), so that the lengths are+ * always aligned. The caller frees "*buffer" with free(); it is NULL when no+ * entry was read.+ *+ * Errors:+ *+ *  - An error of the iterator is stored in "*errptr" as by+ *    rocksdb_iter_get_error: a message that the caller frees. Nothing is+ *    returned then: "*buffer" is NULL and "*count" is zero. "*errptr" must be+ *    NULL on entry.+ *  - If memory runs out with some entries read, they are returned as a short+ *    chunk, and the iterator is on the entry that did not fit. If it runs out+ *    on the first entry, the result is ROCKSDB_HASKELL_NO_MEMORY, with+ *    nothing returned and the iterator where it was.+ *+ * The result is ROCKSDB_HASKELL_OK otherwise, also when "*errptr" is set.+ */+int rocksdb_haskell_iter_chunk(+    rocksdb_iterator_t *iter,+    int backwards,+    size_t max_entries,+    size_t max_bytes,+    const char *prefix,+    size_t prefix_len,+    char **buffer,+    size_t *count,+    unsigned char *more,+    char **errptr);++/* The operation of a record of rocksdb_haskell_writebatch_add_packed. */+#define ROCKSDB_HASKELL_PUT 0+#define ROCKSDB_HASKELL_DELETE 1+#define ROCKSDB_HASKELL_MERGE 2++/*+ * The bytes that come before the key of such a record: a column family+ * handle, two lengths and an operation.+ */+#define ROCKSDB_HASKELL_RECORD_HEADER \+    (sizeof(rocksdb_column_family_handle_t *) + 2 * sizeof(uint32_t) + 1)++/*+ * Add many operations to a write batch in one call.+ *+ * Why: a Haskell program pays for every foreign call that may block, so a+ * batch that is built with rocksdb_writebatch_put_cf pays once for each+ * key. For the 65,000 keys of a Bitcoin block that was more than half of+ * the time that the batch took to write, and nearly all of it when other+ * threads of the program were busy: bench/Batch.hs. This function takes+ * the operations packed one after another in a buffer and makes those+ * calls from C, so that the program pays once for a few thousand.+ *+ * "records" holds "len" bytes: any number of records, with nothing between+ * them and nothing after the last. A record is+ *+ *     rocksdb_column_family_handle_t *family;   NULL: the default one+ *     uint32_t key_len;+ *     uint32_t value_len;                       zero for a delete+ *     unsigned char operation;                  ROCKSDB_HASKELL_PUT, ...+ *     key bytes; value bytes+ *+ * with no padding anywhere: the header is ROCKSDB_HASKELL_RECORD_HEADER+ * bytes, 17 where a pointer has eight, and nothing in it is aligned. The+ * handle and the lengths are as this machine holds them in memory, written+ * with memcpy or its like. Being unaligned is what lets a program write a+ * record where the last one ended, in a buffer of its own, and fill in the+ * lengths once it has serialised the key and the value after the header.+ *+ * The operations are added in the order of the records, each as the+ * function of RocksDB that it names would add it: rocksdb_writebatch_put,+ * rocksdb_writebatch_delete or rocksdb_writebatch_merge for the default+ * column family, and the ones ending in _cf for a handle. RocksDB copies+ * the keys and values, so the buffer is not needed after the call.+ *+ * A merge is only added here. Whether the column family has a merge+ * operator to apply it with is for RocksDB to say, when the batch is+ * written or when the key is read.+ *+ * The buffer is checked before anything is added, and the call never reads+ * outside it. If a record is cut short by the end of the buffer, names an+ * operation that does not exist, or is a delete with a value, the result is+ * ROCKSDB_HASKELL_MALFORMED, the batch is as it was, and "*count" is the+ * number of whole records before the bad one. A handle cannot be checked:+ * one that is not NULL has to be a column family of the database that the+ * batch is written to.+ *+ * The result is ROCKSDB_HASKELL_OK otherwise, and "*count" is the number of+ * operations added.+ */+int rocksdb_haskell_writebatch_add_packed(+    rocksdb_writebatch_t *batch,+    const char *records,+    size_t len,+    size_t *count);++/*+ * The two functions of a merge operator, with the types that+ * rocksdb_mergeoperator_create of rocksdb/c.h asks for.+ *+ * The first argument of both is the state of the operator, which belongs+ * to rocksdb_haskell_options_set_merge_operator: a function must not use+ * it. "key" is the key that is merged, for an operator that needs it.+ *+ * A full merge works out the value of a key: from the value that the key+ * has ("existing_value", which is NULL when it has none) and the+ * "num_operands" merge records written since, oldest first, at+ * "operands_list" with their lengths at "operands_list_length".+ *+ * A partial merge is given merge records only, oldest first, and makes one+ * record of them that does to any value what they did one after another.+ * RocksDB uses it to shorten the list before the value is known, when it+ * compacts files that do not have it.+ *+ * Both return the new bytes in memory from malloc(), which RocksDB frees+ * with free(), their length in "*new_value_length", and 1 in "*success".+ * To fail, a function stores 0 in "*success" and in "*new_value_length",+ * and returns NULL: RocksDB copies that many bytes from the result before+ * it looks at "*success". A read of the key then fails, and so does a+ * compaction that meets it. A partial merge that fails is not an error:+ * RocksDB keeps the records as they are.+ *+ * They are called from the threads of RocksDB, during reads, flushes and+ * compactions, several at a time. They must be plain C: a function that+ * enters the Haskell run-time system from such a thread is not what this+ * is for.+ */+typedef char *(*rocksdb_haskell_full_merge_t)(+    void *state,+    const char *key,+    size_t key_length,+    const char *existing_value,+    size_t existing_value_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length);++typedef char *(*rocksdb_haskell_partial_merge_t)(+    void *state,+    const char *key,+    size_t key_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length);++/*+ * Give the options of a database or of a column family a merge operator+ * made of a name and two functions.+ *+ * Why: rocksdb_mergeoperator_create wants a state with a destructor and a+ * function that returns the name, besides the two that merge. A Haskell+ * program has no C function to give for those, and should not make one+ * that calls back into Haskell, since RocksDB calls them from its own+ * threads. This function makes the operator around a copy of the name,+ * which is all the state it has and is freed with the operator.+ *+ * The options own the operator from then on, and the column families+ * opened with them keep it for as long as they need it: there is nothing+ * for the caller to destroy. RocksDB frees what the two functions return+ * with free(), since the operator is made without a function of its own+ * for that.+ *+ * The result is ROCKSDB_HASKELL_OK, or ROCKSDB_HASKELL_NO_MEMORY with the+ * options as they were.+ */+int rocksdb_haskell_options_set_merge_operator(+    rocksdb_options_t *options,+    const char *name,+    rocksdb_haskell_full_merge_t full_merge,+    rocksdb_haskell_partial_merge_t partial_merge);++#endif
rocksdb-haskell-jprupp.cabal view
@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack  name:           rocksdb-haskell-jprupp-version:        2.7.1+version:        2.8.0 synopsis:       Haskell bindings for RocksDB description:    See README at <https://github.com/jprupp/rocksdb-haskell#readme> category:       Database, FFI@@ -19,6 +19,7 @@     README.md     CHANGELOG.md     AUTHORS+    cbits/rocksdb_haskell.h  source-repository head   type: git@@ -36,6 +37,11 @@   hs-source-dirs:       src   ghc-options: -Wall+  cc-options: -Wall+  include-dirs:+      cbits+  c-sources:+      cbits/rocksdb_haskell.c   extra-libraries:       rocksdb   build-depends:@@ -53,6 +59,9 @@   hs-source-dirs:       test   ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N+  cc-options: -Wall+  c-sources:+      test/cbits/merge_test.c   build-depends:       QuickCheck     , async@@ -61,7 +70,40 @@     , containers     , data-default     , directory+    , filepath     , hspec+    , rocksdb-haskell-jprupp+    , temporary+  default-language: Haskell2010++benchmark batch+  type: exitcode-stdio-1.0+  main-is: Batch.hs+  hs-source-dirs:+      bench+  ghc-options: -Wall -O2 -threaded -rtsopts -with-rtsopts=-N4+  build-depends:+      async+    , base >=4.9 && <5+    , bytestring+    , data-default+    , directory+    , rocksdb-haskell-jprupp+    , temporary+  default-language: Haskell2010++benchmark scan+  type: exitcode-stdio-1.0+  main-is: Scan.hs+  hs-source-dirs:+      bench+  ghc-options: -Wall -O2 -threaded -rtsopts -with-rtsopts=-N4+  build-depends:+      async+    , base >=4.9 && <5+    , bytestring+    , data-default+    , directory     , rocksdb-haskell-jprupp     , temporary   default-language: Haskell2010
src/Database/RocksDB.hs view
@@ -79,6 +79,14 @@ -- to 'withDBCF'. Functions ending in @CF@ act on a given column family; the -- others act on the default column family. --+-- = Merges+--+-- A value can be changed without being read: a merge record ('merge', or+-- 'Merge' in a batch) says how it changes, and the t'MergeOperator' that+-- the 'mergeOperator' of the t'Config' gives the column family works out+-- the value when the key is read or compacted. See t'MergeOperator' for+-- what that is and what it asks of every program that opens the database.+-- -- = Errors -- -- Errors reported by RocksDB are thrown as 'IOError' exceptions created with
src/Database/RocksDB/Base.hs view
@@ -1,552 +1,968 @@ {-# LANGUAGE LambdaCase #-}--- |--- Module      : Database.RocksDB.Base--- Copyright   : (c) 2012-2013 The leveldb-haskell Authors---               (c) 2014-2020 The rocksdb-haskell Authors--- License     : BSD3--- Maintainer  : jprupp@protonmail.ch--- Stability   : experimental--- Portability : non-portable------ RocksDB Haskell binding.------ The API closely follows the C-API of RocksDB. All resources (databases,--- snapshots, iterators) are managed with bracket-style functions such as--- 'withDB', 'withSnapshot' and 'withIter', which release them when the--- supplied action returns or throws. Manual variants such as 'createSnapshot'--- and 'createIterator' exist for cases where the lifetime of a resource cannot--- be expressed as a scope.------ A t'DB' handle is safe to use from many threads at once.--module Database.RocksDB.Base-    ( -- * Exported Types-      DB (..)-    , BatchOp (..)-    , Range-    , ColumnFamily-    , Snapshot-    , ReadOpts--    -- * Options-    , Config (..)--    -- * Basic Database Manipulations-    , withDB-    , withDBCF-    , put-    , putCF-    , delete-    , deleteCF-    , write-    , get-    , getCF-    , multiGet-    , multiGetCF-    , withSnapshot-    , createSnapshot-    , releaseSnapshot--    -- * Administrative Functions-    , Property (..), getProperty, getPropertyCF-    , destroy-    , repair-    , approximateSize--    -- * Iteration-    , module Database.RocksDB.Iterator-    ) where--import           Control.Exception-import           Control.Monad             (forM, when)-import           Data.ByteString           (ByteString)-import qualified Data.ByteString           as BS-import           Data.ByteString.Internal  (ByteString (..))-import qualified Data.ByteString.Unsafe    as BU-import           Data.Int-import           Database.RocksDB.C-import           Database.RocksDB.Internal-import           Database.RocksDB.Iterator-import           Foreign.C.String-import           Foreign.ForeignPtr-import           Foreign.Marshal.Alloc-import           Foreign.Marshal.Array-import           Foreign.Marshal.Utils     (withMany)-import           Foreign.Ptr-import           Foreign.Storable-import           System.Directory---- | Properties exposed by RocksDB. Read them with 'getProperty' or--- 'getPropertyCF'. Every value is returned by RocksDB as text.-data Property-    -- | @rocksdb.num-files-at-level\<N\>@: number of SST files at level @N@-    = NumFilesAtLevel Int-    -- | @rocksdb.stats@: multi-line human-readable statistics-    | Stats-    -- | @rocksdb.sstables@: description of the SST files that make up the-    -- database-    | SSTables-    -- | @rocksdb.estimate-table-readers-mem@: memory for index and filter-    -- blocks of open table files, outside the block cache-    | EstimateTableReadersMem-    -- | @rocksdb.cur-size-all-mem-tables@: memory of all memtables-    | CurSizeAllMemTables-    -- | @rocksdb.block-cache-usage@: memory used by the block cache-    | BlockCacheUsage-    -- | any other property, by its full name-    | PropertyName String-    deriving (Eq, Show)---- | A single operation in an atomic batch written with 'write'.------ Batches are how a program keeps several keys consistent with each other: an--- index and the data it points to, for example, must change together or not at--- all. RocksDB applies a batch atomically and in one write to its log, so a batch--- is also much faster than the same writes made one by one.-data BatchOp = Put !ByteString !ByteString-               -- ^ set a key to a value in the default column family-             | Del !ByteString-               -- ^ delete a key from the default column family-             | PutCF !ColumnFamily !ByteString !ByteString-               -- ^ set a key to a value in a column family-             | DelCF !ColumnFamily !ByteString-               -- ^ delete a key from a column family-             deriving (Eq, Show)---- | Open a database at the given path and pass its handle to a function.------ The handle is closed automatically when the function returns or throws, and--- must not be used afterwards. If 'createIfMissing' is set, the directory is--- created first if needed. Throws an 'IOError' if the database cannot be--- opened, for example if it does not exist and 'createIfMissing' is not set,--- or if it is locked by another process.------ Only the default column family is opened. A database that has other column--- families must be opened with 'withDBCF'.------ Opening is expensive and a database can only be opened by one process at a--- time, so open it once at program start and share the t'DB' between threads for--- the life of the program.-withDB :: FilePath -> Config -> (DB -> IO a) -> IO a-withDB path config f =-    withBlockCache config $ \cache ->-    withOptionsCache cache config $ \opts_ptr ->-    withReadOpts Nothing $ \read_opts ->-    withWriteOpts $ \write_opts ->-    bracket (create_db opts_ptr read_opts write_opts) destroy_db f-  where-    destroy_db db = c_rocksdb_close $ rocksDB db-    create_db opts_ptr read_opts write_opts = do-        when (createIfMissing config) $-            createDirectoryIfMissing True path-        withCString path $ \path_ptr -> do-            db_ptr <- throwIfErr "open" $-                c_rocksdb_open opts_ptr path_ptr-            return DB { rocksDB = db_ptr-                      , columnFamilies = []-                      , readOpts = read_opts-                      , snapshot = Nothing-                      , writeOpts = write_opts-                      }---- | Open a database with column families and pass its handle to a function.------ The second argument configures the database and its default column family.--- The third argument lists the other column families by name, each with its own--- t'Config'. Their handles are stored in 'columnFamilies', in the same order.------ If the database directory is empty, a new database is created along with all--- the listed column families. Otherwise the existing database is opened with--- those column families; if some of them do not exist yet, the open fails unless--- 'createMissingColumnFamilies' is set in the database t'Config'. RocksDB also--- requires every column family that exists in the database to be listed.------ The handle and all column family handles are released when the function--- returns or throws.------ Column families are separate key spaces in one database. They share the--- write-ahead log, so a batch can change several of them atomically, but each--- has its own files, memtables and tuning. Use them to keep data of different--- shapes apart, such as an index with short keys and a table of large values.-withDBCF :: FilePath-         -> Config-         -> [(String, Config)]-         -> (DB -> IO a)-         -> IO a-withDBCF path config cf_cfgs f =-    withBlockCache config $ \cache ->-    withOptionsCache cache config $ \opts_ptr ->-    withOptionsCFCache cache (map snd cf_cfgs) $ \cf_opts ->-    withReadOpts Nothing $ \read_opts ->-    withWriteOpts $ \write_opts ->-    withStrings (map fst cf_cfgs) $ \cf_names ->-    allocaArray (length cf_cfgs + 1) $ \cf_names_array ->-    allocaArray (length cf_cfgs + 1) $ \cf_opts_array ->-    allocaArray (length cf_cfgs + 1) $ \cf_ptrs_array ->-        bracket ( create_db opts_ptr-                            cf_names-                            cf_names_array-                            cf_opts-                            cf_opts_array-                            read_opts-                            cf_ptrs_array-                            write_opts-                ) destroy_db f-  where-    create_new cf_names cf_opts opts_ptr read_opts write_opts = do-        createDirectoryIfMissing True path-        withCString path $ \path_ptr -> do-            db_ptr <- throwIfErr "open" $-                c_rocksdb_open opts_ptr path_ptr-            cfs <- forM (zip cf_names cf_opts) $ \(n, o) ->-                throwIfErr "create_column_family" $-                c_rocksdb_create_column_family-                db_ptr o n-            return DB { rocksDB = db_ptr-                      , columnFamilies = cfs-                      , readOpts = read_opts-                      , snapshot = Nothing-                      , writeOpts = write_opts-                      }-    destroy_db db = do-        mapM_ c_rocksdb_column_family_handle_destroy (columnFamilies db)-        c_rocksdb_close $ rocksDB db-    create_db opts_ptr-              cf_names-              cf_names_array-              cf_opts-              cf_opts_array-              read_opts-              cf_ptrs_array-              write_opts = do-        when (createIfMissing config) $-            createDirectoryIfMissing True path-        listDirectory path >>= \case-            [] -> create_new cf_names cf_opts opts_ptr read_opts write_opts-            _ -> withCString path $ \path_ptr ->-                withCString "default" $ \cf_deflt_name -> do-                    pokeArray cf_names_array (cf_deflt_name : cf_names)-                    pokeArray cf_opts_array (opts_ptr : cf_opts)-                    db_ptr <- throwIfErr "open" $-                        c_rocksdb_open_column_families-                        opts_ptr-                        path_ptr-                        (intToCInt (length cf_cfgs + 1))-                        cf_names_array-                        cf_opts_array-                        cf_ptrs_array-                    (deflt, cfs) <- peekArray (length cf_cfgs + 1) cf_ptrs_array >>= \case-                        d : cs -> return (d, cs)-                        [] -> error "rocksdb_open_column_families returned no handles"-                    -- Every handle must be destroyed before the database is-                    -- closed. The default column family is not kept, so-                    -- release its handle now; the database keeps its own.-                    c_rocksdb_column_family_handle_destroy deflt-                    return DB { rocksDB = db_ptr-                              , columnFamilies = cfs-                              , readOpts = read_opts-                              , snapshot = Nothing-                              , writeOpts = write_opts-                              }---- | Run an action with a snapshot of the database.--- The t'DB' object is not valid after the action ends.------ The t'DB' passed to the action reads from a consistent point-in-time view of--- the database: 'get', 'multiGet' and iterators created from it do not see--- writes made after the snapshot was taken. Writes through it still go to the--- live database.------ Without a snapshot, two reads made one after the other can see different--- states of the database if another thread writes in between. Anything that--- reads several related keys and expects them to agree, such as an index and the--- records it points to, should read through a snapshot.-withSnapshot :: DB -> (DB -> IO a) -> IO a-withSnapshot db f =-    bracket (createSnapshot db) releaseSnapshot (f . fst)---- | Manually create an unmanaged snapshot.--- The returned t'DB' has 'readOpts' configured for the snapshot.--- Use 'releaseSnapshot' to release both the snapshot and its read options.-createSnapshot :: DB -> IO (DB, Snapshot)-createSnapshot db@DB{rocksDB = db_ptr} = do-    snap_ptr <- c_rocksdb_create_snapshot db_ptr-    read_opts <- createReadOpts (Just snap_ptr)-    return (db{readOpts = read_opts, snapshot = Just snap_ptr}, snap_ptr)---- | Function to release an unmanaged snapshot.--- Also releases the read options associated with the snapshot DB.-releaseSnapshot :: (DB, Snapshot) -> IO ()-releaseSnapshot (DB{rocksDB = db_ptr, readOpts = read_opts}, snap_ptr) = do-  destroyReadOpts read_opts-  c_rocksdb_release_snapshot db_ptr snap_ptr---- | Get a property of the default column family, or of the database as a--- whole. Returns 'Nothing' if RocksDB does not know the property.-getProperty :: DB -> Property -> IO (Maybe ByteString)-getProperty DB{rocksDB = db_ptr} p =-    withCString (propertyName p) $ \name ->-    c_rocksdb_property_value db_ptr name >>= propertyResult---- | Get a property of a column family. Many properties, such as the memory--- of memtables and table readers, are kept per column family.-getPropertyCF :: DB -> ColumnFamily -> Property -> IO (Maybe ByteString)-getPropertyCF DB{rocksDB = db_ptr} cf p =-    withCString (propertyName p) $ \name ->-    c_rocksdb_property_value_cf db_ptr cf name >>= propertyResult---- | Turn a property value returned by RocksDB into a 'ByteString', freeing the--- C string.-propertyResult :: CString -> IO (Maybe ByteString)-propertyResult val_ptr-    | val_ptr == nullPtr = return Nothing-    | otherwise = do-          res <- BS.packCString val_ptr-          freeCString val_ptr-          return (Just res)---- | The name RocksDB uses for a property.-propertyName :: Property -> String-propertyName = \case-    NumFilesAtLevel i       -> "rocksdb.num-files-at-level" ++ show i-    Stats                   -> "rocksdb.stats"-    SSTables                -> "rocksdb.sstables"-    EstimateTableReadersMem -> "rocksdb.estimate-table-readers-mem"-    CurSizeAllMemTables     -> "rocksdb.cur-size-all-mem-tables"-    BlockCacheUsage         -> "rocksdb.block-cache-usage"-    PropertyName n          -> n---- | Destroy the given RocksDB database, deleting all of its files. The--- database must not be open. Throws an 'IOError' on failure.-destroy :: FilePath -> Options -> IO ()-destroy path opts_ptr =-    withCString path $ \path_ptr ->-    throwIfErr "destroy" $ c_rocksdb_destroy_db opts_ptr path_ptr---- | Try to recover as much data as possible from a corrupted database. Some--- data may be lost. The database must not be open. Throws an 'IOError' on--- failure.-repair :: FilePath -> Options -> IO ()-repair path opts_ptr =-    withCString path $ \path_ptr ->-    throwIfErr "repair" $ c_rocksdb_repair_db opts_ptr path_ptr----- | A key range given as @(start, limit)@. The start key is included and the--- limit key is excluded.------ TODO: support [Range], like C API does-type Range  = (ByteString, ByteString)---- | Approximate number of bytes of file system space used by the keys in a--- 'Range' of the default column family. Data still in memtables may not be--- counted. Throws an 'IOError' on failure.-approximateSize :: DB -> Range -> IO Int64-approximateSize DB{rocksDB = db_ptr} (from, to) =-    BU.unsafeUseAsCStringLen from $ \(from_ptr, flen) ->-    BU.unsafeUseAsCStringLen to   $ \(to_ptr, tlen)   ->-    withArray [from_ptr]          $ \from_ptrs        ->-    withArray [intToCSize flen]   $ \flen_ptrs        ->-    withArray [to_ptr]            $ \to_ptrs          ->-    withArray [intToCSize tlen]   $ \tlen_ptrs        ->-    alloca                        $ \size_ptr         -> do-        throwIfErr "approximate_sizes" $-            c_rocksdb_approximate_sizes db_ptr 1-                                        from_ptrs flen_ptrs-                                        to_ptrs tlen_ptrs-                                        size_ptr-        fromIntegral <$> peek size_ptr---- | Write a key/value pair to the default column family, replacing any--- previous value. Throws an 'IOError' on failure.-put :: DB -> ByteString -> ByteString -> IO ()-put db = putCommon db Nothing---- | Like 'put', but write to the given column family.-putCF :: DB -> ColumnFamily -> ByteString -> ByteString -> IO ()-putCF db cf = putCommon db (Just cf)---- | Write a key/value pair to the default column family, or to a column family--- if one is given.-putCommon :: DB -> Maybe ColumnFamily -> ByteString -> ByteString -> IO ()-putCommon DB{rocksDB = db_ptr, writeOpts = write_opts} mcf key value =-    BU.unsafeUseAsCStringLen key   $ \(key_ptr, klen) ->-    BU.unsafeUseAsCStringLen value $ \(val_ptr, vlen) ->-        throwIfErr "put" $ case mcf of-            Just cf -> c_rocksdb_put_cf-                      db_ptr write_opts cf-                      key_ptr (intToCSize klen)-                      val_ptr (intToCSize vlen)-            Nothing -> c_rocksdb_put-                      db_ptr write_opts-                      key_ptr (intToCSize klen)-                      val_ptr (intToCSize vlen)---- | Read a value by key from the default column family. Returns 'Nothing' if--- the key is not found. Throws an 'IOError' on failure.-get :: DB -> ByteString -> IO (Maybe ByteString)-get db = getCommon db Nothing---- | Like 'get', but read from the given column family.-getCF :: DB -> ColumnFamily -> ByteString -> IO (Maybe ByteString)-getCF db cf = getCommon db (Just cf)---- | Read a value from the default column family, or from a column family if one--- is given.-getCommon :: DB -> Maybe ColumnFamily -> ByteString -> IO (Maybe ByteString)-getCommon DB{rocksDB = db_ptr, readOpts = read_opts} mcf key =-    BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->-    alloca                       $ \vlen_ptr -> do-        val_ptr <- throwIfErr "get" $-            case mcf of-                Just cf -> c_rocksdb_get_cf-                           db_ptr read_opts cf-                           key_ptr (intToCSize klen) vlen_ptr-                Nothing -> c_rocksdb_get-                           db_ptr read_opts-                           key_ptr (intToCSize klen) vlen_ptr-        if val_ptr == nullPtr-            then return Nothing-            else do-                vlen <- peek vlen_ptr-                res <- BU.unsafePackMallocCStringLen (val_ptr, cSizeToInt vlen)-                return $ Just res---- | Read the values of many keys in one call. RocksDB looks the keys up--- together, sharing the work of finding the files and blocks that hold--- them, which is faster than reading them one by one. The values are in the--- order of the keys, with 'Nothing' for keys that are not found. Duplicate keys--- are allowed. Throws an 'IOError' if any of the lookups fails.------ For a query that needs hundreds of keys at once, such as the balances of many--- addresses, this cuts the time spent finding blocks and crossing into C to a--- fraction of separate 'get' calls.-multiGet :: DB -> [ByteString] -> IO [Maybe ByteString]-multiGet db = multiGetCommon db Nothing---- | 'multiGet' on a column family.-multiGetCF :: DB -> ColumnFamily -> [ByteString] -> IO [Maybe ByteString]-multiGetCF db cf = multiGetCommon db (Just cf)---- | Read many values from the default column family, or from a column family if--- one is given.-multiGetCommon :: DB -> Maybe ColumnFamily -> [ByteString] -> IO [Maybe ByteString]-multiGetCommon _ _ [] = return []-multiGetCommon DB{rocksDB = db_ptr, readOpts = read_opts} mcf keys =-    withMany BU.unsafeUseAsCStringLen keys $ \kls ->-    withArrayLen (map fst kls) $ \n key_ptrs ->-    withArray (map (intToCSize . snd) kls) $ \key_lens ->-    allocaArray n $ \val_ptrs ->-    allocaArray n $ \val_lens ->-    allocaArray n $ \err_ptrs -> do-        case mcf of-            Just cf ->-                withArray (replicate n cf) $ \cfs ->-                c_rocksdb_multi_get_cf-                    db_ptr read_opts cfs (intToCSize n)-                    key_ptrs key_lens val_ptrs val_lens err_ptrs-            Nothing ->-                c_rocksdb_multi_get-                    db_ptr read_opts (intToCSize n)-                    key_ptrs key_lens val_ptrs val_lens err_ptrs-        vals <- peekArray n val_ptrs-        lens <- peekArray n val_lens-        errs <- peekArray n err_ptrs-        -- Own every value and free every error before throwing, so that-        -- none of them leaks.-        res <- forM (zip vals lens) $ \(v, l) ->-            if v == nullPtr-                then return Nothing-                else Just <$> BU.unsafePackMallocCStringLen (v, cSizeToInt l)-        msgs <- forM errs $ \e ->-            if e == nullPtr-                then return Nothing-                else do-                    m <- peekCString e-                    free e-                    return (Just m)-        case [m | Just m <- msgs] of-            m : _ -> throwIO $ userError $ "multi_get: " ++ m-            [] -> return res---- | Delete a key from the default column family. Deleting a key that does not--- exist is not an error. Throws an 'IOError' on failure.-delete :: DB -> ByteString -> IO ()-delete db = deleteCommon db Nothing---- | Like 'delete', but delete from the given column family.-deleteCF :: DB -> ColumnFamily -> ByteString -> IO ()-deleteCF db cf = deleteCommon db (Just cf)---- | Delete a key/value pair from the default column family, or from a column--- family if one is given.-deleteCommon :: DB -> Maybe ColumnFamily -> ByteString -> IO ()-deleteCommon DB{rocksDB = db_ptr, writeOpts = write_opts} mcf key =-    BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->-    throwIfErr "delete" $ case mcf of-    Just cf -> c_rocksdb_delete_cf db_ptr write_opts cf key_ptr (intToCSize klen)-    Nothing -> c_rocksdb_delete db_ptr write_opts key_ptr (intToCSize klen)---- | Perform a batch mutation. All operations in the list are applied atomically:--- either all of them are written or none are. They are applied in order, so a--- later operation on a key overrides an earlier one. Throws an 'IOError' on--- failure.------ The keys and values are kept alive until RocksDB has copied them, because--- they are handed to C as pointers into Haskell memory.-write :: DB -> [BatchOp] -> IO ()-write DB{rocksDB = db_ptr, writeOpts = write_opts} batch =-    bracket-    c_rocksdb_writebatch_create-    c_rocksdb_writebatch_destroy $ \batch_ptr -> do-        mapM_ (batchAdd batch_ptr) batch-        throwIfErr "write" $ c_rocksdb_write db_ptr write_opts batch_ptr-        -- ensure @ByteString@s (and respective shared @CStringLen@s) aren't-        -- GC'ed until here-        mapM_ touch batch-  where-    batchAdd batch_ptr (Put key val) =-        BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->-        BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->-            c_rocksdb_writebatch_put-            batch_ptr-            key_ptr (intToCSize klen)-            val_ptr (intToCSize vlen)-    batchAdd batch_ptr (PutCF cf_ptr key val) =-        BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->-        BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->-            c_rocksdb_writebatch_put_cf-            batch_ptr-            cf_ptr-            key_ptr (intToCSize klen)-            val_ptr (intToCSize vlen)-    batchAdd batch_ptr (Del key) =-        BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->-            c_rocksdb_writebatch_delete-            batch_ptr-            key_ptr (intToCSize klen)-    batchAdd batch_ptr (DelCF cf_ptr key) =-        BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->-            c_rocksdb_writebatch_delete_cf-            batch_ptr-            cf_ptr-            key_ptr (intToCSize klen)-    touch (Put (PS p _ _) (PS p' _ _)) = do-        touchForeignPtr p-        touchForeignPtr p'-    touch (PutCF _ (PS p _ _) (PS p' _ _)) = do-        touchForeignPtr p-        touchForeignPtr p'-    touch (Del (PS p _ _)) =-        touchForeignPtr p-    touch (DelCF _ (PS p _ _)) =-        touchForeignPtr p---- | Marshal a list of strings into C strings that are valid while the function--- runs.-withStrings :: [String] -> ([CString] -> IO a) -> IO a-withStrings ss f =-    go [] ss-  where-    go acc []     = f (reverse acc)-    go acc (x:xs) = withCString x $ \p -> go (p:acc) xs++-- |+-- Module      : Database.RocksDB.Base+-- Copyright   : (c) 2012-2013 The leveldb-haskell Authors+--               (c) 2014-2020 The rocksdb-haskell Authors+-- License     : BSD3+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : non-portable+--+-- RocksDB Haskell binding.+--+-- The API closely follows the C-API of RocksDB. All resources (databases,+-- snapshots, iterators) are managed with bracket-style functions such as+-- 'withDB', 'withSnapshot' and 'withIter', which release them when the+-- supplied action returns or throws. Manual variants such as 'createSnapshot'+-- and 'createIterator' exist for cases where the lifetime of a resource cannot+-- be expressed as a scope.+--+-- A t'DB' handle is safe to use from many threads at once.+module Database.RocksDB.Base+  ( -- * Exported Types+    DB (..),+    BatchOp (..),+    Range,+    ColumnFamily,+    Snapshot,+    ReadOpts,++    -- * Options+    Config (..),+    MergeOperator (..),+    FullMerge,+    PartialMerge,++    -- * Basic Database Manipulations+    withDB,+    withDBCF,+    put,+    putCF,+    delete,+    deleteCF,+    merge,+    mergeCF,+    write,+    get,+    getCF,+    multiGet,+    multiGetCF,+    withSnapshot,+    createSnapshot,+    releaseSnapshot,++    -- * Packed batches+    -- $packed+    writePacked,+    packOps,+    PackedOp (..),+    packedHeaderSize,+    pokePackedHeader,++    -- * Administrative Functions+    Property (..),+    getProperty,+    getPropertyCF,+    destroy,+    repair,+    approximateSize,+    compactRange,+    flush,++    -- * Iteration+    module Database.RocksDB.Iterator,+  )+where++import Control.Exception+import Control.Monad (foldM_, forM, forM_, when)+import Data.ByteString (ByteString)+import qualified Data.ByteString as BS+import Data.ByteString.Internal (ByteString (..))+import qualified Data.ByteString.Internal as BI+import qualified Data.ByteString.Unsafe as BU+import Data.Int+import Data.Maybe (fromMaybe)+import Data.Word (Word32, Word8)+import Database.RocksDB.C+import Database.RocksDB.Internal+import Database.RocksDB.Iterator+import Foreign.C.String+import Foreign.ForeignPtr+import Foreign.Marshal.Alloc+import Foreign.Marshal.Array+import Foreign.Marshal.Utils (copyBytes, withMany)+import Foreign.Ptr+import Foreign.Storable+import System.Directory++-- | Properties exposed by RocksDB. Read them with 'getProperty' or+-- 'getPropertyCF'. Every value is returned by RocksDB as text.+data Property+  = -- | @rocksdb.num-files-at-level\<N\>@: number of SST files at level @N@+    NumFilesAtLevel Int+  | -- | @rocksdb.stats@: multi-line human-readable statistics+    Stats+  | -- | @rocksdb.sstables@: description of the SST files that make up the+    -- database+    SSTables+  | -- | @rocksdb.estimate-table-readers-mem@: memory for index and filter+    -- blocks of open table files, outside the block cache+    EstimateTableReadersMem+  | -- | @rocksdb.cur-size-all-mem-tables@: memory of all memtables+    CurSizeAllMemTables+  | -- | @rocksdb.block-cache-usage@: memory used by the block cache+    BlockCacheUsage+  | -- | any other property, by its full name+    PropertyName String+  deriving (Eq, Show)++-- | A single operation in an atomic batch written with 'write'.+--+-- Batches are how a program keeps several keys consistent with each other: an+-- index and the data it points to, for example, must change together or not at+-- all. RocksDB applies a batch atomically and in one write to its log, so a batch+-- is also much faster than the same writes made one by one.+data BatchOp+  = -- | set a key to a value in the default column family+    Put !ByteString !ByteString+  | -- | delete a key from the default column family+    Del !ByteString+  | -- | set a key to a value in a column family+    PutCF !ColumnFamily !ByteString !ByteString+  | -- | delete a key from a column family+    DelCF !ColumnFamily !ByteString+  | -- | add a merge record to a key of the default column family: the+    -- second byte string is for the t'MergeOperator' of the column family+    -- to combine with the value that the key has. Nothing is read when+    -- the batch is written. RocksDB refuses the batch if the column family+    -- has no merge operator, and 'write' throws. What is left of such a+    -- batch depends on the version: RocksDB 11 keeps none of it, RocksDB+    -- 8.9 keeps the operations that came before the merge. Give every+    -- column family that gets merge records its operator.+    Merge !ByteString !ByteString+  | -- | add a merge record to a key of a column family+    MergeCF !ColumnFamily !ByteString !ByteString+  deriving (Eq, Show)++-- | Open a database at the given path and pass its handle to a function.+--+-- The handle is closed automatically when the function returns or throws, and+-- must not be used afterwards. If 'createIfMissing' is set, the directory is+-- created first if needed. Throws an 'IOError' if the database cannot be+-- opened, for example if it does not exist and 'createIfMissing' is not set,+-- or if it is locked by another process.+--+-- Only the default column family is opened. A database that has other column+-- families must be opened with 'withDBCF'.+--+-- Opening is expensive and a database can only be opened by one process at a+-- time, so open it once at program start and share the t'DB' between threads for+-- the life of the program.+withDB :: FilePath -> Config -> (DB -> IO a) -> IO a+withDB path config f =+  withBlockCache config $ \cache ->+    withOptionsCache cache config $ \opts_ptr ->+      withReadOpts Nothing $ \read_opts ->+        withWriteOpts $ \write_opts ->+          bracket (create_db opts_ptr read_opts write_opts) destroy_db f+  where+    destroy_db db = c_rocksdb_close $ rocksDB db+    create_db opts_ptr read_opts write_opts = do+      when (createIfMissing config) $+        createDirectoryIfMissing True path+      withCString path $ \path_ptr -> do+        db_ptr <-+          throwIfErr "open" $+            c_rocksdb_open opts_ptr path_ptr+        return+          DB+            { rocksDB = db_ptr,+              columnFamilies = [],+              readOpts = read_opts,+              snapshot = Nothing,+              writeOpts = write_opts+            }++-- | Open a database with column families and pass its handle to a function.+--+-- The second argument configures the database and its default column family.+-- The third argument lists the other column families by name, each with its own+-- t'Config'. Their handles are stored in 'columnFamilies', in the same order.+--+-- If the database directory is empty, a new database is created along with all+-- the listed column families. Otherwise the existing database is opened with+-- those column families; if some of them do not exist yet, the open fails unless+-- 'createMissingColumnFamilies' is set in the database t'Config'. RocksDB also+-- requires every column family that exists in the database to be listed.+--+-- The handle and all column family handles are released when the function+-- returns or throws.+--+-- Column families are separate key spaces in one database. They share the+-- write-ahead log, so a batch can change several of them atomically, but each+-- has its own files, memtables and tuning. Use them to keep data of different+-- shapes apart, such as an index with short keys and a table of large values.+withDBCF ::+  FilePath ->+  Config ->+  [(String, Config)] ->+  (DB -> IO a) ->+  IO a+withDBCF path config cf_cfgs f =+  withBlockCache config $ \cache ->+    withOptionsCache cache config $ \opts_ptr ->+      withOptionsCFCache cache (map snd cf_cfgs) $ \cf_opts ->+        withReadOpts Nothing $ \read_opts ->+          withWriteOpts $ \write_opts ->+            withStrings (map fst cf_cfgs) $ \cf_names ->+              allocaArray (length cf_cfgs + 1) $ \cf_names_array ->+                allocaArray (length cf_cfgs + 1) $ \cf_opts_array ->+                  allocaArray (length cf_cfgs + 1) $ \cf_ptrs_array ->+                    bracket+                      ( create_db+                          opts_ptr+                          cf_names+                          cf_names_array+                          cf_opts+                          cf_opts_array+                          read_opts+                          cf_ptrs_array+                          write_opts+                      )+                      destroy_db+                      f+  where+    create_new cf_names cf_opts opts_ptr read_opts write_opts = do+      createDirectoryIfMissing True path+      withCString path $ \path_ptr -> do+        db_ptr <-+          throwIfErr "open" $+            c_rocksdb_open opts_ptr path_ptr+        cfs <- forM (zip cf_names cf_opts) $ \(n, o) ->+          throwIfErr "create_column_family" $+            c_rocksdb_create_column_family+              db_ptr+              o+              n+        return+          DB+            { rocksDB = db_ptr,+              columnFamilies = cfs,+              readOpts = read_opts,+              snapshot = Nothing,+              writeOpts = write_opts+            }+    destroy_db db = do+      mapM_ c_rocksdb_column_family_handle_destroy (columnFamilies db)+      c_rocksdb_close $ rocksDB db+    create_db+      opts_ptr+      cf_names+      cf_names_array+      cf_opts+      cf_opts_array+      read_opts+      cf_ptrs_array+      write_opts = do+        when (createIfMissing config) $+          createDirectoryIfMissing True path+        listDirectory path >>= \case+          [] -> create_new cf_names cf_opts opts_ptr read_opts write_opts+          _ -> withCString path $ \path_ptr ->+            withCString "default" $ \cf_deflt_name -> do+              pokeArray cf_names_array (cf_deflt_name : cf_names)+              pokeArray cf_opts_array (opts_ptr : cf_opts)+              db_ptr <-+                throwIfErr "open" $+                  c_rocksdb_open_column_families+                    opts_ptr+                    path_ptr+                    (intToCInt (length cf_cfgs + 1))+                    cf_names_array+                    cf_opts_array+                    cf_ptrs_array+              (deflt, cfs) <-+                peekArray (length cf_cfgs + 1) cf_ptrs_array >>= \case+                  d : cs -> return (d, cs)+                  [] -> error "rocksdb_open_column_families returned no handles"+              -- Every handle must be destroyed before the database is+              -- closed. The default column family is not kept, so+              -- release its handle now; the database keeps its own.+              c_rocksdb_column_family_handle_destroy deflt+              return+                DB+                  { rocksDB = db_ptr,+                    columnFamilies = cfs,+                    readOpts = read_opts,+                    snapshot = Nothing,+                    writeOpts = write_opts+                  }++-- | Run an action with a snapshot of the database.+-- The t'DB' object is not valid after the action ends.+--+-- The t'DB' passed to the action reads from a consistent point-in-time view of+-- the database: 'get', 'multiGet' and iterators created from it do not see+-- writes made after the snapshot was taken. Writes through it still go to the+-- live database.+--+-- Without a snapshot, two reads made one after the other can see different+-- states of the database if another thread writes in between. Anything that+-- reads several related keys and expects them to agree, such as an index and the+-- records it points to, should read through a snapshot.+withSnapshot :: DB -> (DB -> IO a) -> IO a+withSnapshot db f =+  bracket (createSnapshot db) releaseSnapshot (f . fst)++-- | Manually create an unmanaged snapshot.+-- The returned t'DB' has 'readOpts' configured for the snapshot.+-- Use 'releaseSnapshot' to release both the snapshot and its read options.+createSnapshot :: DB -> IO (DB, Snapshot)+createSnapshot db@DB {rocksDB = db_ptr} = do+  snap_ptr <- c_rocksdb_create_snapshot db_ptr+  read_opts <- createReadOpts (Just snap_ptr)+  return (db {readOpts = read_opts, snapshot = Just snap_ptr}, snap_ptr)++-- | Function to release an unmanaged snapshot.+-- Also releases the read options associated with the snapshot DB.+releaseSnapshot :: (DB, Snapshot) -> IO ()+releaseSnapshot (DB {rocksDB = db_ptr, readOpts = read_opts}, snap_ptr) = do+  destroyReadOpts read_opts+  c_rocksdb_release_snapshot db_ptr snap_ptr++-- | Get a property of the default column family, or of the database as a+-- whole. Returns 'Nothing' if RocksDB does not know the property.+getProperty :: DB -> Property -> IO (Maybe ByteString)+getProperty DB {rocksDB = db_ptr} p =+  withCString (propertyName p) $ \name ->+    c_rocksdb_property_value db_ptr name >>= propertyResult++-- | Get a property of a column family. Many properties, such as the memory+-- of memtables and table readers, are kept per column family.+getPropertyCF :: DB -> ColumnFamily -> Property -> IO (Maybe ByteString)+getPropertyCF DB {rocksDB = db_ptr} cf p =+  withCString (propertyName p) $ \name ->+    c_rocksdb_property_value_cf db_ptr cf name >>= propertyResult++-- | Turn a property value returned by RocksDB into a 'ByteString', freeing the+-- C string.+propertyResult :: CString -> IO (Maybe ByteString)+propertyResult val_ptr+  | val_ptr == nullPtr = return Nothing+  | otherwise = do+      res <- BS.packCString val_ptr+      freeCString val_ptr+      return (Just res)++-- | The name RocksDB uses for a property.+propertyName :: Property -> String+propertyName = \case+  NumFilesAtLevel i -> "rocksdb.num-files-at-level" ++ show i+  Stats -> "rocksdb.stats"+  SSTables -> "rocksdb.sstables"+  EstimateTableReadersMem -> "rocksdb.estimate-table-readers-mem"+  CurSizeAllMemTables -> "rocksdb.cur-size-all-mem-tables"+  BlockCacheUsage -> "rocksdb.block-cache-usage"+  PropertyName n -> n++-- | Destroy the given RocksDB database, deleting all of its files. The+-- database must not be open. Throws an 'IOError' on failure.+destroy :: FilePath -> Options -> IO ()+destroy path opts_ptr =+  withCString path $ \path_ptr ->+    throwIfErr "destroy" $ c_rocksdb_destroy_db opts_ptr path_ptr++-- | Try to recover as much data as possible from a corrupted database. Some+-- data may be lost. The database must not be open. Throws an 'IOError' on+-- failure.+repair :: FilePath -> Options -> IO ()+repair path opts_ptr =+  withCString path $ \path_ptr ->+    throwIfErr "repair" $ c_rocksdb_repair_db opts_ptr path_ptr++-- | A key range given as @(start, limit)@. The start key is included and the+-- limit key is excluded.+--+-- TODO: support [Range], like C API does+type Range = (ByteString, ByteString)++-- | Approximate number of bytes of file system space used by the keys in a+-- 'Range' of the default column family. Data still in memtables may not be+-- counted. Throws an 'IOError' on failure.+approximateSize :: DB -> Range -> IO Int64+approximateSize DB {rocksDB = db_ptr} (from, to) =+  BU.unsafeUseAsCStringLen from $ \(from_ptr, flen) ->+    BU.unsafeUseAsCStringLen to $ \(to_ptr, tlen) ->+      withArray [from_ptr] $ \from_ptrs ->+        withArray [intToCSize flen] $ \flen_ptrs ->+          withArray [to_ptr] $ \to_ptrs ->+            withArray [intToCSize tlen] $ \tlen_ptrs ->+              alloca $ \size_ptr -> do+                throwIfErr "approximate_sizes" $+                  c_rocksdb_approximate_sizes+                    db_ptr+                    1+                    from_ptrs+                    flen_ptrs+                    to_ptrs+                    tlen_ptrs+                    size_ptr+                fromIntegral <$> peek size_ptr++-- | Write again every table file that has keys from the first to the second+-- of the two given, in the default column family or in the one given, and+-- return when it is done. 'Nothing' leaves that end of the range open, so+-- two of them are the whole column family.+--+-- RocksDB compacts by itself, and a program does not need this to keep a+-- database in order. It is for the one thing that only a rewrite does:+-- giving the files that a database already has the layout of its present+-- t'Config', such as a Bloom filter or 'partitionIndexAndFilters', which+-- otherwise reach a file only when RocksDB happens to compact it, and the+-- files of the last level, which hold most of the data, rarely are. Every+-- file in the range is written, those of the last level included.+--+-- It reads and writes all the data in the range, with the background+-- threads of the database ('maxBackgroundJobs'): hours on a large one, and+-- room on the disk for the files being written until the old ones are+-- deleted. The database can be read and written meanwhile. RocksDB reports+-- no error from it; what it could not do stays as it was.+compactRange :: DB -> Maybe ColumnFamily -> Maybe ByteString -> Maybe ByteString -> IO ()+compactRange DB {rocksDB = db_ptr} mcf from to =+  bracket c_rocksdb_compactoptions_create c_rocksdb_compactoptions_destroy $ \opts -> do+    c_rocksdb_compactoptions_set_bottommost_level_compaction opts forceOptimized+    bound from $ \(from_ptr, flen) ->+      bound to $ \(to_ptr, tlen) ->+        case mcf of+          Just cf -> c_rocksdb_compact_range_cf_opt db_ptr cf opts from_ptr flen to_ptr tlen+          Nothing -> c_rocksdb_compact_range_opt db_ptr opts from_ptr flen to_ptr tlen+  where+    -- @BottommostLevelCompaction::kForceOptimized@: the files of the last+    -- level too, but not those that this compaction has just written+    forceOptimized = 3+    bound Nothing f = f (nullPtr, 0)+    bound (Just k) f = BU.unsafeUseAsCStringLen k $ \(p, n) -> f (p, intToCSize n)++-- | Write a key/value pair to the default column family, replacing any+-- previous value. Throws an 'IOError' on failure.+put :: DB -> ByteString -> ByteString -> IO ()+put db = putCommon db Nothing++-- | Like 'put', but write to the given column family.+putCF :: DB -> ColumnFamily -> ByteString -> ByteString -> IO ()+putCF db cf = putCommon db (Just cf)++-- | Write a key/value pair to the default column family, or to a column family+-- if one is given.+putCommon :: DB -> Maybe ColumnFamily -> ByteString -> ByteString -> IO ()+putCommon DB {rocksDB = db_ptr, writeOpts = write_opts} mcf key value =+  BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+    BU.unsafeUseAsCStringLen value $ \(val_ptr, vlen) ->+      throwIfErr "put" $ case mcf of+        Just cf ->+          c_rocksdb_put_cf+            db_ptr+            write_opts+            cf+            key_ptr+            (intToCSize klen)+            val_ptr+            (intToCSize vlen)+        Nothing ->+          c_rocksdb_put+            db_ptr+            write_opts+            key_ptr+            (intToCSize klen)+            val_ptr+            (intToCSize vlen)++-- | Read a value by key from the default column family. Returns 'Nothing' if+-- the key is not found. Throws an 'IOError' on failure.+get :: DB -> ByteString -> IO (Maybe ByteString)+get db = getCommon db Nothing++-- | Like 'get', but read from the given column family.+getCF :: DB -> ColumnFamily -> ByteString -> IO (Maybe ByteString)+getCF db cf = getCommon db (Just cf)++-- | Read a value from the default column family, or from a column family if one+-- is given.+getCommon :: DB -> Maybe ColumnFamily -> ByteString -> IO (Maybe ByteString)+getCommon DB {rocksDB = db_ptr, readOpts = read_opts} mcf key =+  BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+    alloca $ \vlen_ptr -> do+      val_ptr <- throwIfErr "get" $+        case mcf of+          Just cf ->+            c_rocksdb_get_cf+              db_ptr+              read_opts+              cf+              key_ptr+              (intToCSize klen)+              vlen_ptr+          Nothing ->+            c_rocksdb_get+              db_ptr+              read_opts+              key_ptr+              (intToCSize klen)+              vlen_ptr+      if val_ptr == nullPtr+        then return Nothing+        else do+          vlen <- peek vlen_ptr+          res <- BU.unsafePackMallocCStringLen (val_ptr, cSizeToInt vlen)+          return $ Just res++-- | Read the values of many keys in one call. RocksDB looks the keys up+-- together, sharing the work of finding the files and blocks that hold+-- them, which is faster than reading them one by one. The values are in the+-- order of the keys, with 'Nothing' for keys that are not found. Duplicate keys+-- are allowed. Throws an 'IOError' if any of the lookups fails.+--+-- For a query that needs hundreds of keys at once, such as the balances of many+-- addresses, this cuts the time spent finding blocks and crossing into C to a+-- fraction of separate 'get' calls.+multiGet :: DB -> [ByteString] -> IO [Maybe ByteString]+multiGet db = multiGetCommon db Nothing++-- | 'multiGet' on a column family.+multiGetCF :: DB -> ColumnFamily -> [ByteString] -> IO [Maybe ByteString]+multiGetCF db cf = multiGetCommon db (Just cf)++-- | Read many values from the default column family, or from a column family if+-- one is given.+multiGetCommon :: DB -> Maybe ColumnFamily -> [ByteString] -> IO [Maybe ByteString]+multiGetCommon _ _ [] = return []+multiGetCommon DB {rocksDB = db_ptr, readOpts = read_opts} mcf keys =+  withMany BU.unsafeUseAsCStringLen keys $ \kls ->+    withArrayLen (map fst kls) $ \n key_ptrs ->+      withArray (map (intToCSize . snd) kls) $ \key_lens ->+        allocaArray n $ \val_ptrs ->+          allocaArray n $ \val_lens ->+            allocaArray n $ \err_ptrs -> do+              case mcf of+                Just cf ->+                  withArray (replicate n cf) $ \cfs ->+                    c_rocksdb_multi_get_cf+                      db_ptr+                      read_opts+                      cfs+                      (intToCSize n)+                      key_ptrs+                      key_lens+                      val_ptrs+                      val_lens+                      err_ptrs+                Nothing ->+                  c_rocksdb_multi_get+                    db_ptr+                    read_opts+                    (intToCSize n)+                    key_ptrs+                    key_lens+                    val_ptrs+                    val_lens+                    err_ptrs+              vals <- peekArray n val_ptrs+              lens <- peekArray n val_lens+              errs <- peekArray n err_ptrs+              -- Own every value and free every error before throwing, so that+              -- none of them leaks.+              res <- forM (zip vals lens) $ \(v, l) ->+                if v == nullPtr+                  then return Nothing+                  else Just <$> BU.unsafePackMallocCStringLen (v, cSizeToInt l)+              msgs <- forM errs $ \e ->+                if e == nullPtr+                  then return Nothing+                  else do+                    m <- peekCString e+                    free e+                    return (Just m)+              case [m | Just m <- msgs] of+                m : _ -> throwIO $ userError $ "multi_get: " ++ m+                [] -> return res++-- | Delete a key from the default column family. Deleting a key that does not+-- exist is not an error. Throws an 'IOError' on failure.+delete :: DB -> ByteString -> IO ()+delete db = deleteCommon db Nothing++-- | Like 'delete', but delete from the given column family.+deleteCF :: DB -> ColumnFamily -> ByteString -> IO ()+deleteCF db cf = deleteCommon db (Just cf)++-- | Delete a key/value pair from the default column family, or from a column+-- family if one is given.+deleteCommon :: DB -> Maybe ColumnFamily -> ByteString -> IO ()+deleteCommon DB {rocksDB = db_ptr, writeOpts = write_opts} mcf key =+  BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+    throwIfErr "delete" $ case mcf of+      Just cf -> c_rocksdb_delete_cf db_ptr write_opts cf key_ptr (intToCSize klen)+      Nothing -> c_rocksdb_delete db_ptr write_opts key_ptr (intToCSize klen)++-- | Add a merge record to a key of the default column family, for the+-- t'MergeOperator' of the column family to combine with the value that the+-- key has. Nothing is read: the merge is done when the key is read, and+-- when RocksDB compacts the files that hold the record and the value.+--+-- Throws an 'IOError' if the column family has no merge operator, and on+-- any other failure. For many keys at once use a 'Merge' in a batch.+merge :: DB -> ByteString -> ByteString -> IO ()+merge db = mergeCommon db Nothing++-- | Like 'merge', but for a key of the given column family.+mergeCF :: DB -> ColumnFamily -> ByteString -> ByteString -> IO ()+mergeCF db cf = mergeCommon db (Just cf)++-- | Add a merge record to a key of the default column family, or of a+-- column family if one is given.+mergeCommon :: DB -> Maybe ColumnFamily -> ByteString -> ByteString -> IO ()+mergeCommon DB {rocksDB = db_ptr, writeOpts = write_opts} mcf key value =+  BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+    BU.unsafeUseAsCStringLen value $ \(val_ptr, vlen) ->+      throwIfErr "merge" $ case mcf of+        Just cf ->+          c_rocksdb_merge_cf+            db_ptr+            write_opts+            cf+            key_ptr+            (intToCSize klen)+            val_ptr+            (intToCSize vlen)+        Nothing ->+          c_rocksdb_merge+            db_ptr+            write_opts+            key_ptr+            (intToCSize klen)+            val_ptr+            (intToCSize vlen)++-- | Write the memtable of the default column family, or of the one given,+-- to a table file, and return when it is done.+--+-- RocksDB flushes by itself when a memtable is full, and a program does+-- not need this to keep its data: what is in a memtable is in the+-- write-ahead log too. It is for tests and benchmarks that want data in+-- table files, where the data of a database in production is, and with+-- 'disableAutoCompactions' in a file of its own for each flush. Throws an+-- 'IOError' on failure.+flush :: DB -> Maybe ColumnFamily -> IO ()+flush DB {rocksDB = db_ptr} mcf =+  bracket c_rocksdb_flushoptions_create c_rocksdb_flushoptions_destroy $ \opts -> do+    c_rocksdb_flushoptions_set_wait opts 1+    throwIfErr "flush" $ case mcf of+      Just cf -> c_rocksdb_flush_cf db_ptr opts cf+      Nothing -> c_rocksdb_flush db_ptr opts++-- | Perform a batch mutation. All operations in the list are applied atomically:+-- either all of them are written or none are. They are applied in order, so a+-- later operation on a key overrides an earlier one. Throws an 'IOError' on+-- failure.+--+-- The keys and values are kept alive until RocksDB has copied them, because+-- they are handed to C as pointers into Haskell memory.+--+-- Each operation is handed to RocksDB with a foreign call of its own. For a+-- batch of tens of thousands of operations, 'writePacked' hands them over a+-- few thousand at a call.+write :: DB -> [BatchOp] -> IO ()+write DB {rocksDB = db_ptr, writeOpts = write_opts} batch =+  bracket+    c_rocksdb_writebatch_create+    c_rocksdb_writebatch_destroy+    $ \batch_ptr -> do+      mapM_ (batchAdd batch_ptr) batch+      throwIfErr "write" $ c_rocksdb_write db_ptr write_opts batch_ptr+      -- ensure @ByteString@s (and respective shared @CStringLen@s) aren't+      -- GC'ed until here+      mapM_ touch batch+  where+    batchAdd batch_ptr (Put key val) =+      BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+        BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->+          c_rocksdb_writebatch_put+            batch_ptr+            key_ptr+            (intToCSize klen)+            val_ptr+            (intToCSize vlen)+    batchAdd batch_ptr (PutCF cf_ptr key val) =+      BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+        BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->+          c_rocksdb_writebatch_put_cf+            batch_ptr+            cf_ptr+            key_ptr+            (intToCSize klen)+            val_ptr+            (intToCSize vlen)+    batchAdd batch_ptr (Del key) =+      BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+        c_rocksdb_writebatch_delete+          batch_ptr+          key_ptr+          (intToCSize klen)+    batchAdd batch_ptr (DelCF cf_ptr key) =+      BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+        c_rocksdb_writebatch_delete_cf+          batch_ptr+          cf_ptr+          key_ptr+          (intToCSize klen)+    batchAdd batch_ptr (Merge key val) =+      BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+        BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->+          c_rocksdb_writebatch_merge+            batch_ptr+            key_ptr+            (intToCSize klen)+            val_ptr+            (intToCSize vlen)+    batchAdd batch_ptr (MergeCF cf_ptr key val) =+      BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+        BU.unsafeUseAsCStringLen val $ \(val_ptr, vlen) ->+          c_rocksdb_writebatch_merge_cf+            batch_ptr+            cf_ptr+            key_ptr+            (intToCSize klen)+            val_ptr+            (intToCSize vlen)+    touch (Put (PS p _ _) (PS p' _ _)) = do+      touchForeignPtr p+      touchForeignPtr p'+    touch (PutCF _ (PS p _ _) (PS p' _ _)) = do+      touchForeignPtr p+      touchForeignPtr p'+    touch (Del (PS p _ _)) =+      touchForeignPtr p+    touch (DelCF _ (PS p _ _)) =+      touchForeignPtr p+    touch (Merge (PS p _ _) (PS p' _ _)) = do+      touchForeignPtr p+      touchForeignPtr p'+    touch (MergeCF _ (PS p _ _) (PS p' _ _)) = do+      touchForeignPtr p+      touchForeignPtr p'++-- $packed+--+-- 'write' hands a batch to RocksDB one operation at a time, and each is a+-- foreign call of the kind that lets other Haskell threads run meanwhile:+-- the thread gives up its capability for it, and has to get one back. For+-- a batch of a few operations that is nothing. For the 65,000 keys of a+-- Bitcoin block it was more than half of the time of the batch in the+-- benchmark of this package, 26 ms of 48, and 400 ms or more when eight+-- other threads wanted the four capabilities. It comes on top of a+-- 'ByteString' for each key and each value, which is memory that the+-- collector cannot move.+--+-- A packed batch is the same operations written one after another into a+-- few large buffers: /chunks/. 'writePacked' gives each chunk to a small C+-- function of this package, which adds its operations to the batch, and+-- then writes the batch as 'write' does: atomically, with one write.+--+-- There are two ways to make chunks:+--+-- * 'packOps' packs a list of t'BatchOp' into one chunk. It is the simple+--   way, for small batches and for tests.+--+-- * A program that serialises its keys and values itself can do so+--   straight into a buffer of its own, with no 'ByteString' for each: leave+--   'packedHeaderSize' bytes, serialise the key and then the value after+--   them, and fill in the header with 'pokePackedHeader' once the two+--   lengths are known. The next record starts where the value ended.+--+-- A record is a header and then the bytes of the key and of the value, with+-- no padding. The header is the column family handle (a null pointer for+-- the default one), the length of the key and that of the value as 32-bit+-- numbers, and one byte for the operation, all as the machine holds them+-- in memory and none of them aligned. @cbits\/rocksdb_haskell.h@ has the+-- layout for a program that writes records by other means.+--+-- The format is this package's own, and what a chunk becomes is a series of+-- calls to the C API of RocksDB. The alternative was to write RocksDB's own+-- encoding of a batch in Haskell and hand it over whole+-- (@rocksdb_writebatch_create_from@), which saves the copy in C. That+-- encoding is internal to RocksDB, with column family numbers and record+-- types that a binding has no business knowing, and the copy costs a few+-- milliseconds for a batch that takes hundreds to write.+--+-- A chunk holds column family handles, which are pointers: it means+-- something only in the process that made it, and only while the database+-- that the handles came from is open.++-- | What a record of a packed batch does to its key.+data PackedOp+  = -- | set the key to the value, as 'Put' and 'PutCF' do+    PackedPut+  | -- | delete the key, as 'Del' and 'DelCF' do; the record has no value+    PackedDel+  | -- | add a merge record to the key, as 'Merge' and 'MergeCF' do: the+    -- value of the record is for the t'MergeOperator' of the column family+    -- to combine with the value that the key has.+    --+    -- On a column family without a merge operator, RocksDB refuses the+    -- batch when it is written, and 'writePacked' throws. RocksDB 11 then+    -- keeps none of the batch; RocksDB 8.9 keeps the operations that came+    -- before the merge. Do not count on either: a batch that is not all+    -- or nothing is what a batch is there to prevent.+    PackedMerge+  deriving (Eq, Show, Enum, Bounded)++-- | Bytes of the header that comes before the key in a record of a packed+-- batch: 17 where a pointer has eight.+packedHeaderSize :: Int+packedHeaderSize = sizeOf (nullPtr :: ColumnFamily) + 2 * sizeOf (0 :: Word32) + 1++-- | Write the header of a record of a packed batch at a pointer, which+-- needs no alignment: the column family, or 'Nothing' for the default one,+-- the operation, the length of the key and the length of the value. It+-- writes 'packedHeaderSize' bytes. The bytes of the key come right after+-- them, and those of the value right after the key.+--+-- It is for a program that serialises keys and values into its own buffer:+-- see the top of this section. The lengths are only known once that is+-- done, so the header is written last, into the room left for it.+--+-- Throws an 'IOError', and writes nothing, if a length is negative or does+-- not fit in 32 bits, or if a 'PackedDel' is given a value.+pokePackedHeader :: Ptr a -> Maybe ColumnFamily -> PackedOp -> Int -> Int -> IO ()+pokePackedHeader ptr mcf op klen vlen = do+  when (outside klen || outside vlen) $+    throwIO (userError "pokePackedHeader: a length that does not fit in 32 bits")+  when (op == PackedDel && vlen /= 0) $+    throwIO (userError "pokePackedHeader: a delete with a value")+  -- Unaligned writes, in the byte order of the machine, which is how+  -- the C side reads them back with memcpy.+  poke (castPtr ptr) (fromMaybe nullPtr mcf)+  pokeByteOff ptr word (fromIntegral klen :: Word32)+  pokeByteOff ptr (word + 4) (fromIntegral vlen :: Word32)+  pokeByteOff ptr (word + 8) code+  where+    word = sizeOf (nullPtr :: ColumnFamily)+    -- compared as words, which have 32 bits at least, so that the limit is+    -- not a negative number where an 'Int' has 32+    outside n = n < 0 || (fromIntegral n :: Word) > fromIntegral (maxBound :: Word32)+    -- ROCKSDB_HASKELL_PUT, _DELETE and _MERGE of rocksdb_haskell.h+    code :: Word8+    code = case op of+      PackedPut -> 0+      PackedDel -> 1+      PackedMerge -> 2+{-# INLINE pokePackedHeader #-}++-- | Pack the operations of a batch into one chunk for 'writePacked', in+-- their order. @'writePacked' db ['packOps' ops]@ does what @'write' db ops@+-- does.+--+-- This is the simple way to a packed batch: it copies every key and value+-- once more, into a buffer as large as all of them together. Use it for+-- small batches, for the odd operation that a program does not serialise+-- itself, and for tests. A program with a large batch gains more by+-- serialising straight into its own chunks with 'pokePackedHeader'.+--+-- The chunk holds the column family handles of the operations, so it is+-- only good while their database is open. Evaluating it throws an 'IOError'+-- if a key or a value is of four gigabytes or more.+packOps :: [BatchOp] -> ByteString+packOps ops = BI.unsafeCreate (sum (map size ops)) $ \start -> foldM_ pack start ops+  where+    parts (Put k v) = (Nothing, PackedPut, k, v)+    parts (Del k) = (Nothing, PackedDel, k, BS.empty)+    parts (PutCF cf k v) = (Just cf, PackedPut, k, v)+    parts (DelCF cf k) = (Just cf, PackedDel, k, BS.empty)+    parts (Merge k v) = (Nothing, PackedMerge, k, v)+    parts (MergeCF cf k v) = (Just cf, PackedMerge, k, v)+    size op = case parts op of+      (_, _, k, v) -> packedHeaderSize + BS.length k + BS.length v+    -- write the record of an operation, and give where the next one goes+    pack :: Ptr Word8 -> BatchOp -> IO (Ptr Word8)+    pack ptr op = do+      let (mcf, kind, k, v) = parts op+          key_ptr = ptr `plusPtr` packedHeaderSize+          val_ptr = key_ptr `plusPtr` BS.length k+      pokePackedHeader ptr mcf kind (BS.length k) (BS.length v)+      copy k key_ptr+      copy v val_ptr+      return (val_ptr `plusPtr` BS.length v)+    -- memcpy must not be given the pointer of an empty string+    copy bytes to =+      BU.unsafeUseAsCStringLen bytes $ \(from, len) ->+        when (len > 0) $ copyBytes to (castPtr from) len++-- | Write a batch that is given as packed chunks: see the top of this+-- section for what they are and how to make them. All the operations of all+-- the chunks are applied atomically, as one batch and with one write:+-- either all of them are written or none are. They are applied in the order+-- of the chunks, and within a chunk in the order of its records, so a later+-- operation on a key overrides an earlier one.+--+-- Every chunk must hold whole records: a record cannot start in one chunk+-- and end in the next. Where a chunk is cut otherwise does not matter, and+-- an empty chunk or no chunk at all is fine. So a program that packs on+-- several cores gives the chunks of each core, one core after another.+--+-- Use it for batches of thousands of operations, where it saves the+-- foreign call that 'write' makes for each operation, and the 'ByteString'+-- for each key and value if the program packs its own chunks. It makes one+-- foreign call for each chunk, so chunks of a few hundred kilobytes are+-- large enough. For a few operations, 'write' is as fast and simpler.+--+-- Throws an 'IOError', with nothing written, if a chunk is not made of+-- whole records: one that is cut short, an operation that does not exist,+-- a delete with a value. The message names the chunk and the record, both+-- counted from zero. Each chunk is checked in C before any of it is added+-- to the batch, and nothing is read outside it. What cannot be checked is+-- the column family handle of a record: a wrong one crashes the process,+-- as it would in 'PutCF'.+--+-- Also throws an 'IOError' if RocksDB refuses the batch, as 'write' does.+-- That is what happens to a batch with a 'PackedMerge' for a column family+-- that has no 'mergeOperator': RocksDB 11 refuses it when it is written,+-- with nothing written.+writePacked :: DB -> [ByteString] -> IO ()+writePacked DB {rocksDB = db_ptr, writeOpts = write_opts} chunks =+  bracket+    c_rocksdb_writebatch_create+    c_rocksdb_writebatch_destroy+    $ \batch_ptr -> do+      forM_ (zip [0 :: Int ..] chunks) $ \(n, chunk) ->+        -- the chunk cannot move or be freed during the call, and RocksDB+        -- has copied what it needs when the call returns+        BU.unsafeUseAsCStringLen chunk $ \(chunk_ptr, len) ->+          alloca $ \count_ptr -> do+            status <-+              c_rocksdb_haskell_writebatch_add_packed+                batch_ptr+                chunk_ptr+                (intToCSize len)+                count_ptr+            when (status /= 0) $ do+              count <- peek count_ptr+              -- both are counted from zero+              throwIO . userError . unwords $+                [ "writePacked: chunk",+                  show n,+                  "is not made of whole records: record",+                  show (cSizeToInt count),+                  "is malformed or cut short"+                ]+      throwIfErr "write" $ c_rocksdb_write db_ptr write_opts batch_ptr++-- | Marshal a list of strings into C strings that are valid while the function+-- runs.+withStrings :: [String] -> ([CString] -> IO a) -> IO a+withStrings ss f =+  go [] ss+  where+    go acc [] = f (reverse acc)+    go acc (x : xs) = withCString x $ \p -> go (p : acc) xs
src/Database/RocksDB/C.hs view
@@ -1,7 +1,8 @@-{-# LANGUAGE CApiFFI                  #-}-{-# LANGUAGE CPP                      #-}-{-# LANGUAGE EmptyDataDecls           #-}+{-# LANGUAGE CApiFFI #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE EmptyDataDecls #-} {-# LANGUAGE ForeignFunctionInterface #-}+ -- | -- Module      : Database.RocksDB.C -- Copyright   : (c) 2012-2020 The rocksdb-haskell Authors@@ -20,8 +21,13 @@ -- -- The bindings are @safe@ foreign calls. RocksDB calls can take a long time, -- for example when they wait for disk or for a compaction to make room, and a--- @safe@ call lets other Haskell threads keep running meanwhile. The small--- extra cost per call is worth that in a concurrent program.+-- @safe@ call lets other Haskell threads keep running meanwhile. The extra+-- cost per call is worth that in a concurrent program. Where it adds up, in+-- a scan that moves an iterator once for every entry or a batch that is+-- given its keys one by one, the answer is fewer calls and not unsafe ones:+-- 'c_rocksdb_haskell_iter_chunk' reads many entries in one, and+-- 'c_rocksdb_haskell_writebatch_add_packed' adds many operations to a batch+-- in one. -- -- = Keeping the types in step with the header --@@ -37,166 +43,226 @@ -- function with @capi@ instead of @ccall@, as -- 'c_rocksdb_filterpolicy_create_bloom_full' is: the C compiler then reads -- the header that is installed, and converts the argument.- module Database.RocksDB.C-    ( RocksDB-    , Options-    , ReadOpts-    , WriteOpts-    , ColumnFamily-    , SliceTransform-    , Snapshot-    , Iterator-    , WriteBatch-    , FilterPolicy-    , Cache-    , ErrPtr-    , DBName-    , CFName-    , Key-    , Val-    , c_rocksdb_open-    , c_rocksdb_open_column_families-    , c_rocksdb_close-    , c_rocksdb_create_column_family-    , c_rocksdb_drop_column_family-    , c_rocksdb_column_family_handle_destroy-    , c_rocksdb_put-    , c_rocksdb_put_cf-    , c_rocksdb_delete-    , c_rocksdb_delete_cf-    , c_rocksdb_write-    , c_rocksdb_get-    , c_rocksdb_get_cf-    , c_rocksdb_multi_get-    , c_rocksdb_multi_get_cf-    , c_rocksdb_create_snapshot-    , c_rocksdb_release_snapshot-    , c_rocksdb_property_value-    , c_rocksdb_property_value_cf-    , c_rocksdb_approximate_sizes-    , c_rocksdb_destroy_db-    , c_rocksdb_repair_db-    , c_rocksdb_create_iterator-    , c_rocksdb_create_iterator_cf-    , c_rocksdb_iter_destroy-    , c_rocksdb_iter_valid-    , c_rocksdb_iter_seek_to_first-    , c_rocksdb_iter_seek_to_last-    , c_rocksdb_iter_seek-    , c_rocksdb_iter_next-    , c_rocksdb_iter_prev-    , c_rocksdb_iter_key-    , c_rocksdb_iter_value-    , c_rocksdb_iter_get_error-    , c_rocksdb_writebatch_create-    , c_rocksdb_writebatch_destroy-    , c_rocksdb_writebatch_clear-    , c_rocksdb_writebatch_put-    , c_rocksdb_writebatch_put_cf-    , c_rocksdb_writebatch_delete-    , c_rocksdb_writebatch_delete_cf-    , c_rocksdb_options_create-    , c_rocksdb_options_destroy-    , c_rocksdb_options_set_create_if_missing-    , c_rocksdb_options_set_error_if_exists-    , c_rocksdb_options_set_paranoid_checks-    , c_rocksdb_options_set_max_open_files-    , c_rocksdb_options_set_write_buffer_size-    , c_rocksdb_options_set_db_write_buffer_size-    , c_rocksdb_options_set_max_total_wal_size-    , c_rocksdb_options_set_create_missing_column_families-    , c_rocksdb_options_set_prefix_extractor-    , c_rocksdb_options_set_block_based_table_factory-    , c_rocksdb_block_based_options_create-    , c_rocksdb_block_based_options_set_filter_policy-    , c_rocksdb_block_based_options_destroy-    , c_rocksdb_block_based_options_set_block_cache-    , c_rocksdb_block_based_options_set_cache_index_and_filter_blocks-    , c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority-    , c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache-    , c_rocksdb_cache_create_lru-    , c_rocksdb_cache_destroy-    , c_rocksdb_slicetransform_create_fixed_prefix-    , c_rocksdb_slicetransform_destroy-    , c_rocksdb_filterpolicy_destroy-    , c_rocksdb_filterpolicy_create_bloom_full-    , c_rocksdb_readoptions_create-    , c_rocksdb_readoptions_destroy-    , c_rocksdb_readoptions_set_snapshot-    , c_rocksdb_readoptions_set_iterate_upper_bound-    , c_rocksdb_readoptions_set_iterate_lower_bound-    , c_rocksdb_readoptions_set_prefix_same_as_start-    , c_rocksdb_readoptions_set_total_order_seek-    , c_rocksdb_writeoptions_create-    , c_rocksdb_writeoptions_destroy-    , c_rocksdb_free-    ) where+  ( RocksDB,+    Options,+    ReadOpts,+    WriteOpts,+    ColumnFamily,+    SliceTransform,+    Snapshot,+    Iterator,+    WriteBatch,+    FilterPolicy,+    Cache,+    ErrPtr,+    DBName,+    CFName,+    Key,+    Val,+    c_rocksdb_open,+    c_rocksdb_open_column_families,+    c_rocksdb_close,+    c_rocksdb_create_column_family,+    c_rocksdb_drop_column_family,+    c_rocksdb_column_family_handle_destroy,+    c_rocksdb_put,+    c_rocksdb_put_cf,+    c_rocksdb_delete,+    c_rocksdb_delete_cf,+    c_rocksdb_merge,+    c_rocksdb_merge_cf,+    c_rocksdb_write,+    c_rocksdb_get,+    c_rocksdb_get_cf,+    c_rocksdb_multi_get,+    c_rocksdb_multi_get_cf,+    c_rocksdb_create_snapshot,+    c_rocksdb_release_snapshot,+    c_rocksdb_property_value,+    c_rocksdb_property_value_cf,+    c_rocksdb_approximate_sizes,+    CompactOptions,+    c_rocksdb_compactoptions_create,+    c_rocksdb_compactoptions_destroy,+    c_rocksdb_compactoptions_set_bottommost_level_compaction,+    c_rocksdb_compact_range_opt,+    c_rocksdb_compact_range_cf_opt,+    c_rocksdb_destroy_db,+    c_rocksdb_repair_db,+    c_rocksdb_create_iterator,+    c_rocksdb_create_iterator_cf,+    c_rocksdb_iter_destroy,+    c_rocksdb_iter_valid,+    c_rocksdb_iter_seek_to_first,+    c_rocksdb_iter_seek_to_last,+    c_rocksdb_iter_seek,+    c_rocksdb_iter_next,+    c_rocksdb_iter_prev,+    c_rocksdb_iter_key,+    c_rocksdb_iter_value,+    c_rocksdb_iter_get_error,+    c_rocksdb_haskell_iter_chunk,+    c_rocksdb_writebatch_create,+    c_rocksdb_writebatch_destroy,+    c_rocksdb_writebatch_clear,+    c_rocksdb_writebatch_put,+    c_rocksdb_writebatch_put_cf,+    c_rocksdb_writebatch_delete,+    c_rocksdb_writebatch_delete_cf,+    c_rocksdb_writebatch_merge,+    c_rocksdb_writebatch_merge_cf,+    c_rocksdb_haskell_writebatch_add_packed,+    FullMerge,+    PartialMerge,+    c_rocksdb_haskell_options_set_merge_operator,+    c_rocksdb_options_set_uint64add_merge_operator,+    c_rocksdb_options_set_disable_auto_compactions,+    FlushOptions,+    c_rocksdb_flushoptions_create,+    c_rocksdb_flushoptions_destroy,+    c_rocksdb_flushoptions_set_wait,+    c_rocksdb_flush,+    c_rocksdb_flush_cf,+    c_rocksdb_options_create,+    c_rocksdb_options_destroy,+    c_rocksdb_options_set_create_if_missing,+    c_rocksdb_options_set_error_if_exists,+    c_rocksdb_options_set_paranoid_checks,+    c_rocksdb_options_set_max_open_files,+    c_rocksdb_options_set_write_buffer_size,+    c_rocksdb_options_set_db_write_buffer_size,+    c_rocksdb_options_set_max_total_wal_size,+    c_rocksdb_options_set_max_background_jobs,+    c_rocksdb_options_set_create_missing_column_families,+    c_rocksdb_options_set_prefix_extractor,+    c_rocksdb_options_set_block_based_table_factory,+    c_rocksdb_block_based_options_create,+    c_rocksdb_block_based_options_set_filter_policy,+    c_rocksdb_block_based_options_destroy,+    c_rocksdb_block_based_options_set_block_cache,+    c_rocksdb_block_based_options_set_cache_index_and_filter_blocks,+    c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority,+    c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache,+    c_rocksdb_block_based_options_set_index_type,+    c_rocksdb_block_based_options_set_partition_filters,+    c_rocksdb_block_based_options_set_metadata_block_size,+    c_rocksdb_block_based_options_set_pin_top_level_index_and_filter,+    c_rocksdb_cache_create_lru,+    c_rocksdb_cache_destroy,+    c_rocksdb_slicetransform_create_fixed_prefix,+    c_rocksdb_slicetransform_destroy,+    c_rocksdb_filterpolicy_destroy,+    c_rocksdb_filterpolicy_create_bloom_full,+    c_rocksdb_readoptions_create,+    c_rocksdb_readoptions_destroy,+    c_rocksdb_readoptions_set_snapshot,+    c_rocksdb_readoptions_set_iterate_upper_bound,+    c_rocksdb_readoptions_set_iterate_lower_bound,+    c_rocksdb_readoptions_set_prefix_same_as_start,+    c_rocksdb_readoptions_set_total_order_seek,+    c_rocksdb_writeoptions_create,+    c_rocksdb_writeoptions_destroy,+    c_rocksdb_free,+  )+where -import           Foreign-import           Foreign.C.String-import           Foreign.C.Types+import Foreign+import Foreign.C.String+import Foreign.C.Types  -- | Opaque C database object. data LRocksDB+ -- | Opaque C column family handle. data LColumnFamily+ -- | Opaque C iterator. data LIterator+ -- | Opaque C database or column family options. data LOptions+ -- | Opaque C block-based table options. data LBlockBasedOptions+ -- | Opaque C read options. data LReadOpts+ -- | Opaque C snapshot. data LSnapshot+ -- | Opaque C write batch. data LWriteBatch+ -- | Opaque C write options. data LWriteOpts+ -- | Opaque C filter policy. data LFilterPolicy+ -- | Opaque C slice transform (prefix extractor). data LSliceTransform+ -- | Opaque C block cache. data LCache  -- | Pointer to an open database.-type RocksDB            = Ptr LRocksDB+type RocksDB = Ptr LRocksDB+ -- | Pointer to a column family handle.-type ColumnFamily       = Ptr LColumnFamily+type ColumnFamily = Ptr LColumnFamily+ -- | Pointer to database or column family options.-type Options            = Ptr LOptions+type Options = Ptr LOptions+ -- | Pointer to a batch of writes applied atomically.-type WriteBatch         = Ptr LWriteBatch+type WriteBatch = Ptr LWriteBatch+ -- | Pointer to a key transformation, used to extract prefixes.-type SliceTransform     = Ptr LSliceTransform+type SliceTransform = Ptr LSliceTransform+ -- | Pointer to options for reads and iterators.-type ReadOpts           = Ptr LReadOpts+type ReadOpts = Ptr LReadOpts+ -- | Pointer to options for writes.-type WriteOpts          = Ptr LWriteOpts+type WriteOpts = Ptr LWriteOpts++-- | Phantom type of the options of a manual compaction.+data LCompactOptions++-- | Pointer to the options of a manual compaction.+type CompactOptions = Ptr LCompactOptions+ -- | Pointer to a consistent point-in-time view of a database.-type Snapshot           = Ptr LSnapshot+type Snapshot = Ptr LSnapshot+ -- | Pointer to an iterator over keys.-type Iterator           = Ptr LIterator+type Iterator = Ptr LIterator+ -- | Pointer to a filter policy, such as a bloom filter.-type FilterPolicy       = Ptr LFilterPolicy+type FilterPolicy = Ptr LFilterPolicy+ -- | Pointer to block-based table options.-type BlockBasedOptions  = Ptr LBlockBasedOptions+type BlockBasedOptions = Ptr LBlockBasedOptions+ -- | Pointer to a block cache.-type Cache              = Ptr LCache+type Cache = Ptr LCache  -- | Pointer to a location where a C function stores an error message, or leaves @NULL@ on success.-type ErrPtr           = Ptr CString+type ErrPtr = Ptr CString+ -- | Path of a database, as a C string.-type DBName           = CString+type DBName = CString+ -- | Name of a column family, as a C string.-type CFName           = CString+type CFName = CString+ -- | Pointer to the bytes of a key. Its length is passed separately.-type Key              = CString+type Key = CString+ -- | Pointer to the bytes of a value. Its length is passed separately.-type Val              = CString+type Val = CString  -- | Binding to @rocksdb_open@. foreign import ccall safe "rocksdb/c.h rocksdb_open"@@ -204,14 +270,15 @@  -- | Binding to @rocksdb_open_column_families@. foreign import ccall safe "rocksdb/c.h rocksdb_open_column_families"-  c_rocksdb_open_column_families :: Options-                                 -> DBName-                                 -> CInt-                                 -> Ptr CFName-                                 -> Ptr Options-                                 -> Ptr ColumnFamily-                                 -> ErrPtr-                                 -> IO RocksDB+  c_rocksdb_open_column_families ::+    Options ->+    DBName ->+    CInt ->+    Ptr CFName ->+    Ptr Options ->+    Ptr ColumnFamily ->+    ErrPtr ->+    IO RocksDB  -- | Binding to @rocksdb_close@. foreign import ccall safe "rocksdb/c.h rocksdb_close"@@ -219,18 +286,20 @@  -- | Binding to @rocksdb_create_column_family@. foreign import ccall safe "rocksdb/c.h rocksdb_create_column_family"-  c_rocksdb_create_column_family :: RocksDB-                                 -> Options-                                 -> CFName-                                 -> ErrPtr-                                 -> IO ColumnFamily+  c_rocksdb_create_column_family ::+    RocksDB ->+    Options ->+    CFName ->+    ErrPtr ->+    IO ColumnFamily  -- | Binding to @rocksdb_drop_column_family@. foreign import ccall safe "rocksdb/c.h rocksdb_drop_column_family"-  c_rocksdb_drop_column_family :: RocksDB-                               -> ColumnFamily-                               -> ErrPtr-                               -> IO ()+  c_rocksdb_drop_column_family ::+    RocksDB ->+    ColumnFamily ->+    ErrPtr ->+    IO ()  -- | Binding to @rocksdb_column_family_handle_destroy@. foreign import ccall safe "rocksdb/c.h rocksdb_column_family_handle_destroy"@@ -238,94 +307,149 @@  -- | Binding to @rocksdb_put@. foreign import ccall safe "rocksdb/c.h rocksdb_put"-  c_rocksdb_put :: RocksDB-                -> WriteOpts-                -> Key -> CSize-                -> Val -> CSize-                -> ErrPtr-                -> IO ()+  c_rocksdb_put ::+    RocksDB ->+    WriteOpts ->+    Key ->+    CSize ->+    Val ->+    CSize ->+    ErrPtr ->+    IO ()  -- | Binding to @rocksdb_put_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_put_cf"-  c_rocksdb_put_cf :: RocksDB-                   -> WriteOpts-                   -> ColumnFamily-                   -> Key -> CSize-                   -> Val -> CSize-                   -> ErrPtr-                   -> IO ()+  c_rocksdb_put_cf ::+    RocksDB ->+    WriteOpts ->+    ColumnFamily ->+    Key ->+    CSize ->+    Val ->+    CSize ->+    ErrPtr ->+    IO ()  -- | Binding to @rocksdb_delete@. foreign import ccall safe "rocksdb/c.h rocksdb_delete"-  c_rocksdb_delete :: RocksDB-                   -> WriteOpts-                   -> Key -> CSize-                   -> ErrPtr-                   -> IO ()+  c_rocksdb_delete ::+    RocksDB ->+    WriteOpts ->+    Key ->+    CSize ->+    ErrPtr ->+    IO ()  -- | Binding to @rocksdb_delete_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_delete_cf"-  c_rocksdb_delete_cf :: RocksDB-                      -> WriteOpts-                      -> ColumnFamily-                      -> Key -> CSize-                      -> ErrPtr-                      -> IO ()+  c_rocksdb_delete_cf ::+    RocksDB ->+    WriteOpts ->+    ColumnFamily ->+    Key ->+    CSize ->+    ErrPtr ->+    IO () +-- | Binding to @rocksdb_merge@.+foreign import ccall safe "rocksdb/c.h rocksdb_merge"+  c_rocksdb_merge ::+    RocksDB ->+    WriteOpts ->+    Key ->+    CSize ->+    Val ->+    CSize ->+    ErrPtr ->+    IO ()++-- | Binding to @rocksdb_merge_cf@.+foreign import ccall safe "rocksdb/c.h rocksdb_merge_cf"+  c_rocksdb_merge_cf ::+    RocksDB ->+    WriteOpts ->+    ColumnFamily ->+    Key ->+    CSize ->+    Val ->+    CSize ->+    ErrPtr ->+    IO ()+ -- | Binding to @rocksdb_write@. foreign import ccall safe "rocksdb/c.h rocksdb_write"-  c_rocksdb_write :: RocksDB-                  -> WriteOpts-                  -> WriteBatch-                  -> ErrPtr-                  -> IO ()+  c_rocksdb_write ::+    RocksDB ->+    WriteOpts ->+    WriteBatch ->+    ErrPtr ->+    IO ()  -- | Returns NULL if not found. A malloc()ed array otherwise. -- Stores the length of the array in *vallen. -- -- Binding to @rocksdb_get@. foreign import ccall safe "rocksdb/c.h rocksdb_get"-  c_rocksdb_get :: RocksDB-                -> ReadOpts-                -> Key -> CSize-                -> Ptr CSize        -- ^ value length-                -> ErrPtr-                -> IO CString+  c_rocksdb_get ::+    RocksDB ->+    ReadOpts ->+    Key ->+    CSize ->+    -- | value length+    Ptr CSize ->+    ErrPtr ->+    IO CString  -- | Binding to @rocksdb_get_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_get_cf"-  c_rocksdb_get_cf :: RocksDB-                   -> ReadOpts-                   -> ColumnFamily-                   -> Key -> CSize-                   -> Ptr CSize        -- ^ value length-                   -> ErrPtr-                   -> IO CString+  c_rocksdb_get_cf ::+    RocksDB ->+    ReadOpts ->+    ColumnFamily ->+    Key ->+    CSize ->+    -- | value length+    Ptr CSize ->+    ErrPtr ->+    IO CString  -- | Binding to @rocksdb_multi_get@. foreign import ccall safe "rocksdb/c.h rocksdb_multi_get"-  c_rocksdb_multi_get :: RocksDB-                      -> ReadOpts-                      -> CSize        -- ^ number of keys-                      -> Ptr CString  -- ^ keys-                      -> Ptr CSize    -- ^ key sizes-                      -> Ptr CString  -- ^ values-                      -> Ptr CSize    -- ^ values sizes-                      -> ErrPtr-                      -> IO ()+  c_rocksdb_multi_get ::+    RocksDB ->+    ReadOpts ->+    -- | number of keys+    CSize ->+    -- | keys+    Ptr CString ->+    -- | key sizes+    Ptr CSize ->+    -- | values+    Ptr CString ->+    -- | values sizes+    Ptr CSize ->+    ErrPtr ->+    IO ()  -- | Binding to @rocksdb_multi_get_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_multi_get_cf"-  c_rocksdb_multi_get_cf :: RocksDB-                         -> ReadOpts-                         -> Ptr ColumnFamily -- ^ column families-                         -> CSize                   -- ^ number of keys-                         -> Ptr CString             -- ^ keys-                         -> Ptr CSize               -- ^ key sizes-                         -> Ptr CString             -- ^ values-                         -> Ptr CSize               -- ^ values sizes-                         -> ErrPtr-                         -> IO ()+  c_rocksdb_multi_get_cf ::+    RocksDB ->+    ReadOpts ->+    -- | column families+    Ptr ColumnFamily ->+    -- | number of keys+    CSize ->+    -- | keys+    Ptr CString ->+    -- | key sizes+    Ptr CSize ->+    -- | values+    Ptr CString ->+    -- | values sizes+    Ptr CSize ->+    ErrPtr ->+    IO ()  -- | Binding to @rocksdb_create_snapshot@. foreign import ccall safe "rocksdb/c.h rocksdb_create_snapshot"@@ -353,17 +477,20 @@ -- read a pointer that was never passed, and to write through it on failure. -- Older versions do not take the argument, and ignore it. foreign import ccall safe "rocksdb/c.h rocksdb_approximate_sizes"-  c_rocksdb_approximate_sizes :: RocksDB-                              -> CInt-                                 -- ^ num ranges-                              -> Ptr CString -> Ptr CSize-                                 -- ^ range start keys (array)-                              -> Ptr CString -> Ptr CSize-                                 -- ^ range limit keys (array)-                              -> Ptr Word64-                                 -- ^ array of approx. sizes of ranges-                              -> ErrPtr-                              -> IO ()+  c_rocksdb_approximate_sizes ::+    RocksDB ->+    -- | num ranges+    CInt ->+    Ptr CString ->+    -- | range start keys (array)+    Ptr CSize ->+    Ptr CString ->+    -- | range limit keys (array)+    Ptr CSize ->+    -- | array of approx. sizes of ranges+    Ptr Word64 ->+    ErrPtr ->+    IO ()  -- | Binding to @rocksdb_destroy_db@. foreign import ccall safe "rocksdb/c.h rocksdb_destroy_db"@@ -373,23 +500,24 @@ foreign import ccall safe "rocksdb/c.h rocksdb_repair_db"   c_rocksdb_repair_db :: Options -> DBName -> ErrPtr -> IO () - -- -- Iterator --  -- | Binding to @rocksdb_create_iterator@. foreign import ccall safe "rocksdb/c.h rocksdb_create_iterator"-  c_rocksdb_create_iterator :: RocksDB-                            -> ReadOpts-                            -> IO Iterator+  c_rocksdb_create_iterator ::+    RocksDB ->+    ReadOpts ->+    IO Iterator  -- | Binding to @rocksdb_create_iterator_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_create_iterator_cf"-  c_rocksdb_create_iterator_cf :: RocksDB-                               -> ReadOpts-                               -> ColumnFamily-                               -> IO Iterator+  c_rocksdb_create_iterator_cf ::+    RocksDB ->+    ReadOpts ->+    ColumnFamily ->+    IO Iterator  -- | Binding to @rocksdb_iter_destroy@. foreign import ccall safe "rocksdb/c.h rocksdb_iter_destroy"@@ -439,6 +567,35 @@ foreign import ccall safe "rocksdb/c.h rocksdb_iter_get_error"   c_rocksdb_iter_get_error :: Iterator -> ErrPtr -> IO () +-- | Binding to @rocksdb_haskell_iter_chunk@, which is not part of RocksDB: it+-- is this package's own, in @cbits\/rocksdb_haskell.c@, and its header says+-- what it promises. It reads many entries of an iterator in one call.+--+-- The arguments are the iterator, whether to go backwards, the most entries+-- and the most bytes to read, the prefix that keys have to start with and its+-- length, and where to store the buffer with the entries, their number,+-- whether more follow, and an error of the iterator. The result is not zero+-- if there was no memory for the first entry. The caller frees the buffer.+--+-- A safe call, since it moves the iterator and so may wait for the disk. That+-- is the point of it: a scan that moves the iterator from Haskell makes a+-- safe call for every entry, and this makes one for hundreds.+--+-- It is imported with @capi@ so that the C compiler checks the types below+-- against the prototype in the header.+foreign import capi safe "rocksdb_haskell.h rocksdb_haskell_iter_chunk"+  c_rocksdb_haskell_iter_chunk ::+    Iterator ->+    CInt ->+    CSize ->+    CSize ->+    CString ->+    CSize ->+    Ptr CString ->+    Ptr CSize ->+    Ptr CUChar ->+    ErrPtr ->+    IO CInt  -- -- Write batch@@ -458,33 +615,181 @@  -- | Binding to @rocksdb_writebatch_put@. foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_put"-  c_rocksdb_writebatch_put :: WriteBatch-                           -> Key -> CSize-                           -> Val -> CSize-                           -> IO ()+  c_rocksdb_writebatch_put ::+    WriteBatch ->+    Key ->+    CSize ->+    Val ->+    CSize ->+    IO ()  -- | Binding to @rocksdb_writebatch_put_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_put_cf"-  c_rocksdb_writebatch_put_cf :: WriteBatch-                              -> ColumnFamily-                              -> Key -> CSize-                              -> Val -> CSize-                              -> IO ()+  c_rocksdb_writebatch_put_cf ::+    WriteBatch ->+    ColumnFamily ->+    Key ->+    CSize ->+    Val ->+    CSize ->+    IO ()  -- | Binding to @rocksdb_writebatch_delete@. foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_delete"-  c_rocksdb_writebatch_delete :: WriteBatch-                              -> Key -> CSize -> IO ()+  c_rocksdb_writebatch_delete ::+    WriteBatch ->+    Key ->+    CSize ->+    IO ()  -- | Binding to @rocksdb_writebatch_delete_cf@. foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_delete_cf"-  c_rocksdb_writebatch_delete_cf :: WriteBatch-                                 -> ColumnFamily-                                 -> Key-                                 -> CSize-                                 -> IO ()+  c_rocksdb_writebatch_delete_cf ::+    WriteBatch ->+    ColumnFamily ->+    Key ->+    CSize ->+    IO () +-- | Binding to @rocksdb_writebatch_merge@.+foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_merge"+  c_rocksdb_writebatch_merge ::+    WriteBatch ->+    Key ->+    CSize ->+    Val ->+    CSize ->+    IO ()++-- | Binding to @rocksdb_writebatch_merge_cf@.+foreign import ccall safe "rocksdb/c.h rocksdb_writebatch_merge_cf"+  c_rocksdb_writebatch_merge_cf ::+    WriteBatch ->+    ColumnFamily ->+    Key ->+    CSize ->+    Val ->+    CSize ->+    IO ()++-- | Binding to @rocksdb_haskell_writebatch_add_packed@, which is not part of+-- RocksDB: it is this package's own, in @cbits\/rocksdb_haskell.c@, and its+-- header says what it promises and how a record is laid out. It adds many+-- operations to a write batch in one call. --+-- The arguments are the batch, the buffer of packed records and its length,+-- and where to store the number of records. The result is not zero if the+-- buffer does not hold whole and well formed records: the batch is then as+-- it was, and the number is that of the records before the bad one.+--+-- A safe call, like the calls to @rocksdb_writebatch_put_cf@ that it stands+-- for, so that a buffer of megabytes does not hold up the other Haskell+-- threads while RocksDB copies it. A program makes one for a few thousand+-- operations, and no longer one for each.+--+-- It is imported with @capi@ so that the C compiler checks the types below+-- against the prototype in the header.+foreign import capi safe "rocksdb_haskell.h rocksdb_haskell_writebatch_add_packed"+  c_rocksdb_haskell_writebatch_add_packed ::+    WriteBatch ->+    CString ->+    CSize ->+    Ptr CSize ->+    IO CInt++--+-- Merge operators+--++-- | The type of the function of a merge operator that works out the value+-- of a key: the full merge of @rocksdb_mergeoperator_create@.+--+-- Its arguments are a state that it must not use, the key and its length,+-- the value that the key has and its length (a null pointer if it has+-- none), the merge records, oldest first, as an array of pointers and one+-- of lengths, their number, where to store whether it succeeded, and where+-- to store the length of the result. It returns the new value in memory+-- from @malloc@. @cbits\/rocksdb_haskell.h@ says the rest.+type FullMerge =+  Ptr () ->+  CString ->+  CSize ->+  CString ->+  CSize ->+  Ptr CString ->+  Ptr CSize ->+  CInt ->+  Ptr CUChar ->+  Ptr CSize ->+  IO CString++-- | The type of the function of a merge operator that makes one merge+-- record of several: the partial merge of @rocksdb_mergeoperator_create@.+-- Its arguments are those of a t'FullMerge' without the value that the key+-- has.+type PartialMerge =+  Ptr () ->+  CString ->+  CSize ->+  Ptr CString ->+  Ptr CSize ->+  CInt ->+  Ptr CUChar ->+  Ptr CSize ->+  IO CString++-- | Binding to @rocksdb_haskell_options_set_merge_operator@, which is not+-- part of RocksDB: it is this package's own, in+-- @cbits\/rocksdb_haskell.c@. It gives options a merge operator made of a+-- name and two C functions, and keeps a copy of the name for as long as+-- the operator lives. The options own the operator. The result is not+-- zero if there was no memory for it.+--+-- It is imported with @capi@ so that the C compiler checks the types below+-- against the prototype in the header.+foreign import capi safe "rocksdb_haskell.h rocksdb_haskell_options_set_merge_operator"+  c_rocksdb_haskell_options_set_merge_operator ::+    Options ->+    CString ->+    FunPtr FullMerge ->+    FunPtr PartialMerge ->+    IO CInt++-- | Binding to @rocksdb_options_set_uint64add_merge_operator@.+foreign import ccall safe "rocksdb/c.h rocksdb_options_set_uint64add_merge_operator"+  c_rocksdb_options_set_uint64add_merge_operator :: Options -> IO ()++--+-- Flush+--++-- | Phantom type of the options of a flush.+data LFlushOptions++-- | Pointer to the options of a flush.+type FlushOptions = Ptr LFlushOptions++-- | Binding to @rocksdb_flushoptions_create@.+foreign import ccall safe "rocksdb/c.h rocksdb_flushoptions_create"+  c_rocksdb_flushoptions_create :: IO FlushOptions++-- | Binding to @rocksdb_flushoptions_destroy@.+foreign import ccall safe "rocksdb/c.h rocksdb_flushoptions_destroy"+  c_rocksdb_flushoptions_destroy :: FlushOptions -> IO ()++-- | Binding to @rocksdb_flushoptions_set_wait@.+foreign import ccall safe "rocksdb/c.h rocksdb_flushoptions_set_wait"+  c_rocksdb_flushoptions_set_wait :: FlushOptions -> CUChar -> IO ()++-- | Binding to @rocksdb_flush@.+foreign import ccall safe "rocksdb/c.h rocksdb_flush"+  c_rocksdb_flush :: RocksDB -> FlushOptions -> ErrPtr -> IO ()++-- | Binding to @rocksdb_flush_cf@.+foreign import ccall safe "rocksdb/c.h rocksdb_flush_cf"+  c_rocksdb_flush_cf :: RocksDB -> FlushOptions -> ColumnFamily -> ErrPtr -> IO ()++-- -- Options -- @@ -524,15 +829,24 @@ foreign import ccall safe "rocksdb/c.h rocksdb_options_set_max_total_wal_size"   c_rocksdb_options_set_max_total_wal_size :: Options -> Word64 -> IO () +-- | Binding to @rocksdb_options_set_disable_auto_compactions@.+foreign import ccall safe "rocksdb/c.h rocksdb_options_set_disable_auto_compactions"+  c_rocksdb_options_set_disable_auto_compactions :: Options -> CInt -> IO ()++-- | Binding to @rocksdb_options_set_max_background_jobs@.+foreign import ccall safe "rocksdb/c.h rocksdb_options_set_max_background_jobs"+  c_rocksdb_options_set_max_background_jobs :: Options -> CInt -> IO ()+ -- | Binding to @rocksdb_options_set_create_missing_column_families@. foreign import ccall safe "rocksdb/c.h rocksdb_options_set_create_missing_column_families"   c_rocksdb_options_set_create_missing_column_families :: Options -> CBool -> IO ()  -- | Binding to @rocksdb_options_set_prefix_extractor@. foreign import ccall safe "rocksdb/c.h rocksdb_options_set_prefix_extractor"-  c_rocksdb_options_set_prefix_extractor :: Options-                                         -> SliceTransform-                                         -> IO ()+  c_rocksdb_options_set_prefix_extractor ::+    Options ->+    SliceTransform ->+    IO ()  -- -- Block-Based Options@@ -544,45 +858,91 @@  -- | Binding to @rocksdb_block_based_options_destroy@. foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_destroy"-  c_rocksdb_block_based_options_destroy :: BlockBasedOptions-                                        -> IO ()+  c_rocksdb_block_based_options_destroy ::+    BlockBasedOptions ->+    IO ()  -- | Binding to @rocksdb_block_based_options_set_filter_policy@. foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_filter_policy"-  c_rocksdb_block_based_options_set_filter_policy :: BlockBasedOptions-                                                  -> FilterPolicy-                                                  -> IO ()+  c_rocksdb_block_based_options_set_filter_policy ::+    BlockBasedOptions ->+    FilterPolicy ->+    IO ()  -- | Binding to @rocksdb_block_based_options_set_block_cache@. foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_block_cache"-  c_rocksdb_block_based_options_set_block_cache :: BlockBasedOptions-                                                -> Cache-                                                -> IO ()+  c_rocksdb_block_based_options_set_block_cache ::+    BlockBasedOptions ->+    Cache ->+    IO ()  -- | Binding to @rocksdb_block_based_options_set_cache_index_and_filter_blocks@. foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_cache_index_and_filter_blocks"-  c_rocksdb_block_based_options_set_cache_index_and_filter_blocks :: BlockBasedOptions-                                                                  -> CUChar-                                                                  -> IO ()+  c_rocksdb_block_based_options_set_cache_index_and_filter_blocks ::+    BlockBasedOptions ->+    CUChar ->+    IO ()  -- | Binding to @rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority@.-foreign import ccall safe-  "rocksdb/c.h rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority"-  c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority :: BlockBasedOptions-                                                                                     -> CUChar-                                                                                     -> IO ()+foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority"+  c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority ::+    BlockBasedOptions ->+    CUChar ->+    IO () +-- | Binding to @rocksdb_compactoptions_create@.+foreign import ccall safe "rocksdb/c.h rocksdb_compactoptions_create"+  c_rocksdb_compactoptions_create :: IO CompactOptions++-- | Binding to @rocksdb_compactoptions_destroy@.+foreign import ccall safe "rocksdb/c.h rocksdb_compactoptions_destroy"+  c_rocksdb_compactoptions_destroy :: CompactOptions -> IO ()++-- | Binding to @rocksdb_compactoptions_set_bottommost_level_compaction@.+foreign import ccall safe "rocksdb/c.h rocksdb_compactoptions_set_bottommost_level_compaction"+  c_rocksdb_compactoptions_set_bottommost_level_compaction :: CompactOptions -> CUChar -> IO ()++-- | Binding to @rocksdb_compact_range_opt@. A null key leaves that end of+-- the range open.+foreign import ccall safe "rocksdb/c.h rocksdb_compact_range_opt"+  c_rocksdb_compact_range_opt ::+    RocksDB -> CompactOptions -> CString -> CSize -> CString -> CSize -> IO ()++-- | Binding to @rocksdb_compact_range_cf_opt@. A null key leaves that end+-- of the range open.+foreign import ccall safe "rocksdb/c.h rocksdb_compact_range_cf_opt"+  c_rocksdb_compact_range_cf_opt ::+    RocksDB -> ColumnFamily -> CompactOptions -> CString -> CSize -> CString -> CSize -> IO ()++-- | Binding to @rocksdb_block_based_options_set_index_type@.+foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_index_type"+  c_rocksdb_block_based_options_set_index_type :: BlockBasedOptions -> CInt -> IO ()++-- | Binding to @rocksdb_block_based_options_set_partition_filters@.+foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_partition_filters"+  c_rocksdb_block_based_options_set_partition_filters :: BlockBasedOptions -> CUChar -> IO ()++-- | Binding to @rocksdb_block_based_options_set_metadata_block_size@.+foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_metadata_block_size"+  c_rocksdb_block_based_options_set_metadata_block_size :: BlockBasedOptions -> Word64 -> IO ()++-- | Binding to @rocksdb_block_based_options_set_pin_top_level_index_and_filter@.+foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_pin_top_level_index_and_filter"+  c_rocksdb_block_based_options_set_pin_top_level_index_and_filter :: BlockBasedOptions -> CUChar -> IO ()+ -- | Binding to @rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache@. foreign import ccall safe "rocksdb/c.h rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache"-  c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache :: BlockBasedOptions-                                                                            -> CUChar-                                                                            -> IO ()+  c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache ::+    BlockBasedOptions ->+    CUChar ->+    IO ()  -- | Binding to @rocksdb_options_set_block_based_table_factory@. foreign import ccall safe "rocksdb/c.h rocksdb_options_set_block_based_table_factory"-  c_rocksdb_options_set_block_based_table_factory :: Options-                                                  -> BlockBasedOptions-                                                  -> IO ()+  c_rocksdb_options_set_block_based_table_factory ::+    Options ->+    BlockBasedOptions ->+    IO ()  -- -- Block Cache@@ -667,7 +1027,6 @@ -- | Binding to @rocksdb_readoptions_set_total_order_seek@. foreign import ccall safe "rocksdb/c.h rocksdb_readoptions_set_total_order_seek"   c_rocksdb_readoptions_set_total_order_seek :: ReadOpts -> CUChar -> IO ()-  -- -- Write options
src/Database/RocksDB/Internal.hs view
@@ -1,4 +1,6 @@+{-# LANGUAGE LambdaCase #-} {-# LANGUAGE RecordWildCards #-}+ -- | -- Module      : Database.RocksDB.Internal -- Copyright   : (c) 2012-2013 The leveldb-haskell Authors@@ -11,61 +13,63 @@ -- Internal types and helpers shared by the other modules. The t'DB' and t'Config' -- types are re-exported by "Database.RocksDB"; everything else is an -- implementation detail.- module Database.RocksDB.Internal-    ( Config (..)-    , DB (..)+  ( Config (..),+    MergeOperator (..),+    DB (..),      -- * Smart constructors & extractors-    , withOptions-    , withOptionsCache-    , withOptionsCF-    , withOptionsCFCache-    , withBlockCache-    , withReadOpts-    , createReadOpts-    , destroyReadOpts-    , withWriteOpts+    withOptions,+    withOptionsCache,+    withOptionsCF,+    withOptionsCFCache,+    withBlockCache,+    withReadOpts,+    createReadOpts,+    destroyReadOpts,+    withWriteOpts,      -- * Utilities-    , freeCString-    , throwIfErr-    , cSizeToInt-    , intToCSize-    , intToCInt-    , cIntToInt-    , boolToNum-    ) where+    freeCString,+    throwIfErr,+    cSizeToInt,+    intToCSize,+    intToCInt,+    cIntToInt,+    boolToNum,+  )+where -import           Control.Exception-import           Control.Monad-import           Data.Default-import           Data.Maybe        (isJust)-import           Database.RocksDB.C-import           Foreign-import           Foreign.C.String-import           Foreign.C.Types+import Control.Exception+import Control.Monad+import Data.Default+import Data.Maybe (isJust)+import Database.RocksDB.C+import Foreign+import Foreign.C.String+import Foreign.C.Types  -- | Handle to an open database. Obtain one with 'Database.RocksDB.withDB' or -- 'Database.RocksDB.withDBCF'. It is only valid inside the function passed to -- those, and can be shared between threads.-data DB = DB { rocksDB        :: !RocksDB-               -- ^ pointer to the C database object-             , columnFamilies :: ![ColumnFamily]-               -- ^ handles of the column families opened with-               -- 'Database.RocksDB.withDBCF', in the order they were given;-               -- empty for 'Database.RocksDB.withDB'-             , readOpts       :: !ReadOpts-               -- ^ read options used by reads and iterators; set to read from-               -- a snapshot by 'Database.RocksDB.withSnapshot'-             , snapshot       :: !(Maybe Snapshot)-               -- ^ the snapshot that 'readOpts' read from, if any. RocksDB-               -- cannot be asked what snapshot its read options hold, and-               -- 'Database.RocksDB.withIterConfig' has to make read options-               -- of its own that read from the same one.-             , writeOpts      :: !WriteOpts-               -- ^ write options used by all writes-             }+data DB = DB+  { -- | pointer to the C database object+    rocksDB :: !RocksDB,+    -- | handles of the column families opened with+    -- 'Database.RocksDB.withDBCF', in the order they were given;+    -- empty for 'Database.RocksDB.withDB'+    columnFamilies :: ![ColumnFamily],+    -- | read options used by reads and iterators; set to read from+    -- a snapshot by 'Database.RocksDB.withSnapshot'+    readOpts :: !ReadOpts,+    -- | the snapshot that 'readOpts' read from, if any. RocksDB+    -- cannot be asked what snapshot its read options hold, and+    -- 'Database.RocksDB.withIterConfig' has to make read options+    -- of its own that read from the same one.+    snapshot :: !(Maybe Snapshot),+    -- | write options used by all writes+    writeOpts :: !WriteOpts+  }  -- | Configuration of a database or column family. Use 'def' for the defaults -- and override the fields you need:@@ -83,69 +87,224 @@ -- database these grow without bound. Setting 'maxFiles', 'blockCacheSize', -- 'cacheIndexAndFilterBlocks' and 'dbWriteBufferSize' turns that into a fixed -- budget.-data Config = Config { createIfMissing :: !Bool-                       -- ^ create the database if it does not exist-                     , errorIfExists   :: !Bool-                       -- ^ fail to open if the database already exists-                     , paranoidChecks  :: !Bool-                       -- ^ check data aggressively and stop early on any-                       -- corruption-                     , createMissingColumnFamilies :: !Bool-                       -- ^ create the column families passed to-                       -- 'Database.RocksDB.withDBCF' that an existing-                       -- database does not have yet-                     , maxFiles        :: !(Maybe Int)-                       -- ^ maximum number of files RocksDB keeps open-                       -- ('Nothing': RocksDB default, unlimited)-                     , prefixLength    :: !(Maybe Int)-                       -- ^ length in bytes of a fixed key prefix, used by-                       -- bloom filters and prefix seeks-                     , bloomFilter     :: !Bool-                       -- ^ use a full bloom filter with 10 bits per key to-                       -- skip files that cannot contain a key; the filters-                       -- take about 1.25 bytes of memory per key-                     , writeBufferSize :: !(Maybe Int)-                       -- ^ bytes per memtable (RocksDB default 64 MiB);-                       -- per column family-                     , dbWriteBufferSize :: !(Maybe Int)-                       -- ^ bytes for all memtables across all column-                       -- families (RocksDB default unlimited); database-wide,-                       -- ignored in column family configs-                     , blockCacheSize :: !(Maybe Int)-                       -- ^ bytes of a least-recently-used block cache shared-                       -- by the database and all its column families-                       -- ('Nothing': RocksDB gives each column family its-                       -- own default cache, of 32 MiB in RocksDB 8);-                       -- database-wide, ignored in column family configs-                     , cacheIndexAndFilterBlocks :: !Bool-                       -- ^ keep index and filter blocks in the block cache,-                       -- at high priority, with those of level 0 pinned,-                       -- instead of in memory outside it for every open-                       -- file; per column family-                     , maxTotalWalSize :: !(Maybe Int)-                       -- ^ bytes of write-ahead log after which the column-                       -- families holding the oldest log are flushed, so it-                       -- can be deleted (RocksDB default: four times the-                       -- memtables); database-wide, ignored in column family-                       -- configs-                     } deriving (Eq, Show)+data Config = Config+  { -- | create the database if it does not exist+    createIfMissing :: !Bool,+    -- | fail to open if the database already exists+    errorIfExists :: !Bool,+    -- | check data aggressively and stop early on any+    -- corruption+    paranoidChecks :: !Bool,+    -- | create the column families passed to+    -- 'Database.RocksDB.withDBCF' that an existing+    -- database does not have yet+    createMissingColumnFamilies :: !Bool,+    -- | maximum number of files RocksDB keeps open+    -- ('Nothing': RocksDB default, unlimited)+    maxFiles :: !(Maybe Int),+    -- | length in bytes of a fixed key prefix, used by+    -- bloom filters and prefix seeks+    prefixLength :: !(Maybe Int),+    -- | use a full bloom filter with 10 bits per key to+    -- skip files that cannot contain a key; the filters+    -- take about 1.25 bytes of memory per key+    bloomFilter :: !Bool,+    -- | bytes per memtable (RocksDB default 64 MiB);+    -- per column family+    writeBufferSize :: !(Maybe Int),+    -- | bytes for all memtables across all column+    -- families (RocksDB default unlimited); database-wide,+    -- ignored in column family configs+    dbWriteBufferSize :: !(Maybe Int),+    -- | bytes of a least-recently-used block cache shared+    -- by the database and all its column families+    -- ('Nothing': RocksDB gives each column family its+    -- own default cache, of 32 MiB in RocksDB 8);+    -- database-wide, ignored in column family configs+    blockCacheSize :: !(Maybe Int),+    -- | keep index and filter blocks in the block cache,+    -- at high priority, with those of level 0 pinned,+    -- instead of in memory outside it for every open+    -- file; per column family+    cacheIndexAndFilterBlocks :: !Bool,+    -- | write the index and the filter of each table file in+    -- pieces of 4 KiB under a small index of their own, where+    -- each is otherwise one block that grows with the file: on+    -- files of 64 MiB the filter is one to three mebibytes and+    -- the index a few hundred kibibytes. A lookup then loads+    -- the two pieces that it needs and not megabytes, and a+    -- block cache far smaller than all the indices and filters+    -- of the database still serves most lookups from memory.+    -- Without it, a database with more of them than its cache+    -- holds loads and drops them at nearly every lookup: a+    -- millisecond of copying, checksums and decompression+    -- each, on a disk that is idle. The small top-level index+    -- of each open file is kept in the cache. It applies to+    -- the files written from then on; the files that a+    -- database already has keep their layout until they are+    -- compacted, and both kinds are read. It costs one more+    -- lookup in memory for each read. Use it with+    -- 'cacheIndexAndFilterBlocks'; per column family+    partitionIndexAndFilters :: !Bool,+    -- | bytes of write-ahead log after which the column+    -- families holding the oldest log are flushed, so it+    -- can be deleted (RocksDB default: four times the+    -- memtables); database-wide, ignored in column family+    -- configs+    maxTotalWalSize :: !(Maybe Int),+    -- | threads that RocksDB may use at once to flush+    -- memtables and to compact files (RocksDB default 2:+    -- one flush and one compaction). Of @n@ jobs, a+    -- quarter are for flushes and the rest for+    -- compactions, at least one of each. More of them+    -- shorten a backlog of compactions on a disk and+    -- cores that have room for it, and take both from+    -- reads when they have not: on a slow disk a single+    -- compaction already waits for the disk, and a+    -- second one only makes every read wait longer.+    -- Database-wide, ignored in column family configs+    maxBackgroundJobs :: !(Maybe Int),+    -- | how RocksDB works out the value of a key that has+    -- merge records ('Database.RocksDB.Merge'): see+    -- t'MergeOperator'. 'Nothing', the default, for a+    -- column family that is never written with merges; one+    -- that is must be given the same operator every time+    -- the database is opened, by every program that opens+    -- it. Per column family: the one in the configuration+    -- of the database is for the default column family+    mergeOperator :: !(Maybe MergeOperator),+    -- | do not compact the files of the column family in+    -- the background. Files then pile up as memtables are+    -- flushed, and reads get slower with each: it is for+    -- a bulk load that ends with+    -- 'Database.RocksDB.compactRange', and for tests and+    -- benchmarks that need to know which file a key is+    -- in. Per column family+    disableAutoCompactions :: !Bool+  }+  deriving (Eq, Show) +-- | How RocksDB combines the merge records of a key with its value.+--+-- = What a merge is+--+-- A put replaces the value of a key, and a program that wants to change+-- a value has to read it first: add to a counter, add an entry to a list.+-- For a batch of thousands of such changes, those reads are most of the+-- work, and each may wait for the disk.+--+-- A /merge record/ is a third kind of write, next to a put and a delete+-- ('Database.RocksDB.Merge', 'Database.RocksDB.merge'). It says how the+-- value changes and not what it becomes: \"add 5\", \"add this entry\".+-- RocksDB stores the record as it is, without reading anything. The work+-- is done later by the /merge operator/ of the column family: a function+-- that takes the value that the key had and the records written since,+-- oldest first, and gives the value that the key now has. RocksDB calls+-- it when the key is read, and when a compaction brings the records and+-- the value together, after which the key holds a plain value again.+--+-- Readers see none of this: 'Database.RocksDB.get',+-- 'Database.RocksDB.multiGet' and iterators return merged values. What+-- they pay is the merge, for a key that has records pending, and a look+-- into each file that holds one of them.+--+-- A put or a delete after merge records hides them, like any older+-- value. A merge record after a delete, or on a key that was never+-- written, is merged with no value: the operator decides what that means.+--+-- = Which operator+--+-- The operator is part of the meaning of the data, like the format of+-- the values. Every program that opens the database must give the column+-- family the same operator, because RocksDB does not refuse to open one+-- that has merge records without it. This is what RocksDB 11.1 was seen to+-- do then:+--+-- * Merge records that are still in the write-ahead log, not yet flushed+--   to a table file, are __dropped when the database is opened__, without+--   an error. They are lost for good.+--+-- * Merge records in table files are kept, also through a compaction, and+--   are there again for a program that opens the database with the+--   operator. Meanwhile they cannot be read: 'Database.RocksDB.get' throws+--   for a key that has records and no value under them, an iterator stops+--   at such a key with an error, and a 'Database.RocksDB.get' or+--   'Database.RocksDB.multiGet' of a key that has a value under its+--   records __crashes the process__.+--+-- * Writing a merge record is refused.+--+-- A column family opened with another operator gives whatever that+-- operator makes of the records. So a program that starts to write merge+-- records to a database that older versions of it can open should make+-- sure that they no longer can: a column family that they do not know of+-- is enough, as RocksDB refuses to open a database without all its column+-- families, before it reads the write-ahead log.+data MergeOperator+  = -- | RocksDB's own operator for counters. A value and a merge record+    -- are both an unsigned 64-bit number in eight bytes, least significant+    -- byte first, and merging adds them, modulo 2^64. A key without a+    -- value counts from zero, and so does one whose value is not eight+    -- bytes long.+    UInt64Add+  | -- | An operator made of two C functions.+    --+    -- They are C functions and not Haskell ones because RocksDB calls+    -- them from its own threads, during reads, flushes and compactions,+    -- several at once. Import their addresses:+    --+    -- @+    -- foreign import ccall "&my_full_merge" myFullMerge :: FunPtr FullMerge+    -- foreign import ccall "&my_partial_merge" myPartialMerge :: FunPtr PartialMerge+    -- @+    --+    -- @cbits\/rocksdb_haskell.h@ in this package has their C types, as+    -- @rocksdb_haskell_full_merge_t@ and @rocksdb_haskell_partial_merge_t@,+    -- and what they must do: return the result in memory from @malloc@,+    -- which RocksDB frees, and how to report a failure.+    MergeFunctions+      { -- | the name of the operator, which RocksDB writes in its+        -- options file and in every table file. Change it when what+        -- the functions do changes.+        mergeName :: !String,+        -- | the value of a key from the value it had, if any, and its+        -- merge records, oldest first+        fullMerge :: !(FunPtr FullMerge),+        -- | one merge record from several, oldest first, that does what+        -- they did. RocksDB uses it to shorten the list where it has+        -- records and not the value under them. A function that always+        -- fails is allowed: the records are then kept as they are.+        partialMerge :: !(FunPtr PartialMerge)+      }+  deriving (Eq, Show)+ -- | All flags off and all sizes left to RocksDB defaults. instance Default Config where-    def = Config { createIfMissing  = False-                 , errorIfExists    = False-                 , paranoidChecks   = False-                 , createMissingColumnFamilies = False-                 , maxFiles         = Nothing-                 , prefixLength     = Nothing-                 , bloomFilter      = False-                 , writeBufferSize  = Nothing-                 , dbWriteBufferSize = Nothing-                 , blockCacheSize   = Nothing-                 , cacheIndexAndFilterBlocks = False-                 , maxTotalWalSize  = Nothing-                 }+  def =+    Config+      { createIfMissing = False,+        errorIfExists = False,+        paranoidChecks = False,+        createMissingColumnFamilies = False,+        maxFiles = Nothing,+        prefixLength = Nothing,+        bloomFilter = False,+        writeBufferSize = Nothing,+        dbWriteBufferSize = Nothing,+        blockCacheSize = Nothing,+        cacheIndexAndFilterBlocks = False,+        partitionIndexAndFilters = False,+        maxTotalWalSize = Nothing,+        maxBackgroundJobs = Nothing,+        mergeOperator = Nothing,+        disableAutoCompactions = False+      } +-- | @rocksdb_block_based_table_index_type_two_level_index_search@: an index+-- of the pieces of the index.+twoLevelIndexSearch :: CInt+twoLevelIndexSearch = 2+ -- | Create RocksDB options from a t'Config' and pass them to a function. The -- options are destroyed when it returns. withOptions :: Config -> (Options -> IO a) -> IO a@@ -155,46 +314,67 @@ -- any. The options are destroyed when the function returns. withOptionsCache :: Maybe Cache -> Config -> (Options -> IO a) -> IO a withOptionsCache cache Config {..} f = with_opts $ \opts -> do-    when (bloomFilter || isJust cache || cacheIndexAndFilterBlocks) $-        bracket c_rocksdb_block_based_options_create-                c_rocksdb_block_based_options_destroy $ \bo -> do-            when bloomFilter $ do-                fp <- c_rocksdb_filterpolicy_create_bloom_full 10-                c_rocksdb_block_based_options_set_filter_policy bo fp-            forM_ cache $ c_rocksdb_block_based_options_set_block_cache bo-            when cacheIndexAndFilterBlocks $ do-                c_rocksdb_block_based_options_set_cache_index_and_filter_blocks bo 1-                c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority bo 1-                c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache bo 1-            -- the table factory copies the options, filter policy and cache-            c_rocksdb_options_set_block_based_table_factory opts bo-    forM_ prefixLength $ \l -> do-        t <- c_rocksdb_slicetransform_create_fixed_prefix (intToCSize l)-        c_rocksdb_options_set_prefix_extractor opts t-    forM_ maxFiles $-        c_rocksdb_options_set_max_open_files opts . intToCInt-    forM_ writeBufferSize $-        c_rocksdb_options_set_write_buffer_size opts . intToCSize-    forM_ dbWriteBufferSize $-        c_rocksdb_options_set_db_write_buffer_size opts . intToCSize-    forM_ maxTotalWalSize $-        c_rocksdb_options_set_max_total_wal_size opts . fromIntegral-    c_rocksdb_options_set_create_if_missing-        opts (boolToCBool createIfMissing)-    c_rocksdb_options_set_error_if_exists-        opts (boolToCBool errorIfExists)-    c_rocksdb_options_set_paranoid_checks-        opts (boolToCBool paranoidChecks)-    c_rocksdb_options_set_create_missing_column_families-        opts (boolToCBool createMissingColumnFamilies)-    f opts+  when (bloomFilter || isJust cache || cacheIndexAndFilterBlocks || partitionIndexAndFilters)+    $ bracket+      c_rocksdb_block_based_options_create+      c_rocksdb_block_based_options_destroy+    $ \bo -> do+      when bloomFilter $ do+        fp <- c_rocksdb_filterpolicy_create_bloom_full 10+        c_rocksdb_block_based_options_set_filter_policy bo fp+      forM_ cache $ c_rocksdb_block_based_options_set_block_cache bo+      when cacheIndexAndFilterBlocks $ do+        c_rocksdb_block_based_options_set_cache_index_and_filter_blocks bo 1+        c_rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority bo 1+        c_rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache bo 1+      when partitionIndexAndFilters $ do+        c_rocksdb_block_based_options_set_index_type bo twoLevelIndexSearch+        c_rocksdb_block_based_options_set_partition_filters bo 1+        c_rocksdb_block_based_options_set_metadata_block_size bo 4096+        c_rocksdb_block_based_options_set_pin_top_level_index_and_filter bo 1+      -- the table factory copies the options, filter policy and cache+      c_rocksdb_options_set_block_based_table_factory opts bo+  forM_ prefixLength $ \l -> do+    t <- c_rocksdb_slicetransform_create_fixed_prefix (intToCSize l)+    c_rocksdb_options_set_prefix_extractor opts t+  forM_ maxFiles $+    c_rocksdb_options_set_max_open_files opts . intToCInt+  forM_ writeBufferSize $+    c_rocksdb_options_set_write_buffer_size opts . intToCSize+  forM_ dbWriteBufferSize $+    c_rocksdb_options_set_db_write_buffer_size opts . intToCSize+  forM_ maxTotalWalSize $+    c_rocksdb_options_set_max_total_wal_size opts . fromIntegral+  forM_ maxBackgroundJobs $+    c_rocksdb_options_set_max_background_jobs opts . intToCInt+  forM_ mergeOperator $ \case+    UInt64Add -> c_rocksdb_options_set_uint64add_merge_operator opts+    MergeFunctions {..} ->+      withCString mergeName $ \name -> do+        status <- c_rocksdb_haskell_options_set_merge_operator opts name fullMerge partialMerge+        when (status /= 0) $+          throwIO (userError "mergeOperator: no memory for the merge operator")+  when disableAutoCompactions $+    c_rocksdb_options_set_disable_auto_compactions opts 1+  c_rocksdb_options_set_create_if_missing+    opts+    (boolToCBool createIfMissing)+  c_rocksdb_options_set_error_if_exists+    opts+    (boolToCBool errorIfExists)+  c_rocksdb_options_set_paranoid_checks+    opts+    (boolToCBool paranoidChecks)+  c_rocksdb_options_set_create_missing_column_families+    opts+    (boolToCBool createMissingColumnFamilies)+  f opts   where     with_opts =-        bracket+      bracket         c_rocksdb_options_create         c_rocksdb_options_destroy - -- | Create options for several column families at once, in the same order as -- the configurations. withOptionsCF :: [Config] -> ([Options] -> IO a) -> IO a@@ -203,10 +383,10 @@ -- | Options of column families that use this block cache, if any. withOptionsCFCache :: Maybe Cache -> [Config] -> ([Options] -> IO a) -> IO a withOptionsCFCache cache cfgs f =-    go [] cfgs+  go [] cfgs   where     go acc [] = f (reverse acc)-    go acc (c:cs) = withOptionsCache cache c $ \o -> go (o:acc) cs+    go acc (c : cs) = withOptionsCache cache c $ \o -> go (o : acc) cs  -- | The block cache of 'blockCacheSize', if it is set. The database and -- column families that use it keep it alive after this returns.@@ -216,15 +396,16 @@ withBlockCache :: Config -> (Maybe Cache -> IO a) -> IO a withBlockCache Config {blockCacheSize = Nothing} f = f Nothing withBlockCache Config {blockCacheSize = Just n} f =-    bracket (c_rocksdb_cache_create_lru (intToCSize n))-            c_rocksdb_cache_destroy-            (f . Just)+  bracket+    (c_rocksdb_cache_create_lru (intToCSize n))+    c_rocksdb_cache_destroy+    (f . Just)  -- | Create read options, reading from a snapshot if one is given, and pass them -- to a function. They are destroyed when it returns. withReadOpts :: Maybe Snapshot -> (ReadOpts -> IO a) -> IO a withReadOpts maybe_snap_ptr =-    bracket+  bracket     (createReadOpts maybe_snap_ptr)     c_rocksdb_readoptions_destroy @@ -232,9 +413,9 @@ -- Caller is responsible for calling 'destroyReadOpts'. createReadOpts :: Maybe Snapshot -> IO ReadOpts createReadOpts maybe_snap_ptr = do-    read_opts_ptr <- c_rocksdb_readoptions_create-    forM_ maybe_snap_ptr $ c_rocksdb_readoptions_set_snapshot read_opts_ptr-    return read_opts_ptr+  read_opts_ptr <- c_rocksdb_readoptions_create+  forM_ maybe_snap_ptr $ c_rocksdb_readoptions_set_snapshot read_opts_ptr+  return read_opts_ptr  -- | Destroy read options created with 'createReadOpts'. destroyReadOpts :: ReadOpts -> IO ()@@ -244,7 +425,7 @@ -- destroyed when it returns. withWriteOpts :: (WriteOpts -> IO a) -> IO a withWriteOpts =-    bracket+  bracket     c_rocksdb_writeoptions_create     c_rocksdb_writeoptions_destroy @@ -261,18 +442,18 @@ -- freed. throwIfErr :: String -> (ErrPtr -> IO a) -> IO a throwIfErr s f = alloca $ \err_ptr -> do-    poke err_ptr nullPtr-    res  <- f err_ptr-    err_cstr <- peek err_ptr-    when (err_cstr /= nullPtr) $ do-        err <- peekCString err_cstr-        free err_cstr-        throwIO $ userError $ s ++ ": " ++ err-    return res+  poke err_ptr nullPtr+  res <- f err_ptr+  err_cstr <- peek err_ptr+  when (err_cstr /= nullPtr) $ do+    err <- peekCString err_cstr+    free err_cstr+    throwIO $ userError $ s ++ ": " ++ err+  return res  -- | Convert a 'Bool' to a C boolean. boolToCBool :: Bool -> CBool-boolToCBool True  = 1+boolToCBool True = 1 boolToCBool False = 0 {-# INLINE boolToCBool #-} @@ -297,7 +478,7 @@ {-# INLINE cIntToInt #-}  -- | Convert a 'Bool' to @1@ or @0@.-boolToNum :: Num b => Bool -> b-boolToNum True  = fromIntegral (1 :: Int)+boolToNum :: (Num b) => Bool -> b+boolToNum True = fromIntegral (1 :: Int) boolToNum False = fromIntegral (0 :: Int) {-# INLINE boolToNum #-}
src/Database/RocksDB/Iterator.hs view
@@ -73,6 +73,57 @@ --         go [] -- @ --+-- = Reading many entries in one call+--+-- 'iterNext' and 'iterPrev' may wait for the disk, so they are foreign calls+-- of the kind that lets other Haskell threads run meanwhile. That has a+-- price: the thread gives up its capability for the call and has to get one+-- back afterwards, and when more threads want to run than there are+-- capabilities, it waits for it. A scan that moves one step at a time pays+-- that for every entry.+--+-- The run-time system also walks the stack of the thread at every such call.+-- A loop that is not tail recursive, such as one that ends in+-- @(entry :) \<$\> go@, leaves a frame there for every entry read so far, so+-- each step costs more the deeper the scan is.+--+-- 'iterChunk' walks the iterator in C instead, and returns up to some+-- hundreds of entries from one call. 'iterEntries', 'iterEntriesBack',+-- 'iterEntriesPrefix' and 'iterEntriesPrefixBack' are short forms of it for+-- a known number of entries.+--+-- @+-- -- the same scan again, in chunks+-- usersFrom db = 'withIter' db $ \it -> do+--     'iterSeek' it "user:"+--     let go acc = do+--             (entries, more) <- 'iterChunk' it def{'chunkPrefix' = "user:"}+--             let acc' = entries : acc+--             if more then go acc' else return (concat (reverse acc'))+--     go []+-- @+--+-- Seconds for every thread to read 200,000 entries of 52 bytes, on four+-- capabilities and four cores, from the benchmark of this package:+--+-- > scan                                 1 thread   16 threads+-- > single steps, not tail recursive       0.78        3.9+-- > single steps, tail recursive           0.090       0.50+-- > chunks of 256 entries                  0.048       0.25+--+-- So a scan in chunks takes half the time of the best loop of single steps,+-- and a sixteenth of the time of a loop that is not tail recursive. On a+-- machine with more cores than capabilities it gains more, since the threads+-- that are inside the call do not hold a capability: with twenty more cores+-- free, the sixteen threads took 0.12 seconds in chunks and 0.83 in steps.+--+-- Use chunks to read a range. Keep to single steps where each move depends+-- on the entry just read, or where one entry is all that is wanted:+-- 'iterEntry' after a seek reads it with no call that moves the iterator.+--+-- A chunk also reports errors by itself: 'iterChunk' throws if the iterator+-- stopped at one, where single steps need 'iterGetError' to be asked.+-- -- = Safety -- -- The C API crashes the process if an invalid iterator is moved, and the bytes@@ -81,21 +132,25 @@ -- and values are copied into Haskell memory before they are returned. module Database.RocksDB.Iterator   ( Iterator,+     -- * Bracket-style iteration     withIter,     withIterCF,     withIterSnap,     withIterSnapCF,+     -- * Bounded iteration     IterConfig (..),     withIterConfig,     prefixUpperBound,+     -- * Manual iterator management     createIterator,     createIteratorSnap,     createIteratorConfig,     destroyIterator,     destroyReadOpts,+     -- * Iterator operations     iterEntry,     iterFirst,@@ -107,10 +162,18 @@     iterSeek,     iterValid,     iterValue,++    -- * Reading many entries in one call+    iterEntries,+    iterEntriesBack,+    iterEntriesPrefix,+    iterEntriesPrefixBack,+    ChunkConfig (..),+    iterChunk,   ) where -import Control.Exception (bracket, mask_, onException)+import Control.Exception (bracket, finally, mask, mask_, onException, throwIO) import Control.Monad (forM, forM_, when) import Data.ByteString (ByteString) import Data.ByteString qualified as B@@ -121,7 +184,7 @@ import Foreign import Foreign.C.Error (throwErrnoIfNull) import Foreign.C.String (CString)-import Foreign.C.Types (CSize)+import Foreign.C.Types (CChar, CSize)  -- | Create 'Iterator' and use it. --@@ -145,54 +208,54 @@ -- enabling consistent reads across multiple iterators and point queries. -- -- Iterator should not be used after computation ends.-withIterSnap-    :: DB-    -> Maybe Snapshot-    -> (Iterator -> IO a)-    -> IO a+withIterSnap ::+  DB ->+  Maybe Snapshot ->+  (Iterator -> IO a) ->+  IO a withIterSnap db msnap = withIterCommon db msnap Nothing  -- | Create 'Iterator' on a column family with a specific snapshot. -- -- If 'Nothing' is passed for snapshot, the iterator creates its own implicit snapshot. -- If 'Just snapshot' is passed, the iterator uses the provided snapshot.-withIterSnapCF-    :: DB-    -> Maybe Snapshot-    -> ColumnFamily-    -> (Iterator -> IO a)-    -> IO a+withIterSnapCF ::+  DB ->+  Maybe Snapshot ->+  ColumnFamily ->+  (Iterator -> IO a) ->+  IO a withIterSnapCF db msnap cf = withIterCommon db msnap (Just cf)  -- | Create an iterator over the default column family, or over a column family -- if one is given, reading from a snapshot if one is given.-withIterCommon-    :: DB-    -> Maybe Snapshot-    -> Maybe ColumnFamily-    -> (Iterator -> IO a)-    -> IO a-withIterCommon DB{rocksDB = rocks_db, readOpts = read_opts} msnap mcf f =-    case msnap of-        Nothing ->-            -- Use DB's readOpts (iterator creates implicit snapshot)-            bracket (create_iterator read_opts) destroy_iterator f-        Just snap ->-            -- Create temporary ReadOpts with the provided snapshot-            bracket create_read_opts destroy_read_opts $ \ro ->-                bracket (create_iterator ro) destroy_iterator f-          where-            create_read_opts = do-                ro <- c_rocksdb_readoptions_create-                c_rocksdb_readoptions_set_snapshot ro snap-                return ro-            destroy_read_opts = c_rocksdb_readoptions_destroy+withIterCommon ::+  DB ->+  Maybe Snapshot ->+  Maybe ColumnFamily ->+  (Iterator -> IO a) ->+  IO a+withIterCommon DB {rocksDB = rocks_db, readOpts = read_opts} msnap mcf f =+  case msnap of+    Nothing ->+      -- Use DB's readOpts (iterator creates implicit snapshot)+      bracket (create_iterator read_opts) destroy_iterator f+    Just snap ->+      -- Create temporary ReadOpts with the provided snapshot+      bracket create_read_opts destroy_read_opts $ \ro ->+        bracket (create_iterator ro) destroy_iterator f+      where+        create_read_opts = do+          ro <- c_rocksdb_readoptions_create+          c_rocksdb_readoptions_set_snapshot ro snap+          return ro+        destroy_read_opts = c_rocksdb_readoptions_destroy   where     destroy_iterator = c_rocksdb_iter_destroy     create_iterator ro =-        throwErrnoIfNull "create_iterator" $ case mcf of-            Just cf -> c_rocksdb_create_iterator_cf rocks_db ro cf-            Nothing -> c_rocksdb_create_iterator rocks_db ro+      throwErrnoIfNull "create_iterator" $ case mcf of+        Just cf -> c_rocksdb_create_iterator_cf rocks_db ro cf+        Nothing -> c_rocksdb_create_iterator rocks_db ro  -- | Where an iterator stops, and what a seek promises. 'def' asks for -- nothing, which gives the iterator that 'withIter' creates; override the@@ -217,52 +280,52 @@ -- * 'totalOrderSeek' when a database with a 'prefixLength' has to be walked --   from end to end. data IterConfig = IterConfig-    { lowerBound :: !(Maybe ByteString)-      -- ^ the smallest key that the iterator can be at. Going backwards, it-      -- becomes invalid instead of moving to a smaller key; a seek to a-      -- smaller key lands on the first key at or after the bound.-    , upperBound :: !(Maybe ByteString)-      -- ^ a key that the iterator stays below: the bound itself is outside.-      -- Going forwards, the iterator becomes invalid instead of reaching the-      -- bound or a larger key, and 'iterLast' goes to the last key below it.-      -- 'prefixUpperBound' gives the bound that ends a prefix.-    , prefixSameAsStart :: !Bool-      -- ^ stay within the prefix of the key that the iterator was last sent-      -- to: it becomes invalid instead of moving to a key with another-      -- prefix. The prefix is the first 'Database.RocksDB.prefixLength'-      -- bytes of a key. After 'iterSeek' it is the prefix of the key-      -- sought; after 'iterFirst' or 'iterLast', that of the key found, so-      -- that those two give the first or the last prefix of the database-      -- and no more. It has no effect in a database or column family-      -- without a prefix length, nor together with 'totalOrderSeek'.-      ---      -- __Every key has to be at least as long as the prefix__: the keys-      -- sought, and the keys stored. RocksDB takes the prefix of a key-      -- without looking at its length, so with a shorter key it reads the-      -- bytes that follow it in memory, and where the iterator stops is-      -- then anyone's guess. This binding cannot check it for you, because-      -- an 'Iterator' does not know how it was made. If keys can be-      -- shorter, use an 'upperBound', which has no such condition.-    , totalOrderSeek :: !Bool-      -- ^ see every key in order, whatever its prefix, in a database or-      -- column family that has a 'Database.RocksDB.prefixLength'. There, a-      -- plain iterator is only sure to return the keys that share the prefix-      -- of the key it was sent to; what it does once it moves past them, or-      -- after 'iterFirst', is not defined. That is the price of the prefix-      -- filters, which let a seek skip the files that have nothing for a-      -- prefix. This option gives up the filters and gets the order back.-    }-    deriving (Eq, Show)+  { -- | the smallest key that the iterator can be at. Going backwards, it+    -- becomes invalid instead of moving to a smaller key; a seek to a+    -- smaller key lands on the first key at or after the bound.+    lowerBound :: !(Maybe ByteString),+    -- | a key that the iterator stays below: the bound itself is outside.+    -- Going forwards, the iterator becomes invalid instead of reaching the+    -- bound or a larger key, and 'iterLast' goes to the last key below it.+    -- 'prefixUpperBound' gives the bound that ends a prefix.+    upperBound :: !(Maybe ByteString),+    -- | stay within the prefix of the key that the iterator was last sent+    -- to: it becomes invalid instead of moving to a key with another+    -- prefix. The prefix is the first 'Database.RocksDB.prefixLength'+    -- bytes of a key. After 'iterSeek' it is the prefix of the key+    -- sought; after 'iterFirst' or 'iterLast', that of the key found, so+    -- that those two give the first or the last prefix of the database+    -- and no more. It has no effect in a database or column family+    -- without a prefix length, nor together with 'totalOrderSeek'.+    --+    -- __Every key has to be at least as long as the prefix__: the keys+    -- sought, and the keys stored. RocksDB takes the prefix of a key+    -- without looking at its length, so with a shorter key it reads the+    -- bytes that follow it in memory, and where the iterator stops is+    -- then anyone's guess. This binding cannot check it for you, because+    -- an 'Iterator' does not know how it was made. If keys can be+    -- shorter, use an 'upperBound', which has no such condition.+    prefixSameAsStart :: !Bool,+    -- | see every key in order, whatever its prefix, in a database or+    -- column family that has a 'Database.RocksDB.prefixLength'. There, a+    -- plain iterator is only sure to return the keys that share the prefix+    -- of the key it was sent to; what it does once it moves past them, or+    -- after 'iterFirst', is not defined. That is the price of the prefix+    -- filters, which let a seek skip the files that have nothing for a+    -- prefix. This option gives up the filters and gets the order back.+    totalOrderSeek :: !Bool+  }+  deriving (Eq, Show)  -- | No bounds, and seeks as RocksDB does them by default. instance Default IterConfig where-    def =-        IterConfig-            { lowerBound = Nothing-            , upperBound = Nothing-            , prefixSameAsStart = False-            , totalOrderSeek = False-            }+  def =+    IterConfig+      { lowerBound = Nothing,+        upperBound = Nothing,+        prefixSameAsStart = False,+        totalOrderSeek = False+      }  -- | The smallest key that is greater than every key that starts with the -- given prefix: the 'upperBound' of a scan of the prefix.@@ -276,9 +339,9 @@ -- Just "user;" prefixUpperBound :: ByteString -> Maybe ByteString prefixUpperBound prefix =-    case B.unsnoc (B.dropWhileEnd (== 0xff) prefix) of-        Nothing -> Nothing-        Just (start, byte) -> Just (B.snoc start (byte + 1))+  case B.unsnoc (B.dropWhileEnd (== 0xff) prefix) of+    Nothing -> Nothing+    Just (start, byte) -> Just (B.snoc start (byte + 1))  -- | Create an 'Iterator' that stops where an t'IterConfig' says, over the -- default column family or over a column family if one is given, and use it.@@ -297,7 +360,7 @@ -- Iterator should not be used after computation ends. withIterConfig :: DB -> Maybe ColumnFamily -> IterConfig -> (Iterator -> IO a) -> IO a withIterConfig db mcf cfg f =-    bracket (createIteratorConfig db mcf cfg) snd (f . fst)+  bracket (createIteratorConfig db mcf cfg) snd (f . fst)  -- | Manually create an unmanaged iterator like that of 'withIterConfig'. --@@ -310,34 +373,34 @@ -- For automatic resource management, prefer 'withIterConfig'. createIteratorConfig :: DB -> Maybe ColumnFamily -> IterConfig -> IO (Iterator, IO ()) createIteratorConfig db mcf cfg = mask_ $ do-    ro <- c_rocksdb_readoptions_create-    lower <- bound ro c_rocksdb_readoptions_set_iterate_lower_bound (lowerBound cfg)-    upper <- bound ro c_rocksdb_readoptions_set_iterate_upper_bound (upperBound cfg)-    let release = do-            c_rocksdb_readoptions_destroy ro-            mapM_ free lower-            mapM_ free upper-    it <- flip onException release $ do-        forM_ (snapshot db) (c_rocksdb_readoptions_set_snapshot ro)-        when (prefixSameAsStart cfg) $ c_rocksdb_readoptions_set_prefix_same_as_start ro 1-        when (totalOrderSeek cfg) $ c_rocksdb_readoptions_set_total_order_seek ro 1-        createIteratorWithOpts db ro mcf-    return (it, c_rocksdb_iter_destroy it >> release)+  ro <- c_rocksdb_readoptions_create+  lower <- bound ro c_rocksdb_readoptions_set_iterate_lower_bound (lowerBound cfg)+  upper <- bound ro c_rocksdb_readoptions_set_iterate_upper_bound (upperBound cfg)+  let release = do+        c_rocksdb_readoptions_destroy ro+        mapM_ free lower+        mapM_ free upper+  it <- flip onException release $ do+    forM_ (snapshot db) (c_rocksdb_readoptions_set_snapshot ro)+    when (prefixSameAsStart cfg) $ c_rocksdb_readoptions_set_prefix_same_as_start ro 1+    when (totalOrderSeek cfg) $ c_rocksdb_readoptions_set_total_order_seek ro 1+    createIteratorWithOpts db ro mcf+  return (it, c_rocksdb_iter_destroy it >> release)   where     -- A copy of the bound outside the Haskell heap, where it does not move     -- and lives until it is freed. One byte is asked for at least, so that     -- an empty bound has a pointer too.     bound ::-        ReadOpts ->-        (ReadOpts -> CString -> CSize -> IO ()) ->-        Maybe ByteString ->-        IO (Maybe CString)+      ReadOpts ->+      (ReadOpts -> CString -> CSize -> IO ()) ->+      Maybe ByteString ->+      IO (Maybe CString)     bound ro set mkey = forM mkey $ \key -> do-        let len = B.length key-        ptr <- mallocBytes (max 1 len)-        BU.unsafeUseAsCString key $ \src -> copyBytes ptr src len-        set ro ptr (intToCSize len)-        return ptr+      let len = B.length key+      ptr <- mallocBytes (max 1 len)+      BU.unsafeUseAsCString key $ \src -> copyBytes ptr src len+      set ro ptr (intToCSize len)+      return ptr  -- | Manually create unmanaged iterator, over the default column family or over -- a column family if one is given. It uses the 'readOpts' of the t'DB', so if@@ -345,10 +408,10 @@ -- snapshot. It must be released with 'destroyIterator', and must not outlive -- the t'DB'. createIterator :: DB -> Maybe ColumnFamily -> IO Iterator-createIterator DB{rocksDB = rocks_db, readOpts = read_opts} mcf =-    throwErrnoIfNull "create_iterator" $ case mcf of-        Just cf -> c_rocksdb_create_iterator_cf rocks_db read_opts cf-        Nothing -> c_rocksdb_create_iterator rocks_db read_opts+createIterator DB {rocksDB = rocks_db, readOpts = read_opts} mcf =+  throwErrnoIfNull "create_iterator" $ case mcf of+    Just cf -> c_rocksdb_create_iterator_cf rocks_db read_opts cf+    Nothing -> c_rocksdb_create_iterator rocks_db read_opts  -- | Manually create unmanaged iterator on a specific snapshot. --@@ -360,29 +423,29 @@ -- and 'destroyReadOpts' for the read options. -- -- For automatic resource management, prefer 'withIterSnap'.-createIteratorSnap-    :: DB-    -> Maybe Snapshot-    -> Maybe ColumnFamily-    -> IO (Iterator, Maybe ReadOpts)+createIteratorSnap ::+  DB ->+  Maybe Snapshot ->+  Maybe ColumnFamily ->+  IO (Iterator, Maybe ReadOpts) createIteratorSnap db msnap mcf = case msnap of-    Nothing -> (, Nothing) <$> createIterator db mcf-    Just snap -> do-        ro <- c_rocksdb_readoptions_create-        c_rocksdb_readoptions_set_snapshot ro snap-        it <- createIteratorWithOpts db ro mcf-        return (it, Just ro)+  Nothing -> (,Nothing) <$> createIterator db mcf+  Just snap -> do+    ro <- c_rocksdb_readoptions_create+    c_rocksdb_readoptions_set_snapshot ro snap+    it <- createIteratorWithOpts db ro mcf+    return (it, Just ro)  -- | Internal: create iterator with explicit ReadOpts.-createIteratorWithOpts-    :: DB-    -> ReadOpts-    -> Maybe ColumnFamily-    -> IO Iterator-createIteratorWithOpts DB{rocksDB = rocks_db} ro mcf =-    throwErrnoIfNull "create_iterator" $ case mcf of-        Just cf -> c_rocksdb_create_iterator_cf rocks_db ro cf-        Nothing -> c_rocksdb_create_iterator rocks_db ro+createIteratorWithOpts ::+  DB ->+  ReadOpts ->+  Maybe ColumnFamily ->+  IO Iterator+createIteratorWithOpts DB {rocksDB = rocks_db} ro mcf =+  throwErrnoIfNull "create_iterator" $ case mcf of+    Just cf -> c_rocksdb_create_iterator_cf rocks_db ro cf+    Nothing -> c_rocksdb_create_iterator rocks_db ro  -- | Destroy unmanaged iterator created with 'createIterator' or -- 'createIteratorSnap'. It must not be used afterwards.@@ -393,16 +456,16 @@ -- function returns /true/ iff the iterator is valid. iterValid :: Iterator -> IO Bool iterValid iter_ptr = do-    x <- c_rocksdb_iter_valid iter_ptr-    return (x /= 0)+  x <- c_rocksdb_iter_valid iter_ptr+  return (x /= 0)  -- | Position at the first key in the source that is at or past target. The -- iterator is /valid/ after this call iff the source contains an entry that -- comes at or past target. iterSeek :: Iterator -> ByteString -> IO () iterSeek iter_ptr key =-    BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->-        c_rocksdb_iter_seek iter_ptr key_ptr (intToCSize klen)+  BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) ->+    c_rocksdb_iter_seek iter_ptr key_ptr (intToCSize klen)  -- | Position at the first key in the source. The iterator is /valid/ after this -- call iff the source is not empty.@@ -425,8 +488,8 @@ -- which is the right trade for a Haskell binding. iterNext :: Iterator -> IO () iterNext it = do-    valid <- iterValid it-    when valid $ c_rocksdb_iter_next it+  valid <- iterValid it+  when valid $ c_rocksdb_iter_next it  -- | Moves to the previous entry in the source. After this call, 'iterValid' is -- /true/ iff the iterator was not positioned at the first entry in the source.@@ -436,8 +499,8 @@ -- 'iterFirst' or 'iterLast' instead. iterPrev :: Iterator -> IO () iterPrev it = do-    valid <- iterValid it-    when valid $ c_rocksdb_iter_prev it+  valid <- iterValid it+  when valid $ c_rocksdb_iter_prev it  -- | Return the key for the current entry if the iterator is currently -- positioned at an entry, ie. 'iterValid'.@@ -456,9 +519,9 @@ -- at an entry, ie. 'iterValid'. iterEntry :: Iterator -> IO (Maybe (ByteString, ByteString)) iterEntry it = do-    mkey <- iterKey it-    mval <- iterValue it-    return $ (,) <$> mkey <*> mval+  mkey <- iterKey it+  mval <- iterValue it+  return $ (,) <$> mkey <*> mval  -- | Check for errors --@@ -468,31 +531,230 @@ -- apart. Returns 'Nothing' if there was no error. iterGetError :: Iterator -> IO (Maybe ByteString) iterGetError iter_ptr = alloca $ \err_ptr -> do-    poke err_ptr nullPtr-    c_rocksdb_iter_get_error iter_ptr err_ptr-    err_str <- peek err_ptr-    if err_str == nullPtr-        then return Nothing-        else Just <$> BU.unsafePackMallocCString err_str+  poke err_ptr nullPtr+  c_rocksdb_iter_get_error iter_ptr err_ptr+  err_str <- peek err_ptr+  if err_str == nullPtr+    then return Nothing+    else Just <$> BU.unsafePackMallocCString err_str +-- | The next entries, at most as many as asked for, from the one the+-- iterator is on and going forwards. The iterator is left on the entry after+-- the last one returned, so that calling again continues the scan. --+-- Fewer entries than asked for means that the iterator reached the end, of+-- the data or of its bounds, and is no longer valid. A count of zero or less+-- returns nothing and does not move the iterator.+--+-- It returns what that many rounds of 'iterEntry' and 'iterNext' would, from+-- one foreign call for all of them; see the module header for what that+-- saves. Every entry is held in memory at once, twice for a moment, so ask+-- for no more than fit: to read a range of unknown size, call it in a loop,+-- or use 'iterChunk', which also takes a limit in bytes.+--+-- Throws an 'IOError' if the iterator stopped because of an error.+iterEntries :: Iterator -> Int -> IO [(ByteString, ByteString)]+iterEntries it n = fst <$> iterChunk it whole {chunkEntries = n}++-- | Like 'iterEntries', going backwards: the entry that the iterator is on+-- comes first and those before it follow, in descending order. The iterator+-- is left on the entry before the last one returned.+iterEntriesBack :: Iterator -> Int -> IO [(ByteString, ByteString)]+iterEntriesBack it n = fst <$> iterChunk it whole {chunkEntries = n, chunkBackwards = True}++-- | Like 'iterEntries', for the keys that start with a prefix: it also stops+-- at the first entry whose key does not, and leaves the iterator on that+-- entry, which is not returned. Fewer entries than asked for means that the+-- keys with the prefix have all been read.+--+-- It does not seek: send the iterator to the prefix with 'iterSeek' first.+--+-- @+-- -- the first hundred users+-- 'iterSeek' it "user:"+-- users <- 'iterEntriesPrefix' it "user:" 100+-- @+iterEntriesPrefix :: Iterator -> ByteString -> Int -> IO [(ByteString, ByteString)]+iterEntriesPrefix it prefix n =+  fst <$> iterChunk it whole {chunkEntries = n, chunkPrefix = prefix}++-- | Like 'iterEntriesPrefix', going backwards as 'iterEntriesBack' does. The+-- iterator has to be on the last key of the prefix to begin with: 'iterLast'+-- puts it there if its 'upperBound' is the 'prefixUpperBound' of the prefix.+iterEntriesPrefixBack :: Iterator -> ByteString -> Int -> IO [(ByteString, ByteString)]+iterEntriesPrefixBack it prefix n =+  fst <$> iterChunk it whole {chunkEntries = n, chunkPrefix = prefix, chunkBackwards = True}++-- | A chunk with no limit on its bytes and no prefix, for the functions that+-- promise a number of entries.+whole :: ChunkConfig+whole = def {chunkBytes = maxBound}++-- | What one call of 'iterChunk' reads: how much, which keys, and in which+-- direction. 'def' reads forwards, any key, up to 256 entries or 1 MiB;+-- override the fields you need:+--+-- @+-- def { 'chunkPrefix' = "user:" }+-- @+--+-- The two limits are there for different reasons. 'chunkEntries' spreads+-- the cost of the foreign call over many entries. In the benchmark of this+-- package, chunks of 16 take up to a fifth longer than chunks of 256, and chunks+-- of 1024 are no faster; a larger chunk only makes a reader that stops early+-- pay for more entries that it does not want. 'chunkBytes' bounds the+-- memory: a chunk is held twice for a moment, in the buffer that C fills and+-- in the 'ByteString's copied from it, and 256 values can be of any size.+data ChunkConfig = ChunkConfig+  { -- | the most entries to read. Zero or less reads nothing.+    chunkEntries :: !Int,+    -- | stop before the keys and values read take more bytes than this. The+    -- first entry is read whatever its size, so a chunk is only empty where+    -- there is nothing to read, and a scan gets past an entry that is+    -- larger than the limit.+    chunkBytes :: !Int,+    -- | read only entries whose key starts with these bytes, and stop at the+    -- first one that does not. Empty, the default, for any key.+    --+    -- The iterator still has to move to that first key to see that it is+    -- outside, as in a loop written by hand, and that can be slow where+    -- deleted keys follow the range: see the module header. An 'upperBound'+    -- or 'prefixSameAsStart' stops the iterator before it, and this prefix+    -- is then not needed, though it does no harm.+    chunkPrefix :: !ByteString,+    -- | go backwards, with 'iterPrev', and return the entries in descending+    -- order.+    chunkBackwards :: !Bool+  }+  deriving (Eq, Show)++-- | Forwards, any key, up to 256 entries or 1 MiB of keys and values.+instance Default ChunkConfig where+  def =+    ChunkConfig+      { chunkEntries = 256,+        chunkBytes = 1024 * 1024,+        chunkPrefix = B.empty,+        chunkBackwards = False+      }++-- | Read the next entries, as a t'ChunkConfig' says, from the one the+-- iterator is on, and tell whether more follow.+--+-- This is the function to build a scan on. The iterator is walked in C, in+-- one foreign call for the whole chunk, where a loop of 'iterEntry' and+-- 'iterNext' makes several for every entry; the module header has the+-- figures.+--+-- @+-- -- every entry whose key starts with "user:"+-- 'iterSeek' it "user:"+-- let go acc = do+--         (entries, more) <- 'iterChunk' it def{'chunkPrefix' = "user:"}+--         let acc' = entries : acc+--         if more then go acc' else return (concat (reverse acc'))+-- go []+-- @+--+-- The chunk ends at the first of: the end of the data; a bound of the+-- iterator, from 'upperBound', 'lowerBound' or 'prefixSameAsStart', which+-- all work as they do with 'iterNext' and 'iterPrev'; a key that does not+-- start with 'chunkPrefix'; 'chunkEntries' entries; 'chunkBytes' bytes.+--+-- __Where the iterator is left:__ on the first entry that was not returned,+-- or not valid if there is none. So the next call continues where this one+-- stopped, 'iterEntry' shows the entry that comes next, and after a chunk+-- that a prefix ended, the iterator is on the first key outside the prefix.+--+-- __The flag__ is 'True' if the iterator is left on an entry whose key+-- starts with 'chunkPrefix': the next call would return something. It is+-- 'False' when the scan is over, which saves the call that would find that+-- out by returning nothing.+--+-- The entries are ordinary 'ByteString's, each a copy of its own that the+-- garbage collector frees on its own.+--+-- Unlike the functions that move one step, this one throws an 'IOError' if+-- the iterator stopped because of an error, such as a corrupt block or a+-- failed read: there is no need to ask 'iterGetError' after it, and a scan+-- cannot take an error for the end of the data. It also throws if there is+-- no memory for a single entry.+iterChunk :: Iterator -> ChunkConfig -> IO ([(ByteString, ByteString)], Bool)+iterChunk it cfg =+  BU.unsafeUseAsCStringLen (chunkPrefix cfg) $ \(prefix_ptr, prefix_len) ->+    alloca $ \buffer_ptr ->+      alloca $ \count_ptr ->+        alloca $ \more_ptr ->+          -- Nothing may interrupt between the call that allocates the+          -- buffer and the handler that frees it.+          mask $ \restore -> do+            poke buffer_ptr nullPtr+            status <-+              throwIfErr "iterChunk" $+                c_rocksdb_haskell_iter_chunk+                  it+                  (if chunkBackwards cfg then 1 else 0)+                  (limit (chunkEntries cfg))+                  (limit (chunkBytes cfg))+                  prefix_ptr+                  (intToCSize prefix_len)+                  buffer_ptr+                  count_ptr+                  more_ptr+            buffer <- peek buffer_ptr+            flip finally (free buffer) $ do+              when (status /= 0) $+                throwIO (userError "iterChunk: no memory for an entry")+              count <- peek count_ptr+              more <- peek more_ptr+              entries <- restore (unpackChunk buffer (cSizeToInt count))+              return (entries, more /= 0)+  where+    -- a limit below zero is a limit of zero, and not a huge unsigned number+    limit = intToCSize . max 0++-- -- Internal -- +-- | Copy so many entries out of a buffer filled by+-- 'c_rocksdb_haskell_iter_chunk', in the order they are in. Each entry is+-- two lengths, the key, the value, and padding up to the size of a length;+-- see @cbits\/rocksdb_haskell.h@.+--+-- Every key and value becomes a 'ByteString' of its own. Slices of the one+-- buffer would save the copy, and keep the whole chunk alive for as long as+-- any one key of it is.+unpackChunk :: Ptr CChar -> Int -> IO [(ByteString, ByteString)]+unpackChunk buffer = go [] 0+  where+    word = sizeOf (0 :: CSize)+    go acc _ 0 = return (reverse acc)+    go acc offset n = do+      key_len <- cSizeToInt <$> peekByteOff buffer offset+      value_len <- cSizeToInt <$> peekByteOff buffer (offset + word)+      let key_ptr = buffer `plusPtr` (offset + 2 * word)+          value_ptr = key_ptr `plusPtr` key_len+          size = 2 * word + key_len + value_len+          padded = (size + word - 1) `div` word * word+      key <- B.packCStringLen (key_ptr, key_len)+      value <- B.packCStringLen (value_ptr, value_len)+      go ((key, value) : acc) (offset + padded) (n - 1 :: Int)+ -- | Read a C string from the iterator using the given accessor, if the -- iterator is valid. The string is copied, since it belongs to the iterator. iterString ::-    Iterator ->-    (Iterator -> Ptr CSize -> IO CString) ->-    IO (Maybe ByteString)+  Iterator ->+  (Iterator -> Ptr CSize -> IO CString) ->+  IO (Maybe ByteString) iterString it f = do-    valid <- iterValid it-    if valid-        then alloca $ \len_ptr -> do-            str_ptr <- f it len_ptr-            if str_ptr == nullPtr-                then return Nothing-                else do-                    len <- peek len_ptr-                    Just <$> B.packCStringLen (str_ptr, cSizeToInt len)-        else return Nothing+  valid <- iterValid it+  if valid+    then alloca $ \len_ptr -> do+      str_ptr <- f it len_ptr+      if str_ptr == nullPtr+        then return Nothing+        else do+          len <- peek len_ptr+          Just <$> B.packCStringLen (str_ptr, cSizeToInt len)+    else return Nothing
test/Spec.hs view
@@ -1,4 +1,6 @@+{-# LANGUAGE ForeignFunctionInterface #-} {-# LANGUAGE ImportQualifiedPost #-}+{-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TupleSections #-} @@ -26,17 +28,27 @@  import Control.Concurrent import Control.Concurrent.Async-import Control.Exception (bracket)+import Control.Exception (bracket, evaluate, try) import Control.Monad+import Data.Bits (shiftR) import Data.ByteString (ByteString) import Data.ByteString qualified as BS import Data.ByteString.Char8 qualified as C+import Data.ByteString.Internal qualified as BI import Data.Default (def)+import Data.List (isInfixOf, isPrefixOf, isSuffixOf) import Data.Map.Strict (Map) import Data.Map.Strict qualified as Map import Data.Maybe+import Data.Word (Word64, Word8) import Database.RocksDB+import Foreign.Marshal.Utils (copyBytes)+import Foreign.Ptr (FunPtr, Ptr, castPtr, plusPtr) import GHC.Clock (getMonotonicTime)+import System.Directory (listDirectory)+import System.FilePath ((</>))+import System.IO (IOMode (ReadWriteMode), SeekMode (AbsoluteSeek), hSeek, withBinaryFile)+import System.IO.Error (ioeGetErrorString) import System.IO.Temp import Test.Hspec import Test.Hspec.QuickCheck (modifyMaxSuccess, prop)@@ -48,6 +60,10 @@ main = hspec $ do   around (withTunedDB ["one", "two", "tree"]) tunedSpec   around withPlainDB plainSpec+  around withPlainDB chunkSpec+  chunkErrorSpec+  packedSpec+  mergeSpec   boundSpec   openingSpec   tuningSpec@@ -260,6 +276,14 @@       withIterConfig db Nothing def {prefixSameAsStart = True} $ \itr -> do         iterSeek itr "bbb2"         map fst <$> scan iterPrev itr `shouldReturn` ["bbb2", "bbb1"]+    it "stays within the prefix when read in chunks" $ \db -> do+      fill db+      withIterConfig db Nothing def {prefixSameAsStart = True} $ \itr -> do+        iterSeek itr "bbb"+        iterChunk itr def `shouldReturn` ([("bbb1", "bbb1"), ("bbb2", "bbb2")], False)+        iterValid itr `shouldReturn` False+        iterSeek itr "ccc"+        iterEntriesBack itr 10 `shouldReturn` [("ccc1", "ccc1")]     it "stays within the prefix when the data is in table files" $ \db -> do       fill db       -- enough writes to flush the keys out of memory, where the prefix@@ -597,6 +621,719 @@           forwards itr `shouldReturn` [("m", "changed"), ("n", "20")]  -------------------------------------------------------------------------------+-- Reading many entries in one call+-------------------------------------------------------------------------------++-- | The bytes that the keys and values of some entries take, which is what+-- 'chunkBytes' limits.+sizeOf :: [(ByteString, ByteString)] -> Int+sizeOf entries = sum [BS.length k + BS.length v | (k, v) <- entries]++-- | Read chunk after chunk until one says that no more follow, and give the+-- chunks. Fails if a chunk is empty when the one before promised more: the+-- loop would never end.+chunksOf :: Iterator -> ChunkConfig -> IO [[(ByteString, ByteString)]]+chunksOf itr cfg = go []+  where+    go acc = do+      (entries, more) <- iterChunk itr cfg+      if more+        then do+          entries `shouldNotBe` []+          go (entries : acc)+        else return (reverse (entries : acc))++chunkSpec :: SpecWith DB+chunkSpec = describe "Reading many entries in one call" $ do+  let keys = ["a1", "a2", "b1", "b2", "c1"] :: [ByteString]+      fill db = forM_ keys $ \k -> put db k k+      same = map (\k -> (k, k))+      -- ten entries of 108 bytes each: a key of 8 and a value of 100+      wide = [(key i, BS.replicate 100 (fromIntegral i)) | i <- [0 .. 9]]+  it "reads the next entries and leaves the iterator after them" $ \db -> do+    fill db+    withIter db $ \itr -> do+      iterFirst itr+      iterEntries itr 2 `shouldReturn` same ["a1", "a2"]+      iterKey itr `shouldReturn` Just "b1"+      iterEntries itr 100 `shouldReturn` same ["b1", "b2", "c1"]+      iterValid itr `shouldReturn` False+      iterEntries itr 100 `shouldReturn` []+  it "reads backwards, in descending order" $ \db -> do+    fill db+    withIter db $ \itr -> do+      iterLast itr+      iterEntriesBack itr 2 `shouldReturn` same ["c1", "b2"]+      iterKey itr `shouldReturn` Just "b1"+      iterEntriesBack itr 100 `shouldReturn` same ["b1", "a2", "a1"]+      iterValid itr `shouldReturn` False+  it "reads nothing, and stays, when asked for no entries" $ \db -> do+    fill db+    withIter db $ \itr -> do+      iterSeek itr "b1"+      iterEntries itr 0 `shouldReturn` []+      iterEntries itr (-3) `shouldReturn` []+      iterEntriesBack itr minBound `shouldReturn` []+      iterChunk itr def {chunkEntries = 0} `shouldReturn` ([], True)+      iterKey itr `shouldReturn` Just "b1"+  it "stops at the first key outside the prefix, and stays on it" $ \db -> do+    fill db+    withIter db $ \itr -> do+      iterSeek itr "b"+      iterEntriesPrefix itr "b" 100 `shouldReturn` same ["b1", "b2"]+      iterKey itr `shouldReturn` Just "c1"+      iterEntriesPrefix itr "b" 100 `shouldReturn` []+      iterKey itr `shouldReturn` Just "c1"+      iterSeek itr "b2"+      iterEntriesPrefixBack itr "b" 100 `shouldReturn` same ["b2", "b1"]+      iterKey itr `shouldReturn` Just "a2"+  it "takes a key equal to the prefix, and not one shorter than it" $ \db -> do+    forM_ ["a", "ab", "abc", "b"] $ \k -> put db k ""+    withIter db $ \itr -> do+      iterFirst itr+      iterEntriesPrefix itr "ab" 100 `shouldReturn` []+      iterSeek itr "ab"+      map fst <$> iterEntriesPrefix itr "ab" 100 `shouldReturn` ["ab", "abc"]+  it "tells whether more entries follow" $ \db -> do+    fill db+    withIter db $ \itr -> do+      let cfg = def {chunkPrefix = "a"}+      iterFirst itr+      iterChunk itr cfg {chunkEntries = 1} `shouldReturn` (same ["a1"], True)+      -- the limit is reached where the prefix ends: nothing follows+      iterFirst itr+      iterChunk itr cfg {chunkEntries = 2} `shouldReturn` (same ["a1", "a2"], False)+      iterChunk itr cfg `shouldReturn` ([], False)+      -- and where the data ends+      iterSeek itr "c1"+      iterChunk itr def {chunkEntries = 1} `shouldReturn` (same ["c1"], False)+  it "stops before the limit of bytes" $ \db -> do+    forM_ wide $ uncurry (put db)+    withIter db $ \itr -> do+      let sizes bytes = do+            iterFirst itr+            map length <$> chunksOf itr def {chunkBytes = bytes}+      -- two entries are 216 bytes+      sizes 250 `shouldReturn` [2, 2, 2, 2, 2]+      sizes 216 `shouldReturn` [2, 2, 2, 2, 2]+      sizes 215 `shouldReturn` [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]+      sizes 500 `shouldReturn` [4, 4, 2]+      sizes maxBound `shouldReturn` [10]+  it "reads one entry at least, whatever the limit of bytes" $ \db -> do+    forM_ wide $ uncurry (put db)+    withIter db $ \itr -> forM_ [0, 1, 107, -1, minBound] $ \bytes -> do+      iterFirst itr+      chunks <- chunksOf itr def {chunkBytes = bytes}+      chunks `shouldBe` map (: []) wide+  it "gets past an entry larger than the limit of bytes" $ \db -> do+    -- three megabytes between small entries, with a limit of one+    let big = BS.replicate (3 * 1024 * 1024) 7+        entries = [("a", "small"), ("b", big), ("c", "small"), ("d", "small")]+    forM_ entries $ uncurry (put db)+    withIter db $ \itr -> do+      iterFirst itr+      chunks <- chunksOf itr def+      map (map fst) chunks `shouldBe` [["a"], ["b"], ["c", "d"]]+      concat chunks `shouldBe` entries+      iterLast itr+      back <- chunksOf itr def {chunkBackwards = True}+      map (map fst) back `shouldBe` [["d", "c"], ["b"], ["a"]]+  it "reads across many chunks what single steps read" $ \db -> do+    let entries = [(key i, C.pack (show i)) | i <- [0 .. 9999]]+    write db (map (uncurry Put) entries)+    withIter db $ \itr -> forM_ [1, 7, 256, 10000, 20000] $ \n -> do+      iterFirst itr+      chunks <- chunksOf itr def {chunkEntries = n}+      concat chunks `shouldBe` entries+      map length (init chunks) `shouldSatisfy` all (== n)+      iterLast itr+      back <- chunksOf itr def {chunkEntries = n, chunkBackwards = True}+      concat back `shouldBe` reverse entries+  it "stops at the bounds of the iterator" $ \db -> do+    fill db+    withIterConfig db Nothing def {lowerBound = Just "a2", upperBound = Just "b2"} $ \itr -> do+      iterFirst itr+      iterChunk itr def `shouldReturn` (same ["a2", "b1"], False)+      iterValid itr `shouldReturn` False+      iterLast itr+      iterChunk itr def {chunkBackwards = True} `shouldReturn` (same ["b1", "a2"], False)+      iterValid itr `shouldReturn` False+  it "reads a prefix backwards from the bound that ends it" $ \db -> do+    fill db+    withIterConfig db Nothing def {upperBound = prefixUpperBound "a"} $ \itr -> do+      iterLast itr+      iterEntriesPrefixBack itr "a" 100 `shouldReturn` same ["a2", "a1"]+  it "returns nothing where there is nothing" $ \db -> do+    withIter db $ \itr -> do+      iterFirst itr+      iterChunk itr def `shouldReturn` ([], False)+      iterEntries itr 10 `shouldReturn` []+    fill db+    withIter db $ \itr -> do+      -- not sent anywhere yet+      iterChunk itr def `shouldReturn` ([], False)+      -- a prefix that no key has+      iterSeek itr "b3"+      iterChunk itr def {chunkPrefix = "b3"} `shouldReturn` ([], False)+      iterKey itr `shouldReturn` Just "c1"+      -- past the last key+      iterSeek itr "d"+      iterChunk itr def `shouldReturn` ([], False)+      iterChunk itr def {chunkBackwards = True} `shouldReturn` ([], False)+  it "keeps bytes that C takes for the end of a string, and empty keys and values" $ \db -> do+    let entries = [("", "empty key"), ("\0", ""), ("\0\0a", "\0b\0"), ("\0\255", "\255\0")]+    forM_ entries $ uncurry (put db)+    withIter db $ \itr -> do+      iterFirst itr+      iterEntries itr 100 `shouldReturn` entries+      iterSeek itr "\0"+      iterEntriesPrefix itr "\0\0" 100 `shouldReturn` []+      iterEntriesPrefix itr "\0" 100 `shouldReturn` drop 1 entries+  it "stays within its column family" $ \db -> do+    other <- firstFamily db+    fill db+    putCF db other "b9" "other"+    withIterCF db other $ \itr -> do+      iterFirst itr+      iterEntries itr 100 `shouldReturn` [("b9", "other")]+  it "returns entries that outlive the iterator" $ \db -> do+    let entries = [(key i, BS.replicate 50 (fromIntegral i)) | i <- [0 .. 999]]+    write db (map (uncurry Put) entries)+    got <- withIter db $ \itr -> iterFirst itr >> iterEntries itr 1000+    -- the heap is churned before the entries are looked at+    sum (map BS.length [key j | j <- [1 .. 20000]]) `shouldBe` 160000+    got `shouldBe` entries+  it "reads the same from many threads at once" $ \db -> do+    let entries = [(key i, C.pack (show i)) | i <- [0 .. 19999]]+    write db (map (uncurry Put) entries)+    results <- forConcurrently [1 .. 16 :: Int] $ \n ->+      withIter db $ \itr -> do+        iterFirst itr+        concat <$> chunksOf itr def {chunkEntries = n * 10}+    forM_ results (`shouldBe` entries)++-- | An iterator that meets a block it cannot read is no longer valid, which+-- is also how it looks at the end of the data. A scan in chunks has to tell+-- the two apart by itself, since its caller gets a list and nothing else.+chunkErrorSpec :: Spec+chunkErrorSpec = describe "Reading many entries from a damaged file" $ do+  it "throws, where single steps look as if the data had ended" $+    withTempPath $ \path -> do+      -- compacted, so that the entries are in a table file+      withDB path plain $ \db -> do+        forM_ [0 .. 999] $ \i -> put db (key i) (BS.replicate 100 (fromIntegral i))+        compactRange db Nothing Nothing Nothing+      tables <- filter (".sst" `isSuffixOf`) <$> listDirectory path+      tables `shouldNotBe` []+      -- garbage over the start of each file, where the first block of+      -- entries is: its checksum no longer matches+      forM_ tables $ \table ->+        withBinaryFile (path </> table) ReadWriteMode $ \h -> do+          hSeek h AbsoluteSeek 100+          BS.hPut h (BS.replicate 100 0xa5)+      withDB path plain $ \db -> withIter db $ \itr -> do+        iterFirst itr+        iterValid itr `shouldReturn` False+        iterGetError itr >>= (`shouldSatisfy` isJust)+        iterFirst itr+        iterChunk itr def `shouldThrow` anyIOException+        iterFirst itr+        iterEntries itr 10 `shouldThrow` anyIOException++-------------------------------------------------------------------------------+-- Packed batches+-------------------------------------------------------------------------------++-- | A write to a column family that is named by its number, so that the same+-- operation can be given to two databases: zero is the default family, and+-- the others are those of 'columnFamilies' from one. 'Nothing' deletes.+data FamilyOp = FamilyOp Int ByteString (Maybe ByteString)++-- | The operation of a batch for a database.+batchOp :: DB -> FamilyOp -> BatchOp+batchOp _ (FamilyOp 0 k (Just v)) = Put k v+batchOp _ (FamilyOp 0 k Nothing) = Del k+batchOp db (FamilyOp n k mv) = case drop (n - 1) (columnFamilies db) of+  cf : _ -> maybe (DelCF cf k) (PutCF cf k) mv+  [] -> error ("no column family " <> show n)++-- | Every entry of every column family of a database, the default one+-- first.+contents :: DB -> IO [[(ByteString, ByteString)]]+contents db = (:) <$> withIter db forwards <*> forM (columnFamilies db) (\cf -> withIterCF db cf forwards)++-- | Two new databases with the same three column families besides the+-- default: one to write with 'write', and one with 'writePacked'.+withTwoDBs :: (DB -> DB -> IO a) -> IO a+withTwoDBs go = new (new . go)+  where+    families = map (,plain) ["one", "two", "three"]+    new f = withTempPath $ \path -> withDBCF path plain {createMissingColumnFamilies = True} families f++-- | Require that the operations, cut into chunks of so many, leave what+-- they leave when they are written with 'write': in every column family.+shouldWriteAs :: Int -> [FamilyOp] -> IO ()+shouldWriteAs size ops = withTwoDBs $ \expected packed -> do+  write expected (map (batchOp expected) ops)+  writePacked packed (map packOps (groupsOf size (map (batchOp packed) ops)))+  written <- contents packed+  wanted <- contents expected+  map length written `shouldBe` map length wanted+  written `shouldBe` wanted++-- | A list in pieces of a length, the last one shorter.+groupsOf :: Int -> [a] -> [[a]]+groupsOf _ [] = []+groupsOf n xs = let (a, b) = splitAt n xs in a : groupsOf n b++-- | A chunk with one record, written as a program that packs its own+-- chunks does: room for the header, the key and the value after it, and+-- then the header. The record starts at an odd address, since records are+-- not aligned.+record :: Maybe ColumnFamily -> PackedOp -> ByteString -> ByteString -> ByteString+record mcf op k v =+  BS.drop 1 . BI.unsafeCreate (1 + size) $ \buffer -> do+    let start = buffer `plusPtr` 1 :: Ptr Word8+        key_ptr = start `plusPtr` packedHeaderSize+    copy k key_ptr+    copy v (key_ptr `plusPtr` BS.length k)+    pokePackedHeader start mcf op (BS.length k) (BS.length v)+  where+    size = packedHeaderSize + BS.length k + BS.length v+    copy bytes to =+      BS.useAsCStringLen bytes $ \(from, len) -> copyBytes to (castPtr from) len++-- | Replace the bytes at an offset.+patch :: Int -> [Word8] -> ByteString -> ByteString+patch at bytes whole = BS.take at whole <> BS.pack bytes <> BS.drop (at + length bytes) whole++-- | A batch that a small C function of the package hands to RocksDB, a few+-- thousand operations at a call. It has to do what 'write' does with the+-- same operations, and it reads a buffer that the caller made: so it must+-- not believe what the buffer says about its own lengths.+packedSpec :: Spec+packedSpec = describe "Packed batches" $ do+  let several =+        [ FamilyOp 0 "a" (Just "default"),+          FamilyOp 1 "a" (Just "one"),+          FamilyOp 2 "a" (Just "two"),+          FamilyOp 3 "a" (Just "three"),+          FamilyOp 0 "empty value" (Just ""),+          FamilyOp 2 "" (Just "empty key"),+          FamilyOp 3 "zero\0inside" (Just "\0\0"),+          FamilyOp 1 "gone" (Just "v"),+          FamilyOp 1 "gone" Nothing,+          FamilyOp 2 "never there" Nothing+        ]+      -- twenty thousand operations over four families: every key is set,+      -- every third one is set again and every seventh deleted+      many =+        concat+          [ [FamilyOp (i `mod` 4) (key i) (Just (C.pack (show i)))]+              <> [FamilyOp (i `mod` 4) (key i) (Just "again") | i `mod` 3 == 0]+              <> [FamilyOp (i `mod` 4) (key i) Nothing | i `mod` 7 == 0]+          | i <- [1 .. 14000]+          ]+  it "writes what the same operations written with write leave, in every column family" $ do+    shouldWriteAs (length several) several+    withTwoDBs $ \_ db -> do+      writePacked db [packOps (map (batchOp db) several)]+      contents db+        `shouldReturn` [ [("a", "default"), ("empty value", "")],+                         [("a", "one")],+                         [("", "empty key"), ("a", "two")],+                         [("a", "three"), ("zero\0inside", "\0\0")]+                       ]+  it "applies operations in order, within a chunk and from one chunk to the next" $ do+    let ops =+          [ FamilyOp 0 "k" (Just "first"),+            FamilyOp 0 "k" (Just "second"),+            FamilyOp 1 "gone" (Just "v"),+            FamilyOp 1 "gone" Nothing,+            FamilyOp 1 "back" Nothing,+            FamilyOp 1 "back" (Just "v")+          ]+    forM_ [1, 2, 3, 6] $ \size -> do+      shouldWriteAs size ops+      withTwoDBs $ \_ db -> do+        writePacked db (map packOps (groupsOf size (map (batchOp db) ops)))+        contents db `shouldReturn` [[("k", "second")], [("back", "v")], [], []]+  it "accepts no chunk, and empty chunks among the others" $+    withTwoDBs $ \_ db -> do+      writePacked db []+      writePacked db [BS.empty]+      contents db `shouldReturn` [[], [], [], []]+      writePacked db [BS.empty, packOps [Put "a" "1"], BS.empty, packOps [], packOps [Put "b" "2"]]+      contents db `shouldReturn` [[("a", "1"), ("b", "2")], [], [], []]+  it "writes a large batch, in one chunk and over many" $ do+    length many `shouldSatisfy` (> 20000)+    shouldWriteAs (length many) many+    shouldWriteAs 97 many+    shouldWriteAs 1 (take 3000 many)+  it "takes records that a program wrote into a buffer of its own" $+    withTwoDBs $ \_ db -> do+      other <- firstFamily db+      packedHeaderSize `shouldBe` BS.length (packOps [Put "" ""])+      record Nothing PackedPut "key" "value" `shouldBe` packOps [Put "key" "value"]+      record (Just other) PackedPut "key" "" `shouldBe` packOps [PutCF other "key" ""]+      record (Just other) PackedDel "key" "" `shouldBe` packOps [DelCF other "key"]+      put db "old" "v"+      writePacked+        db+        [ record Nothing PackedPut "a" "1" <> record (Just other) PackedPut "b" "2",+          record Nothing PackedDel "old" ""+        ]+      contents db `shouldReturn` [[("a", "1")], [("b", "2")], [], []]+  it "refuses a header that cannot be: a length that is negative, a delete with a value" $ do+    evaluate (record Nothing PackedPut "k" "v") `shouldReturn` packOps [Put "k" "v"]+    evaluate (record Nothing PackedDel "k" "v") `shouldThrow` anyIOException+    BI.create 64 (\p -> pokePackedHeader p Nothing PackedPut (-1) 0) `shouldThrow` anyIOException+    BI.create 64 (\p -> pokePackedHeader p Nothing PackedPut 0 (-1)) `shouldThrow` anyIOException+  describe "A chunk that is not made of whole records" $ do+    let good = packOps [Put "before" "v"]+        -- two records of 17 bytes of header, a key of 3 and a value of 5+        chunk = packOps [Put "aaa" "11111", Put "bbb" "22222"]+        size = BS.length chunk `div` 2+        operation = packedHeaderSize - 1+        key_len = packedHeaderSize - 9+        value_len = packedHeaderSize - 5+        -- the chunk is refused, by this package and not by RocksDB, with+        -- the number of its first bad record, and nothing of the batch is+        -- written: not the good chunks around it either+        refused n bad = withTwoDBs $ \_ db -> do+          writePacked db [good, bad, good] `shouldThrow` \e ->+            ("writePacked: chunk 1 " `isPrefixOf` ioeGetErrorString e)+              && ((": record " <> show (n :: Int) <> " is") `isInfixOf` ioeGetErrorString e)+          writePacked db [bad] `shouldThrow` anyIOException+          contents db `shouldReturn` [[], [], [], []]+          -- and the database is still good for a batch+          writePacked db [good]+          contents db `shouldReturn` [[("before", "v")], [], [], []]+    it "is written when nothing is wrong with it" $+      withTwoDBs $ \_ db -> do+        writePacked db [good, chunk, good]+        contents db `shouldReturn` [[("aaa", "11111"), ("bbb", "22222"), ("before", "v")], [], [], []]+    it "is refused when it ends within a value" $+      refused 1 (BS.init chunk)+    it "is refused when it ends within a key" $+      refused 1 (BS.take (size + packedHeaderSize + 1) chunk)+    it "is refused when it ends within a header, or right after one" $ do+      refused 1 (BS.take (size + 1) chunk)+      refused 1 (BS.take (size + packedHeaderSize - 1) chunk)+      refused 1 (BS.take (size + packedHeaderSize) chunk)+      refused 0 (BS.take 1 chunk)+    it "is refused when bytes follow its last record" $+      refused 2 (chunk <> "\0")+    it "is refused when a length says more than there is" $ do+      refused 1 (patch (size + key_len) [9, 0, 0, 0] chunk)+      refused 1 (patch (size + value_len) [6, 0, 0, 0] chunk)+      refused 0 (patch key_len [255, 255, 255, 255] chunk)+      refused 0 (patch value_len [255, 255, 255, 255] chunk)+      refused 0 (patch key_len [255, 255, 255, 255] (patch value_len [255, 255, 255, 255] chunk))+    it "is refused when it names an operation that does not exist" $ do+      refused 1 (patch (size + operation) [3] chunk)+      refused 0 (patch operation [255] chunk)+    it "is refused when a delete has a value" $+      refused 1 (patch (size + operation) [1] chunk)+  describe "A merge" $ do+    -- What a merge does to a value is for the tests of merge records. Here:+    -- that a packed record is taken for a merge and handed to RocksDB as+    -- one. RocksDB 11 says so itself, by refusing the batch for want of a+    -- merge operator. A RocksDB that took such a batch would have a merge+    -- record under the key, and no way to read it.+    it "is handed to RocksDB as a merge, in the default column family and in another" $+      withTwoDBs $ \_ db -> do+        other <- firstFamily db+        let asMerge chunks reading = do+              written <- try (writePacked db chunks)+              case written of+                Left e -> do+                  ioeGetErrorString e `shouldSatisfy` (not . ("writePacked" `isPrefixOf`))+                  ioeGetErrorString e `shouldSatisfy` ("erge" `isInfixOf`)+                  -- the merge was not written; the put that came before+                  -- it in the batch was not either with RocksDB 11, and+                  -- was with RocksDB 8.9+                  contents db >>= (`shouldSatisfy` (`elem` [[[], [], [], []], [[("a", "1")], [], [], []]]))+                Right () -> reading `shouldThrow` anyIOException+        asMerge [packOps [Put "a" "1"], record Nothing PackedMerge "k" "v"] (get db "k")+        asMerge [record (Just other) PackedMerge "k" "v", packOps [Put "a" "1"]] (getCF db other "k")+        record Nothing PackedMerge "k" "v" `shouldBe` packOps [Merge "k" "v"]+        record (Just other) PackedMerge "k" "v" `shouldBe` packOps [MergeCF other "k" "v"]++-------------------------------------------------------------------------------+-- Merge records and merge operators+-------------------------------------------------------------------------------++-- The merge operator of the tests, in @test\/cbits\/merge_test.c@: the+-- value of a key is the value it had and its merge records, oldest first,+-- joined with commas. The value shows which records were merged and in+-- which order, which a sum would not.+foreign import ccall "&rocksdb_haskell_test_join_full" joinFull :: FunPtr FullMerge++foreign import ccall "&rocksdb_haskell_test_join_partial" joinPartial :: FunPtr PartialMerge++foreign import ccall "&rocksdb_haskell_test_no_partial" noPartial :: FunPtr PartialMerge++-- | Join with commas.+joining :: MergeOperator+joining = MergeFunctions {mergeName = "test-join", fullMerge = joinFull, partialMerge = joinPartial}++-- | Join with commas, with a partial merge that always fails: RocksDB has+-- to keep every record until it has the value under them.+joiningWhole :: MergeOperator+joiningWhole = MergeFunctions {mergeName = "test-join-whole", fullMerge = joinFull, partialMerge = noPartial}++-- | A number as RocksDB's own operator for counters reads and writes it:+-- eight bytes, the least significant first.+counter :: Word64 -> ByteString+counter n = BS.pack [fromIntegral (n `shiftR` (8 * i)) | i <- [0 .. 7]]++-- | The configuration of a column family, or of the database, with a merge+-- operator. Nothing is compacted behind the back of a test, so that each+-- flush leaves a table file of its own and merge records stay where they+-- were written until the test compacts.+merging :: Maybe MergeOperator -> Config+merging op = plain {createMissingColumnFamilies = True, mergeOperator = op, disableAutoCompactions = True}++-- | The families of 'withMergeDB' besides the default.+mergeFamilies :: [(String, Config)]+mergeFamilies =+  [ ("count", merging (Just UInt64Add)),+    ("whole", merging (Just joiningWhole)),+    ("none", merging Nothing)+  ]++-- | A database whose default column family joins with commas, with three+-- more: one that counts, one that joins and cannot merge records without+-- the value, and one without a merge operator.+withMergeDB :: (DB -> ColumnFamily -> ColumnFamily -> ColumnFamily -> IO a) -> IO a+withMergeDB go = withTempPath $ \path -> openMergeDB path go++openMergeDB :: FilePath -> (DB -> ColumnFamily -> ColumnFamily -> ColumnFamily -> IO a) -> IO a+openMergeDB path go =+  withDBCF path (merging (Just joining)) mergeFamilies $ \db -> case columnFamilies db of+    [count, whole, none] -> go db count whole none+    cfs -> fail ("expected three column families, got " <> show (length cfs))++-- | Flush every column family of a database.+flushAll :: DB -> IO ()+flushAll db = flush db Nothing >> mapM_ (flush db . Just) (columnFamilies db)++-- | Compact every column family of a database, to its last level.+compactAll :: DB -> IO ()+compactAll db = do+  compactRange db Nothing Nothing Nothing+  forM_ (columnFamilies db) $ \cf -> compactRange db (Just cf) Nothing Nothing++-- | A merge record says how a value changes, and the merge operator of the+-- column family works out the value when it is read or compacted. What can+-- go wrong is the order of the records, what a put or a delete among them+-- hides, and a value that depends on where its parts happen to be: in the+-- memtable, in several table files, or compacted into one.+mergeSpec :: Spec+mergeSpec = describe "Merge records" $ do+  it "adds to a counter, from zero where the key has no value" $+    withMergeDB $ \db count _ _ -> do+      mergeCF db count "k" (counter 5)+      getCF db count "k" `shouldReturn` Just (counter 5)+      mergeCF db count "k" (counter 7)+      getCF db count "k" `shouldReturn` Just (counter 12)+      putCF db count "set" (counter 100)+      mergeCF db count "set" (counter 1)+      getCF db count "set" `shouldReturn` Just (counter 101)+      -- modulo 2^64, so a record can take away as well+      mergeCF db count "set" (counter (negate 101))+      getCF db count "set" `shouldReturn` Just (counter 0)+      getCF db count "absent" `shouldReturn` Nothing+  it "gives a custom operator the value and the records, oldest first" $+    withMergeDB $ \db _ _ _ -> do+      merge db "new" "a"+      get db "new" `shouldReturn` Just "a"+      merge db "new" "b"+      merge db "new" "c"+      get db "new" `shouldReturn` Just "a,b,c"+      put db "old" "x"+      merge db "old" "y"+      get db "old" `shouldReturn` Just "x,y"+      -- a value of no bytes is a value, and a record of no bytes a record+      put db "empty" ""+      merge db "empty" ""+      get db "empty" `shouldReturn` Just ","+  it "hides the records before a put or a delete, and applies those after" $+    withMergeDB $ \db _ _ _ -> do+      merge db "put" "a" >> put db "put" "x" >> merge db "put" "b"+      get db "put" `shouldReturn` Just "x,b"+      merge db "del" "a" >> delete db "del"+      get db "del" `shouldReturn` Nothing+      merge db "del" "b"+      get db "del" `shouldReturn` Just "b"+      put db "last" "x" >> merge db "last" "a" >> delete db "last"+      get db "last" `shouldReturn` Nothing+      -- and the same when each write is in a table file of its own+      forM_ [merge db "files" "a", put db "files" "x", merge db "files" "b", delete db "files", merge db "files" "c"] $+        \step -> step >> flush db Nothing+      get db "files" `shouldReturn` Just "c"+      merge db "files" "d"+      get db "files" `shouldReturn` Just "c,d"+  it "takes merges in a batch, among puts and deletes, written either way" $ do+    let ops count whole =+          [ Put "a" "1",+            Merge "a" "2",+            MergeCF count "n" (counter 3),+            Merge "b" "x",+            MergeCF whole "a" "w",+            Merge "a" "3",+            MergeCF count "n" (counter 4),+            Put "gone" "v",+            Merge "gone" "w",+            Del "gone",+            MergeCF whole "a" "v"+          ]+        wanted = [[("a", "1,2,3"), ("b", "x")], [("n", counter 7)], [("a", "w,v")], []]+    withMergeDB $ \db count whole _ -> do+      write db (ops count whole)+      contents db `shouldReturn` wanted+    withMergeDB $ \db count whole _ -> do+      writePacked db [packOps (ops count whole)]+      contents db `shouldReturn` wanted+    withMergeDB $ \db count whole _ -> do+      writePacked db (map packOps (groupsOf 2 (ops count whole)))+      contents db `shouldReturn` wanted+  it "gives the same values from the memtable, from table files and after a compaction" $+    withMergeDB $ \db count whole _ -> do+      let keys = ["both", "records", "value"] :: [ByteString]+          wanted =+            [ [("both", "x,a,b,c"), ("records", "a,b,c"), ("value", "x")],+              [("both", counter 16), ("records", counter 6), ("value", counter 10)],+              [("both", "x,a,b,c"), ("records", "a,b,c"), ("value", "x")],+              []+            ]+          -- every way of reading, in every family+          agree = do+            contents db `shouldReturn` wanted+            forM_ (zip (Nothing : map Just (columnFamilies db)) wanted) $ \(mcf, entries) -> do+              let values = map (`lookup` entries) keys+              case mcf of+                Nothing -> do+                  mapM (get db) keys `shouldReturn` values+                  multiGet db keys `shouldReturn` values+                  withIter db backwards `shouldReturn` reverse entries+                  withIter db (\itr -> iterFirst itr >> iterEntries itr 10) `shouldReturn` entries+                Just cf -> do+                  mapM (getCF db cf) keys `shouldReturn` values+                  multiGetCF db cf keys `shouldReturn` values+                  withIterCF db cf backwards `shouldReturn` reverse entries+                  withIterCF db cf (\itr -> iterFirst itr >> iterEntries itr 10) `shouldReturn` entries+          each k v n = do+            merge db k v+            mergeCF db whole k v+            mergeCF db count k (counter n)+      forM_ ["both", "value"] $ \k -> do+        put db k "x"+        putCF db whole k "x"+        putCF db count k (counter 10)+      flushAll db+      forM_ [("a", 1), ("b", 2)] $ \(v, n) -> do+        each "both" v n+        each "records" v n+        flushAll db+      -- the last record of each key is in the memtable, the others in a+      -- table file each, and the value in another+      each "both" "c" 3+      each "records" "c" 3+      agree+      withSnapshot db $ \snap -> do+        merge db "both" "later"+        get snap "both" `shouldReturn` Just "x,a,b,c"+        delete db "both" >> put db "both" "x,a,b,c"+      flushAll db+      agree+      compactAll db+      agree+      -- and what is merged after the compaction is merged with its result+      each "both" "d" 4+      get db "both" `shouldReturn` Just "x,a,b,c,d"+      getCF db whole "both" `shouldReturn` Just "x,a,b,c,d"+      getCF db count "both" `shouldReturn` Just (counter 20)+  it "keeps its values when the database is closed and opened again" $+    withTempPath $ \path -> do+      openMergeDB path $ \db count whole _ -> do+        put db "k" "x" >> merge db "k" "a"+        flushAll db+        -- these are in the write-ahead log only+        merge db "k" "b"+        mergeCF db count "n" (counter 2)+        mergeCF db whole "k" "w"+      openMergeDB path $ \db _ _ _ ->+        contents db `shouldReturn` [[("k", "x,a,b")], [("n", counter 2)], [("k", "w")], []]+  it "throws at the read of a key whose merge fails, and at no other" $+    withMergeDB $ \db _ _ _ -> do+      merge db "bad" "a" >> merge db "bad" "!" >> merge db "bad" "b"+      put db "good" "x" >> merge db "good" "y"+      get db "bad" `shouldThrow` anyIOException+      get db "good" `shouldReturn` Just "x,y"+      withIter db (\itr -> iterFirst itr >> iterEntries itr 10) `shouldThrow` anyIOException+      -- a put hides the record that cannot be merged+      put db "bad" "mended"+      get db "bad" `shouldReturn` Just "mended"+      withIter db forwards `shouldReturn` [("bad", "mended"), ("good", "x,y")]+  it "refuses a merge in a column family that has no merge operator" $+    withMergeDB $ \db _ _ none -> do+      mergeCF db none "k" "v" `shouldThrow` anyIOException+      write db [Put "a" "1", MergeCF none "k" "v"] `shouldThrow` anyIOException+      writePacked db [packOps [Put "a" "1", MergeCF none "k" "v"]] `shouldThrow` anyIOException+      -- the merge is nowhere; RocksDB 8.9 keeps the put that came before+      -- it in the refused batch, RocksDB 11 does not+      contents db >>= (`shouldSatisfy` (`elem` [[[], [], [], []], [[("a", "1")], [], [], []]]))+  -- A database is opened without the merge operator that its records need+  -- by a program that does not know of them. RocksDB opens it all the+  -- same. This is what RocksDB 11 does with records that are in table+  -- files, as far as it can be tried: a read of a key that has a value+  -- under its records, with 'get' or 'multiGet', crashes the process.+  it "cannot read merge records in table files without their operator, and keeps them" $+    withTempPath $ \path -> do+      let without = [(name, cfg {mergeOperator = Nothing}) | (name, cfg) <- mergeFamilies]+      openMergeDB path $ \db _ _ _ -> do+        put db "plain" "x"+        merge db "records" "a" >> merge db "records" "b"+        flushAll db+      withDBCF path (merging Nothing) without $ \db -> do+        get db "plain" `shouldReturn` Just "x"+        get db "records" `shouldThrow` anyIOException+        withIter db (\itr -> iterFirst itr >> iterEntries itr 10) `shouldThrow` anyIOException+        merge db "records" "c" `shouldThrow` anyIOException+        -- it can still be written, flushed and compacted+        put db "more" "y"+        flushAll db+        compactAll db+      openMergeDB path $ \db _ _ _ ->+        withIter db forwards `shouldReturn` [("more", "y"), ("plain", "x"), ("records", "a,b")]+  -- Records that are still in the write-ahead log fare worse. RocksDB 11+  -- drops them when it opens the database without the operator, and says+  -- nothing: they are lost for the program that has the operator too. It+  -- refuses to open a database without all its column families, and does+  -- so before it reads the log: a column family that the other program+  -- does not know of keeps the records safe.+  it "loses the merge records of the log to an open without their operator, unless the open is refused" $ do+    let without = [(name, cfg {mergeOperator = Nothing}) | (name, cfg) <- mergeFamilies]+        -- a value in a table file, and merge records that are in the log+        -- and in no table file when the database is closed+        records path = openMergeDB path $ \db _ _ _ -> do+          put db "value" "x"+          flush db Nothing+          merge db "value" "a" >> merge db "new" "b"+    withTempPath $ \path -> do+      records path+      withDBCF path (merging Nothing) without $ \_ -> return ()+      openMergeDB path $ \db _ _ _ ->+        withIter db forwards `shouldReturn` [("value", "x")]+    withTempPath $ \path -> do+      records path+      withDBCF path (merging Nothing) (take 2 without) (\_ -> return ()) `shouldThrow` anyIOException+      openMergeDB path $ \db _ _ _ ->+        withIter db forwards `shouldReturn` [("new", "b"), ("value", "x,a")]++------------------------------------------------------------------------------- -- Opening and closing ------------------------------------------------------------------------------- @@ -742,11 +1479,56 @@         multiGetCF db one (map key [1, 10000, 20000])           `shouldReturn` replicate 3 (Just value)         getProperty db BlockCacheUsage >>= (`shouldSatisfy` positive)+  it "loads a piece of the index and of the filter for a lookup, not all of them" $ do+    -- One table file of 200,000 keys has a filter of some 250 kB and an+    -- index of about as much. A lookup in a file that the cache does not+    -- know loads both whole, or two pieces of 4 KiB.+    let size = 64 * 1024 * 1024 :: Int+        whole =+          tuned+            { errorIfExists = False,+              bloomFilter = True,+              blockCacheSize = Just size,+              cacheIndexAndFilterBlocks = True+            }+        pieces = whole {partitionIndexAndFilters = True}+        value = C.replicate 20 'v'+        afterOneLookup cfg =+          withTempPath $ \path -> do+            withDBCF path cfg [("one", cfg)] $ \db -> do+              one <- firstFamily db+              forM_ [1 .. 200000] $ \i -> putCF db one (key i) value+            -- opening it writes what was in memory to a file of the first+            -- level, all of whose index and filter the cache keeps: the+            -- compaction moves it down, where they are loaded as needed+            withDBCF path cfg [("one", cfg)] $ \db -> do+              one <- firstFamily db+              compactRange db (Just one) Nothing Nothing+            withDBCF path cfg [("one", cfg)] $ \db -> do+              one <- firstFamily db+              getCF db one (key 100000) `shouldReturn` Just value+              usage <- getProperty db BlockCacheUsage+              return (maybe 0 (read . C.unpack) usage :: Int)+    held <- afterOneLookup whole+    heldInPieces <- afterOneLookup pieces+    putStrLn $ "block cache after one lookup: " <> show held <> " bytes whole, " <> show heldInPieces <> " in pieces"+    held `shouldSatisfy` (> 200 * 1024)+    heldInPieces `shouldSatisfy` (< held `div` 4)   it "opens with a bound on the write-ahead log" $     withTempPath $ \path ->       withDBCF path tuned {maxTotalWalSize = Just (1024 * 1024)} [("one", tuned)] $ \db -> do         put db "key" "value"         get db "key" `shouldReturn` Just "value"+  it "opens with the number of background jobs that it is given" $+    withTempPath $ \path -> do+      let setting = "max_background_jobs="+          -- RocksDB writes the options that it runs with to a file+          jobs = do+            files <- filter ("OPTIONS-" `isPrefixOf`) <$> listDirectory path+            concat <$> forM files (fmap (settings . lines) . readFile . (path </>))+          settings = filter (setting `isPrefixOf`) . map (dropWhile (== ' '))+      withDB path plain {maxBackgroundJobs = Just 5} $ \db -> put db "key" "value"+      jobs >>= (`shouldSatisfy` elem (setting <> "5"))   it "works with few open files and paranoid checks" $     withTempPath $ \path -> do       let strict =@@ -784,6 +1566,39 @@   n <- frequency [(1, return 0), (4, choose (1, 24))]   BS.pack <$> vectorOf n arbitrary +-- | A prefix that keys of 'genKey' often start with, and often the empty+-- one, which every key starts with.+genPrefix :: Gen ByteString+genPrefix = do+  n <- frequency [(2, return 0), (2, return 1), (1, return 2)]+  BS.pack <$> vectorOf n (elements [0, 1, 97, 255])++-- | Chunks small enough that a scan of a few entries takes several, with a+-- limit of bytes that is often below the size of one entry: 'genKey' and+-- 'genValue' give entries of up to 27 bytes.+genChunk :: Gen ChunkConfig+genChunk = do+  entries <- choose (1, 5)+  bytes <- frequency [(1, return 0), (3, choose (1, 60)), (1, return maxBound)]+  prefix <- genPrefix+  back <- arbitrary+  return+    ChunkConfig+      { chunkEntries = entries,+        chunkBytes = bytes,+        chunkPrefix = prefix,+        chunkBackwards = back+      }++-- | An iterator with no bound, one, or two that leave a range between them.+genBounds :: Gen IterConfig+genBounds = do+  a <- genKey+  b <- genKey+  lower <- elements [Nothing, Just (min a b)]+  upper <- elements [Nothing, Just (max a b)]+  return def {lowerBound = lower, upperBound = upper}+ genOps :: Gen [Op] genOps = listOf $ frequency [(3, Set <$> genKey <*> genValue), (1, Unset <$> genKey)] @@ -795,6 +1610,43 @@ opKey (Set k _) = k opKey (Unset k) = k +-- | A step of a history with merge records: a write, a merge record for the+-- operator that joins with commas, or a flush of the memtable to a table+-- file of its own.+data Step+  = Write Op+  | Join ByteString ByteString+  | Flush+  deriving (Show)++stepKey :: Step -> Maybe ByteString+stepKey (Write op) = Just (opKey op)+stepKey (Join k _) = Just k+stepKey Flush = Nothing++-- | Histories in which keys often have several merge records, with puts,+-- deletes and flushes among them. The records are letters: the one byte+-- that makes the operator of the tests fail is not among them.+genMergeOps :: Gen [Step]+genMergeOps =+  listOf . frequency $+    [ (2, Write <$> (Set <$> genKey <*> genValue)),+      (1, Write . Unset <$> genKey),+      (5, Join <$> genKey <*> (BS.pack <$> listOf (elements [97 .. 122]))),+      (2, return Flush)+    ]++-- | What the database should hold after a history, with the operator that+-- joins with commas: a record on a key without a value is the value, and+-- on a key with one it follows a comma.+joinModel :: [Step] -> Map ByteString ByteString+joinModel = foldl apply Map.empty+  where+    apply m (Write (Set k v)) = Map.insert k v m+    apply m (Write (Unset k)) = Map.delete k m+    apply m (Join k v) = Map.insertWith (\new old -> old <> "," <> new) k v m+    apply m Flush = m+ -- | What the database should hold after the operations, in order. model :: [Op] -> Map ByteString ByteString model = foldl apply Map.empty@@ -840,6 +1692,39 @@     forAllOps $ \ops -> inNewDB $ \db -> do       write db (batchOf ops)       shouldHold db (map opKey ops) (model ops)+  prop "after one packed batch, however it is cut into chunks" $+    forAllOps $ \ops -> forAll (choose (1, 5)) $ \size -> inNewDB $ \db -> do+      writePacked db (map packOps (groupsOf size (batchOf ops)))+      shouldHold db (map opKey ops) (model ops)+  prop "with merge records among the writes, in memtables, in table files and compacted" $+    forAll genMergeOps $ \steps -> ioProperty $ withTempPath $ \path ->+      withDB path (merging (Just joining)) $ \db -> do+        let keys = [k | Just k <- map stepKey steps]+            wanted = joinModel steps+        forM_ steps $ \case+          Write (Set k v) -> put db k v+          Write (Unset k) -> delete db k+          Join k v -> merge db k v+          Flush -> flush db Nothing+        shouldHold db keys wanted+        flush db Nothing+        shouldHold db keys wanted+        compactRange db Nothing Nothing Nothing+        shouldHold db keys wanted+  prop "with merge records in one batch, plain or packed" $+    forAll genMergeOps $ \steps -> forAll arbitrary $ \packed -> ioProperty $ withTempPath $ \path ->+      withDB path (merging (Just joining)) $ \db -> do+        let batch =+              [ op+              | step <- steps,+                op <- case step of+                  Write (Set k v) -> [Put k v]+                  Write (Unset k) -> [Del k]+                  Join k v -> [Merge k v]+                  Flush -> []+              ]+        if packed then writePacked db (map packOps (groupsOf 3 batch)) else write db batch+        shouldHold db [k | Just k <- map stepKey steps] (joinModel steps)   prop "after the database is closed and opened again" $     forAllOps $ \ops -> ioProperty $ withTempPath $ \path -> do       withDB path plain $ \db -> runOps db ops@@ -861,6 +1746,58 @@       withIter db $ \itr -> forM_ targets $ \target -> do         iterSeek itr target         iterEntry itr `shouldReturn` Map.lookupGE target (model ops)+  prop "in chunks, as one step at a time" $+    forAllOps $ \ops -> forAll genChunk $ \cfg -> forAll genBounds $ \bounds ->+      forAll genKey $ \start -> inNewDB $ \db -> do+        runOps db ops+        withIterConfig db Nothing bounds $ \itr -> do+          let step = if chunkBackwards cfg then iterPrev else iterNext+              place = do+                iterSeek itr start+                valid <- iterValid itr+                when (chunkBackwards cfg && not valid) (iterLast itr)+          -- what single steps read from there to the end, and the part of+          -- it that the chunks are to return+          place+          rest <- scan step itr+          let wanted = takeWhile ((chunkPrefix cfg `BS.isPrefixOf`) . fst) rest+          place+          let go done = do+                (entries, more) <- iterChunk itr cfg+                let done' = done + length entries+                    left = drop done' wanted+                -- the entries that come next, within both limits+                entries `shouldBe` take (length entries) (drop done wanted)+                length entries `shouldSatisfy` (<= chunkEntries cfg)+                when (length entries > 1) $+                  sizeOf entries `shouldSatisfy` (<= chunkBytes cfg)+                -- and not fewer than the limits allow+                when (length entries < chunkEntries cfg) $+                  sizeOf (entries <> take 1 left) `shouldSatisfy` (\n -> null left || n > chunkBytes cfg)+                -- the flag says whether anything is left+                more `shouldBe` not (null left)+                -- the iterator is on the first entry that was not returned+                iterEntry itr `shouldReturn` listToMaybe (drop done' rest)+                when more $ do+                  entries `shouldNotBe` []+                  go done'+          go 0+          iterGetError itr `shouldReturn` Nothing+  prop "a number of entries at once, as one step at a time" $+    forAllOps $ \ops -> forAll (choose (0, 8)) $ \n -> forAll arbitrary $ \back ->+      forAll genPrefix $ \prefix -> forAll genKey $ \start -> inNewDB $ \db -> do+        runOps db ops+        withIter db $ \itr -> do+          let step = if back then iterPrev else iterNext+          iterSeek itr start+          rest <- scan step itr+          iterSeek itr start+          (if back then iterEntriesBack else iterEntries) itr n `shouldReturn` take n rest+          iterEntry itr `shouldReturn` listToMaybe (drop n rest)+          let wanted = take n (takeWhile ((prefix `BS.isPrefixOf`) . fst) rest)+          iterSeek itr start+          (if back then iterEntriesPrefixBack else iterEntriesPrefix) itr prefix n `shouldReturn` wanted+          iterEntry itr `shouldReturn` listToMaybe (drop (length wanted) rest)   prop "through a snapshot, whatever is written afterwards" $     forAllOps $ \earlier -> forAllOps $ \later -> inNewDB $ \db -> do       let keys = map opKey (earlier <> later)
+ test/cbits/merge_test.c view
@@ -0,0 +1,134 @@+/*+ * A merge operator for the tests of rocksdb-haskell-jprupp: it joins byte+ * strings with commas.+ *+ * The value of a key is the value it had, if any, then its merge records+ * from the oldest to the newest, with a comma between each two. So the+ * value shows which records were merged and in which order, which a sum+ * would not. A record of the single byte '!' makes a merge fail, to see+ * what RocksDB does then.+ *+ * The functions have the types of rocksdb_haskell_full_merge_t and+ * rocksdb_haskell_partial_merge_t in cbits/rocksdb_haskell.h.+ */++#include <stddef.h>+#include <stdlib.h>+#include <string.h>++/* Report a failure the way the header says to. */+static char *fail(unsigned char *success, size_t *new_value_length)+{+    *success = 0;+    *new_value_length = 0;+    return NULL;+}++/*+ * Join a first byte string, if there is one, and some more, with commas.+ */+static char *join(+    const char *first,+    size_t first_length,+    const char *const *list,+    const size_t *lengths,+    int count,+    unsigned char *success,+    size_t *new_value_length)+{+    size_t size = first == NULL ? 0 : first_length;+    size_t used = 0;+    char *joined;+    int i;++    for (i = 0; i < count; i++) {+        if (lengths[i] == 1 && list[i][0] == '!') {+            return fail(success, new_value_length);+        }+        /* a comma before every part but the first */+        size += lengths[i] + (first != NULL || i > 0 ? 1 : 0);+    }+    /* malloc may return NULL for no bytes, which is not a failure here */+    joined = malloc(size > 0 ? size : 1);+    if (joined == NULL) {+        return fail(success, new_value_length);+    }+    if (first != NULL && first_length > 0) {+        memcpy(joined, first, first_length);+        used = first_length;+    }+    for (i = 0; i < count; i++) {+        if (first != NULL || i > 0) {+            joined[used] = ',';+            used += 1;+        }+        if (lengths[i] > 0) {+            memcpy(joined + used, list[i], lengths[i]);+            used += lengths[i];+        }+    }+    *success = 1;+    *new_value_length = used;+    return joined;+}++char *rocksdb_haskell_test_join_full(+    void *state,+    const char *key,+    size_t key_length,+    const char *existing_value,+    size_t existing_value_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length)+{+    (void)state;+    (void)key;+    (void)key_length;+    return join(+        existing_value,+        existing_value_length,+        operands_list,+        operands_list_length,+        num_operands,+        success,+        new_value_length);+}++char *rocksdb_haskell_test_join_partial(+    void *state,+    const char *key,+    size_t key_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length)+{+    (void)state;+    (void)key;+    (void)key_length;+    return join(NULL, 0, operands_list, operands_list_length, num_operands, success, new_value_length);+}++/* A partial merge that always fails: RocksDB then keeps the records. */+char *rocksdb_haskell_test_no_partial(+    void *state,+    const char *key,+    size_t key_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length)+{+    (void)state;+    (void)key;+    (void)key_length;+    (void)operands_list;+    (void)operands_list_length;+    (void)num_operands;+    return fail(success, new_value_length);+}