diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,3 @@
+1.0.0
+
+- Initial release
diff --git a/LICENSE b/LICENSE
new file mode 100644
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,30 @@
+Copyright (c) 2024, Gabriella Gonzalez
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of Gabriella Gonzalez nor the names of other
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/README.md b/README.md
new file mode 100644
--- /dev/null
+++ b/README.md
@@ -0,0 +1,29 @@
+# `tiktoken`
+
+This is a Haskell implementation of
+[`tiktoken`](https://github.com/openai/tiktoken), but just the tokenization
+logic.  In other words, given an existing encoding (like `cl100k_base`) you
+can tokenize a string (into smaller strings or token ranks).
+
+This means that you can't (yet) use this package to create your own new
+encodings, but you can use it to consume encodings.  In particular, this comes
+in handy for prompt engineering where you want to use as much of the available
+prompt tokens as possible (which requires accurately counting tokens).
+
+Encoding speed is ≈2.6-3.1 MB/s on an M1 MacBook Pro (using only one core since
+this package does not yet support parallel tokenization):
+
+```
+All
+  Encode 10 MB of Wikipedia
+    r50k_base:   OK (23.88s)
+      3.356 s ± 151 ms
+    p50k_base:   OK (10.39s)
+      3.445 s ±  31 ms
+    p50k_edit:   OK (11.13s)
+      3.693 s ± 240 ms
+    cl100k_base: OK (11.16s)
+      3.685 s ± 143 ms
+    o200k_base:  OK (11.01s)
+      3.648 s ± 134 ms
+```
diff --git a/benchmark/Main.hs b/benchmark/Main.hs
new file mode 100644
--- /dev/null
+++ b/benchmark/Main.hs
@@ -0,0 +1,32 @@
+{-# LANGUAGE BlockArguments #-}
+
+module Main where
+
+import System.FilePath ((</>))
+import Test.Tasty.Bench (Benchmark)
+import Tiktoken (Encoding)
+
+import qualified Control.DeepSeq as DeepSeq
+import qualified Control.Exception as Exception
+import qualified Data.ByteString as ByteString
+import qualified Test.Tasty.Bench as Bench
+import qualified Tiktoken
+
+main :: IO ()
+main = do
+    Bench.defaultMain
+        [ Bench.env (ByteString.readFile "tasty/sample.txt") \text -> do
+            let benchmark name encoding = do
+                    let loadEncoding = Exception.evaluate (DeepSeq.force encoding)
+
+                    Bench.env loadEncoding \encoding -> do
+                        Bench.bench name (Bench.nf (Tiktoken.toTokens encoding) text)
+
+            Bench.bgroup "Encode 10 MiB of Wikipedia"
+                [ benchmark "r50k_base" Tiktoken.r50k_base
+                , benchmark "p50k_base" Tiktoken.p50k_base
+                , benchmark "p50k_edit" Tiktoken.p50k_edit
+                , benchmark "cl100k_base" Tiktoken.cl100k_base
+                , benchmark "o200k_base" Tiktoken.o200k_base
+                ]
+        ]
diff --git a/src/Tiktoken.hs b/src/Tiktoken.hs
new file mode 100644
--- /dev/null
+++ b/src/Tiktoken.hs
@@ -0,0 +1,534 @@
+{-# LANGUAGE BlockArguments        #-}
+{-# LANGUAGE DeriveAnyClass        #-}
+{-# LANGUAGE DeriveGeneric         #-}
+{-# LANGUAGE DerivingStrategies    #-}
+{-# LANGUAGE DuplicateRecordFields #-}
+{-# LANGUAGE NamedFieldPuns        #-}
+{-# LANGUAGE NumericUnderscores    #-}
+{-# LANGUAGE OverloadedStrings     #-}
+{-# LANGUAGE OverloadedLists       #-}
+{-# LANGUAGE QuasiQuotes           #-}
+{-# LANGUAGE RecordWildCards       #-}
+
+-- | You can use this module to convert back and forth between a `ByteString`
+--   and its corresponding tokens using an existing encoding like @cl100k_base@
+--   or @o200k_base@
+--
+--   Example usage:
+--
+-- @
+-- {-# LANGUAGE OverloadedStrings #-}
+--
+-- import "Tiktoken" (`o200k_base`, toTokens, toRanks)
+--
+-- main :: `IO` ()
+-- main = do
+--     -- `Just` [\"El\",\" perro\",\" come\",\" las\",\" man\",\"z\",\"anas\"]
+--     `print` (`toTokens` `o200k_base` \"El perro come las manzanas\")
+--
+--     -- `Just` [4422,96439,3063,1996,873,89,14457]
+--     `print` (`toRanks` `o200k_base` \"El perro come las manzanas\")
+-- @
+module Tiktoken
+    ( -- * Encoding
+      Encoding
+    , tiktokenToEncoding
+    , addSpecialTokens
+
+      -- * Stock Encodings
+    , r50k_base
+    , p50k_base
+    , p50k_edit
+    , cl100k_base
+    , o200k_base
+
+      -- * Tokenization
+    , toTokens
+    , toRanks
+    , toTokensAndRanks
+
+      -- * Detokenization
+    , fromTokens
+    , fromRanks
+    ) where
+
+import Control.Applicative ((<|>))
+import Control.DeepSeq (NFData)
+import Control.Monad.ST (ST)
+import Control.Monad.Trans.Class (lift)
+import Data.ByteString (ByteString)
+import Data.Function (on)
+import Data.HashMap.Strict (HashMap)
+import Data.IntMap (IntMap, Key)
+import Data.List.NonEmpty (NonEmpty(..))
+import Data.Map (Map)
+import Data.Text (Text)
+import Data.Vector (MVector, Vector, (!?))
+import Data.Void (Void)
+import Data.Word (Word8)
+import GHC.Generics (Generic)
+import System.FilePath ((</>))
+import Text.Megaparsec (ParseErrorBundle, ParsecT)
+import Text.RawString.QQ (r)
+
+import qualified Control.Exception as Exception
+import qualified Data.ByteString as ByteString
+import qualified Data.ByteString.Base64 as Base64.Encoding
+import qualified Data.ByteString.Char8 as Char8
+import qualified Data.HashMap.Strict as HashMap
+import qualified Data.IntMap.Strict as IntMap
+import qualified Data.List as List
+import qualified Data.List.NonEmpty as NonEmpty
+import qualified Data.Map as Map
+import qualified Data.Ord as Ord
+import qualified Data.Text as Text
+import qualified Data.Text.Encoding as Text.Encoding
+import qualified Data.Text.IO as Text.IO
+import qualified Data.Vector as Vector
+import qualified Data.Vector.Mutable as Vector.Mutable
+import qualified Paths_tiktoken as Paths
+import qualified System.IO.Unsafe as Unsafe
+import qualified Text.Megaparsec as Megaparsec
+import qualified Text.Megaparsec.Char as Megaparsec.Char
+import qualified Text.Regex.PCRE.Light as Regex
+
+{-| This is an efficient internal representation of an encoding like
+    @cl100k_base@, @p50k_edit@, or @o200k_base@
+-}
+data Encoding = Encoding
+    { encode :: HashMap ByteString Int
+    , decode :: Vector ByteString
+    , specialTokens :: Map ByteString Int
+    , regex :: ByteString
+    } deriving stock (Generic)
+      deriving anyclass (NFData)
+
+parseToken :: ParsecT Void Text m ByteString
+parseToken = do
+    base64Text <- Megaparsec.takeWhileP (Just "Base64 character") (/= ' ')
+
+    let base64Bytes = Text.Encoding.encodeUtf8 base64Text
+
+    token <- case Base64.Encoding.decodeBase64Untyped base64Bytes of
+        Left text -> fail (Text.unpack text)
+        Right token -> return token
+
+    -- We don't bother parsing the token ID because the tokens are always stored
+    -- in sequential order by token ID.  We could *not* assume this but this
+    -- would not only make the parsing slower but it would also require using
+    -- a `HashMap` instead of a `Vector` to handle potential gaps in the token
+    -- ID sequence.  It's much more efficient to make this simplifying
+    -- assumption.
+
+    _ <- Megaparsec.takeWhileP (Just "Base64 character") (/= '\n')
+
+    _ <- Megaparsec.Char.char '\n'
+
+    return token
+
+parseDecode :: ParsecT Void Text (ST s) (MVector s ByteString)
+parseDecode = do
+    -- 100,000 is the size of the largest commonly-used encoding at the time of
+    -- this writing (`cl100k_base`) and it's not that expensive to pre-allocate
+    -- a `Vector` that big, so let's go wild and start with a large allocation.
+    let initialSize = 100_000
+
+    initialVector <- lift (Vector.Mutable.new initialSize)
+
+    let loop index vector
+            | index < size = do
+                let success = do
+                        token <- parseToken
+
+                        lift (Vector.Mutable.write vector index token)
+
+                        loop (index + 1) vector
+
+                let failure = do
+                        return (Vector.Mutable.take index vector)
+
+                success <|> failure
+                
+            | otherwise = do
+                largerVector <- lift (Vector.Mutable.grow vector size)
+
+                loop index largerVector
+          where
+            size = Vector.Mutable.length vector
+
+    loop 0 initialVector
+
+-- | Create an `Encoding` from regular expression and an ordered set of tokens
+tokensToEncoding
+    :: ByteString
+    -- ^ Regular expression used for coarse-grained splitting of the input
+    -> Vector ByteString
+    -- ^ The tokens in sequential order of their token IDs
+    -> Encoding
+tokensToEncoding regex decode = Encoding{..}
+  where
+    encode = HashMap.fromList (Vector.toList (Vector.imap adapt decode))
+      where
+        adapt index token = (token, index)
+
+    specialTokens = mempty
+
+-- | Parse an encoding from the @.tiktoken@ file format
+tiktokenToEncoding
+    :: ByteString
+    -- ^ Regular expression used for coarse-grained splitting of the input
+    -> Text
+    -- ^ The contents of the @.tiktoken@ file
+    -> Either (ParseErrorBundle Text Void) Encoding
+tiktokenToEncoding regex text =
+    fmap (tokensToEncoding regex)
+        (Vector.createT (Megaparsec.runParserT parseDecode "" text))
+
+-- | Add special tokens to a base `Encoding`
+addSpecialTokens :: Map ByteString Int -> Encoding -> Encoding
+addSpecialTokens tokens Encoding{ specialTokens = oldSpecialTokens, .. } =
+    Encoding{ specialTokens = Map.union tokens oldSpecialTokens, .. }
+
+_ENDOFTEXT :: ByteString
+_ENDOFTEXT = "<|endoftext|>"
+
+_FIM_PREFIX :: ByteString
+_FIM_PREFIX = "<|fim_prefix|>"
+
+_FIM_MIDDLE :: ByteString
+_FIM_MIDDLE = "<|fim_middle|>"
+
+_FIM_SUFFIX :: ByteString
+_FIM_SUFFIX = "<|fim_suffix|>"
+
+_ENDOFPROMPT :: ByteString
+_ENDOFPROMPT = "<|endofprompt|>"
+
+loadEncoding :: FilePath -> ByteString -> Map ByteString Int -> IO Encoding
+loadEncoding file regex specialTokens = do
+    dataDirectory <- Paths.getDataDir
+
+    text <- Text.IO.readFile (dataDirectory </> file)
+
+    encoding <- case tiktokenToEncoding regex text of
+        Left exception -> Exception.throwIO exception
+        Right encoding -> return encoding
+
+    return (addSpecialTokens specialTokens encoding)
+
+-- | @r50k_base@ `Encoding`
+r50k_base :: Encoding
+r50k_base =
+    Unsafe.unsafePerformIO
+        (loadEncoding "r50k_base.tiktoken" regex [ (_ENDOFTEXT, 50256) ])
+  where
+    regex =
+        [r|'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+|]
+{-# NOINLINE r50k_base #-}
+
+-- | @p50k_base@ `Encoding`
+p50k_base :: Encoding
+p50k_base =
+    Unsafe.unsafePerformIO
+        (loadEncoding "p50k_base.tiktoken" regex [ (_ENDOFTEXT, 50256) ])
+  where
+    regex =
+        [r|'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+|]
+{-# NOINLINE p50k_base #-}
+
+-- | @p50k_edit@ `Encoding`
+p50k_edit :: Encoding
+p50k_edit =
+    Unsafe.unsafePerformIO
+        (loadEncoding "p50k_base.tiktoken"
+            regex
+            [ (_ENDOFTEXT , 50256)
+            , (_FIM_PREFIX, 50281)
+            , (_FIM_MIDDLE, 50282)
+            , (_FIM_SUFFIX, 50283)
+            ] 
+        )
+  where
+    regex =
+        [r|'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+|]
+{-# NOINLINE p50k_edit #-}
+
+-- | @cl100k_base@ `Encoding`
+cl100k_base :: Encoding
+cl100k_base =
+    Unsafe.unsafePerformIO
+        (loadEncoding "cl100k_base.tiktoken"
+            regex
+            [ (_ENDOFTEXT  , 100257)
+            , (_FIM_PREFIX , 100258)
+            , (_FIM_MIDDLE , 100259)
+            , (_FIM_SUFFIX , 100260)
+            , (_ENDOFPROMPT, 100276)
+            ]
+        )
+  where
+    regex =
+        [r|'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+|]
+{-# NOINLINE cl100k_base #-}
+
+-- | @o200k_base@ `Encoding`
+o200k_base :: Encoding
+o200k_base =
+    Unsafe.unsafePerformIO
+        (loadEncoding "o200k_base.tiktoken"
+            regex
+            [ (_ENDOFTEXT  , 199999)
+            , (_ENDOFPROMPT, 200018)
+            ]
+        )
+  where
+    regex =
+        Char8.intercalate "|"
+            [ [r|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|]
+            , [r|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|]
+            , [r|\p{N}{1,3}|]
+            , [r| ?[^\s\p{L}\p{N}]+[\r\n/]*|]
+            , [r|\s*[\r\n]+|]
+            , [r|\s+(?!\S)|]
+            , [r|\s+|]
+            ]
+{-# NOINLINE o200k_base #-}
+
+minimumBy :: (a -> a -> Ordering) -> IntMap a -> Maybe (Int, a)
+minimumBy comparison intMap
+    | IntMap.null intMap =
+        Nothing
+    | otherwise =
+        Just (List.minimumBy (comparison `on` snd) (IntMap.toList intMap))
+
+drop1 :: [a] -> [a]
+drop1 (_ : xs) = xs
+drop1      []  = []
+
+{-| This is basically the same thing as `Maybe Int` except with an `Ord`
+    instance that treats `Ranked` values as less than `Unranked` values
+-}
+data Ranked = Ranked Int | Unranked
+    deriving (Eq, Ord)
+
+data Chunk = Chunk
+    { rank  :: Int
+      -- ^ Rank of this chunk
+    , rank2 :: Ranked
+      -- ^ Rank of this chunk combined with the next chunk
+    }
+
+{-| This corresponds to the `_byte_pair_merge` function in the upstream `tiktoken`
+    package:
+
+    https://github.com/openai/tiktoken/blob/c0ba74c238d18b4824c25f3c27fc8698055b9a76/src/lib.rs#L18-L74
+
+    The intermediate data structure is an `IntMap` instead of a `Vector` but other
+    than that the algorithm is essentially identical.
+-}
+bytePairEncode
+    :: HashMap ByteString Int -> ByteString -> Maybe [(Int, ByteString)]
+bytePairEncode hashMap bytes
+    | Just rank <- HashMap.lookup bytes hashMap =
+        Just [ (rank, bytes) ]
+    | ByteString.null bytes =
+        pure []
+    | otherwise = do
+        -- In practice this should always return a `Just` because all of
+        -- OpenAI's encodings are defined all bytes, but in theory the user
+        -- could create an `Encoding` that doesn't satisfy that invariant, so
+        -- we still need to handle that case.
+        let lookupByte :: Word8 -> Maybe Int
+            lookupByte word8 = HashMap.lookup (ByteString.singleton word8) hashMap
+
+        let toChunk w0 w1 = do
+                rank <- lookupByte w0
+
+                let rank2 = lookupSlice (ByteString.pack [ w0, w1 ])
+
+                pure Chunk{ rank, rank2 }
+
+        initChunks <- do
+            sequence (ByteString.zipWith toChunk bytes (ByteString.tail bytes))
+
+        lastChunk <- do
+            rank <- lookupByte (ByteString.last bytes)
+
+            pure Chunk{ rank, rank2 = Unranked }
+
+        {- Unlike the upstream `tiktoken` we do not use `Vector (Key, Ranked)`
+           as our intermediate datastructure, but rather something more like
+           `IntMap Ranked` (technically `IntMap Chunk`, which is just a tiny
+           optimization).
+
+           This makes it cheaper to delete keys without having to rebuild a
+           `Vector` each time, but at the expense of neighbor lookups (e.g.
+           `lookupLT` / `lookupGT`) being more expensive.
+        -}
+        let initialMap :: IntMap Chunk
+            initialMap =
+                IntMap.fromList (zip [0 ..] (initChunks <> [ lastChunk ]))
+
+        let keyValues = IntMap.toAscList (loop initialMap)
+
+        pure do
+            let adapt (index, Chunk{ rank }) nextIndex =
+                    (rank, slice index nextIndex)
+
+            zipWith adapt keyValues (drop1 (map fst keyValues) <> [ size ])
+  where
+    size :: Int
+    size = ByteString.length bytes
+
+    lookupSlice :: ByteString -> Ranked
+    lookupSlice b = case HashMap.lookup b hashMap of
+        Nothing  -> Unranked
+        Just int -> Ranked int
+
+    slice :: Int -> Int -> ByteString
+    slice begin end = ByteString.take (end - begin) (ByteString.drop begin bytes)
+
+    loop :: IntMap Chunk -> IntMap Chunk
+    loop chunks0 = case minimumBy (Ord.comparing rank2) chunks0 of
+        Just (index, Chunk{ rank2 = Ranked ranked }) -> loop chunks3
+          where
+            chunks1 = rerank index ranked chunks0
+
+            chunks2 = case IntMap.lookupLT index chunks1 of
+                Just (prevIndex, Chunk{ rank = prevRanked }) ->
+                    rerank prevIndex prevRanked chunks1
+                _ ->
+                    chunks1
+
+            chunks3 = case IntMap.lookupGT index chunks2 of
+                -- In theory we should never hit the `Nothing` case here because
+                -- the `rank2` field can only be `Ranked` if there is a `Chunk`
+                -- following this one.
+                Nothing ->
+                    error "Tiktoken.bytePairEncode: Internal error - a ranked byte pair is missing the second byte in the pair"
+                Just (nextIndex, _) -> IntMap.delete nextIndex chunks2
+
+        _ ->
+            chunks0
+
+    rerank :: Key -> Int -> IntMap Chunk -> IntMap Chunk
+    rerank index0 rank chunks = IntMap.insert index0 newChunk chunks
+      where
+        maybeIndex3 = do
+            (index1, _) <- IntMap.lookupGT index0 chunks
+
+            (index2, _) <- IntMap.lookupGT index1 chunks
+
+            pure case IntMap.lookupGT index2 chunks of
+                Just (index3, _) -> index3
+                Nothing          -> size
+
+        rank2 = case maybeIndex3 of
+            Nothing     -> Unranked
+            Just index3 -> lookupSlice (slice index0 index3)
+
+        newChunk = Chunk{ rank, rank2 }
+
+{-| Split a `ByteString` into smaller `ByteString`s, each of which are
+    successive longest possible matches to the provided regular expression
+-}
+splitUsingRegex
+    :: ByteString
+    -- ^ Regex to match
+    -> ByteString
+    -- ^ Bytes to split into chunks
+    -> Maybe [ByteString]
+splitUsingRegex pattern = loop Prelude.id
+  where
+    loop diff bytes
+        | ByteString.null bytes =
+            Just (diff [])
+        | otherwise =
+            case Regex.match regex bytes [ Regex.exec_no_utf8_check ] of
+                Just (prefix : _) ->
+                    let suffix = ByteString.drop (ByteString.length prefix) bytes
+                    in  loop (diff . (prefix :)) suffix
+                _ -> Nothing
+
+    regex = Regex.compile pattern [ Regex.utf8 ]
+
+{-| Divide up the input into coarse-grained chunks based on the provided splitting
+    regular expression before doing the final byte pair encoding
+-}
+bytePairEncodeWithSplitting :: Encoding -> ByteString -> Maybe [(Int, ByteString)]
+bytePairEncodeWithSplitting Encoding{..} bytes = do
+    chunks <- splitUsingRegex regex bytes
+
+    tokenss <- traverse (bytePairEncode encode) chunks
+
+    pure (concat tokenss)
+
+{-| Split a `ByteString` into smaller `ByteString`s separated by the given
+    separator
+-}
+splitOnSeparator
+    :: ByteString
+    -- ^ Separator
+    -> ByteString
+    -- ^ `ByteString` to separate
+    -> NonEmpty ByteString
+splitOnSeparator separator initialBytes = initialPrefix :| loop initialSuffix
+  where
+    split = ByteString.breakSubstring separator
+
+    (initialPrefix, initialSuffix) = split initialBytes
+
+    loop bytes
+        | ByteString.null bytes = []
+        | otherwise             = prefix : loop suffix
+      where
+        rest = ByteString.drop (ByteString.length separator) bytes
+
+        (prefix, suffix) = split rest
+
+-- | Tokenizer that is special-token-aware
+toTokensAndRanks :: Encoding -> ByteString -> Maybe [(Int, ByteString)]
+toTokensAndRanks encoding@Encoding{..} initialBytes =
+    foldr cons nil (Map.toList specialTokens) initialBytes
+  where
+    cons (token, rank) tokenizer bytes = do
+        fmap joinSegments (traverse tokenizer (splitOnSeparator token bytes))
+      where
+        joinSegments =
+              concat
+            . NonEmpty.toList
+            . NonEmpty.intersperse [ (rank, token) ]
+
+    nil bytes = bytePairEncodeWithSplitting encoding bytes
+
+{-| Use an `Encoding` to tokenize a `ByteString` into smaller `ByteString`s
+
+    This only fails if you provide an `Encoding` that cannot rank all possible
+    1-byte sequences
+-}
+toTokens :: Encoding -> ByteString -> Maybe [ByteString]
+toTokens = fmap (fmap (fmap (fmap snd))) toTokensAndRanks
+
+{-| Use an `Encoding` to tokenize a `ByteString` into ranks
+
+    This only fails if you provide an `Encoding` that cannot rank all possible
+    1-byte sequences
+-}
+toRanks :: Encoding -> ByteString -> Maybe [Int]
+toRanks = fmap (fmap (fmap (fmap fst))) toTokensAndRanks
+
+{-| Combine a sequence of `ByteString` tokens back into a `ByteString`
+
+    This is just a synonym for @"Data.ByteString".`ByteString.concat`@ (no
+    `Encoding` necessary), provided solely for consistency/convenience.
+-}
+fromTokens :: [ByteString] -> ByteString
+fromTokens = ByteString.concat
+
+{-| Convert a sequence of ranks back into a `ByteString`
+
+    This will fail if you supply any ranks which are not recognized by the
+    `Encoding`.
+-}
+fromRanks :: Encoding -> [Int] -> Maybe ByteString
+fromRanks Encoding{..} vector = fmap fromTokens (traverse (decode !?) vector)
diff --git a/tasty/Main.hs b/tasty/Main.hs
new file mode 100644
--- /dev/null
+++ b/tasty/Main.hs
@@ -0,0 +1,24 @@
+{-# LANGUAGE BlockArguments #-}
+
+module Main where
+
+import qualified Data.ByteString as ByteString
+import qualified Data.Foldable as Foldable
+import qualified Data.Text as Text
+import qualified Test.Tasty as Tasty
+import qualified Test.Tasty.Silver as Silver
+
+import qualified Tiktoken
+
+main :: IO ()
+main = do
+    Tasty.defaultMain do
+        let actualTokens = do
+                bytes <- ByteString.readFile "tasty/sample.txt"
+                case Tiktoken.toRanks Tiktoken.cl100k_base bytes of
+                    Nothing     -> fail "Encoding failed"
+                    Just tokens -> return tokens
+
+        let render = Text.unlines . map (Text.pack . show) . Foldable.toList
+                
+        Silver.goldenVsAction "golden" "tasty/tokens.golden" actualTokens render
diff --git a/tasty/sample.txt b/tasty/sample.txt
new file mode 100644
# file too large to diff: tasty/sample.txt
diff --git a/tasty/tokens.golden b/tasty/tokens.golden
new file mode 100644
# file too large to diff: tasty/tokens.golden
diff --git a/tiktoken.cabal b/tiktoken.cabal
new file mode 100644
--- /dev/null
+++ b/tiktoken.cabal
@@ -0,0 +1,65 @@
+cabal-version:      2.4
+name:               tiktoken
+version:            1.0.0
+synopsis:           Haskell implementation of tiktoken
+description:        This packages only implements tokenization.  In other words,
+                    given an existing encoding (`cl100k_base`) you can tokenize
+                    an input.
+license:            BSD-3-Clause
+license-file:       LICENSE
+author:             Gabriella Gonzalez
+maintainer:         GenuineGabriella@gmail.com
+
+extra-source-files: README.md
+                  , CHANGELOG.md
+                  , tasty/sample.txt
+                  , tasty/tokens.golden
+                  , tasty/tokens.golden
+data-dir:           data
+
+library
+    build-depends:    base                 >= 4.18.0.0 && < 5
+                    , base64               >= 1.0      && < 1.1
+                    , bytestring           >= 0.11.3.0
+                    , containers           >= 0.5.0.0
+                    , deepseq              >= 1.4.0.0
+                    , filepath
+                    , megaparsec                          < 9.7
+                    , pcre-light           >= 0.2
+                    , raw-strings-qq     
+                    , text
+                    , transformers         >= 0.2.0.0
+                    , unordered-containers
+                    , vector                              < 0.14
+    hs-source-dirs:   src
+    exposed-modules:  Tiktoken
+    other-modules:    Paths_tiktoken
+    autogen-modules:  Paths_tiktoken
+    ghc-options:      -Wall
+    default-language: Haskell2010
+
+test-suite tasty
+    type:             exitcode-stdio-1.0
+    build-depends:    base
+                    , bytestring
+                    , tiktoken
+                    , tasty
+                    , tasty-silver < 3.4
+                    , text
+    hs-source-dirs:   tasty
+    main-is:          Main.hs
+    ghc-options:      -Wall
+    default-language: Haskell2010
+
+benchmark benchmark
+    hs-source-dirs:   benchmark
+    main-is:          Main.hs
+    type:             exitcode-stdio-1.0
+    build-depends:    base
+                    , bytestring
+                    , deepseq
+                    , filepath
+                    , tasty-bench < 0.4
+                    , tiktoken
+    ghc-options:      -with-rtsopts=-A32m -fproc-alignment=64
+    default-language: Haskell2010