tiktoken (empty) → 1.0.0
raw patch · 9 files changed
+717/−0 lines, 9 filesdep +basedep +base64dep +bytestring
Dependencies added: base, base64, bytestring, containers, deepseq, filepath, megaparsec, pcre-light, raw-strings-qq, tasty, tasty-bench, tasty-silver, text, tiktoken, transformers, unordered-containers, vector
Files
- CHANGELOG.md +3/−0
- LICENSE +30/−0
- README.md +29/−0
- benchmark/Main.hs +32/−0
- src/Tiktoken.hs +534/−0
- tasty/Main.hs +24/−0
- tasty/sample.txt too large to diff
- tasty/tokens.golden too large to diff
- tiktoken.cabal +65/−0
+ CHANGELOG.md view
@@ -0,0 +1,3 @@+1.0.0++- Initial release
+ LICENSE view
@@ -0,0 +1,30 @@+Copyright (c) 2024, Gabriella Gonzalez++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.++ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following+ disclaimer in the documentation and/or other materials provided+ with the distribution.++ * Neither the name of Gabriella Gonzalez nor the names of other+ contributors may be used to endorse or promote products derived+ from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ README.md view
@@ -0,0 +1,29 @@+# `tiktoken`++This is a Haskell implementation of+[`tiktoken`](https://github.com/openai/tiktoken), but just the tokenization+logic. In other words, given an existing encoding (like `cl100k_base`) you+can tokenize a string (into smaller strings or token ranks).++This means that you can't (yet) use this package to create your own new+encodings, but you can use it to consume encodings. In particular, this comes+in handy for prompt engineering where you want to use as much of the available+prompt tokens as possible (which requires accurately counting tokens).++Encoding speed is ≈2.6-3.1 MB/s on an M1 MacBook Pro (using only one core since+this package does not yet support parallel tokenization):++```+All+ Encode 10 MB of Wikipedia+ r50k_base: OK (23.88s)+ 3.356 s ± 151 ms+ p50k_base: OK (10.39s)+ 3.445 s ± 31 ms+ p50k_edit: OK (11.13s)+ 3.693 s ± 240 ms+ cl100k_base: OK (11.16s)+ 3.685 s ± 143 ms+ o200k_base: OK (11.01s)+ 3.648 s ± 134 ms+```
+ benchmark/Main.hs view
@@ -0,0 +1,32 @@+{-# LANGUAGE BlockArguments #-}++module Main where++import System.FilePath ((</>))+import Test.Tasty.Bench (Benchmark)+import Tiktoken (Encoding)++import qualified Control.DeepSeq as DeepSeq+import qualified Control.Exception as Exception+import qualified Data.ByteString as ByteString+import qualified Test.Tasty.Bench as Bench+import qualified Tiktoken++main :: IO ()+main = do+ Bench.defaultMain+ [ Bench.env (ByteString.readFile "tasty/sample.txt") \text -> do+ let benchmark name encoding = do+ let loadEncoding = Exception.evaluate (DeepSeq.force encoding)++ Bench.env loadEncoding \encoding -> do+ Bench.bench name (Bench.nf (Tiktoken.toTokens encoding) text)++ Bench.bgroup "Encode 10 MiB of Wikipedia"+ [ benchmark "r50k_base" Tiktoken.r50k_base+ , benchmark "p50k_base" Tiktoken.p50k_base+ , benchmark "p50k_edit" Tiktoken.p50k_edit+ , benchmark "cl100k_base" Tiktoken.cl100k_base+ , benchmark "o200k_base" Tiktoken.o200k_base+ ]+ ]
+ src/Tiktoken.hs view
@@ -0,0 +1,534 @@+{-# LANGUAGE BlockArguments #-}+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE DerivingStrategies #-}+{-# LANGUAGE DuplicateRecordFields #-}+{-# LANGUAGE NamedFieldPuns #-}+{-# LANGUAGE NumericUnderscores #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE OverloadedLists #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE RecordWildCards #-}++-- | You can use this module to convert back and forth between a `ByteString`+-- and its corresponding tokens using an existing encoding like @cl100k_base@+-- or @o200k_base@+--+-- Example usage:+--+-- @+-- {-# LANGUAGE OverloadedStrings #-}+--+-- import "Tiktoken" (`o200k_base`, toTokens, toRanks)+--+-- main :: `IO` ()+-- main = do+-- -- `Just` [\"El\",\" perro\",\" come\",\" las\",\" man\",\"z\",\"anas\"]+-- `print` (`toTokens` `o200k_base` \"El perro come las manzanas\")+--+-- -- `Just` [4422,96439,3063,1996,873,89,14457]+-- `print` (`toRanks` `o200k_base` \"El perro come las manzanas\")+-- @+module Tiktoken+ ( -- * Encoding+ Encoding+ , tiktokenToEncoding+ , addSpecialTokens++ -- * Stock Encodings+ , r50k_base+ , p50k_base+ , p50k_edit+ , cl100k_base+ , o200k_base++ -- * Tokenization+ , toTokens+ , toRanks+ , toTokensAndRanks++ -- * Detokenization+ , fromTokens+ , fromRanks+ ) where++import Control.Applicative ((<|>))+import Control.DeepSeq (NFData)+import Control.Monad.ST (ST)+import Control.Monad.Trans.Class (lift)+import Data.ByteString (ByteString)+import Data.Function (on)+import Data.HashMap.Strict (HashMap)+import Data.IntMap (IntMap, Key)+import Data.List.NonEmpty (NonEmpty(..))+import Data.Map (Map)+import Data.Text (Text)+import Data.Vector (MVector, Vector, (!?))+import Data.Void (Void)+import Data.Word (Word8)+import GHC.Generics (Generic)+import System.FilePath ((</>))+import Text.Megaparsec (ParseErrorBundle, ParsecT)+import Text.RawString.QQ (r)++import qualified Control.Exception as Exception+import qualified Data.ByteString as ByteString+import qualified Data.ByteString.Base64 as Base64.Encoding+import qualified Data.ByteString.Char8 as Char8+import qualified Data.HashMap.Strict as HashMap+import qualified Data.IntMap.Strict as IntMap+import qualified Data.List as List+import qualified Data.List.NonEmpty as NonEmpty+import qualified Data.Map as Map+import qualified Data.Ord as Ord+import qualified Data.Text as Text+import qualified Data.Text.Encoding as Text.Encoding+import qualified Data.Text.IO as Text.IO+import qualified Data.Vector as Vector+import qualified Data.Vector.Mutable as Vector.Mutable+import qualified Paths_tiktoken as Paths+import qualified System.IO.Unsafe as Unsafe+import qualified Text.Megaparsec as Megaparsec+import qualified Text.Megaparsec.Char as Megaparsec.Char+import qualified Text.Regex.PCRE.Light as Regex++{-| This is an efficient internal representation of an encoding like+ @cl100k_base@, @p50k_edit@, or @o200k_base@+-}+data Encoding = Encoding+ { encode :: HashMap ByteString Int+ , decode :: Vector ByteString+ , specialTokens :: Map ByteString Int+ , regex :: ByteString+ } deriving stock (Generic)+ deriving anyclass (NFData)++parseToken :: ParsecT Void Text m ByteString+parseToken = do+ base64Text <- Megaparsec.takeWhileP (Just "Base64 character") (/= ' ')++ let base64Bytes = Text.Encoding.encodeUtf8 base64Text++ token <- case Base64.Encoding.decodeBase64Untyped base64Bytes of+ Left text -> fail (Text.unpack text)+ Right token -> return token++ -- We don't bother parsing the token ID because the tokens are always stored+ -- in sequential order by token ID. We could *not* assume this but this+ -- would not only make the parsing slower but it would also require using+ -- a `HashMap` instead of a `Vector` to handle potential gaps in the token+ -- ID sequence. It's much more efficient to make this simplifying+ -- assumption.++ _ <- Megaparsec.takeWhileP (Just "Base64 character") (/= '\n')++ _ <- Megaparsec.Char.char '\n'++ return token++parseDecode :: ParsecT Void Text (ST s) (MVector s ByteString)+parseDecode = do+ -- 100,000 is the size of the largest commonly-used encoding at the time of+ -- this writing (`cl100k_base`) and it's not that expensive to pre-allocate+ -- a `Vector` that big, so let's go wild and start with a large allocation.+ let initialSize = 100_000++ initialVector <- lift (Vector.Mutable.new initialSize)++ let loop index vector+ | index < size = do+ let success = do+ token <- parseToken++ lift (Vector.Mutable.write vector index token)++ loop (index + 1) vector++ let failure = do+ return (Vector.Mutable.take index vector)++ success <|> failure+ + | otherwise = do+ largerVector <- lift (Vector.Mutable.grow vector size)++ loop index largerVector+ where+ size = Vector.Mutable.length vector++ loop 0 initialVector++-- | Create an `Encoding` from regular expression and an ordered set of tokens+tokensToEncoding+ :: ByteString+ -- ^ Regular expression used for coarse-grained splitting of the input+ -> Vector ByteString+ -- ^ The tokens in sequential order of their token IDs+ -> Encoding+tokensToEncoding regex decode = Encoding{..}+ where+ encode = HashMap.fromList (Vector.toList (Vector.imap adapt decode))+ where+ adapt index token = (token, index)++ specialTokens = mempty++-- | Parse an encoding from the @.tiktoken@ file format+tiktokenToEncoding+ :: ByteString+ -- ^ Regular expression used for coarse-grained splitting of the input+ -> Text+ -- ^ The contents of the @.tiktoken@ file+ -> Either (ParseErrorBundle Text Void) Encoding+tiktokenToEncoding regex text =+ fmap (tokensToEncoding regex)+ (Vector.createT (Megaparsec.runParserT parseDecode "" text))++-- | Add special tokens to a base `Encoding`+addSpecialTokens :: Map ByteString Int -> Encoding -> Encoding+addSpecialTokens tokens Encoding{ specialTokens = oldSpecialTokens, .. } =+ Encoding{ specialTokens = Map.union tokens oldSpecialTokens, .. }++_ENDOFTEXT :: ByteString+_ENDOFTEXT = "<|endoftext|>"++_FIM_PREFIX :: ByteString+_FIM_PREFIX = "<|fim_prefix|>"++_FIM_MIDDLE :: ByteString+_FIM_MIDDLE = "<|fim_middle|>"++_FIM_SUFFIX :: ByteString+_FIM_SUFFIX = "<|fim_suffix|>"++_ENDOFPROMPT :: ByteString+_ENDOFPROMPT = "<|endofprompt|>"++loadEncoding :: FilePath -> ByteString -> Map ByteString Int -> IO Encoding+loadEncoding file regex specialTokens = do+ dataDirectory <- Paths.getDataDir++ text <- Text.IO.readFile (dataDirectory </> file)++ encoding <- case tiktokenToEncoding regex text of+ Left exception -> Exception.throwIO exception+ Right encoding -> return encoding++ return (addSpecialTokens specialTokens encoding)++-- | @r50k_base@ `Encoding`+r50k_base :: Encoding+r50k_base =+ Unsafe.unsafePerformIO+ (loadEncoding "r50k_base.tiktoken" regex [ (_ENDOFTEXT, 50256) ])+ where+ regex =+ [r|'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+|]+{-# NOINLINE r50k_base #-}++-- | @p50k_base@ `Encoding`+p50k_base :: Encoding+p50k_base =+ Unsafe.unsafePerformIO+ (loadEncoding "p50k_base.tiktoken" regex [ (_ENDOFTEXT, 50256) ])+ where+ regex =+ [r|'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+|]+{-# NOINLINE p50k_base #-}++-- | @p50k_edit@ `Encoding`+p50k_edit :: Encoding+p50k_edit =+ Unsafe.unsafePerformIO+ (loadEncoding "p50k_base.tiktoken"+ regex+ [ (_ENDOFTEXT , 50256)+ , (_FIM_PREFIX, 50281)+ , (_FIM_MIDDLE, 50282)+ , (_FIM_SUFFIX, 50283)+ ] + )+ where+ regex =+ [r|'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+|]+{-# NOINLINE p50k_edit #-}++-- | @cl100k_base@ `Encoding`+cl100k_base :: Encoding+cl100k_base =+ Unsafe.unsafePerformIO+ (loadEncoding "cl100k_base.tiktoken"+ regex+ [ (_ENDOFTEXT , 100257)+ , (_FIM_PREFIX , 100258)+ , (_FIM_MIDDLE , 100259)+ , (_FIM_SUFFIX , 100260)+ , (_ENDOFPROMPT, 100276)+ ]+ )+ where+ regex =+ [r|'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+|]+{-# NOINLINE cl100k_base #-}++-- | @o200k_base@ `Encoding`+o200k_base :: Encoding+o200k_base =+ Unsafe.unsafePerformIO+ (loadEncoding "o200k_base.tiktoken"+ regex+ [ (_ENDOFTEXT , 199999)+ , (_ENDOFPROMPT, 200018)+ ]+ )+ where+ regex =+ Char8.intercalate "|"+ [ [r|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|]+ , [r|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|]+ , [r|\p{N}{1,3}|]+ , [r| ?[^\s\p{L}\p{N}]+[\r\n/]*|]+ , [r|\s*[\r\n]+|]+ , [r|\s+(?!\S)|]+ , [r|\s+|]+ ]+{-# NOINLINE o200k_base #-}++minimumBy :: (a -> a -> Ordering) -> IntMap a -> Maybe (Int, a)+minimumBy comparison intMap+ | IntMap.null intMap =+ Nothing+ | otherwise =+ Just (List.minimumBy (comparison `on` snd) (IntMap.toList intMap))++drop1 :: [a] -> [a]+drop1 (_ : xs) = xs+drop1 [] = []++{-| This is basically the same thing as `Maybe Int` except with an `Ord`+ instance that treats `Ranked` values as less than `Unranked` values+-}+data Ranked = Ranked Int | Unranked+ deriving (Eq, Ord)++data Chunk = Chunk+ { rank :: Int+ -- ^ Rank of this chunk+ , rank2 :: Ranked+ -- ^ Rank of this chunk combined with the next chunk+ }++{-| This corresponds to the `_byte_pair_merge` function in the upstream `tiktoken`+ package:++ https://github.com/openai/tiktoken/blob/c0ba74c238d18b4824c25f3c27fc8698055b9a76/src/lib.rs#L18-L74++ The intermediate data structure is an `IntMap` instead of a `Vector` but other+ than that the algorithm is essentially identical.+-}+bytePairEncode+ :: HashMap ByteString Int -> ByteString -> Maybe [(Int, ByteString)]+bytePairEncode hashMap bytes+ | Just rank <- HashMap.lookup bytes hashMap =+ Just [ (rank, bytes) ]+ | ByteString.null bytes =+ pure []+ | otherwise = do+ -- In practice this should always return a `Just` because all of+ -- OpenAI's encodings are defined all bytes, but in theory the user+ -- could create an `Encoding` that doesn't satisfy that invariant, so+ -- we still need to handle that case.+ let lookupByte :: Word8 -> Maybe Int+ lookupByte word8 = HashMap.lookup (ByteString.singleton word8) hashMap++ let toChunk w0 w1 = do+ rank <- lookupByte w0++ let rank2 = lookupSlice (ByteString.pack [ w0, w1 ])++ pure Chunk{ rank, rank2 }++ initChunks <- do+ sequence (ByteString.zipWith toChunk bytes (ByteString.tail bytes))++ lastChunk <- do+ rank <- lookupByte (ByteString.last bytes)++ pure Chunk{ rank, rank2 = Unranked }++ {- Unlike the upstream `tiktoken` we do not use `Vector (Key, Ranked)`+ as our intermediate datastructure, but rather something more like+ `IntMap Ranked` (technically `IntMap Chunk`, which is just a tiny+ optimization).++ This makes it cheaper to delete keys without having to rebuild a+ `Vector` each time, but at the expense of neighbor lookups (e.g.+ `lookupLT` / `lookupGT`) being more expensive.+ -}+ let initialMap :: IntMap Chunk+ initialMap =+ IntMap.fromList (zip [0 ..] (initChunks <> [ lastChunk ]))++ let keyValues = IntMap.toAscList (loop initialMap)++ pure do+ let adapt (index, Chunk{ rank }) nextIndex =+ (rank, slice index nextIndex)++ zipWith adapt keyValues (drop1 (map fst keyValues) <> [ size ])+ where+ size :: Int+ size = ByteString.length bytes++ lookupSlice :: ByteString -> Ranked+ lookupSlice b = case HashMap.lookup b hashMap of+ Nothing -> Unranked+ Just int -> Ranked int++ slice :: Int -> Int -> ByteString+ slice begin end = ByteString.take (end - begin) (ByteString.drop begin bytes)++ loop :: IntMap Chunk -> IntMap Chunk+ loop chunks0 = case minimumBy (Ord.comparing rank2) chunks0 of+ Just (index, Chunk{ rank2 = Ranked ranked }) -> loop chunks3+ where+ chunks1 = rerank index ranked chunks0++ chunks2 = case IntMap.lookupLT index chunks1 of+ Just (prevIndex, Chunk{ rank = prevRanked }) ->+ rerank prevIndex prevRanked chunks1+ _ ->+ chunks1++ chunks3 = case IntMap.lookupGT index chunks2 of+ -- In theory we should never hit the `Nothing` case here because+ -- the `rank2` field can only be `Ranked` if there is a `Chunk`+ -- following this one.+ Nothing ->+ error "Tiktoken.bytePairEncode: Internal error - a ranked byte pair is missing the second byte in the pair"+ Just (nextIndex, _) -> IntMap.delete nextIndex chunks2++ _ ->+ chunks0++ rerank :: Key -> Int -> IntMap Chunk -> IntMap Chunk+ rerank index0 rank chunks = IntMap.insert index0 newChunk chunks+ where+ maybeIndex3 = do+ (index1, _) <- IntMap.lookupGT index0 chunks++ (index2, _) <- IntMap.lookupGT index1 chunks++ pure case IntMap.lookupGT index2 chunks of+ Just (index3, _) -> index3+ Nothing -> size++ rank2 = case maybeIndex3 of+ Nothing -> Unranked+ Just index3 -> lookupSlice (slice index0 index3)++ newChunk = Chunk{ rank, rank2 }++{-| Split a `ByteString` into smaller `ByteString`s, each of which are+ successive longest possible matches to the provided regular expression+-}+splitUsingRegex+ :: ByteString+ -- ^ Regex to match+ -> ByteString+ -- ^ Bytes to split into chunks+ -> Maybe [ByteString]+splitUsingRegex pattern = loop Prelude.id+ where+ loop diff bytes+ | ByteString.null bytes =+ Just (diff [])+ | otherwise =+ case Regex.match regex bytes [ Regex.exec_no_utf8_check ] of+ Just (prefix : _) ->+ let suffix = ByteString.drop (ByteString.length prefix) bytes+ in loop (diff . (prefix :)) suffix+ _ -> Nothing++ regex = Regex.compile pattern [ Regex.utf8 ]++{-| Divide up the input into coarse-grained chunks based on the provided splitting+ regular expression before doing the final byte pair encoding+-}+bytePairEncodeWithSplitting :: Encoding -> ByteString -> Maybe [(Int, ByteString)]+bytePairEncodeWithSplitting Encoding{..} bytes = do+ chunks <- splitUsingRegex regex bytes++ tokenss <- traverse (bytePairEncode encode) chunks++ pure (concat tokenss)++{-| Split a `ByteString` into smaller `ByteString`s separated by the given+ separator+-}+splitOnSeparator+ :: ByteString+ -- ^ Separator+ -> ByteString+ -- ^ `ByteString` to separate+ -> NonEmpty ByteString+splitOnSeparator separator initialBytes = initialPrefix :| loop initialSuffix+ where+ split = ByteString.breakSubstring separator++ (initialPrefix, initialSuffix) = split initialBytes++ loop bytes+ | ByteString.null bytes = []+ | otherwise = prefix : loop suffix+ where+ rest = ByteString.drop (ByteString.length separator) bytes++ (prefix, suffix) = split rest++-- | Tokenizer that is special-token-aware+toTokensAndRanks :: Encoding -> ByteString -> Maybe [(Int, ByteString)]+toTokensAndRanks encoding@Encoding{..} initialBytes =+ foldr cons nil (Map.toList specialTokens) initialBytes+ where+ cons (token, rank) tokenizer bytes = do+ fmap joinSegments (traverse tokenizer (splitOnSeparator token bytes))+ where+ joinSegments =+ concat+ . NonEmpty.toList+ . NonEmpty.intersperse [ (rank, token) ]++ nil bytes = bytePairEncodeWithSplitting encoding bytes++{-| Use an `Encoding` to tokenize a `ByteString` into smaller `ByteString`s++ This only fails if you provide an `Encoding` that cannot rank all possible+ 1-byte sequences+-}+toTokens :: Encoding -> ByteString -> Maybe [ByteString]+toTokens = fmap (fmap (fmap (fmap snd))) toTokensAndRanks++{-| Use an `Encoding` to tokenize a `ByteString` into ranks++ This only fails if you provide an `Encoding` that cannot rank all possible+ 1-byte sequences+-}+toRanks :: Encoding -> ByteString -> Maybe [Int]+toRanks = fmap (fmap (fmap (fmap fst))) toTokensAndRanks++{-| Combine a sequence of `ByteString` tokens back into a `ByteString`++ This is just a synonym for @"Data.ByteString".`ByteString.concat`@ (no+ `Encoding` necessary), provided solely for consistency/convenience.+-}+fromTokens :: [ByteString] -> ByteString+fromTokens = ByteString.concat++{-| Convert a sequence of ranks back into a `ByteString`++ This will fail if you supply any ranks which are not recognized by the+ `Encoding`.+-}+fromRanks :: Encoding -> [Int] -> Maybe ByteString+fromRanks Encoding{..} vector = fmap fromTokens (traverse (decode !?) vector)
+ tasty/Main.hs view
@@ -0,0 +1,24 @@+{-# LANGUAGE BlockArguments #-}++module Main where++import qualified Data.ByteString as ByteString+import qualified Data.Foldable as Foldable+import qualified Data.Text as Text+import qualified Test.Tasty as Tasty+import qualified Test.Tasty.Silver as Silver++import qualified Tiktoken++main :: IO ()+main = do+ Tasty.defaultMain do+ let actualTokens = do+ bytes <- ByteString.readFile "tasty/sample.txt"+ case Tiktoken.toRanks Tiktoken.cl100k_base bytes of+ Nothing -> fail "Encoding failed"+ Just tokens -> return tokens++ let render = Text.unlines . map (Text.pack . show) . Foldable.toList+ + Silver.goldenVsAction "golden" "tasty/tokens.golden" actualTokens render
+ tasty/sample.txt view
file too large to diff
+ tasty/tokens.golden view
file too large to diff
+ tiktoken.cabal view
@@ -0,0 +1,65 @@+cabal-version: 2.4+name: tiktoken+version: 1.0.0+synopsis: Haskell implementation of tiktoken+description: This packages only implements tokenization. In other words,+ given an existing encoding (`cl100k_base`) you can tokenize+ an input.+license: BSD-3-Clause+license-file: LICENSE+author: Gabriella Gonzalez+maintainer: GenuineGabriella@gmail.com++extra-source-files: README.md+ , CHANGELOG.md+ , tasty/sample.txt+ , tasty/tokens.golden+ , tasty/tokens.golden+data-dir: data++library+ build-depends: base >= 4.18.0.0 && < 5+ , base64 >= 1.0 && < 1.1+ , bytestring >= 0.11.3.0+ , containers >= 0.5.0.0+ , deepseq >= 1.4.0.0+ , filepath+ , megaparsec < 9.7+ , pcre-light >= 0.2+ , raw-strings-qq + , text+ , transformers >= 0.2.0.0+ , unordered-containers+ , vector < 0.14+ hs-source-dirs: src+ exposed-modules: Tiktoken+ other-modules: Paths_tiktoken+ autogen-modules: Paths_tiktoken+ ghc-options: -Wall+ default-language: Haskell2010++test-suite tasty+ type: exitcode-stdio-1.0+ build-depends: base+ , bytestring+ , tiktoken+ , tasty+ , tasty-silver < 3.4+ , text+ hs-source-dirs: tasty+ main-is: Main.hs+ ghc-options: -Wall+ default-language: Haskell2010++benchmark benchmark+ hs-source-dirs: benchmark+ main-is: Main.hs+ type: exitcode-stdio-1.0+ build-depends: base+ , bytestring+ , deepseq+ , filepath+ , tasty-bench < 0.4+ , tiktoken+ ghc-options: -with-rtsopts=-A32m -fproc-alignment=64+ default-language: Haskell2010