diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,23 @@ +1.0.2 + +- [Correctly handle gaps in ranks](https://github.com/Gabriella439/tiktoken/commit/cc82cb192b1f22a2185257b3a1045e2aaf25a4e8) + + The old implementation assumed that encoding don't have gaps in their ranks, + but some do (especially near the end, typically reserved for special tokens). + This change fixes the internal implementation to correctly handle those gaps. + +- [Fix `o200k_base` regex to match upstream](https://github.com/Gabriella439/tiktoken/commit/fa70dddb431af5a71b243325cf9ef72061721e99) + + The upstream `tiktoken` package uses a flavor of regex that subtly differs + from the Haskell `pcre-light` package. Specifically, they differ in whether + they treat ideographic space (U+3000) as whitespace (which this change fixes). + + There may be other differences yet to be uncovered, but this is the only one + that has arisen so far when comparing to upstream on a large corpus of text. + 1.0.1 -- Small fixes to documentation +- [Small fixes to documentation](https://github.com/Gabriella439/tiktoken/commit/bb0eaf724a8120e7cd2ada2ad09835369bac02bc) 1.0.0 diff --git a/src/Tiktoken.hs b/src/Tiktoken.hs --- a/src/Tiktoken.hs +++ b/src/Tiktoken.hs @@ -52,10 +52,8 @@ , fromRanks ) where -import Control.Applicative ((<|>)) +import Control.Applicative (many) import Control.DeepSeq (NFData) -import Control.Monad.ST (ST) -import Control.Monad.Trans.Class (lift) import Data.ByteString (ByteString) import Data.Function (on) import Data.HashMap.Strict (HashMap) @@ -63,10 +61,10 @@ import Data.List.NonEmpty (NonEmpty(..)) import Data.Map (Map) import Data.Text (Text) -import Data.Vector (MVector, Vector, (!?)) import Data.Void (Void) import Data.Word (Word8) import GHC.Generics (Generic) +import Prelude hiding (lookup) import System.FilePath (()) import Text.Megaparsec (ParseErrorBundle, ParsecT) import Text.RawString.QQ (r) @@ -84,12 +82,11 @@ import qualified Data.Text as Text import qualified Data.Text.Encoding as Text.Encoding import qualified Data.Text.IO as Text.IO -import qualified Data.Vector as Vector -import qualified Data.Vector.Mutable as Vector.Mutable import qualified Paths_tiktoken as Paths import qualified System.IO.Unsafe as Unsafe import qualified Text.Megaparsec as Megaparsec import qualified Text.Megaparsec.Char as Megaparsec.Char +import qualified Text.Megaparsec.Char.Lexer as Megaparsec.Lexer import qualified Text.Regex.PCRE.Light as Regex {-| This is an efficient internal representation of an encoding like @@ -97,13 +94,13 @@ -} data Encoding = Encoding { encode :: HashMap ByteString Int - , decode :: Vector ByteString + , decode :: IntMap ByteString , specialTokens :: Map ByteString Int , regex :: ByteString } deriving stock (Generic) deriving anyclass (NFData) -parseToken :: ParsecT Void Text m ByteString +parseToken :: ParsecT Void Text m (Int, ByteString) parseToken = do base64Text <- Megaparsec.takeWhileP (Just "Base64 character") (/= ' ') @@ -113,63 +110,26 @@ Left text -> fail (Text.unpack text) Right token -> return token - -- We don't bother parsing the token ID because the tokens are always stored - -- in sequential order by token ID. We could *not* assume this but this - -- would not only make the parsing slower but it would also require using - -- a `HashMap` instead of a `Vector` to handle potential gaps in the token - -- ID sequence. It's much more efficient to make this simplifying - -- assumption. + _ <- Megaparsec.Char.char ' ' - _ <- Megaparsec.takeWhileP (Just "Base64 character") (/= '\n') + rank <- Megaparsec.Lexer.decimal _ <- Megaparsec.Char.char '\n' - return token - -parseDecode :: ParsecT Void Text (ST s) (MVector s ByteString) -parseDecode = do - -- 100,000 is the size of the largest commonly-used encoding at the time of - -- this writing (`cl100k_base`) and it's not that expensive to pre-allocate - -- a `Vector` that big, so let's go wild and start with a large allocation. - let initialSize = 100_000 - - initialVector <- lift (Vector.Mutable.new initialSize) - - let loop index vector - | index < size = do - let success = do - token <- parseToken - - lift (Vector.Mutable.write vector index token) - - loop (index + 1) vector - - let failure = do - return (Vector.Mutable.take index vector) - - success <|> failure - - | otherwise = do - largerVector <- lift (Vector.Mutable.grow vector size) - - loop index largerVector - where - size = Vector.Mutable.length vector - - loop 0 initialVector + return (rank, token) -- | Create an `Encoding` from regular expression and an ordered set of tokens tokensToEncoding :: ByteString -- ^ Regular expression used for coarse-grained splitting of the input - -> Vector ByteString + -> IntMap ByteString -- ^ The tokens in sequential order of their token IDs -> Encoding tokensToEncoding regex decode = Encoding{..} where - encode = HashMap.fromList (Vector.toList (Vector.imap adapt decode)) + encode = HashMap.fromList (map swap (IntMap.toList decode)) where - adapt index token = (token, index) + swap (rank, token) = (token, rank) specialTokens = mempty @@ -180,9 +140,11 @@ -> Text -- ^ The contents of the @.tiktoken@ file -> Either (ParseErrorBundle Text Void) Encoding -tiktokenToEncoding regex text = - fmap (tokensToEncoding regex) - (Vector.createT (Megaparsec.runParserT parseDecode "" text)) +tiktokenToEncoding regex text = Megaparsec.runParser parser "" text + where + parser = do + keyValues <- many parseToken + pure (tokensToEncoding regex (IntMap.fromList keyValues)) -- | Add special tokens to a base `Encoding` addSpecialTokens :: Map ByteString Int -> Encoding -> Encoding @@ -282,15 +244,24 @@ ] ) where + -- Note: This is not the same as the upstream pattern: + -- + -- https://github.com/openai/tiktoken/blob/c0ba74c238d18b4824c25f3c27fc8698055b9a76/tiktoken_ext/openai_public.py#L101-L111 + -- + -- The reason why is that the `pcre-light` package doesn't treat an + -- ideographic space (U+3000) as whitespace whereas Python's `regex` package + -- does, so all of the rules below have to basically replace all occurrences + -- of `\s` with `\s\x{3000}`. There might be other discrepancies, but + -- that's the only one I've found so far. regex = Char8.intercalate "|" [ [r|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|] , [r|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|] , [r|\p{N}{1,3}|] - , [r| ?[^\s\p{L}\p{N}]+[\r\n/]*|] - , [r|\s*[\r\n]+|] - , [r|\s+(?!\S)|] - , [r|\s+|] + , [r| ?[^\s\x{3000}\p{L}\p{N}]+[\r\n/]*|] + , [r|[\s\x{3000}]*[\r\n]+|] + , [r|[\s\x{3000}]+(?![^\s\x{3000}])|] + , [r|[\s\x{3000}]+|] ] {-# NOINLINE o200k_base #-} @@ -318,13 +289,14 @@ -- ^ Rank of this chunk combined with the next chunk } -{-| This corresponds to the `_byte_pair_merge` function in the upstream `tiktoken` +{-| This corresponds to the @_byte_pair_merge@ function in the upstream @tiktoken@ package: https://github.com/openai/tiktoken/blob/c0ba74c238d18b4824c25f3c27fc8698055b9a76/src/lib.rs#L18-L74 - The intermediate data structure is an `IntMap` instead of a `Vector` but other - than that the algorithm is essentially identical. + The intermediate data structure is an `IntMap` instead of a + `Data.Vector.Vector` but other than that the algorithm is essentially + identical. -} bytePairEncode :: HashMap ByteString Int -> ByteString -> Maybe [(Int, ByteString)] @@ -335,7 +307,7 @@ pure [] | otherwise = do -- In practice this should always return a `Just` because all of - -- OpenAI's encodings are defined all bytes, but in theory the user + -- OpenAI's encodings are defined for all bytes, but in theory the user -- could create an `Encoding` that doesn't satisfy that invariant, so -- we still need to handle that case. let lookupByte :: Word8 -> Maybe Int @@ -390,13 +362,13 @@ loop :: IntMap Chunk -> IntMap Chunk loop chunks0 = case minimumBy (Ord.comparing rank2) chunks0 of - Just (index, Chunk{ rank2 = Ranked ranked }) -> loop chunks3 + Just (index, Chunk{ rank2 = Ranked rank }) -> loop chunks3 where - chunks1 = rerank index ranked chunks0 + chunks1 = rerank index rank chunks0 chunks2 = case IntMap.lookupLT index chunks1 of - Just (prevIndex, Chunk{ rank = prevRanked }) -> - rerank prevIndex prevRanked chunks1 + Just (prevIndex, Chunk{ rank = prevRank }) -> + rerank prevIndex prevRank chunks1 _ -> chunks1 @@ -406,7 +378,8 @@ -- following this one. Nothing -> error "Tiktoken.bytePairEncode: Internal error - a ranked byte pair is missing the second byte in the pair" - Just (nextIndex, _) -> IntMap.delete nextIndex chunks2 + Just (nextIndex, _) -> + IntMap.delete nextIndex chunks2 _ -> chunks0 @@ -450,7 +423,7 @@ in loop (diff . (prefix :)) suffix _ -> Nothing - regex = Regex.compile pattern [ Regex.utf8 ] + regex = Regex.compile pattern [ Regex.utf8, Regex.anchored ] {-| Divide up the input into coarse-grained chunks based on the provided splitting regular expression before doing the final byte pair encoding @@ -536,4 +509,6 @@ `Encoding`. -} fromRanks :: Encoding -> [Int] -> Maybe ByteString -fromRanks Encoding{..} vector = fmap fromTokens (traverse (decode !?) vector) +fromRanks Encoding{..} vector = fmap fromTokens (traverse lookup vector) + where + lookup rank = IntMap.lookup rank decode diff --git a/tasty/Main.hs b/tasty/Main.hs --- a/tasty/Main.hs +++ b/tasty/Main.hs @@ -2,23 +2,74 @@ module Main where +import Data.Text (Text) +import Test.QuickCheck.Instances () +import Test.Tasty.QuickCheck (Property, (===)) +import Tiktoken (Encoding) + import qualified Data.ByteString as ByteString import qualified Data.Foldable as Foldable import qualified Data.Text as Text +import qualified Data.Text.Encoding as Text.Encoding import qualified Test.Tasty as Tasty +import qualified Test.Tasty.QuickCheck as Tasty.QuickCheck import qualified Test.Tasty.Silver as Silver - import qualified Tiktoken +roundTrip :: Encoding -> Text -> Property +roundTrip encoding text = Just bytes === result + where + bytes = Text.Encoding.encodeUtf8 (Text.take 100 text) + + result = do + ranks <- Tiktoken.toRanks encoding bytes + + Tiktoken.fromRanks encoding ranks + main :: IO () main = do Tasty.defaultMain do - let actualTokens = do - bytes <- ByteString.readFile "tasty/sample.txt" - case Tiktoken.toRanks Tiktoken.cl100k_base bytes of + let actual encode input encoding = do + bytes <- ByteString.readFile input + case encode encoding bytes of Nothing -> fail "Encoding failed" Just tokens -> return tokens - let render = Text.unlines . map (Text.pack . show) . Foldable.toList + let normalRanks = actual Tiktoken.toRanks "tasty/sample/input.txt" + let specialRanks = actual Tiktoken.toRanks "tasty/special/input.txt" + + let normalTokens = + actual Tiktoken.toTokens "tasty/tokenization/input.txt" + + let render :: Show a => [a] -> Text + render = Text.unlines . map (Text.pack . show) . Foldable.toList - Silver.goldenVsAction "golden" "tasty/tokens.golden" actualTokens render + Tasty.testGroup "tests" + [ Tasty.testGroup "golden" + [ Silver.goldenVsAction "r50k_base" "tasty/sample/r50k_base.golden" (normalRanks Tiktoken.r50k_base) render + , Silver.goldenVsAction "p50k_base" "tasty/sample/p50k_base.golden" (normalRanks Tiktoken.p50k_base) render + , Silver.goldenVsAction "cl100k_base" "tasty/sample/cl100k_base.golden" (normalRanks Tiktoken.cl100k_base) render + , Silver.goldenVsAction "o200k_base" "tasty/sample/o200k_base.golden" (normalRanks Tiktoken.o200k_base) render + ] + , Tasty.testGroup "special tokens" + [ Silver.goldenVsAction "r50k_base" "tasty/special/r50k_base.golden" (specialRanks Tiktoken.r50k_base) render + , Silver.goldenVsAction "p50k_base" "tasty/special/p50k_base.golden" (specialRanks Tiktoken.p50k_base) render + , Silver.goldenVsAction "p50k_edit" "tasty/special/p50k_edit.golden" (specialRanks Tiktoken.p50k_edit) render + , Silver.goldenVsAction "cl100k_base" "tasty/special/cl100k_base.golden" (specialRanks Tiktoken.cl100k_base) render + , Silver.goldenVsAction "o200k_base" "tasty/special/o200k_base.golden" (specialRanks Tiktoken.o200k_base) render + ] + , Tasty.testGroup "tokenization" + [ Silver.goldenVsAction "r50k_base" "tasty/tokenization/r50k_base.golden" (normalTokens Tiktoken.r50k_base) render + , Silver.goldenVsAction "p50k_base" "tasty/tokenization/p50k_base.golden" (normalTokens Tiktoken.p50k_base) render + , Silver.goldenVsAction "cl100k_base" "tasty/tokenization/cl100k_base.golden" (normalTokens Tiktoken.cl100k_base) render + , Silver.goldenVsAction "o200k_base" "tasty/tokenization/o200k_base.golden" (normalTokens Tiktoken.o200k_base) render + ] + , Tasty.testGroup "roundtrip" + [ Tasty.QuickCheck.testProperty "r50k_base" (roundTrip Tiktoken.r50k_base) + , Tasty.QuickCheck.testProperty "p50k_base" (roundTrip Tiktoken.p50k_base) + , Tasty.QuickCheck.testProperty "p50k_edit" (roundTrip Tiktoken.p50k_edit) + , Tasty.QuickCheck.testProperty "cl100k_base" (roundTrip Tiktoken.cl100k_base) + , Tasty.QuickCheck.testProperty "o200k_base" (roundTrip Tiktoken.cl100k_base) + ] + + ] diff --git a/tasty/sample.txt b/tasty/sample.txt deleted file mode 100644 # file too large to diff: tasty/sample.txt diff --git a/tasty/sample/cl100k_base.golden b/tasty/sample/cl100k_base.golden new file mode 100644 # file too large to diff: tasty/sample/cl100k_base.golden diff --git a/tasty/sample/input.txt b/tasty/sample/input.txt new file mode 100644 # file too large to diff: tasty/sample/input.txt diff --git a/tasty/sample/o200k_base.golden b/tasty/sample/o200k_base.golden new file mode 100644 # file too large to diff: tasty/sample/o200k_base.golden diff --git a/tasty/sample/p50k_base.golden b/tasty/sample/p50k_base.golden new file mode 100644 # file too large to diff: tasty/sample/p50k_base.golden diff --git a/tasty/sample/r50k_base.golden b/tasty/sample/r50k_base.golden new file mode 100644 # file too large to diff: tasty/sample/r50k_base.golden diff --git a/tasty/special/cl100k_base.golden b/tasty/special/cl100k_base.golden new file mode 100644 --- /dev/null +++ b/tasty/special/cl100k_base.golden @@ -0,0 +1,10 @@ +100257 +271 +100258 +271 +100259 +271 +100260 +271 +100276 +198 diff --git a/tasty/special/input.txt b/tasty/special/input.txt new file mode 100644 --- /dev/null +++ b/tasty/special/input.txt @@ -0,0 +1,9 @@ +<|endoftext|> + +<|fim_prefix|> + +<|fim_middle|> + +<|fim_suffix|> + +<|endofprompt|> diff --git a/tasty/special/o200k_base.golden b/tasty/special/o200k_base.golden new file mode 100644 --- /dev/null +++ b/tasty/special/o200k_base.golden @@ -0,0 +1,22 @@ +199999 +279 +27 +91 +103473 +33197 +91 +3037 +27 +91 +103473 +155207 +91 +3037 +27 +91 +103473 +87556 +91 +3037 +200018 +198 diff --git a/tasty/special/p50k_base.golden b/tasty/special/p50k_base.golden new file mode 100644 --- /dev/null +++ b/tasty/special/p50k_base.golden @@ -0,0 +1,43 @@ +50256 +198 +198 +27 +91 +69 +320 +62 +40290 +91 +29 +198 +198 +27 +91 +69 +320 +62 +27171 +91 +29 +198 +198 +27 +91 +69 +320 +62 +37333 +844 +91 +29 +198 +198 +27 +91 +437 +1659 +16963 +457 +91 +29 +198 diff --git a/tasty/special/p50k_edit.golden b/tasty/special/p50k_edit.golden new file mode 100644 --- /dev/null +++ b/tasty/special/p50k_edit.golden @@ -0,0 +1,18 @@ +50256 +628 +50281 +628 +50282 +628 +50283 +198 +198 +27 +91 +437 +1659 +16963 +457 +91 +29 +198 diff --git a/tasty/special/r50k_base.golden b/tasty/special/r50k_base.golden new file mode 100644 --- /dev/null +++ b/tasty/special/r50k_base.golden @@ -0,0 +1,43 @@ +50256 +198 +198 +27 +91 +69 +320 +62 +40290 +91 +29 +198 +198 +27 +91 +69 +320 +62 +27171 +91 +29 +198 +198 +27 +91 +69 +320 +62 +37333 +844 +91 +29 +198 +198 +27 +91 +437 +1659 +16963 +457 +91 +29 +198 diff --git a/tasty/tokenization/cl100k_base.golden b/tasty/tokenization/cl100k_base.golden new file mode 100644 --- /dev/null +++ b/tasty/tokenization/cl100k_base.golden @@ -0,0 +1,96 @@ +"Lorem" +" ipsum" +" dolor" +" sit" +" amet" +"," +" consectetur" +" adipiscing" +" elit" +"," +" sed" +" do" +" eiusmod" +" tempor" +" incididunt" +" ut" +" labore" +" et" +" dolore" +" magna" +" aliqua" +"." +" Ut" +" enim" +" ad" +" minim" +" veniam" +"," +" quis" +" nostr" +"ud" +" exercitation" +" ullam" +"co" +" labor" +"is" +" nisi" +" ut" +" aliqu" +"ip" +" ex" +" ea" +" commodo" +" consequat" +"." +" Duis" +" aute" +" ir" +"ure" +" dolor" +" in" +" repreh" +"enderit" +" in" +" volupt" +"ate" +" velit" +" esse" +" c" +"illum" +" dolore" +" eu" +" fug" +"iat" +" nulla" +" pari" +"atur" +"." +" Except" +"eur" +" sint" +" occ" +"aec" +"at" +" cupid" +"atat" +" non" +" pro" +"ident" +"," +" sunt" +" in" +" culpa" +" qui" +" offic" +"ia" +" deser" +"unt" +" moll" +"it" +" anim" +" id" +" est" +" labor" +"um" +".\n" diff --git a/tasty/tokenization/input.txt b/tasty/tokenization/input.txt new file mode 100644 --- /dev/null +++ b/tasty/tokenization/input.txt @@ -0,0 +1,1 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. diff --git a/tasty/tokenization/o200k_base.golden b/tasty/tokenization/o200k_base.golden new file mode 100644 --- /dev/null +++ b/tasty/tokenization/o200k_base.golden @@ -0,0 +1,87 @@ +"Lorem" +" ipsum" +" dolor" +" sit" +" amet" +"," +" consectetur" +" adipiscing" +" elit" +"," +" sed" +" do" +" eiusmod" +" tempor" +" incididunt" +" ut" +" labore" +" et" +" dolore" +" magna" +" aliqua" +"." +" Ut" +" enim" +" ad" +" minim" +" veniam" +"," +" quis" +" nost" +"rud" +" exercitation" +" ullam" +"co" +" laboris" +" nisi" +" ut" +" aliqu" +"ip" +" ex" +" ea" +" commodo" +" consequat" +"." +" Duis" +" aute" +" ir" +"ure" +" dolor" +" in" +" reprehenderit" +" in" +" voluptate" +" velit" +" esse" +" c" +"illum" +" dolore" +" eu" +" fugiat" +" nulla" +" pariatur" +"." +" Ex" +"cepteur" +" sint" +" occaec" +"at" +" cupid" +"atat" +" non" +" pro" +"ident" +"," +" sunt" +" in" +" culpa" +" qui" +" officia" +" deserunt" +" moll" +"it" +" anim" +" id" +" est" +" laborum" +".\n" diff --git a/tasty/tokenization/p50k_base.golden b/tasty/tokenization/p50k_base.golden new file mode 100644 --- /dev/null +++ b/tasty/tokenization/p50k_base.golden @@ -0,0 +1,154 @@ +"L" +"orem" +" " +"ips" +"um" +" d" +"olor" +" sit" +" am" +"et" +"," +" con" +"sect" +"et" +"ur" +" adip" +"is" +"cing" +" el" +"it" +"," +" sed" +" do" +" e" +"ius" +"mod" +" tempor" +" inc" +"id" +"id" +"unt" +" ut" +" lab" +"ore" +" et" +" d" +"ol" +"ore" +" mag" +"na" +" al" +"iqu" +"a" +"." +" Ut" +" en" +"im" +" ad" +" minim" +" ven" +"iam" +"," +" qu" +"is" +" nost" +"r" +"ud" +" exerc" +"itation" +" u" +"ll" +"am" +"co" +" labor" +"is" +" n" +"isi" +" ut" +" al" +"iqu" +"ip" +" ex" +" e" +"a" +" commod" +"o" +" consequ" +"at" +"." +" Du" +"is" +" a" +"ute" +" ir" +"ure" +" d" +"olor" +" in" +" rep" +"re" +"he" +"nder" +"it" +" in" +" vol" +"upt" +"ate" +" vel" +"it" +" es" +"se" +" c" +"ill" +"um" +" d" +"ol" +"ore" +" e" +"u" +" fug" +"iat" +" null" +"a" +" par" +"i" +"atur" +"." +" Except" +"eur" +" s" +"int" +" occ" +"a" +"ec" +"at" +" cup" +"id" +"at" +"at" +" non" +" pro" +"ident" +"," +" s" +"unt" +" in" +" cul" +"pa" +" qui" +" offic" +"ia" +" des" +"er" +"unt" +" m" +"oll" +"it" +" anim" +" id" +" est" +" labor" +"um" +"." +"\n" diff --git a/tasty/tokenization/r50k_base.golden b/tasty/tokenization/r50k_base.golden new file mode 100644 --- /dev/null +++ b/tasty/tokenization/r50k_base.golden @@ -0,0 +1,154 @@ +"L" +"orem" +" " +"ips" +"um" +" d" +"olor" +" sit" +" am" +"et" +"," +" con" +"sect" +"et" +"ur" +" adip" +"is" +"cing" +" el" +"it" +"," +" sed" +" do" +" e" +"ius" +"mod" +" tempor" +" inc" +"id" +"id" +"unt" +" ut" +" lab" +"ore" +" et" +" d" +"ol" +"ore" +" mag" +"na" +" al" +"iqu" +"a" +"." +" Ut" +" en" +"im" +" ad" +" minim" +" ven" +"iam" +"," +" qu" +"is" +" nost" +"r" +"ud" +" exerc" +"itation" +" u" +"ll" +"am" +"co" +" labor" +"is" +" n" +"isi" +" ut" +" al" +"iqu" +"ip" +" ex" +" e" +"a" +" commod" +"o" +" consequ" +"at" +"." +" Du" +"is" +" a" +"ute" +" ir" +"ure" +" d" +"olor" +" in" +" rep" +"re" +"he" +"nder" +"it" +" in" +" vol" +"upt" +"ate" +" vel" +"it" +" es" +"se" +" c" +"ill" +"um" +" d" +"ol" +"ore" +" e" +"u" +" fug" +"iat" +" null" +"a" +" par" +"i" +"atur" +"." +" Except" +"eur" +" s" +"int" +" occ" +"a" +"ec" +"at" +" cup" +"id" +"at" +"at" +" non" +" pro" +"ident" +"," +" s" +"unt" +" in" +" cul" +"pa" +" qui" +" offic" +"ia" +" des" +"er" +"unt" +" m" +"oll" +"it" +" anim" +" id" +" est" +" labor" +"um" +"." +"\n" diff --git a/tasty/tokens.golden b/tasty/tokens.golden deleted file mode 100644 # file too large to diff: tasty/tokens.golden diff --git a/tiktoken.cabal b/tiktoken.cabal --- a/tiktoken.cabal +++ b/tiktoken.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: tiktoken -version: 1.0.1 +version: 1.0.2 synopsis: Haskell implementation of tiktoken description: This packages only implements tokenization. In other words, given an existing encoding (`cl100k_base`) you can tokenize @@ -12,9 +12,8 @@ extra-source-files: README.md , CHANGELOG.md - , tasty/sample.txt - , tasty/tokens.golden - , tasty/tokens.golden + , tasty/**/*.txt + , tasty/**/*.golden data-dir: data library @@ -28,9 +27,7 @@ , pcre-light >= 0.2 , raw-strings-qq , text - , transformers >= 0.2.0.0 , unordered-containers - , vector < 0.14 hs-source-dirs: src exposed-modules: Tiktoken other-modules: Paths_tiktoken @@ -42,10 +39,12 @@ type: exitcode-stdio-1.0 build-depends: base , bytestring - , tiktoken + , quickcheck-instances , tasty + , tasty-quickcheck , tasty-silver < 3.4 , text + , tiktoken hs-source-dirs: tasty main-is: Main.hs ghc-options: -Wall