llama-cpp-haskell 0.2 → 0.2.1

raw patch · 3 files changed

+66/−18 lines, 3 filesdep +http-clientPVP ok

version bump matches the API change (PVP)

Dependencies added: http-client

API changes (from Hackage documentation)

Files

• Llama.hs +38/−2
• Main.hs +25/−14
• llama-cpp-haskell.cabal +3/−2

@@ -4,6 +4,7 @@  import Conduit import Data.Aeson+import Data.ByteString.Lazy (ByteString) import Data.Text (Text) import Data.Word import GHC.Generics@@ -36,6 +37,18 @@   } deriving (Show, Generic) instance FromJSON LlamaApplyTemplateResponse +data LlamaTokenizeRequest = LlamaTokenizeRequest+  { content :: Text+  , add_special :: Bool+  , parse_special :: Bool+  } deriving (Show, Generic)+instance ToJSON LlamaTokenizeRequest++newtype LlamaTokenizeResponse = LlamaTokenizeResponse+  { tokens :: [Token]+  } deriving (Show, Generic)+instance FromJSON LlamaTokenizeResponse+ newtype LlamaDetokenizeRequest = LlamaDetokenizeRequest   { tokens :: [Token]   } deriving (Show, Generic)@@ -63,15 +76,24 @@ type Token = Word32 type URL = String +-- |Apply the LLM tempate to produce a raw LLM prompt from the role-content pairs+applyTemplateSimple :: URL -> LlamaApplyTemplateRequest -> IO (Maybe Text)+applyTemplateSimple = applyTemplateGeneral httpLBS++-- |Like `applyTemplateSimple` but with user-supplied `Manager` applyTemplate :: URL -> Manager -> LlamaApplyTemplateRequest -> IO (Maybe Text)-applyTemplate url manager input = do+applyTemplate url manager = applyTemplateGeneral (`httpLbs` manager) url++-- |Like `applyTemplateSimple` but with user-supplied fetcher+applyTemplateGeneral :: (ToJSON p) => (Request -> IO (Response ByteString)) -> [Char] -> p -> IO (Maybe Text)+applyTemplateGeneral fetch url input = do   let request = parseRequest_ $ url ++ "/apply-template"       body = encode input       req = request { method = "POST"                     , requestBody = RequestBodyLBS body                     , requestHeaders = [("Content-Type", "application/json")]                     }-  response <- httpLbs req manager+  response <- fetch req   case decode (responseBody response) of     Just (LlamaApplyTemplateResponse text) -> return (Just text)     Nothing -> do@@ -103,6 +125,20 @@                     , requestHeaders = [("Content-Type", "application/json")]                     }   pure $ httpSource req getResponseBody .| eventConduit++tokenize :: URL -> LlamaTokenizeRequest -> IO (Maybe [Token])+tokenize url input = do+  let request = parseRequest_ $ url ++ "/tokenize"+      req = request { method = "POST"+                    , requestBody = RequestBodyLBS $ encode input+                    , requestHeaders = [("Content-Type", "application/json")]+                    }+  response <- httpLBS req+  case decode (responseBody response) of+    Just (LlamaTokenizeResponse result) -> return (Just result)+    Nothing -> do+      liftIO $ hPutStrLn stderr "Failed to decode Llama response"+      return Nothing  detokenize :: URL -> [Token] -> IO (Maybe Text) detokenize url input = do

@@ -22,7 +22,8 @@   { systemPrompt :: w ::: Text <?> "The system prompt to use" <!> "You are a helpful assistant."   , url :: w ::: String <?> "llama-server URL" <!> "http://localhost:8080"   , streaming :: w ::: Bool <?> "use to stream output from the LLM"-  , stripThinking :: w ::: Bool <?> "remove \"</think>\" and everything that occurs before it, only works in non-streaming mode"+  , stripThinking :: w ::: Bool <?> "remove \"</think>\" and everything that occurs before it"+  , templateOnly :: w ::: Bool <?> "only apply the chat template without running LLM completion"   } deriving (Generic)  instance ParseRecord (Options Wrapped) where@@ -36,16 +37,26 @@   let request = [ LlamaMessage System $ systemPrompt opts                 , LlamaMessage User input                 ]-  case streaming opts of-    False -> do-      response <- llamaTemplated (url opts) (LlamaApplyTemplateRequest request)-      case response of-        Nothing -> T.hPutStrLn stderr "Got no response from the server." >> exitFailure-        Just r -> T.putStrLn $ if stripThinking opts then snd $ T.breakOnEnd "</think>" r else r-    True -> do-      hSetBuffering stdout NoBuffering-      conduit <- llamaTemplatedStreaming (url opts) (LlamaApplyTemplateRequest request)-      runResourceT $ do-        list <- lazyConsume conduit-        liftIO $ mapM_ (T.putStr . LS.content) $ list-      T.putStrLn ""+  if templateOnly opts then do+    response <- applyTemplateSimple (url opts) (LlamaApplyTemplateRequest request)+    case response of+      Nothing -> T.hPutStrLn stderr "Got no response from the server." >> exitFailure+      Just r -> T.putStrLn $ if stripThinking opts then snd $ T.breakOnEnd "</think>" r else r+  else+    case streaming opts of+      False -> do+        response <- llamaTemplated (url opts) (LlamaApplyTemplateRequest request)+        case response of+          Nothing -> T.hPutStrLn stderr "Got no response from the server." >> exitFailure+          Just r -> T.putStrLn $ if stripThinking opts then snd $ T.breakOnEnd "</think>" r else r+      True -> do+        hSetBuffering stdout NoBuffering+        conduit <- llamaTemplatedStreaming (url opts) (LlamaApplyTemplateRequest request)+        runResourceT $ do+          list <- lazyConsume conduit+          liftIO $ mapM_ T.putStr $ (if stripThinking opts then dropSep "</think>" else id) $ map LS.content list+        T.putStrLn ""++dropSep :: Eq a => a -> [a] -> [a]+dropSep _ [] = []+dropSep a (x:xs) = if a == x then xs else dropSep a xs

@@ -1,6 +1,6 @@ cabal-version:      2.2 name:               llama-cpp-haskell-version:            0.2+version:            0.2.1 synopsis:           Haskell bindings for the llama.cpp llama-server and a simple CLI description:        This is the interface that allows one to interface with llama-server RPC API using Haskell concepts. It also includes a `llamacall` binary to do it from your favorite command line shell and use it in scripting. license:            AGPL-3.0-only@@ -20,7 +20,7 @@ Source-repository this   type:              git   location:          https://github.com/l29ah/llama-cpp-haskell.git-  tag:               0.2+  tag:               0.2.1  common stuff     ghc-options: -Wall@@ -29,6 +29,7 @@                     , conduit ^>= 1.3.5                     , conduit-extra ^>= 1.3.7                     , exceptions ^>= 0.10.9+                    , http-client ^>= 0.7.19                     , http-conduit ^>= 2.3                     , aeson ^>= 2.2                     , text ^>= 2.1