diff --git a/.gitignore b/.gitignore new file mode 100644 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# stack uses this directory for build artifacts +/.stack-work/ + +# cabal stuff +dist-newstyle + +# Made by 'hasktags --ctags .' +tags diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +1.3 (2025-03-27) + + * Initial release diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,15 @@ +Copyright (c) 2024, Dino Morelli + +Permission to use, copy, modify, and/or distribute this software +for any purpose with or without fee is hereby granted, provided +that the above copyright notice and this permission notice appear +in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA +OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# genai-lib + + +## Synopsis + +Library and command line tool for interacting with various generative AI LLMs + + +## Description + +Library and command line tool for performing completions and chats with various +generative AI LLMs (Large Language Models). Works today with Ollama and some +work has been done to support OpenAI in the future. + +This project is very much in-progress and incomplete. + + +## Getting source + +Source code is available from codeberg at the [genai-lib](https://codeberg.org/dinofp/genai-lib.git) project page. + + +## Contact + +Dino Morelli diff --git a/genai-lib.cabal b/genai-lib.cabal new file mode 100644 --- /dev/null +++ b/genai-lib.cabal @@ -0,0 +1,100 @@ +cabal-version: 2.2 + +name: genai-lib +version: 1.3 +synopsis: Library and command line tool for interacting with various generative + AI LLMs +description: Library and command line tool for performing completions and chats + with various generative AI LLMs (Large Language Models). +author: Dino Morelli +maintainer: dino@ui3.info +copyright: 2024 Dino Morelli +category: Unclassified +license: ISC +license-file: LICENSE +build-type: Simple +extra-source-files: + .gitignore + README.md + stack.yaml + stack.yaml.lock +extra-doc-files: + CHANGELOG.md + +source-repository head + type: git + location: https://codeberg.org/dinofp/genai-lib + +common lang + default-language: Haskell2010 + default-extensions: + BangPatterns + DeriveFoldable + DeriveFunctor + DeriveGeneric + DeriveLift + DeriveTraversable + EmptyCase + FlexibleContexts + FlexibleInstances + GeneralizedNewtypeDeriving + ImportQualifiedPost + InstanceSigs + KindSignatures + LambdaCase + MultiParamTypeClasses + MultiWayIf + NamedFieldPuns + NumericUnderscores + OverloadedStrings + ScopedTypeVariables + StandaloneDeriving + TupleSections + ghc-options: + -fwarn-tabs + -Wall + -Wcompat + -Wincomplete-record-updates + -Wincomplete-uni-patterns + -Wpartial-fields + -Wredundant-constraints + build-depends: + aeson >= 2.1.2 && < 2.2 + , base >= 3 && < 5 + , formatting >= 7.2.0 && < 7.3 + , text >= 1.2.5 && < 3 + +library + import: lang + exposed-modules: + GenAILib.Common + GenAILib.HTTP + GenAILib.Ollama + GenAILib.System.Log + hs-source-dirs: + src/lib + build-depends: + bytestring >= 0.11.4 && < 0.12 + , hslogger >= 1.3.1 && < 1.4 + , http-client >= 0.7.16 && < 0.8 + , servant >= 0.20.1 && < 0.21 + , servant-client >= 0.20 && < 0.21 + , servant-client-core >= 0.20 && < 0.21 + , string-conv >= 0.2.0 && < 0.3 + +executable genai + import: lang + main-is: Main.hs + hs-source-dirs: + src/app + other-modules: + GenAI.Common + GenAI.Opts + Paths_genai_lib + autogen-modules: + Paths_genai_lib + build-depends: + heredoc >= 0.2.0 && < 0.3 + , genai-lib + , optparse-applicative >= 0.17.1 && < 0.19 + , prettyprinter >= 1.7.1 && < 1.8 diff --git a/src/app/GenAI/Common.hs b/src/app/GenAI/Common.hs new file mode 100644 --- /dev/null +++ b/src/app/GenAI/Common.hs @@ -0,0 +1,39 @@ +{-# LANGUAGE OverloadedRecordDot #-} + +module GenAI.Common + where + +import Data.Aeson (Value (Object)) +import Data.Aeson.Types (emptyObject) +import Data.Text.Lazy qualified as TL +import GenAILib.Common (LLMOptions (..), Model, Prompt (..), RawOutput) +import GenAILib.Ollama (Host, OllamaRequest (..), Stream, System) +import GenAILib.System.Log (Priority (DEBUG)) + + +data CLIOptions = CLIOptions + { host :: Host + , system :: Maybe System + , llmOptions :: LLMOptions + , stream :: Stream + , rawOutput :: RawOutput + , verbose :: Verbose + , model :: Model + } + + +newtype Verbose = Verbose Bool + deriving Show + + +verbosityToPriority :: Verbose -> Maybe Priority +verbosityToPriority (Verbose True) = Just DEBUG +verbosityToPriority (Verbose False) = Nothing + + +mkLLMRequest :: CLIOptions -> TL.Text -> OllamaRequest +mkLLMRequest opts promptText = OllamaRequest opts.model opts.system + (Prompt promptText) opts.stream (wrapMaybe opts.llmOptions.v) + where + wrapMaybe o@(Object _) = if o == emptyObject then Nothing else Just o + wrapMaybe _ = Nothing diff --git a/src/app/GenAI/Opts.hs b/src/app/GenAI/Opts.hs new file mode 100644 --- /dev/null +++ b/src/app/GenAI/Opts.hs @@ -0,0 +1,154 @@ +{-# LANGUAGE DuplicateRecordFields, OverloadedRecordDot, QuasiQuotes #-} + +module GenAI.Opts + ( parseOpts + ) + where + +import Data.Text.Lazy (pack) +import Data.Version (showVersion) +import Formatting ((%+), format, formatToString, string) +import Formatting.ShortFormatters (s) +import GenAILib.Common (Model (..), RawOutput (..), convertOptions) +import GenAILib.Ollama (Host (..), Stream (..), + System (..), defaultHost, hostFromString) +import Options.Applicative +import Paths_genai_lib (version) +import Prettyprinter (pretty) +import System.Environment (getProgName) +import Text.Heredoc (here) + +import GenAI.Common (CLIOptions (..), Verbose (..)) + + +readHost :: ReadM Host +readHost = eitherReader (\hostPortStr -> + maybe + (Left $ formatToString + ("Unable to parse" %+ string %+ "into host and port parts") hostPortStr) + Right + $ hostFromString hostPortStr + ) + + +{- HLINT ignore "Functor law" -} +parser :: Parser CLIOptions +parser = CLIOptions + <$> option readHost + ( long "host" + <> short 'H' + <> metavar "HOST:PORT" + <> help "Host and port where ollama serve is running" + <> showDefault + <> value defaultHost + ) + -- NOTE: The inner <$> is for Maybe, the outer <$> is for Parser + <*> ( (System . pack <$>) <$> optional ( strOption + ( long "system" + <> short 's' + <> metavar "STR" + <> help "System parameter" + ) + )) + <*> ( convertOptions <$> many ( strOption + ( long "option" + <> short 'o' + <> metavar "KEY:VALUE" + <> help "Options for the LLM, repeat for each option pair. See LLM OPTIONS below" + ) + )) + <*> ( Stream <$> switch + ( long "stream" + <> help "Response will be returned as a stream of objects" + ) + ) + <*> ( RawOutput <$> switch + ( long "raw-output" + <> short 'r' + <> help "Output the entire JSON response from the LLM" + ) + ) + <*> ( Verbose <$> switch + ( long "verbose" + <> short 'v' + <> help "Enable verbose output. Note: All logging goes to stderr" + ) + ) + <*> ( Model . pack <$> argument str + ( metavar "MODEL_ID" + <> help "Model identifier, see available models with `ollama list`" + ) + ) + + +versionHelper :: String -> Parser (a -> a) +versionHelper progName = + infoOption (formatToString (s %+ s) progName (showVersion version)) $ mconcat + [ long "version" + , help "Show version information" + , hidden + ] + + +parseOpts :: IO CLIOptions +parseOpts = do + pn <- getProgName + execParser $ info (parser <**> helper <**> versionHelper pn) + ( header (formatToString (s %+ "- Command line tool for interacting with an ollama server") pn) + <> footer' + ) + + +footer' :: InfoMod a +footer' = footerDoc . Just . pretty . format content . showVersion $ version + where content = [here|OVERVIEW + +This software expects the prompt string on STDIN and will respond with the LLM +response on STDOUT + +example usage + + $ echo "Why is the sky blue?" | genai some-fancy-model + +WHY WAS THIS DONE? + +ollama ships with command-line ability to submit prompts and do completions: + + $ ollama MODEL [PROMPT] ... + +but as far as I can see there's no easy way to use this method to change +options like the temperature and seed. This software allows greater control +over the options. + + $ echo "Tell me a story" | genai some-other-model -o temperature:0.2 + +LLM OPTIONS + +Some settings are communicated to the LLM as a map of options. Use them like this + + -o temperature:0.2 + +Some useful options + + option name description default + ------------------------------------------------------------------------------------- + seed Sets the random number seed to use for generation. Setting 0 + this to a specific number will make the model generate the + same text for the same prompt. + temperature The temperature of the model. Increasing the temperature 0.8 + will make the model answer more creatively. + Range 0.0 to 1.0 + top_k Reduces the probability of generating nonsense. A higher 40 + value (e.g. 100) will give more diverse answers, while a + lower value (e.g. 10) will be more conservative. + Range 0 to 100 + top_p Works together with top-k. A higher value (e.g., 0.95) will 0.9 + lead to more diverse text, while a lower value (e.g., 0.5) + will generate more focused and conservative text. + Range unknown, guessing 0.0 to 1.0 + +For the complete list of LLM options, see + +https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values + +Version|] %+ s %+ " Dino Morelli " diff --git a/src/app/Main.hs b/src/app/Main.hs new file mode 100644 --- /dev/null +++ b/src/app/Main.hs @@ -0,0 +1,22 @@ +{-# LANGUAGE OverloadedRecordDot #-} + +import Data.Text.Lazy.IO qualified as TL +import GenAILib.HTTP (display, doCompletion) +import GenAILib.System.Log (infoM, initLogging, lname) +import System.IO (stdin) + +import GenAI.Common (CLIOptions (host, rawOutput, verbose), mkLLMRequest, + verbosityToPriority) +import GenAI.Opts (parseOpts) + + +main :: IO () +main = do + opts <- parseOpts + initLogging . verbosityToPriority . verbose $ opts + infoM lname "Logging configured" + infoM lname "Args parsed" + + promptText <- TL.hGetContents stdin + let or' = mkLLMRequest opts promptText + doCompletion opts.host or' >>= display opts.rawOutput diff --git a/src/lib/GenAILib/Common.hs b/src/lib/GenAILib/Common.hs new file mode 100644 --- /dev/null +++ b/src/lib/GenAILib/Common.hs @@ -0,0 +1,58 @@ +{-# LANGUAGE DuplicateRecordFields #-} + +module GenAILib.Common + where + +import Control.Arrow ((&&&)) +import Control.Monad (when) +import Data.Aeson (Key, ToJSON, Value (Number, String), object, toJSON) +import Data.Aeson.Key (fromString) +import Data.Aeson.Types (Pair) +import Data.Maybe (mapMaybe) +import Data.String.Conv (toS) +import Data.Text.Lazy qualified as TL +import GHC.Generics (Generic) + + +newtype Model = Model TL.Text + deriving Generic + +instance ToJSON Model + + +newtype Prompt = Prompt TL.Text + deriving Generic + +instance ToJSON Prompt + + +splitAtColon :: String -> Maybe (String, String) +splitAtColon combinedStr = do + let (leftSide, rightSide) = takeWhile (/= ':') &&& dropWhile (/= ':') $ combinedStr + when (null rightSide) Nothing + pure (leftSide, tail rightSide) + + +newtype RawOutput = RawOutput Bool + + +newtype LLMOptions = LLMOptions { v :: Value } + deriving Generic + +instance ToJSON LLMOptions + + +-- For debugging the LLM options parsing, unused most of the time +debugOptsJSON :: LLMOptions -> Value +debugOptsJSON = toJSON + +convertOptions :: [String] -> LLMOptions +convertOptions = LLMOptions . object . pairs' + +pairs' :: [String] -> [Pair] +pairs' = map convertTypes . mapMaybe splitAtColon + +convertTypes :: (String, String) -> (Key, Value) +-- Turns out the only non-number option we're interested in for Ollama is "stop" +convertTypes (keystr@"stop", valstr) = (fromString keystr, String . toS $ valstr) +convertTypes (keystr, valstr) = (fromString keystr, Number . read $ valstr) diff --git a/src/lib/GenAILib/HTTP.hs b/src/lib/GenAILib/HTTP.hs new file mode 100644 --- /dev/null +++ b/src/lib/GenAILib/HTTP.hs @@ -0,0 +1,97 @@ +{-# LANGUAGE DataKinds, TypeOperators #-} + +module GenAILib.HTTP + ( display + , doChat + , doCompletion + ) + where + +import Data.Aeson (FromJSON, ToJSON, Value (Object, String), decode, encode) +import Data.Aeson.KeyMap qualified as A +import Data.ByteString.Lazy qualified as BL +import Data.Maybe (fromMaybe) +import Data.Proxy (Proxy (..)) +import Data.String.Conv (toS) +import Data.Text.IO qualified as TS +import Formatting ((%+), formatToString, string) +import Network.HTTP.Client (defaultManagerSettings, managerResponseTimeout, + newManager, responseTimeoutMicro) +import Servant.API ((:<|>) (..), (:>), JSON, Post, ReqBody) +import Servant.Client (BaseUrl (..), Scheme (Http), ClientM, + mkClientEnv, runClientM) +import Servant.Client.Core (clientIn) +import System.Exit (exitFailure) + +import GenAILib.Common (RawOutput (..)) +import GenAILib.Ollama (Host (..)) +import GenAILib.System.Log (debugM, emergencyM, errorM, lname) + + +defaultHttpTimeout :: Int +defaultHttpTimeout = 1_200_000_000 -- In microseconds, this is 2 minutes + + +{- NOTE: The reason we've chosen Aeson's Value type for the response is we want + to do one of these things with these responses: + + 1. Display only the LLM response text (in the JSON Value's 'response' or 'messages.content' field) + 2. Pretty-print the entire JSON response ( | jq . ) + 3. Dump the entire JSON response as-is ( | jq -r|--raw-output . ) + + We don't need to model the entire Ollama response JSON document to achieve these goals. +-} +type API + = "api" :> "generate" :> ReqBody '[JSON] Value :> Post '[JSON] Value + :<|> "api" :> "chat" :> ReqBody '[JSON] Value :> Post '[JSON] Value + +-- A Proxy to our API +api :: Proxy API +api = Proxy + +clientM :: Proxy ClientM +clientM = Proxy + +completion :: Value -> ClientM Value +chat :: Value -> ClientM Value +(completion :<|> chat) = api `clientIn` clientM + + +doCompletion :: ToJSON j => Host -> j -> IO Value +doCompletion = doRequest completion + +doChat :: ToJSON j => Host -> j -> IO Value +doChat = doRequest chat + +doRequest :: (FromJSON f, ToJSON t) => (f -> ClientM Value) -> Host -> t -> IO Value +doRequest client' (Host hostName port) bodyData = do + debugM lname . toS . encode $ bodyData + let + bodyDoc = fromMaybe (error "Something went wrong with JSON-ifying the POST data") + . decode . encode $ bodyData + manager' <- newManager $ defaultManagerSettings + { managerResponseTimeout = responseTimeoutMicro defaultHttpTimeout } + eres <- runClientM (client' bodyDoc) + (mkClientEnv manager' (BaseUrl Http (toS hostName) port "")) + case eres of + Left err -> logAndExit $ show err + Right v@(Object _) -> pure v + Right somethingElse -> logAndExit $ show somethingElse + + +logAndExit :: String -> IO a +logAndExit msg = do + errorM lname $ formatToString ("Can't continue:" %+ string) msg + exitFailure + + +display :: RawOutput -> Value -> IO () + +display (RawOutput True) v = BL.putStr $ encode v <> "\n" + +display (RawOutput False) (Object keyMap) = case A.lookup "response" keyMap of + Just (String t) -> TS.putStrLn t + Just somethingElse -> emergencyM lname $ show somethingElse + Nothing -> emergencyM lname $ show keyMap + +display _ somethingElse = emergencyM lname $ show somethingElse diff --git a/src/lib/GenAILib/Ollama.hs b/src/lib/GenAILib/Ollama.hs new file mode 100644 --- /dev/null +++ b/src/lib/GenAILib/Ollama.hs @@ -0,0 +1,72 @@ +{-# LANGUAGE DuplicateRecordFields #-} + +module GenAILib.Ollama + ( Host (..) + , OllamaRequest (..) + , Stream (..) + , System (..) + , defaultHost + , hostFromString + ) + where + +import Data.Aeson (ToJSON, Value, defaultOptions, genericToEncoding, + genericToJSON, omitNothingFields, toEncoding, toJSON) +import Data.Aeson qualified as Aeson +import Data.String.Conv (toS) +import Data.Text.Lazy qualified as TL +import Formatting ((%), formatToString, int, text) +import GHC.Generics (Generic) +import Text.Read (readMaybe) + +import GenAILib.Common (Model, Prompt, splitAtColon) + + +newtype Stream = Stream Bool + deriving Generic + +instance ToJSON Stream + + +{- ^ This data structure is used to construct the POST body for an Ollama REST call +-} +data OllamaRequest = OllamaRequest + { model :: Model + , system :: Maybe System + , prompt :: Prompt + , stream :: Stream + , options :: Maybe Value + } + deriving Generic + +customOptions :: Aeson.Options +customOptions = defaultOptions { omitNothingFields = True } + +instance ToJSON OllamaRequest where + toJSON = genericToJSON customOptions + toEncoding = genericToEncoding customOptions + + +data Host = Host TL.Text Int + +-- I don't normally do this because it harms debugging when we want to see the +-- actual Haskell data structure serialized BUT this plays nicer with +-- optparse-applicative's showDefault +instance Show Host where + show (Host hostName port) = formatToString (text % ":" % int) hostName port + + +defaultHost :: Host +defaultHost = Host "localhost" 11434 + + +hostFromString :: String -> Maybe Host +hostFromString combinedStr = do + (hostName, portStr) <- splitAtColon combinedStr + Host (toS hostName) <$> readMaybe portStr + + +newtype System = System TL.Text + deriving Generic + +instance ToJSON System diff --git a/src/lib/GenAILib/System/Log.hs b/src/lib/GenAILib/System/Log.hs new file mode 100644 --- /dev/null +++ b/src/lib/GenAILib/System/Log.hs @@ -0,0 +1,47 @@ +module GenAILib.System.Log + ( initLogging, lname + , logTest + + -- Re-exported from System.Log.Logger + , Priority (..), debugM, infoM, noticeM, warningM, errorM + , criticalM, alertM, emergencyM + ) + where + +import Data.Maybe (fromMaybe) +import System.IO (stderr) +import System.Log.Formatter (simpleLogFormatter) +import System.Log.Handler (setFormatter) +import System.Log.Handler.Simple (streamHandler) +import System.Log.Logger + + +lname :: String +lname = "normal-output" + + +initLogging :: Maybe Priority -> IO () +initLogging mLogPriority = do + let logPriority = fromMaybe NOTICE mLogPriority + + -- Remove the root logger's default handler that writes every + -- message to stderr! + updateGlobalLogger rootLoggerName removeHandler + + errH <- flip setFormatter (simpleLogFormatter "[$time : $prio] $msg") + <$> streamHandler stderr DEBUG + updateGlobalLogger lname $ setHandlers [errH] + updateGlobalLogger lname $ setLevel logPriority + + +-- Test function to generate every kind of log message +logTest :: IO () +logTest = do + debugM lname "log test message DEBUG 1 of 8" + infoM lname "log test message INFO 2 of 8" + noticeM lname "log test message NOTICE 3 of 8" + warningM lname "log test message WARNING 4 of 8" + errorM lname "log test message ERROR 5 of 8" + criticalM lname "log test message CRITICAL 6 of 8" + alertM lname "log test message ALERT 7 of 8" + emergencyM lname "log test message EMERGENCY 8 of 8" diff --git a/stack.yaml b/stack.yaml new file mode 100644 --- /dev/null +++ b/stack.yaml @@ -0,0 +1,68 @@ +# This file was automatically generated by 'stack init' +# +# Some commonly used options have been documented as comments in this file. +# For advanced use and comprehensive documentation of the format, please see: +# https://docs.haskellstack.org/en/stable/yaml_configuration/ + +# A 'specific' Stackage snapshot or a compiler version. +# A snapshot resolver dictates the compiler version and the set of packages +# to be used for project dependencies. For example: +# +# snapshot: lts-22.28 +# snapshot: nightly-2024-07-05 +# snapshot: ghc-9.6.6 +# +# The location of a snapshot can be provided as a file or url. Stack assumes +# a snapshot provided as a file might change, whereas a url resource does not. +# +# snapshot: ./custom-snapshot.yaml +# snapshot: https://example.com/snapshots/2024-01-01.yaml +# snapshot: +# url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/22/34.yaml +snapshot: lts-22.6 + +# User packages to be built. +# Various formats can be used as shown in the example below. +# +# packages: +# - some-directory +# - https://example.com/foo/bar/baz-0.0.2.tar.gz +# subdirs: +# - auto-update +# - wai +packages: +- . +# Dependency packages to be pulled from upstream that are not in the snapshot. +# These entries can reference officially published versions as well as +# forks / in-progress versions pinned to a git hash. For example: +# +# extra-deps: +# - acme-missiles-0.3 +# - git: https://github.com/commercialhaskell/stack.git +# commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a +# +# extra-deps: [] + +# Override default flag values for project packages and extra-deps +# flags: {} + +# Extra package databases containing global packages +# extra-package-dbs: [] + +# Control whether we use the GHC we find on the path +# system-ghc: true +# +# Require a specific version of Stack, using version ranges +# require-stack-version: -any # Default +# require-stack-version: ">=3.1" +# +# Override the architecture used by Stack, especially useful on Windows +# arch: i386 +# arch: x86_64 +# +# Extra directories used by Stack for building +# extra-include-dirs: [/path/to/dir] +# extra-lib-dirs: [/path/to/dir] +# +# Allow a newer minor version of GHC than the snapshot specifies +# compiler-check: newer-minor diff --git a/stack.yaml.lock b/stack.yaml.lock new file mode 100644 --- /dev/null +++ b/stack.yaml.lock @@ -0,0 +1,12 @@ +# This file was autogenerated by Stack. +# You should not edit this file by hand. +# For more information, please see the documentation at: +# https://docs.haskellstack.org/en/stable/lock_files + +packages: [] +snapshots: +- completed: + sha256: 1b4c2669e26fa828451830ed4725e4d406acc25a1fa24fcc039465dd13d7a575 + size: 714100 + url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/22/6.yaml + original: lts-22.6