packages feed

kenshou-check-0.1.0.0: src/Kenshou/Check/Ledger/Read.hs

module Kenshou.Check.Ledger.Read
  ( FactSource (..),
    SegmentRef (..),
    LedgerSet (..),
    LedgerReadError (..),
    openSegment,
    discoverLedgers,
    foldFacts,
  )
where

import Control.Exception (Exception, catch, throwIO)
import Control.Monad (foldM)
import Data.Aeson (eitherDecodeStrict')
import Data.ByteString qualified as ByteString
import Data.ByteString.Char8 qualified as ByteString.Char8
import Data.List (sort)
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Check.Fact (Fact)
import Kenshou.Check.Ledger (LedgerHeader)
import Kenshou.Core.Canonical (sha256Hex)
import System.Directory (doesDirectoryExist, doesFileExist, listDirectory)
import System.FilePath (makeRelative, takeExtension, (</>))
import System.IO
import System.IO.Error (isEOFError)

newtype FactSource = FactSource {next :: IO (Maybe Fact)}

data SegmentRef = SegmentRef
  { path :: !FilePath,
    relativePath :: !FilePath,
    sha256 :: !Text,
    bytes :: !Integer
  }
  deriving stock (Eq, Show)

newtype LedgerSet = LedgerSet {segments :: [SegmentRef]}
  deriving stock (Eq, Show)

newtype LedgerReadError = LedgerReadError Text deriving stock (Eq, Show)

instance Exception LedgerReadError

openSegment :: FilePath -> IO (LedgerHeader, FactSource)
openSegment path = do
  handle <- openBinaryFile path ReadMode
  hSetBuffering handle (BlockBuffering (Just (64 * 1024)))
  endsWithNewline <- fileEndsWithNewline handle
  headerLine <- ByteString.Char8.hGetLine handle
  header <- either (throwIO . LedgerReadError . Text.pack) pure (eitherDecodeStrict' headerLine)
  pure (header, FactSource (readNext handle endsWithNewline))
  where
    readNext handle endsWithNewline = do
      done <- hIsEOF handle
      if done
        then hClose handle >> pure Nothing
        else do
          let readLine = do
                line <- ByteString.Char8.hGetLine handle
                atEnd <- hIsEOF handle
                if atEnd && not endsWithNewline
                  then hClose handle >> pure Nothing
                  else case eitherDecodeStrict' line of
                    Right fact -> pure (Just fact)
                    Left err -> hClose handle >> throwIO (LedgerReadError (Text.pack path <> ": " <> Text.pack err))
          readLine `catch` \err ->
            if isEOFError err && not endsWithNewline
              then hClose handle >> pure Nothing
              else throwIO (err :: IOError)

discoverLedgers :: FilePath -> IO LedgerSet
discoverLedgers root = do
  exists <- doesDirectoryExist root
  if not exists
    then pure (LedgerSet [])
    else do
      names <- sort <$> listDirectory root
      paths <- filterMFile [root </> name | name <- names, takeExtension name == ".jsonl"]
      LedgerSet <$> traverse describe paths
  where
    filterMFile [] = pure []
    filterMFile (path : rest) = do
      file <- doesFileExist path
      others <- filterMFile rest
      pure (if file then path : others else others)
    describe path = do
      contents <- ByteString.readFile path
      pure (SegmentRef path (makeRelative root path) (sha256Hex contents) (fromIntegral (ByteString.length contents)))

foldFacts :: LedgerSet -> state -> (state -> Fact -> IO state) -> IO state
foldFacts ledger initial step = foldM foldSegment initial ledger.segments
  where
    foldSegment state segment = do
      (_, source) <- openSegment segment.path
      go state source
    go state source =
      source.next >>= \case
        Nothing -> pure state
        Just fact -> step state fact >>= (`go` source)

fileEndsWithNewline :: Handle -> IO Bool
fileEndsWithNewline handle = do
  size <- hFileSize handle
  if size == 0
    then pure False
    else do
      hSeek handle SeekFromEnd (-1)
      byte <- ByteString.hGet handle 1
      hSeek handle AbsoluteSeek 0
      pure (byte == ByteString.Char8.pack "\n")