packages feed

hs-hath-1.3.0: app/Migrate.hs

-- | migrate-to-r2: upload an SQLite or Filesystem cache to an R2 bucket.
module Migrate ( main ) where

import           Database.SQLite.Simple ( Only(..), fold_, query_, withConnection )

import           Hath.Cache.R2          ( R2Connection(..)
                                        , fileURIToKey
                                        , mkR2Connection
                                        , putR2Object
                                        , runR2
                                        )
import           Hath.Config            ( R2Config(..) )
import           Hath.FileURI           ( FileURI, parseFileURI )

import           Network.Minio          ( defaultPutObjectOptions )
import qualified Network.Minio          as Minio

import           Options.Applicative

import           Relude

import           System.Directory       ( doesDirectoryExist, doesFileExist, listDirectory )
import           System.FilePath        ( (</>) )

data MigrateOptions
  = MigrateOptions
  { optSourceType :: !SourceType
  , optSourcePath :: !FilePath
  , optR2Endpoint :: !Text
  , optR2Bucket   :: !Text
  , optDryRun     :: !Bool
  , optBatchSize  :: !Int
  , optVerbose    :: !Bool
  }

data SourceType = SQLiteSource | FilesystemSource

parseOptions :: Parser MigrateOptions
parseOptions
  = MigrateOptions
  <$> option
    parseSourceType
    (long "source-type" <> short 't' <> metavar "TYPE" <> help "Source type: sqlite or filesystem")
  <*> strOption
    (long "source"
     <> short 's'
     <> metavar "PATH"
     <> help "Source path (SQLite db file or cache directory)")
  <*> strOption
    (long "endpoint"
     <> short 'e'
     <> metavar "URL"
     <> help "R2 endpoint URL (e.g., https://xxx.r2.cloudflarestorage.com)")
  <*> strOption (long "bucket" <> short 'b' <> metavar "NAME" <> help "R2 bucket name")
  <*> switch
    (long "dry-run" <> short 'n' <> help "Show what would be migrated without actually uploading")
  <*> option
    auto
    (long "batch-size"
     <> metavar "N"
     <> value 100
     <> showDefault
     <> help "Number of files to process per batch")
  <*> switch (long "verbose" <> short 'v' <> help "Enable verbose output")

parseSourceType :: ReadM SourceType
parseSourceType = eitherReader $ \case
  "sqlite"     -> Right SQLiteSource
  "filesystem" -> Right FilesystemSource
  "fs"         -> Right FilesystemSource
  other        -> Left $ "Unknown source type: " <> other <> ". Use 'sqlite' or 'filesystem'"

optsInfo :: ParserInfo MigrateOptions
optsInfo
  = info
    (parseOptions <**> helper)
    (fullDesc
     <> progDesc "Migrate cache files to Cloudflare R2"
     <> header "migrate-to-r2 - Cache migration tool for hs-hath")

main :: IO ()
main = do
  opts <- execParser optsInfo
  mkR2Connection R2Config { r2Endpoint = optR2Endpoint opts, r2Bucket = optR2Bucket opts } >>= \case
    Left err -> do
      putTextLn $ "Error creating R2 connection: " <> err
      exitFailure
    Right r2 -> do
      putTextLn $ "Connected to R2: " <> optR2Endpoint opts <> "/" <> optR2Bucket opts
      tally <- case optSourceType opts of
        SQLiteSource     -> migrateSQLite opts r2
        FilesystemSource -> migrateFilesystem opts r2
      putTextLn "\nMigration complete:"
      putTextLn $ "  Migrated: " <> show (migrated tally)
      putTextLn $ "  Errors: " <> show (failed tally)

data Tally = Tally { migrated :: !Int, failed :: !Int }

migrateOne :: MigrateOptions -> Text -> (FileURI -> IO (Either Text ())) -> Tally -> IO Tally
migrateOne opts fileId upload tally
  | optDryRun opts = tally <$ say ("  Would upload: " <> fileId)
  | otherwise
    = maybe (pure $ Left "not a file id") upload (parseFileURI (encodeUtf8 fileId)) >>= \case
      Left err -> tally { failed = failed tally + 1 }
        <$ say ("  Error uploading " <> fileId <> ": " <> err)
      Right () -> tally { migrated = migrated tally + 1 } <$ say ("  Uploaded: " <> fileId)
  where
    say = when (optVerbose opts) . putTextLn

-- | Rows are streamed, so the database is never in memory as a whole.
migrateSQLite :: MigrateOptions -> R2Connection -> IO Tally
migrateSQLite opts r2 = withConnection (optSourcePath opts) $ \conn -> do
  putTextLn $ "Migrating from SQLite: " <> toText (optSourcePath opts)
  total <- maybe (0 :: Int) fromOnly . listToMaybe <$> query_ conn "SELECT COUNT(*) FROM files"
  putTextLn $ "Total files to migrate: " <> show total
  when (optDryRun opts) $ putTextLn "[DRY RUN] Would upload the following files:"
  fmap fst
    $ fold_ conn "SELECT file_id, bytes FROM files" ( Tally 0 0, 0 :: Int )
    $ \( tally, done ) ( fileId, bytes ) -> do
      tally' <- migrateOne opts fileId (\uri -> putR2Object r2 uri bytes) tally
      when ((done + 1) `mod` optBatchSize opts == 0)
        $ putTextLn
        $ "  Progress: " <> show (done + 1) <> "/" <> show total
      pure ( tally', done + 1 )

-- | The cache is @root\/xxxx\/\<file id>@, xxxx being the first four characters of the id.
migrateFilesystem :: MigrateOptions -> R2Connection -> IO Tally
migrateFilesystem opts r2 = do
  putTextLn $ "Migrating from filesystem: " <> toText root
  unlessM (doesDirectoryExist root) $ do
    putTextLn $ "Error: Directory does not exist: " <> toText root
    exitFailure
  when (optDryRun opts) $ putTextLn "[DRY RUN] Would upload the following files:"
  dirs <- filterM (doesDirectoryExist . (root </>)) =<< listDirectory root
  foldlM migrateDirectory (Tally 0 0) dirs
  where
    root = optSourcePath opts

    migrateDirectory tally dir = foldlM (migrateFile dir) tally =<< listDirectory (root </> dir)

    migrateFile dir tally name = do
      let path       = root </> dir </> name
          upload uri
            = runR2 r2
            $ Minio.fPutObject (r2ConnBucket r2) (fileURIToKey uri) path defaultPutObjectOptions
      ifM (doesFileExist path) (migrateOne opts (toText name) upload tally) (pure tally)