rehoo (empty) → 0.1.0

raw patch · 5 files changed

+184/−0 lines, 5 filesdep +SafeSemaphoredep +basedep +cmdargssetup-changed

Dependencies added: SafeSemaphore, base, cmdargs, hslogger, monad-loops, regex-posix, shelly, split, system-filepath, text

Files

• LICENSE +28/−0
• Main.hs +117/−0
• README.md +11/−0
• Setup.hs +0/−0
• rehoo.cabal +28/−0

@@ -0,0 +1,28 @@+Copyright (c) 2003-2009, John Wiegley.  All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are+met:++- Redistributions of source code must retain the above copyright+  notice, this list of conditions and the following disclaimer.++- Redistributions in binary form must reproduce the above copyright+  notice, this list of conditions and the following disclaimer in the+  documentation and/or other materials provided with the distribution.++- Neither the name of New Artisans LLC nor the names of its+  contributors may be used to endorse or promote products derived from+  this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

@@ -0,0 +1,117 @@+{-# LANGUAGE DeriveDataTypeable #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ExtendedDefaultRules #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++module Main where++import           Control.Applicative+import           Control.Concurrent+import qualified Control.Concurrent.MSem as MSem+import           Control.Exception+import           Control.Monad hiding (sequence)+import           Data.Foldable+import qualified Data.List as L+import           Data.List.Split+import           Data.Text.Lazy as T hiding (filter, map, chunksOf)+import           Data.Traversable+import           Filesystem.Path+import           GHC.Conc+import           Prelude hiding (FilePath, sequence)+import           Shelly+import           System.Console.CmdArgs+import           System.Environment (getArgs, withArgs)+import           System.IO hiding (FilePath)++default (Text)++version :: String+version       = "0.1.0"++copyright :: String+copyright     = "2012"++reHooSummary :: String+reHooSummary = "rehoo v" ++ version ++ ", (C) John Wiegley " ++ copyright++data Rehoo = Rehoo+    { chunks  :: Int+    , jobs    :: Int+    , outfile :: String+    , dir     :: String }+    deriving (Data, Typeable, Show, Eq)++reHoo :: Rehoo+reHoo = Rehoo+    { chunks  = def &= typ "INT"+                &= help "Process INT .hoo's at a time (def: 16)"+    , jobs    = def &= name "j" &= typ "INT"+                &= help "Run INT hoogle combine's (def: # of capabilities)"+    , outfile = def &= typFile   &= help "Output file (defaults to default.hoo)"+    , dir     = def &= args &= typDir } &=+    summary reHooSummary &=+    program "rehoo" &=+    help "Rebuild default.hoo from many .hoo files in the given directory"++main :: IO ()+main = do+  -- process command-line options+  mainArgs <- getArgs+  opts     <- withArgs (if L.null mainArgs then ["--help"] else mainArgs)+                       (cmdArgs reHoo)++  caps <- GHC.Conc.getNumCapabilities++  let jobs'   = max (jobs opts) caps+      chunks' = max (chunks opts) 16++  _ <- GHC.Conc.setNumCapabilities jobs'++  hoos <- shelly $ filter <$> pure (`hasExtension` "hoo")+                          <*> (ls . fromText . T.pack . dir $ opts)++  putStrLn $ "Running with " ++ show jobs'+          ++ " workers and " ++ show chunks'+          ++ " sized chunks per worker"++  pool     <- MSem.new jobs'+  tempPath <- processHoos pool chunks' hoos++  shelly $ verbosely $+    mv tempPath $ fromText $ case outfile opts of+                               "" -> "default.hoo"+                               x  -> T.pack x++processHoos :: MSem.MSem Int -> Int -> [FilePath] -> IO FilePath+processHoos pool size hoos+  | L.length hoos > size =+    -- Split the list into 'size' sized chunks, then fork off a thread to+    -- recursively process each chunk.  The results are collected in series+    -- from MVars that each contain the final pathname of the subjob.+    bracket (traverse forkProcessHoos (chunksOf size hoos) >>=+             traverse takeMVar)+            (shelly . verbosely . traverse_ rm)+            (processHoos pool size)++  | otherwise = do+    -- Now that we have a list of files < size elements long, and we are+    -- already in our own thread, we can start the expensive process of+    -- running hoogle combine with the output going to a temp file.+    (tempPath, hndl) <- openTempFile "." "rehoo.hoo"+    hClose hndl++    MSem.with pool $+      shelly $ verbosely $+        run_ "hoogle" ( "combine" : "--outfile" : T.pack tempPath+                      : map toTextIgnore hoos)++    return . fromText . T.pack $ tempPath++  where+    forkProcessHoos :: [FilePath] -> IO (MVar FilePath)+    forkProcessHoos xs = do+      mVar <- newEmptyMVar+      _    <- forkIO $ processHoos pool size xs >>= putMVar mVar+      return mVar++-- Main.hs (rehoo) ends here

@@ -0,0 +1,11 @@+rehoo rebuilds `default.hoo` in parallel from an arbitrarily large number of+`.hoo` files.++Usage:++    rehoo -j8 .+    +This finds all `.hoo` files in the current directory, and results in a new+`default.hoo` in the same directory.  The aim is to be as quick at this as+possible, while minimizing the hit on system resources (like open file+handles).

@@ -0,0 +1,28 @@+Name: rehoo++Version:  0.1.0+Synopsis: Rebuild default.hoo from many .hoo files in the current directory++Description: Rebuild default.hoo from many .hoo files in the current directory++Homepage:           https://github.com/jwiegley/rehoo+License:            BSD3+License-file:       LICENSE+Author:             John Wiegley+Maintainer:         John Wiegley <johnw@newartisans.com>+Category:           Development+Build-type:         Simple+Cabal-version:      >= 1.8++Extra-Source-Files: README.md++Executable rehoo+    Main-is: Main.hs+    Ghc-options: -threaded++    Build-depends: base >= 4 && < 5, shelly, cmdargs, hslogger, regex-posix,+                   system-filepath, text, monad-loops, split, SafeSemaphore++Source-repository head+  type:     git+  location: https://github.com/jwiegley/rehoo