hadoop-tools 0.5 → 0.6
raw patch · 3 files changed
+127/−36 lines, 3 filesdep +unixdep ~hadoop-rpc
Dependencies added: unix
Dependency ranges changed: hadoop-rpc
Files
- hadoop-tools.cabal +8/−5
- src/Chmod.hs +7/−11
- src/Main.hs +112/−20
hadoop-tools.cabal view
@@ -1,5 +1,5 @@ name: hadoop-tools-version: 0.5+version: 0.6 synopsis: Fast command line tools for working with Hadoop.@@ -7,9 +7,11 @@ description: hh - Blazing fast interaction with HDFS .- Currently we only support v7 of the RPC protocol (< CDH5).+ If built against hadoop-rpc-1.x.x.x then these tools support v9 of the+ Hadoop RPC protocol (CDH 5.x and above). .- Support for v9 (>= CDH5) is coming soon.+ Earlier versions (< 0.6) can be installed using+ --constraint=hadoop-rpc==0.1.1.1 if you need v7 support. . > hh cat - Print the contents of a file to stdout > hh cd - Change working directory@@ -42,7 +44,7 @@ main-is: Main.hs hs-source-dirs: src - ghc-options: -funbox-strict-fields+ ghc-options: -funbox-strict-fields -Wall ghc-prof-options: -auto-all -caf-all other-modules:@@ -58,7 +60,7 @@ , configurator >= 0.3 , exceptions >= 0.6 , filepath >= 1.3- , hadoop-rpc >= 0.1.1.1+ , hadoop-rpc >= 1.0.0.1 , old-locale >= 1.0 , optparse-applicative >= 0.11 , protobuf >= 0.2.0.4@@ -68,6 +70,7 @@ , text >= 1.1 , time >= 1.4 , transformers >= 0.4+ , unix >= 2.7 , vector >= 0.10 test-suite test
src/Chmod.hs view
@@ -9,19 +9,18 @@ import Control.Applicative import Control.Monad (guard, msum) import qualified Data.Attoparsec.ByteString.Char8 as Atto-import qualified Data.Attoparsec.Combinator as Atto import Data.Bits import qualified Data.ByteString.Char8 as B-import Data.Char (ord)+import Data.Char (ord, isOctDigit) import Data.List (foldl') import Data.Maybe (mapMaybe)-import Data.Word (Word16, Word32)+import Data.Word (Word16) import Data.Hadoop.Types (FileType(..)) ------------------------------------------------------------------------ -data ChmodWho = Chmod_u | Chmod_g | Chmod_o | Chmod_a +data ChmodWho = Chmod_u | Chmod_g | Chmod_o | Chmod_a deriving (Show, Eq) data ChmodWhat = Chmod_r | Chmod_w | Chmod_x | Chmod_X | Chmod_s | Chmod_t@@ -78,13 +77,12 @@ ] octal :: Atto.Parser Word16- octal = B.foldl' step 0 `fmap` Atto.takeWhile1 isDig+ octal = B.foldl' step 0 `fmap` Atto.takeWhile1 isOctDigit where- isDig w = w >= '0' && w <= '7' step a w = a * 8 + fromIntegral (ord w - 48) applyChmod :: FileType -> [Chmod] -> Word16 -> Word16-applyChmod filetype cs old = foldl' f old cs+applyChmod filetype = flip (foldl' f) where f :: Word16 -> Chmod -> Word16 f _ (SetOctal new) = new@@ -96,8 +94,7 @@ f old (SetMinusWho who src) = minus who old (extract src old) set :: ChmodWho -> Word16 -> Word16 -> Word16- set who old new = (old .&. (complement (mask who))) .|.- setWho who new+ set who old new = (old .&. complement (mask who)) .|. setWho who new plus :: ChmodWho -> Word16 -> Word16 -> Word16 plus who old new = old .|. setWho who new@@ -115,7 +112,7 @@ setWho :: ChmodWho -> Word16 -> Word16 setWho Chmod_a new = foldl' (.|.) 0 $- map (\w -> setWho w new) [Chmod_u, Chmod_g, Chmod_o]+ map (`setWho` new) [Chmod_u, Chmod_g, Chmod_o] setWho who new = new `shiftL` s who extract :: ChmodWho -> Word16 -> Word16@@ -139,4 +136,3 @@ s Chmod_u = 6 s Chmod_g = 3 s Chmod_o = 0-
src/Main.hs view
@@ -4,15 +4,14 @@ module Main (main) where import Control.Concurrent.STM-import Control.Exception (SomeException, throwIO, bracket, fromException)+import Control.Exception (SomeException, throwIO, fromException) import Control.Monad import Control.Monad.Catch (handle, throwM) import Control.Monad.IO.Class (MonadIO, liftIO) import qualified Data.Attoparsec.ByteString.Char8 as Atto-import Data.Bits ((.&.), shiftR)+import Data.Bits ((.&.), shiftL, shiftR) import Data.ByteString (ByteString) import qualified Data.ByteString.Char8 as B-import Data.Char (ord) import Data.Foldable (foldMap) import Data.Maybe (fromMaybe) import qualified Data.Text as T@@ -28,21 +27,23 @@ import Data.Configurator.Types (Worth(..)) import Options.Applicative hiding (Success) import System.Environment (getEnv)+import System.Exit (exitFailure) import qualified System.FilePath as FilePath import qualified System.FilePath.Posix as Posix import System.IO import System.IO.Unsafe (unsafePerformIO) import System.Locale (defaultTimeLocale)+import System.Posix.User (GroupEntry(..), getGroups, getGroupEntryForID) import Text.PrettyPrint.Boxes hiding ((<>), (//)) -import Data.Hadoop.Configuration (getHadoopConfig)+import Data.Hadoop.Configuration (getHadoopConfig, getHadoopUser) import Data.Hadoop.HdfsPath import Data.Hadoop.Types import Network.Hadoop.Hdfs hiding (runHdfs) import Network.Hadoop.Read import Chmod-import qualified Glob as Glob+import qualified Glob import Paths_hadoop_tools (version) @@ -53,10 +54,11 @@ cmd <- execParser optsParser case cmd of SubIO io -> io- SubHdfs hdfs -> handle printError (runHdfs hdfs)+ SubHdfs hdfs -> handle exitError (runHdfs hdfs) where optsParser = info (helper <*> options) (fullDesc <> header "hh - Blazing fast interaction with HDFS")+ exitError err = printError err >> exitFailure runHdfs :: Hdfs a -> IO a runHdfs hdfs = do@@ -88,6 +90,12 @@ getHdfsUser :: IO (Maybe User) getHdfsUser = C.load [Optional configPath] >>= flip C.lookup "hdfs.user" +getGroupNames :: IO [Group]+getGroupNames = do+ groups <- getGroups+ entries <- mapM getGroupEntryForID groups+ return $ map (T.pack . groupName) entries+ getNameNode :: IO (Maybe NameNode) getNameNode = do cfg <- C.load [Optional configPath]@@ -111,8 +119,8 @@ return (home `FilePath.combine` ".hhwd") {-# NOINLINE workingDirConfigPath #-} -getDefaultWorkingDir :: MonadIO m => m HdfsPath-getDefaultWorkingDir = liftIO $ (("/user" </>) . T.encodeUtf8 . hcUser) <$> getConfig+getHomeDir :: MonadIO m => m HdfsPath+getHomeDir = liftIO $ (("/user" </>) . T.encodeUtf8 . hcUser) <$> getConfig getWorkingDir :: MonadIO m => m HdfsPath getWorkingDir = liftIO $ handle onError@@ -120,7 +128,7 @@ <$> B.readFile workingDirConfigPath where onError :: SomeException -> IO HdfsPath- onError = const getDefaultWorkingDir+ onError = const getHomeDir setWorkingDir :: MonadIO m => HdfsPath -> m () setWorkingDir path = liftIO $ B.writeFile workingDirConfigPath@@ -146,6 +154,12 @@ go xs (y : ys) = go (y : xs) ys go xs [] = (xs, []) +dropFileName :: HdfsPath -> HdfsPath+dropFileName = B.pack . Posix.dropFileName . B.unpack++takeFileName :: HdfsPath -> HdfsPath+takeFileName = B.pack . Posix.takeFileName . B.unpack+ ------------------------------------------------------------------------ data SubCommand = SubCommand@@ -176,6 +190,9 @@ , subPwd , subRemove , subRename+ , subTest+ , subTestNewer+ , subTestOlder , subVersion ] @@ -199,7 +216,7 @@ where go = cd <$> optional (argument bstr (completeDir <> help "the directory to change to")) cd mpath = SubHdfs $ do- path <- getAbsolute =<< maybe getDefaultWorkingDir return mpath+ path <- getAbsolute =<< maybe getHomeDir return mpath _ <- getListingOrFail path setWorkingDir path @@ -236,9 +253,9 @@ subFind :: SubCommand subFind = SubCommand "find" "Recursively search a directory tree" go where- go = find <$> (optional (argument bstr (completeDir <> help "the path to recursively search")))- <*> (optional (option bstr (long "name" <> metavar "FILENAME"- <> help "the file name to match")))+ go = find <$> optional (argument bstr (completeDir <> help "the path to recursively search"))+ <*> optional (option bstr (long "name" <> metavar "FILENAME"+ <> help "the file name to match")) find mpath mexpr = SubHdfs $ do matcher <- liftIO (mkMatcher mexpr) printFindResults (fromMaybe "" mpath) matcher@@ -258,9 +275,7 @@ let dst = fromMaybe (Posix.takeFileName $ B.unpack src) mdst absSrc <- getAbsolute src mReadHandle <- openRead absSrc- let doRead readHandle = liftIO $ bracket- (openFile dst WriteMode)- (hClose)+ let doRead readHandle = liftIO $ withFile dst WriteMode (\writeHandle -> hdfsMapM_ (B.hPut writeHandle) readHandle) maybe (return ()) doRead mReadHandle @@ -290,10 +305,21 @@ where go = rm <$> argument bstr (completePath <> help "the file/directory to remove") <*> switch (short 'r' <> help "recursively remove the whole file hierarchy")- rm path recursive = SubHdfs $ do+ <*> switch (short 's' <> long "skipTrash" <> help "immediately delete, bypassing trash")+ rm path recursive skipTrash = SubHdfs $ do absPath <- getAbsolute path- ok <- delete recursive absPath- unless ok $ liftIO . B.putStrLn $ "Failed to remove: " <> absPath+ if skipTrash || ".Trash" `elem` B.split '/' absPath+ then do+ ok <- delete recursive absPath+ unless ok $ liftIO . B.putStrLn $ "Failed to remove: " <> absPath+ else do+ home <- getHomeDir+ let absDst = home </> ".Trash" </> "Current" <> absPath+ absDstDir = dropFileName absDst+ ok <- mkdirs True absDstDir+ unless ok $ liftIO . B.putStrLn $ "Failed to make trash folder: " <> absDstDir+ rename True absPath absDst+ liftIO . B.putStrLn . B.unwords $ ["Moved:", absPath, "to trash at:", absDst] subRename :: SubCommand subRename = SubCommand "mv" "Rename a file or directory" go@@ -304,7 +330,73 @@ mv src dst force = SubHdfs $ do absSrc <- getAbsolute src absDst <- getAbsolute dst- rename force absSrc absDst+ mSrcType <- fmap fsFileType <$> getFileInfo absSrc+ mDstType <- fmap fsFileType <$> getFileInfo absDst+ let absDst' = if (mSrcType, mDstType) == (Just File, Just Dir)+ then absDst </> takeFileName src+ else absDst+ rename force absSrc absDst'++subTest :: SubCommand+subTest = SubCommand "test" "If file exists, has zero length, is a directory then return 0, else return 1" go+ where+ go = test <$> argument bstr (completePath <> help "file/directory")+ <*> switch (short 'e' <> help "Test exists")+ <*> switch (short 'z' <> help "Test is zero length")+ <*> switch (short 'd' <> help "Test is a directory")+ <*> switch (short 'f' <> help "Test is a regular file")+ <*> switch (short 'l' <> help "Test is a symbolic link")+ <*> switch (short 'r' <> help "Test read permission is granted")+ <*> switch (short 'w' <> help "Test write permission is granted")+ <*> switch (short 'x' <> help "Test exectute permission is granted")+ test path e z d f l r w x = SubHdfs $ do+ absPath <- getAbsolute path+ minfo <- getFileInfo absPath+ user <- liftIO getHadoopUser+ groups <- liftIO getGroupNames+ case minfo of+ Nothing -> fail $ unwords ["No such file/directory", B.unpack absPath]+ Just fs@FileStatus{..} -> do+ when (d && fsFileType /= Dir) $ fail . unwords $ ["Not a directory"]+ when (f && fsFileType /= File) $ fail . unwords $ ["Not a regular file"]+ when (l && fsFileType /= SymLink) $ fail . unwords $ ["Not a regular file"]+ when (z && fsLength /= 0) $ fail . unwords $ ["Not zero length"]+ when (r && not (hasPerm user groups fs 4)) $ fail . unwords $ ["Not readable"]+ when (w && not (hasPerm user groups fs 2)) $ fail . unwords $ ["Not writable"]+ when (x && not (hasPerm user groups fs 1)) $ fail . unwords $ ["Not executable"]+ where+ hasPerm user groups FileStatus{..} p = (fsOwner == user && (fsPermission .&. (p `shiftL` 6)) /= 0) ||+ (fsGroup `elem` groups && (fsPermission .&. (p `shiftL` 3)) /= 0) ||+ ((fsPermission .&. p) /= 0)++subTestNewer :: SubCommand+subTestNewer = SubCommand "test-newer" "file1 is newer (modification time) than file2" go+ where+ go = testNewer <$> argument bstr (completePath <> help "file/directory")+ <*> argument bstr (completePath <> help "file/directory")+ <*> pure False++subTestOlder :: SubCommand+subTestOlder = SubCommand "test-older" "file1 is older (modification time) than file2" go+ where+ go = testNewer <$> argument bstr (completePath <> help "file/directory")+ <*> argument bstr (completePath <> help "file/directory")+ <*> pure True++testNewer :: HdfsPath -> HdfsPath -> Bool -> SubMethod+testNewer path1 path2 older = SubHdfs $ do+ absPath1 <- getAbsolute path1+ absPath2 <- getAbsolute path2+ minfo1 <- getFileInfo absPath1+ minfo2 <- getFileInfo absPath2+ case (minfo1, minfo2, older) of+ (Just fs1, Just fs2, False) ->+ when (fsModificationTime fs1 < fsModificationTime fs2) $+ fail . unwords $ [B.unpack path1, "is older than", B.unpack path2]+ (Just fs1, Just fs2, True) ->+ when (fsModificationTime fs1 > fsModificationTime fs2) $+ fail . unwords $ [B.unpack path1, "is newer than", B.unpack path2]+ _ -> fail $ unwords ["No such file/directory"] subVersion :: SubCommand subVersion = SubCommand "version" "Show version information" go