diff --git a/hadoop-tools.cabal b/hadoop-tools.cabal --- a/hadoop-tools.cabal +++ b/hadoop-tools.cabal @@ -1,5 +1,5 @@ name: hadoop-tools -version: 0.5 +version: 0.6 synopsis: Fast command line tools for working with Hadoop. @@ -7,9 +7,11 @@ description: hh - Blazing fast interaction with HDFS . - Currently we only support v7 of the RPC protocol (< CDH5). + If built against hadoop-rpc-1.x.x.x then these tools support v9 of the + Hadoop RPC protocol (CDH 5.x and above). . - Support for v9 (>= CDH5) is coming soon. + Earlier versions (< 0.6) can be installed using + --constraint=hadoop-rpc==0.1.1.1 if you need v7 support. . > hh cat - Print the contents of a file to stdout > hh cd - Change working directory @@ -42,7 +44,7 @@ main-is: Main.hs hs-source-dirs: src - ghc-options: -funbox-strict-fields + ghc-options: -funbox-strict-fields -Wall ghc-prof-options: -auto-all -caf-all other-modules: @@ -58,7 +60,7 @@ , configurator >= 0.3 , exceptions >= 0.6 , filepath >= 1.3 - , hadoop-rpc >= 0.1.1.1 + , hadoop-rpc >= 1.0.0.1 , old-locale >= 1.0 , optparse-applicative >= 0.11 , protobuf >= 0.2.0.4 @@ -68,6 +70,7 @@ , text >= 1.1 , time >= 1.4 , transformers >= 0.4 + , unix >= 2.7 , vector >= 0.10 test-suite test diff --git a/src/Chmod.hs b/src/Chmod.hs --- a/src/Chmod.hs +++ b/src/Chmod.hs @@ -9,19 +9,18 @@ import Control.Applicative import Control.Monad (guard, msum) import qualified Data.Attoparsec.ByteString.Char8 as Atto -import qualified Data.Attoparsec.Combinator as Atto import Data.Bits import qualified Data.ByteString.Char8 as B -import Data.Char (ord) +import Data.Char (ord, isOctDigit) import Data.List (foldl') import Data.Maybe (mapMaybe) -import Data.Word (Word16, Word32) +import Data.Word (Word16) import Data.Hadoop.Types (FileType(..)) ------------------------------------------------------------------------ -data ChmodWho = Chmod_u | Chmod_g | Chmod_o | Chmod_a +data ChmodWho = Chmod_u | Chmod_g | Chmod_o | Chmod_a deriving (Show, Eq) data ChmodWhat = Chmod_r | Chmod_w | Chmod_x | Chmod_X | Chmod_s | Chmod_t @@ -78,13 +77,12 @@ ] octal :: Atto.Parser Word16 - octal = B.foldl' step 0 `fmap` Atto.takeWhile1 isDig + octal = B.foldl' step 0 `fmap` Atto.takeWhile1 isOctDigit where - isDig w = w >= '0' && w <= '7' step a w = a * 8 + fromIntegral (ord w - 48) applyChmod :: FileType -> [Chmod] -> Word16 -> Word16 -applyChmod filetype cs old = foldl' f old cs +applyChmod filetype = flip (foldl' f) where f :: Word16 -> Chmod -> Word16 f _ (SetOctal new) = new @@ -96,8 +94,7 @@ f old (SetMinusWho who src) = minus who old (extract src old) set :: ChmodWho -> Word16 -> Word16 -> Word16 - set who old new = (old .&. (complement (mask who))) .|. - setWho who new + set who old new = (old .&. complement (mask who)) .|. setWho who new plus :: ChmodWho -> Word16 -> Word16 -> Word16 plus who old new = old .|. setWho who new @@ -115,7 +112,7 @@ setWho :: ChmodWho -> Word16 -> Word16 setWho Chmod_a new = foldl' (.|.) 0 $ - map (\w -> setWho w new) [Chmod_u, Chmod_g, Chmod_o] + map (`setWho` new) [Chmod_u, Chmod_g, Chmod_o] setWho who new = new `shiftL` s who extract :: ChmodWho -> Word16 -> Word16 @@ -139,4 +136,3 @@ s Chmod_u = 6 s Chmod_g = 3 s Chmod_o = 0 - diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -4,15 +4,14 @@ module Main (main) where import Control.Concurrent.STM -import Control.Exception (SomeException, throwIO, bracket, fromException) +import Control.Exception (SomeException, throwIO, fromException) import Control.Monad import Control.Monad.Catch (handle, throwM) import Control.Monad.IO.Class (MonadIO, liftIO) import qualified Data.Attoparsec.ByteString.Char8 as Atto -import Data.Bits ((.&.), shiftR) +import Data.Bits ((.&.), shiftL, shiftR) import Data.ByteString (ByteString) import qualified Data.ByteString.Char8 as B -import Data.Char (ord) import Data.Foldable (foldMap) import Data.Maybe (fromMaybe) import qualified Data.Text as T @@ -28,21 +27,23 @@ import Data.Configurator.Types (Worth(..)) import Options.Applicative hiding (Success) import System.Environment (getEnv) +import System.Exit (exitFailure) import qualified System.FilePath as FilePath import qualified System.FilePath.Posix as Posix import System.IO import System.IO.Unsafe (unsafePerformIO) import System.Locale (defaultTimeLocale) +import System.Posix.User (GroupEntry(..), getGroups, getGroupEntryForID) import Text.PrettyPrint.Boxes hiding ((<>), (//)) -import Data.Hadoop.Configuration (getHadoopConfig) +import Data.Hadoop.Configuration (getHadoopConfig, getHadoopUser) import Data.Hadoop.HdfsPath import Data.Hadoop.Types import Network.Hadoop.Hdfs hiding (runHdfs) import Network.Hadoop.Read import Chmod -import qualified Glob as Glob +import qualified Glob import Paths_hadoop_tools (version) @@ -53,10 +54,11 @@ cmd <- execParser optsParser case cmd of SubIO io -> io - SubHdfs hdfs -> handle printError (runHdfs hdfs) + SubHdfs hdfs -> handle exitError (runHdfs hdfs) where optsParser = info (helper <*> options) (fullDesc <> header "hh - Blazing fast interaction with HDFS") + exitError err = printError err >> exitFailure runHdfs :: Hdfs a -> IO a runHdfs hdfs = do @@ -88,6 +90,12 @@ getHdfsUser :: IO (Maybe User) getHdfsUser = C.load [Optional configPath] >>= flip C.lookup "hdfs.user" +getGroupNames :: IO [Group] +getGroupNames = do + groups <- getGroups + entries <- mapM getGroupEntryForID groups + return $ map (T.pack . groupName) entries + getNameNode :: IO (Maybe NameNode) getNameNode = do cfg <- C.load [Optional configPath] @@ -111,8 +119,8 @@ return (home `FilePath.combine` ".hhwd") {-# NOINLINE workingDirConfigPath #-} -getDefaultWorkingDir :: MonadIO m => m HdfsPath -getDefaultWorkingDir = liftIO $ (("/user" ) . T.encodeUtf8 . hcUser) <$> getConfig +getHomeDir :: MonadIO m => m HdfsPath +getHomeDir = liftIO $ (("/user" ) . T.encodeUtf8 . hcUser) <$> getConfig getWorkingDir :: MonadIO m => m HdfsPath getWorkingDir = liftIO $ handle onError @@ -120,7 +128,7 @@ <$> B.readFile workingDirConfigPath where onError :: SomeException -> IO HdfsPath - onError = const getDefaultWorkingDir + onError = const getHomeDir setWorkingDir :: MonadIO m => HdfsPath -> m () setWorkingDir path = liftIO $ B.writeFile workingDirConfigPath @@ -146,6 +154,12 @@ go xs (y : ys) = go (y : xs) ys go xs [] = (xs, []) +dropFileName :: HdfsPath -> HdfsPath +dropFileName = B.pack . Posix.dropFileName . B.unpack + +takeFileName :: HdfsPath -> HdfsPath +takeFileName = B.pack . Posix.takeFileName . B.unpack + ------------------------------------------------------------------------ data SubCommand = SubCommand @@ -176,6 +190,9 @@ , subPwd , subRemove , subRename + , subTest + , subTestNewer + , subTestOlder , subVersion ] @@ -199,7 +216,7 @@ where go = cd <$> optional (argument bstr (completeDir <> help "the directory to change to")) cd mpath = SubHdfs $ do - path <- getAbsolute =<< maybe getDefaultWorkingDir return mpath + path <- getAbsolute =<< maybe getHomeDir return mpath _ <- getListingOrFail path setWorkingDir path @@ -236,9 +253,9 @@ subFind :: SubCommand subFind = SubCommand "find" "Recursively search a directory tree" go where - go = find <$> (optional (argument bstr (completeDir <> help "the path to recursively search"))) - <*> (optional (option bstr (long "name" <> metavar "FILENAME" - <> help "the file name to match"))) + go = find <$> optional (argument bstr (completeDir <> help "the path to recursively search")) + <*> optional (option bstr (long "name" <> metavar "FILENAME" + <> help "the file name to match")) find mpath mexpr = SubHdfs $ do matcher <- liftIO (mkMatcher mexpr) printFindResults (fromMaybe "" mpath) matcher @@ -258,9 +275,7 @@ let dst = fromMaybe (Posix.takeFileName $ B.unpack src) mdst absSrc <- getAbsolute src mReadHandle <- openRead absSrc - let doRead readHandle = liftIO $ bracket - (openFile dst WriteMode) - (hClose) + let doRead readHandle = liftIO $ withFile dst WriteMode (\writeHandle -> hdfsMapM_ (B.hPut writeHandle) readHandle) maybe (return ()) doRead mReadHandle @@ -290,10 +305,21 @@ where go = rm <$> argument bstr (completePath <> help "the file/directory to remove") <*> switch (short 'r' <> help "recursively remove the whole file hierarchy") - rm path recursive = SubHdfs $ do + <*> switch (short 's' <> long "skipTrash" <> help "immediately delete, bypassing trash") + rm path recursive skipTrash = SubHdfs $ do absPath <- getAbsolute path - ok <- delete recursive absPath - unless ok $ liftIO . B.putStrLn $ "Failed to remove: " <> absPath + if skipTrash || ".Trash" `elem` B.split '/' absPath + then do + ok <- delete recursive absPath + unless ok $ liftIO . B.putStrLn $ "Failed to remove: " <> absPath + else do + home <- getHomeDir + let absDst = home ".Trash" "Current" <> absPath + absDstDir = dropFileName absDst + ok <- mkdirs True absDstDir + unless ok $ liftIO . B.putStrLn $ "Failed to make trash folder: " <> absDstDir + rename True absPath absDst + liftIO . B.putStrLn . B.unwords $ ["Moved:", absPath, "to trash at:", absDst] subRename :: SubCommand subRename = SubCommand "mv" "Rename a file or directory" go @@ -304,7 +330,73 @@ mv src dst force = SubHdfs $ do absSrc <- getAbsolute src absDst <- getAbsolute dst - rename force absSrc absDst + mSrcType <- fmap fsFileType <$> getFileInfo absSrc + mDstType <- fmap fsFileType <$> getFileInfo absDst + let absDst' = if (mSrcType, mDstType) == (Just File, Just Dir) + then absDst takeFileName src + else absDst + rename force absSrc absDst' + +subTest :: SubCommand +subTest = SubCommand "test" "If file exists, has zero length, is a directory then return 0, else return 1" go + where + go = test <$> argument bstr (completePath <> help "file/directory") + <*> switch (short 'e' <> help "Test exists") + <*> switch (short 'z' <> help "Test is zero length") + <*> switch (short 'd' <> help "Test is a directory") + <*> switch (short 'f' <> help "Test is a regular file") + <*> switch (short 'l' <> help "Test is a symbolic link") + <*> switch (short 'r' <> help "Test read permission is granted") + <*> switch (short 'w' <> help "Test write permission is granted") + <*> switch (short 'x' <> help "Test exectute permission is granted") + test path e z d f l r w x = SubHdfs $ do + absPath <- getAbsolute path + minfo <- getFileInfo absPath + user <- liftIO getHadoopUser + groups <- liftIO getGroupNames + case minfo of + Nothing -> fail $ unwords ["No such file/directory", B.unpack absPath] + Just fs@FileStatus{..} -> do + when (d && fsFileType /= Dir) $ fail . unwords $ ["Not a directory"] + when (f && fsFileType /= File) $ fail . unwords $ ["Not a regular file"] + when (l && fsFileType /= SymLink) $ fail . unwords $ ["Not a regular file"] + when (z && fsLength /= 0) $ fail . unwords $ ["Not zero length"] + when (r && not (hasPerm user groups fs 4)) $ fail . unwords $ ["Not readable"] + when (w && not (hasPerm user groups fs 2)) $ fail . unwords $ ["Not writable"] + when (x && not (hasPerm user groups fs 1)) $ fail . unwords $ ["Not executable"] + where + hasPerm user groups FileStatus{..} p = (fsOwner == user && (fsPermission .&. (p `shiftL` 6)) /= 0) || + (fsGroup `elem` groups && (fsPermission .&. (p `shiftL` 3)) /= 0) || + ((fsPermission .&. p) /= 0) + +subTestNewer :: SubCommand +subTestNewer = SubCommand "test-newer" "file1 is newer (modification time) than file2" go + where + go = testNewer <$> argument bstr (completePath <> help "file/directory") + <*> argument bstr (completePath <> help "file/directory") + <*> pure False + +subTestOlder :: SubCommand +subTestOlder = SubCommand "test-older" "file1 is older (modification time) than file2" go + where + go = testNewer <$> argument bstr (completePath <> help "file/directory") + <*> argument bstr (completePath <> help "file/directory") + <*> pure True + +testNewer :: HdfsPath -> HdfsPath -> Bool -> SubMethod +testNewer path1 path2 older = SubHdfs $ do + absPath1 <- getAbsolute path1 + absPath2 <- getAbsolute path2 + minfo1 <- getFileInfo absPath1 + minfo2 <- getFileInfo absPath2 + case (minfo1, minfo2, older) of + (Just fs1, Just fs2, False) -> + when (fsModificationTime fs1 < fsModificationTime fs2) $ + fail . unwords $ [B.unpack path1, "is older than", B.unpack path2] + (Just fs1, Just fs2, True) -> + when (fsModificationTime fs1 > fsModificationTime fs2) $ + fail . unwords $ [B.unpack path1, "is newer than", B.unpack path2] + _ -> fail $ unwords ["No such file/directory"] subVersion :: SubCommand subVersion = SubCommand "version" "Show version information" go