diff --git a/bash-completion b/bash-completion new file mode 100644 --- /dev/null +++ b/bash-completion @@ -0,0 +1,14 @@ +#!/bin/bash +_hh() +{ + local cmdline + CMDLINE=(--bash-completion-index $COMP_CWORD) + + for arg in ${COMP_WORDS[@]}; do + CMDLINE=(${CMDLINE[@]} --bash-completion-word $arg) + done + + COMPREPLY=( $(hh "${CMDLINE[@]}") ) +} + +complete -o nospace -F _hh hh diff --git a/hadoop-tools.cabal b/hadoop-tools.cabal --- a/hadoop-tools.cabal +++ b/hadoop-tools.cabal @@ -1,5 +1,5 @@ name: hadoop-tools -version: 0.4.0.1 +version: 0.5 synopsis: Fast command line tools for working with Hadoop. @@ -33,6 +33,9 @@ build-type: Simple cabal-version: >= 1.10 +extra-source-files: + bash-completion + executable hh default-language: Haskell2010 @@ -44,6 +47,7 @@ other-modules: Glob + Chmod Paths_hadoop_tools build-depends: @@ -56,7 +60,7 @@ , filepath >= 1.3 , hadoop-rpc >= 0.1.1.1 , old-locale >= 1.0 - , optparse-applicative >= 0.10 + , optparse-applicative >= 0.11 , protobuf >= 0.2.0.4 , regex-pcre-builtin >= 0.94 , split >= 0.2 @@ -65,3 +69,21 @@ , time >= 1.4 , transformers >= 0.4 , vector >= 0.10 + +test-suite test + default-language: Haskell2010 + type: + exitcode-stdio-1.0 + hs-source-dirs: + tests,src + main-is: + test.hs + build-depends: + base >= 4 && < 5 + , attoparsec + , bytestring >= 0.10 + , hadoop-rpc >= 0.1.1.1 + , tasty >= 0.7 + , tasty-hunit + , tasty-quickcheck + , vector diff --git a/src/Chmod.hs b/src/Chmod.hs new file mode 100644 --- /dev/null +++ b/src/Chmod.hs @@ -0,0 +1,142 @@ +module Chmod ( + Chmod(..) + , ChmodWho(..) + , ChmodWhat(..) + , parseChmod + , applyChmod + ) where + +import Control.Applicative +import Control.Monad (guard, msum) +import qualified Data.Attoparsec.ByteString.Char8 as Atto +import qualified Data.Attoparsec.Combinator as Atto +import Data.Bits +import qualified Data.ByteString.Char8 as B +import Data.Char (ord) +import Data.List (foldl') +import Data.Maybe (mapMaybe) +import Data.Word (Word16, Word32) + +import Data.Hadoop.Types (FileType(..)) + +------------------------------------------------------------------------ + +data ChmodWho = Chmod_u | Chmod_g | Chmod_o | Chmod_a + deriving (Show, Eq) +data ChmodWhat = Chmod_r | Chmod_w | Chmod_x | Chmod_X + | Chmod_s | Chmod_t + deriving (Show, Eq) + +data Chmod = SetOctal Word16 + | SetEqual ChmodWho [ChmodWhat] + | SetPlus ChmodWho [ChmodWhat] + | SetMinus ChmodWho [ChmodWhat] + | SetEqualWho ChmodWho ChmodWho + | SetPlusWho ChmodWho ChmodWho + | SetMinusWho ChmodWho ChmodWho + deriving (Show, Eq) + +parseChmod :: Atto.Parser [Chmod] +parseChmod = do + cs <- Atto.sepBy chmod1 (Atto.char ',') + guard (cs /= []) + return cs + where + chmod1 = msum + [ SetOctal <$> octal + , SetEqual <$> ugoa <* Atto.char '=' <*> Atto.many1 rwx + , SetPlus <$> ugoa <* Atto.char '+' <*> Atto.many1 rwx + , SetMinus <$> ugoa <* Atto.char '-' <*> Atto.many1 rwx + , SetEqualWho <$> ugoa <* Atto.char '=' <*> ugo + , SetPlusWho <$> ugoa <* Atto.char '+' <*> ugo + , SetMinusWho <$> ugoa <* Atto.char '-' <*> ugo + ] + + ugo :: Atto.Parser ChmodWho + ugo = msum + [ pure Chmod_u <* Atto.char 'u' + , pure Chmod_g <* Atto.char 'g' + , pure Chmod_o <* Atto.char 'o' + ] + + ugoa :: Atto.Parser ChmodWho + ugoa = msum + [ ugo + , pure Chmod_a <* Atto.char 'a' + ] + + rwx :: Atto.Parser ChmodWhat + rwx = msum + [ pure Chmod_r <* Atto.char 'r' + , pure Chmod_w <* Atto.char 'w' + , pure Chmod_x <* Atto.char 'x' + , pure Chmod_X <* Atto.char 'X' + {- + , pure Chmod_s <* Atto.char 's' + , pure Chmod_t <* Atto.char 't' + -} + ] + + octal :: Atto.Parser Word16 + octal = B.foldl' step 0 `fmap` Atto.takeWhile1 isDig + where + isDig w = w >= '0' && w <= '7' + step a w = a * 8 + fromIntegral (ord w - 48) + +applyChmod :: FileType -> [Chmod] -> Word16 -> Word16 +applyChmod filetype cs old = foldl' f old cs + where + f :: Word16 -> Chmod -> Word16 + f _ (SetOctal new) = new + f old (SetEqual who ws) = set who old (foldRWX old ws) + f old (SetPlus who ws) = plus who old (foldRWX old ws) + f old (SetMinus who ws) = minus who old (foldRWX old ws) + f old (SetEqualWho who src) = set who old (extract src old) + f old (SetPlusWho who src) = plus who old (extract src old) + f old (SetMinusWho who src) = minus who old (extract src old) + + set :: ChmodWho -> Word16 -> Word16 -> Word16 + set who old new = (old .&. (complement (mask who))) .|. + setWho who new + + plus :: ChmodWho -> Word16 -> Word16 -> Word16 + plus who old new = old .|. setWho who new + + minus :: ChmodWho -> Word16 -> Word16 -> Word16 + minus who old new = old `xor` setWho who new + + foldRWX old = foldl' (\old what -> old .|. b what) 0 . mapMaybe (hX old) + + o3 u g o = u*64 + g*8 + o + + mask :: ChmodWho -> Word16 + mask Chmod_a = o3 7 7 7 + mask who = 7 `shiftL` s who + + setWho :: ChmodWho -> Word16 -> Word16 + setWho Chmod_a new = foldl' (.|.) 0 $ + map (\w -> setWho w new) [Chmod_u, Chmod_g, Chmod_o] + setWho who new = new `shiftL` s who + + extract :: ChmodWho -> Word16 -> Word16 + extract who old = (mask who .&. old) `shiftR` s who + + -- handle X + hX :: Word16 -> ChmodWhat -> Maybe ChmodWhat + hX old Chmod_X = if filetype == Dir || old .&. o3 1 1 1 /= 0 + then Just Chmod_x + else Nothing + hX _ what = Just what + + -- Bit to set for what + b :: ChmodWhat -> Word16 + b Chmod_r = 4 + b Chmod_w = 2 + b Chmod_x = 1 + + -- Number of bits to shift for who + s :: ChmodWho -> Int + s Chmod_u = 6 + s Chmod_g = 3 + s Chmod_o = 0 + diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -41,7 +41,9 @@ import Network.Hadoop.Hdfs hiding (runHdfs) import Network.Hadoop.Read +import Chmod import qualified Glob as Glob + import Paths_hadoop_tools (version) ------------------------------------------------------------------------ @@ -183,8 +185,8 @@ completeDir :: Mod ArgumentFields a completeDir = completer (fileCompletion (== Dir)) <> metavar "DIRECTORY" -bstr :: Monad m => String -> m ByteString -bstr x = B.pack `liftM` str x +bstr :: ReadM ByteString +bstr = B.pack <$> str subCat :: SubCommand subCat = SubCommand "cat" "Print the contents of a file to stdout" go @@ -209,17 +211,9 @@ chmod modeS path = either (\_ -> error $ "Unknown mode" ++ B.unpack modeS) (\mode -> SubHdfs $ modifyPerms mode path) - (Atto.parseOnly parseMode modeS) - - parseMode = octal - - octal :: Atto.Parser Word16 - octal = B.foldl' step 0 `fmap` Atto.takeWhile1 isDig - where - isDig w = w >= '0' && w <= '7' - step a w = a * 8 + fromIntegral (ord w - 48) + (Atto.parseOnly parseChmod modeS) - modifyPerms :: Word16 -> HdfsPath -> Hdfs () + modifyPerms :: [Chmod] -> HdfsPath -> Hdfs () modifyPerms mode path = do absPath <- getAbsolute path minfo <- getFileInfo absPath @@ -231,7 +225,7 @@ liftIO . putStrLn . unwords $ ["OLD:", formatMode fsFileType fsPermission] liftIO . putStrLn . unwords $ ["NEW:", formatMode fsFileType mode] -} - setPermissions (fromIntegral mode) absPath + setPermissions (fromIntegral $ applyChmod fsFileType mode fsPermission) absPath subDiskUsage :: SubCommand subDiskUsage = SubCommand "du" "Show the amount of space used by file or directory" go diff --git a/tests/test.hs b/tests/test.hs new file mode 100644 --- /dev/null +++ b/tests/test.hs @@ -0,0 +1,94 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Main where + +import Test.Tasty +import Test.Tasty.HUnit +import Test.Tasty.QuickCheck as QC + +import Data.Attoparsec.ByteString as Atto +import Data.ByteString (ByteString) +import qualified Data.ByteString.Char8 as B +import Data.Either +import Data.Word (Word16, Word32) + +import Data.Hadoop.Types (FileType(..)) + +import Chmod + +------------------------------------------------------------ + +main = defaultMain tests + +------------------------------------------------------------ + +tests :: TestTree +tests = testGroup "Tests" [unitTests] + +unitTests = testGroup "Unit tests" + [ testChmod "755" [SetOctal 493] + , testChmod "0755" [SetOctal 493] + , testChmod "u=r" [SetEqual Chmod_u [Chmod_r]] + , testChmodShouldFail "kittens" + + , testApplyChmod File [SetEqual Chmod_o [Chmod_r]] + (oct 0 0 0) (oct 0 0 4) + , testApplyChmod File [SetEqual Chmod_o [Chmod_r, Chmod_w]] + (oct 0 0 0) (oct 0 0 6) + , testApplyChmod File [SetEqual Chmod_g [Chmod_r]] + (oct 0 0 0) (oct 0 4 0) + , testApplyChmod File [SetPlus Chmod_o [Chmod_w]] + (oct 0 0 4) (oct 0 0 6) + , testApplyChmod File [SetPlus Chmod_o [Chmod_w,Chmod_r]] + (oct 0 0 1) (oct 0 0 7) + , testApplyChmod File [SetMinus Chmod_o [Chmod_w]] + (oct 0 0 6) (oct 0 0 4) + , testApplyChmod File [SetMinus Chmod_o [Chmod_w,Chmod_r]] + (oct 0 0 7) (oct 0 0 1) + , testApplyChmod File [SetEqualWho Chmod_g Chmod_o] + (oct 0 0 1) (oct 0 1 1) + , testApplyChmod File [SetPlusWho Chmod_g Chmod_o] + (oct 0 2 1) (oct 0 3 1) + , testApplyChmod File [SetMinusWho Chmod_g Chmod_o] + (oct 0 3 1) (oct 0 2 1) + , testApplyChmod File [SetMinusWho Chmod_g Chmod_o] + (oct 0 3 1) (oct 0 2 1) + , testApplyChmod File [SetEqual Chmod_a [Chmod_r]] + (oct 0 0 0) (oct 4 4 4) + , testApplyChmod File [SetPlus Chmod_a [Chmod_r]] + (oct 0 1 0) (oct 4 5 4) + , testApplyChmod File [SetMinus Chmod_a [Chmod_x]] + (oct 7 5 5) (oct 6 4 4) + , testApplyChmod File [SetEqualWho Chmod_a Chmod_u] + (oct 6 4 4) (oct 6 6 6) + , testApplyChmod File [SetPlusWho Chmod_a Chmod_u] + (oct 7 5 5) (oct 7 7 7) + , testApplyChmod File [SetMinusWho Chmod_a Chmod_o] + (oct 7 5 5) (oct 2 0 0) + + , testApplyChmod File [SetPlus Chmod_o [Chmod_X]] + (oct 0 1 0) (oct 0 1 1) + , testApplyChmod Dir [SetPlus Chmod_o [Chmod_X]] + (oct 0 0 0) (oct 0 0 1) + , testApplyChmod Dir [SetPlus Chmod_o [Chmod_X]] + (oct 0 1 0) (oct 0 1 1) + ] + +oct u g o = u*64 + g*8 + o + +testChmod :: ByteString -> [Chmod] -> TestTree +testChmod input expected = + testCase (unwords ["Parse chmod", B.unpack input]) $ + parseOnly parseChmod input @?= Right expected + +testChmodShouldFail :: ByteString -> TestTree +testChmodShouldFail input = + testCase (unwords ["Parse chmod", B.unpack input]) $ + assertBool "Failed to catch invalid chmod" + (isLeft $ parseOnly parseChmod input) + +testApplyChmod :: FileType -> [Chmod] -> Word16 -> Word16 -> TestTree +testApplyChmod filetype input old expected = + testCase (unwords ["Apply chmod", show input, "to", show old]) $ + applyChmod filetype input old @?= expected +