hadoop-tools 0.4.0.1 → 0.5
raw patch · 5 files changed
+281/−15 lines, 5 filesdep +tastydep +tasty-hunitdep +tasty-quickcheckdep ~attoparsecdep ~basedep ~optparse-applicative
Dependencies added: tasty, tasty-hunit, tasty-quickcheck
Dependency ranges changed: attoparsec, base, optparse-applicative, vector
Files
- bash-completion +14/−0
- hadoop-tools.cabal +24/−2
- src/Chmod.hs +142/−0
- src/Main.hs +7/−13
- tests/test.hs +94/−0
+ bash-completion view
@@ -0,0 +1,14 @@+#!/bin/bash+_hh()+{+ local cmdline+ CMDLINE=(--bash-completion-index $COMP_CWORD)++ for arg in ${COMP_WORDS[@]}; do+ CMDLINE=(${CMDLINE[@]} --bash-completion-word $arg)+ done++ COMPREPLY=( $(hh "${CMDLINE[@]}") )+}++complete -o nospace -F _hh hh
hadoop-tools.cabal view
@@ -1,5 +1,5 @@ name: hadoop-tools-version: 0.4.0.1+version: 0.5 synopsis: Fast command line tools for working with Hadoop.@@ -33,6 +33,9 @@ build-type: Simple cabal-version: >= 1.10 +extra-source-files:+ bash-completion+ executable hh default-language: Haskell2010 @@ -44,6 +47,7 @@ other-modules: Glob+ Chmod Paths_hadoop_tools build-depends:@@ -56,7 +60,7 @@ , filepath >= 1.3 , hadoop-rpc >= 0.1.1.1 , old-locale >= 1.0- , optparse-applicative >= 0.10+ , optparse-applicative >= 0.11 , protobuf >= 0.2.0.4 , regex-pcre-builtin >= 0.94 , split >= 0.2@@ -65,3 +69,21 @@ , time >= 1.4 , transformers >= 0.4 , vector >= 0.10++test-suite test+ default-language: Haskell2010+ type:+ exitcode-stdio-1.0+ hs-source-dirs:+ tests,src+ main-is:+ test.hs+ build-depends:+ base >= 4 && < 5+ , attoparsec+ , bytestring >= 0.10+ , hadoop-rpc >= 0.1.1.1+ , tasty >= 0.7+ , tasty-hunit+ , tasty-quickcheck+ , vector
+ src/Chmod.hs view
@@ -0,0 +1,142 @@+module Chmod (+ Chmod(..)+ , ChmodWho(..)+ , ChmodWhat(..)+ , parseChmod+ , applyChmod+ ) where++import Control.Applicative+import Control.Monad (guard, msum)+import qualified Data.Attoparsec.ByteString.Char8 as Atto+import qualified Data.Attoparsec.Combinator as Atto+import Data.Bits+import qualified Data.ByteString.Char8 as B+import Data.Char (ord)+import Data.List (foldl')+import Data.Maybe (mapMaybe)+import Data.Word (Word16, Word32)++import Data.Hadoop.Types (FileType(..))++------------------------------------------------------------------------++data ChmodWho = Chmod_u | Chmod_g | Chmod_o | Chmod_a + deriving (Show, Eq)+data ChmodWhat = Chmod_r | Chmod_w | Chmod_x | Chmod_X+ | Chmod_s | Chmod_t+ deriving (Show, Eq)++data Chmod = SetOctal Word16+ | SetEqual ChmodWho [ChmodWhat]+ | SetPlus ChmodWho [ChmodWhat]+ | SetMinus ChmodWho [ChmodWhat]+ | SetEqualWho ChmodWho ChmodWho+ | SetPlusWho ChmodWho ChmodWho+ | SetMinusWho ChmodWho ChmodWho+ deriving (Show, Eq)++parseChmod :: Atto.Parser [Chmod]+parseChmod = do+ cs <- Atto.sepBy chmod1 (Atto.char ',')+ guard (cs /= [])+ return cs+ where+ chmod1 = msum+ [ SetOctal <$> octal+ , SetEqual <$> ugoa <* Atto.char '=' <*> Atto.many1 rwx+ , SetPlus <$> ugoa <* Atto.char '+' <*> Atto.many1 rwx+ , SetMinus <$> ugoa <* Atto.char '-' <*> Atto.many1 rwx+ , SetEqualWho <$> ugoa <* Atto.char '=' <*> ugo+ , SetPlusWho <$> ugoa <* Atto.char '+' <*> ugo+ , SetMinusWho <$> ugoa <* Atto.char '-' <*> ugo+ ]++ ugo :: Atto.Parser ChmodWho+ ugo = msum+ [ pure Chmod_u <* Atto.char 'u'+ , pure Chmod_g <* Atto.char 'g'+ , pure Chmod_o <* Atto.char 'o'+ ]++ ugoa :: Atto.Parser ChmodWho+ ugoa = msum+ [ ugo+ , pure Chmod_a <* Atto.char 'a'+ ]++ rwx :: Atto.Parser ChmodWhat+ rwx = msum+ [ pure Chmod_r <* Atto.char 'r'+ , pure Chmod_w <* Atto.char 'w'+ , pure Chmod_x <* Atto.char 'x'+ , pure Chmod_X <* Atto.char 'X'+ {-+ , pure Chmod_s <* Atto.char 's'+ , pure Chmod_t <* Atto.char 't'+ -}+ ]++ octal :: Atto.Parser Word16+ octal = B.foldl' step 0 `fmap` Atto.takeWhile1 isDig+ where+ isDig w = w >= '0' && w <= '7'+ step a w = a * 8 + fromIntegral (ord w - 48)++applyChmod :: FileType -> [Chmod] -> Word16 -> Word16+applyChmod filetype cs old = foldl' f old cs+ where+ f :: Word16 -> Chmod -> Word16+ f _ (SetOctal new) = new+ f old (SetEqual who ws) = set who old (foldRWX old ws)+ f old (SetPlus who ws) = plus who old (foldRWX old ws)+ f old (SetMinus who ws) = minus who old (foldRWX old ws)+ f old (SetEqualWho who src) = set who old (extract src old)+ f old (SetPlusWho who src) = plus who old (extract src old)+ f old (SetMinusWho who src) = minus who old (extract src old)++ set :: ChmodWho -> Word16 -> Word16 -> Word16+ set who old new = (old .&. (complement (mask who))) .|.+ setWho who new++ plus :: ChmodWho -> Word16 -> Word16 -> Word16+ plus who old new = old .|. setWho who new++ minus :: ChmodWho -> Word16 -> Word16 -> Word16+ minus who old new = old `xor` setWho who new++ foldRWX old = foldl' (\old what -> old .|. b what) 0 . mapMaybe (hX old)++ o3 u g o = u*64 + g*8 + o++ mask :: ChmodWho -> Word16+ mask Chmod_a = o3 7 7 7+ mask who = 7 `shiftL` s who++ setWho :: ChmodWho -> Word16 -> Word16+ setWho Chmod_a new = foldl' (.|.) 0 $+ map (\w -> setWho w new) [Chmod_u, Chmod_g, Chmod_o]+ setWho who new = new `shiftL` s who++ extract :: ChmodWho -> Word16 -> Word16+ extract who old = (mask who .&. old) `shiftR` s who++ -- handle X+ hX :: Word16 -> ChmodWhat -> Maybe ChmodWhat+ hX old Chmod_X = if filetype == Dir || old .&. o3 1 1 1 /= 0+ then Just Chmod_x+ else Nothing+ hX _ what = Just what++ -- Bit to set for what+ b :: ChmodWhat -> Word16+ b Chmod_r = 4+ b Chmod_w = 2+ b Chmod_x = 1++ -- Number of bits to shift for who+ s :: ChmodWho -> Int+ s Chmod_u = 6+ s Chmod_g = 3+ s Chmod_o = 0+
src/Main.hs view
@@ -41,7 +41,9 @@ import Network.Hadoop.Hdfs hiding (runHdfs) import Network.Hadoop.Read +import Chmod import qualified Glob as Glob+ import Paths_hadoop_tools (version) ------------------------------------------------------------------------@@ -183,8 +185,8 @@ completeDir :: Mod ArgumentFields a completeDir = completer (fileCompletion (== Dir)) <> metavar "DIRECTORY" -bstr :: Monad m => String -> m ByteString-bstr x = B.pack `liftM` str x+bstr :: ReadM ByteString+bstr = B.pack <$> str subCat :: SubCommand subCat = SubCommand "cat" "Print the contents of a file to stdout" go@@ -209,17 +211,9 @@ chmod modeS path = either (\_ -> error $ "Unknown mode" ++ B.unpack modeS) (\mode -> SubHdfs $ modifyPerms mode path)- (Atto.parseOnly parseMode modeS)-- parseMode = octal-- octal :: Atto.Parser Word16- octal = B.foldl' step 0 `fmap` Atto.takeWhile1 isDig- where- isDig w = w >= '0' && w <= '7'- step a w = a * 8 + fromIntegral (ord w - 48)+ (Atto.parseOnly parseChmod modeS) - modifyPerms :: Word16 -> HdfsPath -> Hdfs ()+ modifyPerms :: [Chmod] -> HdfsPath -> Hdfs () modifyPerms mode path = do absPath <- getAbsolute path minfo <- getFileInfo absPath@@ -231,7 +225,7 @@ liftIO . putStrLn . unwords $ ["OLD:", formatMode fsFileType fsPermission] liftIO . putStrLn . unwords $ ["NEW:", formatMode fsFileType mode] -}- setPermissions (fromIntegral mode) absPath+ setPermissions (fromIntegral $ applyChmod fsFileType mode fsPermission) absPath subDiskUsage :: SubCommand subDiskUsage = SubCommand "du" "Show the amount of space used by file or directory" go
+ tests/test.hs view
@@ -0,0 +1,94 @@+{-# LANGUAGE OverloadedStrings #-}++module Main where ++import Test.Tasty+import Test.Tasty.HUnit+import Test.Tasty.QuickCheck as QC++import Data.Attoparsec.ByteString as Atto+import Data.ByteString (ByteString)+import qualified Data.ByteString.Char8 as B+import Data.Either+import Data.Word (Word16, Word32)++import Data.Hadoop.Types (FileType(..))++import Chmod++------------------------------------------------------------++main = defaultMain tests++------------------------------------------------------------++tests :: TestTree+tests = testGroup "Tests" [unitTests]++unitTests = testGroup "Unit tests"+ [ testChmod "755" [SetOctal 493]+ , testChmod "0755" [SetOctal 493]+ , testChmod "u=r" [SetEqual Chmod_u [Chmod_r]]+ , testChmodShouldFail "kittens"+ + , testApplyChmod File [SetEqual Chmod_o [Chmod_r]]+ (oct 0 0 0) (oct 0 0 4)+ , testApplyChmod File [SetEqual Chmod_o [Chmod_r, Chmod_w]]+ (oct 0 0 0) (oct 0 0 6)+ , testApplyChmod File [SetEqual Chmod_g [Chmod_r]]+ (oct 0 0 0) (oct 0 4 0)+ , testApplyChmod File [SetPlus Chmod_o [Chmod_w]]+ (oct 0 0 4) (oct 0 0 6)+ , testApplyChmod File [SetPlus Chmod_o [Chmod_w,Chmod_r]]+ (oct 0 0 1) (oct 0 0 7)+ , testApplyChmod File [SetMinus Chmod_o [Chmod_w]]+ (oct 0 0 6) (oct 0 0 4)+ , testApplyChmod File [SetMinus Chmod_o [Chmod_w,Chmod_r]]+ (oct 0 0 7) (oct 0 0 1)+ , testApplyChmod File [SetEqualWho Chmod_g Chmod_o]+ (oct 0 0 1) (oct 0 1 1)+ , testApplyChmod File [SetPlusWho Chmod_g Chmod_o]+ (oct 0 2 1) (oct 0 3 1)+ , testApplyChmod File [SetMinusWho Chmod_g Chmod_o]+ (oct 0 3 1) (oct 0 2 1)+ , testApplyChmod File [SetMinusWho Chmod_g Chmod_o]+ (oct 0 3 1) (oct 0 2 1)+ , testApplyChmod File [SetEqual Chmod_a [Chmod_r]]+ (oct 0 0 0) (oct 4 4 4)+ , testApplyChmod File [SetPlus Chmod_a [Chmod_r]]+ (oct 0 1 0) (oct 4 5 4)+ , testApplyChmod File [SetMinus Chmod_a [Chmod_x]]+ (oct 7 5 5) (oct 6 4 4)+ , testApplyChmod File [SetEqualWho Chmod_a Chmod_u]+ (oct 6 4 4) (oct 6 6 6)+ , testApplyChmod File [SetPlusWho Chmod_a Chmod_u]+ (oct 7 5 5) (oct 7 7 7)+ , testApplyChmod File [SetMinusWho Chmod_a Chmod_o]+ (oct 7 5 5) (oct 2 0 0)++ , testApplyChmod File [SetPlus Chmod_o [Chmod_X]]+ (oct 0 1 0) (oct 0 1 1)+ , testApplyChmod Dir [SetPlus Chmod_o [Chmod_X]]+ (oct 0 0 0) (oct 0 0 1)+ , testApplyChmod Dir [SetPlus Chmod_o [Chmod_X]]+ (oct 0 1 0) (oct 0 1 1)+ ]++oct u g o = u*64 + g*8 + o++testChmod :: ByteString -> [Chmod] -> TestTree+testChmod input expected =+ testCase (unwords ["Parse chmod", B.unpack input]) $+ parseOnly parseChmod input @?= Right expected++testChmodShouldFail :: ByteString -> TestTree+testChmodShouldFail input =+ testCase (unwords ["Parse chmod", B.unpack input]) $+ assertBool "Failed to catch invalid chmod"+ (isLeft $ parseOnly parseChmod input)++testApplyChmod :: FileType -> [Chmod] -> Word16 -> Word16 -> TestTree+testApplyChmod filetype input old expected =+ testCase (unwords ["Apply chmod", show input, "to", show old]) $+ applyChmod filetype input old @?= expected+