smap 0.1.0 → 0.2.0

raw patch · 4 files changed

+124/−131 lines, 4 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• README.md +11/−6
• smap.cabal +6/−6
• src/Smap/Commands.hs +51/−73
• src/Smap/Flags.hs +56/−46

@@ -88,16 +88,20 @@ Jane Doe ``` -#### xor - Symmetric difference+It's worth noting that both **int** and **sub** treat their first argument as a *stream*, not a *set*. This means that they won't deduplicate values from their first argument. In practice you will find that this is the most useful arrangement. You can always use `smap cat` to turn a stream into a set. -Patients who only have a cold or mumps, but not both: +To put this all together, let's find patients who only have a cold or mumps, but not both:+ ```bash-$ smap xor has_cold has_mumps+$ smap sub <(smap cat has_cold has_mumps) <(smap int has_cold has_mumps) Carol Carell John Smith ``` ++If you haven't seen the `<(command)` syntax before, it's a very useful shell tool called [process substitution](https://www.tldp.org/LDP/abs/html/process-sub.html).+ ### Advanced usage (maps)  When using `smap` with sets, the behavior is pretty straightforward. It gets a bit more complicated when@@ -120,7 +124,7 @@  To understand the above: -* `<(cut -f 2 -d ' ' patients)` gets a list of all the patients' last names and creates a virtual file with this list. See [bash process substitution](https://www.tldp.org/LDP/abs/html/process-sub.html).+* `<(cut -f 2 -d ' ' patients)` gets a list of all the patients' last names and creates a pipe containing this list.  * `+<(cut -f 2 -d ' ' patients),patients` constructs a stream where the keys are the last names and the values are the whole names.  `cat` deduplicates by key, so if we see a second (or third, or fourth, etc.) person from a given family we don't print them out.@@ -146,5 +150,6 @@ ### Approximate mode  If you're processing lots of lines and running up against memory limits, -you can use the `--approximate` option to keep track of a 64-bit hash -of each line instead of the entire line.+you can use the `--approximate` or `-a` option to keep track of a 64-bit hash +of each line instead of the entire line. You can also use +`--approx-with-key` or `-k` if you want to specify the SipHash key.

@@ -4,12 +4,12 @@ -- -- see: https://github.com/sol/hpack ----- hash: f9b3c8428fd796fc574a982c19dc5a090f0a419b37f6c465b3466cea621b02fc+-- hash: db4e5b07ac8350e9046998e591625af97e33131e0c2b52007a60ad66cb2f65a6  name:           smap-version:        0.1.0+version:        0.2.0 synopsis:       A command line tool for working with sets and maps-description:    Please see the README on GitHub at <https://github.com/wyager/smap>+description:    Please see the README on GitHub below or at <https://github.com/wyager/smap> category:       Text homepage:       https://github.com/wyager/smap#readme bug-reports:    https://github.com/wyager/smap/issues@@ -35,7 +35,7 @@       Paths_smap   hs-source-dirs:       src-  default-extensions: TupleSections RankNTypes ScopedTypeVariables LambdaCase BangPatterns+  default-extensions: TupleSections RankNTypes ScopedTypeVariables LambdaCase BangPatterns KindSignatures DataKinds   ghc-options: -O2 -Wall -fexpose-all-unfoldings -optP-Wno-nonportable-include-path   build-depends:       attoparsec@@ -59,7 +59,7 @@       Paths_smap   hs-source-dirs:       app-  default-extensions: TupleSections RankNTypes ScopedTypeVariables LambdaCase BangPatterns+  default-extensions: TupleSections RankNTypes ScopedTypeVariables LambdaCase BangPatterns KindSignatures DataKinds   ghc-options: -rtsopts -Wall -O2 -fspecialize-aggressively -optP-Wno-nonportable-include-path   build-depends:       attoparsec@@ -85,7 +85,7 @@       Paths_smap   hs-source-dirs:       test-  default-extensions: TupleSections RankNTypes ScopedTypeVariables LambdaCase BangPatterns+  default-extensions: TupleSections RankNTypes ScopedTypeVariables LambdaCase BangPatterns KindSignatures DataKinds   ghc-options: -rtsopts   build-depends:       attoparsec

@@ -1,7 +1,7 @@ module Smap.Commands (run) where  import Prelude hiding (filter, subtract, init, sin)-import qualified Data.HashMap.Strict as Map (insert, empty, foldrWithKey, alter, member)+import qualified Data.HashMap.Strict as Map (insert, empty, member) import Data.HashMap.Strict as Map (HashMap) import Data.Hashable (Hashable) import Data.ByteString.Char8 (ByteString)@@ -11,106 +11,84 @@ import Control.Monad (foldM) import Control.Monad.Trans.Class (lift) import Data.Strict.Tuple (Pair((:!:)))+import Data.List.NonEmpty (NonEmpty((:|))) import qualified Control.Monad.Trans.Resource as Resource import Control.Monad.IO.Class (MonadIO)-import Crypto.MAC.SipHash (SipKey(..), SipHash(..), hash)-import Data.Word (Word64)+import Crypto.MAC.SipHash (SipHash(..), hash) import Smap.Flags   ( Hdl(Std, File)-  , SetDescriptor(Keyed, UnKeyed)-  , Command(Union, Subtract, Intersect, Xor)+  , Descriptor(Keyed, UnKeyed)+  , Command(Union, Subtract, Intersect)   , Accuracy(Approximate, Exact)   ) -force :: Monad m => S.Stream (BS8.ByteString m) m r -> S.Stream (S.Of ByteString) m r-force = S.mapsM BS8.toStrict--duplicate :: forall a m . Monad m => S.Stream (S.Of a) m () -> Stream m a a-duplicate = P.map (\x -> x :!: x)- type Stream m k v = S.Stream (S.Of (Pair k v)) m ()  type SetOperation-  =  forall k-   . (Hashable k, Eq k)-  => [Stream (Resource.ResourceT IO) k ByteString] -- Input maps-  -> Stream (Resource.ResourceT IO) k ByteString -- Output map+  =  forall key+   . (Hashable key, Eq key)+  => NonEmpty (Stream (Resource.ResourceT IO) key ByteString) -- Input maps+  -> Stream (Resource.ResourceT IO) key ByteString -- Output map  cat :: SetOperation cat streams = foldM filter Map.empty streams *> return ()  where   filter seen = P.foldM_ filter' (return seen) return . S.hoist lift    -- for some strange reason I can't import alterF from Data.HashMap.Strict-  filter' (seen :: HashMap k ()) (bs :!: v) = -    if bs `Map.member` seen-      then return seen-      else P.yield (bs :!: v) >> return (Map.insert bs () seen)+  filter' (seen :: HashMap k ()) (bs :!: v) = if bs `Map.member` seen+    then return seen+    else P.yield (bs :!: v) >> return (Map.insert bs () seen) -filterFirstSetWith :: (Bool -> Bool) -> SetOperation-filterFirstSetWith _                []           = return ()-filterFirstSetWith includeIfPresent (first : seconds) = do-  subtract <- lift $ collects seconds-  P.filter (\(k :!: _) -> includeIfPresent (k `Map.member` subtract)) first+filterStreamWith :: (Bool -> Bool) -> SetOperation+filterStreamWith includeIfPresent (first :| seconds) = do+  second <- lift $ collects seconds+  P.filter (\(k :!: _) -> includeIfPresent (k `Map.member` second)) first  where   collects = foldM collect Map.empty   collect subs = P.fold_ (\s (k :!: _) -> Map.insert k () s) subs id  sub :: SetOperation-sub = filterFirstSetWith not+sub = filterStreamWith not -intersect :: SetOperation-intersect = filterFirstSetWith id+int :: SetOperation+int = filterStreamWith id -xor :: SetOperation-xor = go Map.empty+load :: (MonadIO m, Resource.MonadResource m) => Descriptor ty -> Stream m ByteString ByteString+load descriptor = case descriptor of+  UnKeyed hdl       -> P.map (\x -> x :!: x) (linesOf hdl)+  Keyed keys values -> S.zipsWith'+    (\q (k P.:> ks) (v P.:> vs) -> (k :!: v) P.:> (q ks vs))+    (linesOf keys)+    (linesOf values)  where-  go parity []       = Map.foldrWithKey (\k v -> (P.yield (k :!: v) >>)) (return ()) parity-  go parity (x : xs) = do-    parity' <- lift $ P.fold_ toggle parity id x-    go parity' xs-    where toggle hm (k :!: v) = Map.alter (maybe (Just v) (const Nothing)) k hm---keyMap :: Monad m => (k1 -> k2) -> Stream m k1 v -> Stream m k2 v-keyMap f = P.map (\(k :!: v) -> (f k :!: v))--sip :: SipKey -> ByteString -> Word64-sip key bs = let SipHash h = hash key bs in h--hin :: (MonadIO m, Resource.MonadResource m) => Hdl -> BS8.ByteString m ()-hin Std         = BS8.stdin-hin (File path) = BS8.readFile path--sin :: (MonadIO m, Resource.MonadResource m) => SetDescriptor -> Stream m ByteString ByteString-sin (UnKeyed hdl  ) = duplicate $ force $ BS8.lines $ hin hdl-sin (Keyed hks hvs) = S.zipsWith'-  (\q (k P.:> ks) (v P.:> vs) -> (k :!: v) P.:> (q ks vs))-  (f hks)-  (f hvs)-  where f = force . BS8.lines . hin--hout :: (MonadIO m, Resource.MonadResource m) => Hdl -> BS8.ByteString m a -> m a-hout Std         = BS8.stdout-hout (File path) = BS8.writeFile path+  linesOf = S.mapsM BS8.toStrict . BS8.lines . hin+  hin Std         = BS8.stdin+  hin (File path) = BS8.readFile path -format :: Monad m => Stream m k ByteString -> BS8.ByteString m ()-format = BS8.unlines . S.maps (\((_k :!: v) P.:> r) -> BS8.fromStrict v >> return r)+withAccuracy+  :: Accuracy+  -> SetOperation+  -> NonEmpty (Stream (Resource.ResourceT IO) ByteString ByteString)+  -> Hdl+  -> IO ()+withAccuracy accuracy op inputs output = case accuracy of+  Exact           -> approximateWith id+  Approximate key -> approximateWith (sip key)+ where+  format = BS8.unlines . S.maps (\((_k :!: v) P.:> r) -> BS8.fromStrict v >> return r)+  hout Std         = BS8.stdout+  hout (File path) = BS8.writeFile path+  sip key bs = let SipHash h = hash key bs in h+  keyMap f = P.map (\(k :!: v) -> (f k :!: v))+  approximateWith approximator =+    Resource.runResourceT $ hout output $ format $ op $ fmap (keyMap approximator) inputs  run :: Command -> IO () run cmd = case cmd of-  Subtract accuracy p ms o -> withAccuracy accuracy sub (p : ms) o-  Intersect accuracy is o  -> withAccuracy accuracy intersect is o-  Xor       accuracy is o  -> withAccuracy accuracy xor is o-  Union     accuracy is o  -> withAccuracy accuracy cat inputs o+  Subtract  acc p ms o -> withAccuracy acc sub (load p :| fmap load ms) o+  Intersect acc i is o -> withAccuracy acc int (load i :| fmap load is) o+  Union acc is o       -> withAccuracy acc cat (fmap load inputs) o    where     inputs = case is of-      [] -> [UnKeyed Std]-      xs -> xs- where-  withAccuracy accuracy (op :: SetOperation) inputs output = case accuracy of-    Exact           -> approximateWith id-    Approximate key -> approximateWith (sip key)-   where-    approximateWith approximator =-      Resource.runResourceT $ hout output $ format $ op $ fmap (keyMap approximator . sin) inputs-+      []       -> UnKeyed Std :| []+      (x : xs) -> x :| xs

@@ -1,7 +1,10 @@+-- Command line flag parsing + module Smap.Flags   ( Hdl(Std, File)-  , SetDescriptor(Keyed, UnKeyed)-  , Command(Union, Subtract, Intersect, Xor)+  , Descriptor(Keyed, UnKeyed)+  , InputType(Set, Stream)+  , Command(Union, Subtract, Intersect)   , Accuracy(Approximate, Exact)   , command   )@@ -16,16 +19,20 @@ data Hdl = Std -- stdin/stdout          | File FilePath -data SetDescriptor = Keyed Hdl Hdl -- Keys are in the first file, values in the second (map behavior)-                   | UnKeyed Hdl -- key = value (set behavior)+-- Just to help us keep track of how a descriptor is used+data InputType = Set -- Duplicates are discarded+               | Stream -- Duplicates are not discarded +data Descriptor (ty :: InputType)+   = Keyed Hdl Hdl -- Keys are in the first file, values in the second (map behavior)+   | UnKeyed Hdl -- key = value (set behavior)+ data Accuracy = Approximate SipKey -- Don't keep the actual value as a key, just its hash               | Exact -- Keep the actual value as a key -data Command = Union Accuracy [SetDescriptor] Hdl-             | Subtract Accuracy SetDescriptor [SetDescriptor] Hdl-             | Intersect Accuracy [SetDescriptor] Hdl-             | Xor Accuracy [SetDescriptor] Hdl+data Command = Union Accuracy [Descriptor 'Set] Hdl+             | Subtract Accuracy (Descriptor 'Stream) [Descriptor 'Set] Hdl+             | Intersect Accuracy (Descriptor 'Stream) [Descriptor 'Set] Hdl  hdl :: A.Parser Hdl hdl = stdin <|> path@@ -33,23 +40,14 @@   stdin = Std <$ A.char '-'   path  = File . Text.unpack <$> A.takeWhile (/= ',') -setExplanation :: String-setExplanation = "Use '-' for stdin. Use +keyfile,valfile for separate keys and values."--multiSetExplanation :: String-multiSetExplanation = "Can specify 0 or more files. " ++ setExplanation- outHdl :: O.Parser Hdl outHdl = O.option   (aToO hdl)   (O.metavar "OUTFILE" <> O.short 'o' <> O.long "out" <> O.value Std <> O.help "Defaults to stdout."   ) -inFiles :: O.Parser [SetDescriptor]-inFiles = many $ O.argument (aToO setDescriptor) (O.metavar "FILE*" <> O.help multiSetExplanation)--setDescriptor :: A.Parser SetDescriptor-setDescriptor =+descriptor :: A.Parser (Descriptor ty)+descriptor =   (keyed <|> unkeyed)     <*    (A.endOfInput A.<?> "more filepath characters than expected")     A.<?> "Could not parse filepath"@@ -62,28 +60,51 @@     v <- hdl     return (Keyed k v) + aToO :: A.Parser a -> O.ReadM a aToO p = O.eitherReader (A.parseOnly p . Text.pack) +stream :: O.Parser (Descriptor 'Stream)+stream = O.argument+  (aToO descriptor)+  (O.metavar "STREAM" <> O.help+    "Stream source file. Use '-' for stdin. Use +keyfile,valfile for separate keys and values."+  )++sets :: O.Parser [Descriptor 'Set]+sets = many $ O.argument+  (aToO descriptor)+  (  O.metavar "SET*"+  <> O.help+       "Set/map source files (0 or more). Use '-' for stdin. Use +keyfile,valfile for separate keys and values."+  )+ accuracy :: O.Parser Accuracy-accuracy = approx <|> exact+accuracy = approx <|> approxWithKey <|> exact  where   approx = O.flag'     (Approximate (SipKey 0 0))-    (O.short 'a' <> O.long "approximate" <> O.help-      (concat-        [ "For deduplication, store a 64-bit siphash rather than the whole line. "-        , "Can save memory, depending on the set operation"-        ]-      )+    (  O.short 'a'+    <> O.long "approximate"+    <> O.help+         "For deduplication, store a 64-bit siphash rather than the whole line. Can save memory, depending on the set operation"     )++  approxWithKey = O.option+    (fmap+      (\i -> Approximate $ SipKey (fromIntegral $ i `div` (2 ^ (64 :: Integer))) (fromIntegral i))+      (O.auto :: O.ReadM Integer)+    )+    (O.short 'k' <> O.long "approx-with-key" <> O.metavar "KEY" <> O.help+      "like --approximate, but takes a 128-bit SipHash key. Example: -k 0x1234..."+    )   exact = pure Exact  catCommand :: O.Mod O.CommandFields Command catCommand = O.command "cat" $ O.info   (value O.<**> O.helper)   (O.fullDesc <> O.progDesc "Set union. Output everything from the input sets, deduplicated.")-  where value = Union <$> accuracy <*> inFiles <*> outHdl+  where value = Union <$> accuracy <*> sets <*> outHdl  subCommand :: O.Mod O.CommandFields Command subCommand = O.command "sub" $ O.info@@ -96,36 +117,25 @@       ]     )   )- where-  value = Subtract <$> accuracy <*> plus <*> minuses <*> outHdl-  plus  = O.argument (aToO setDescriptor) (O.metavar "PLUSFILE" <> O.help setExplanation)-  minuses =-    many $ O.argument (aToO setDescriptor) (O.metavar "MINUSFILE*" <> O.help multiSetExplanation)+  where value = Subtract <$> accuracy <*> stream <*> sets <*> outHdl  intersectCommand :: O.Mod O.CommandFields Command intersectCommand = O.command "int" $ O.info   (value O.<**> O.helper)   (O.fullDesc <> O.progDesc-    "Set intersection. Output values which are in all input sets. Deduplicates values."-  )-  where value = Intersect <$> accuracy <*> inFiles <*> outHdl--xorCommand :: O.Mod O.CommandFields Command-xorCommand = O.command "xor" $ O.info-  (value O.<**> O.helper)-  (O.fullDesc <> O.progDesc     (concat-      [ "Set xor/symmetric difference. "-      , "Given files A and B, output all values which are A or B but not both. "-      , "Given more than 2 files, output everything that appears in an odd number of files."+      [ "Set intersection. "+      , "Given stream A and sets B,C,..., output values in A "+      , "only if they are in at least one of B,C,... "+      , "Does not deduplicate values from A."       ]     )   )-  where value = Xor <$> accuracy <*> inFiles <*> outHdl+  where value = Intersect <$> accuracy <*> stream <*> sets <*> outHdl  command :: IO Command command = O.execParser   (O.info-    ((O.subparser (catCommand <> subCommand <> intersectCommand <> xorCommand)) O.<**> O.helper)-    (O.fullDesc <> O.progDesc "Use -h for help. Available commands: cat, sub, int, xor.")+    ((O.subparser (catCommand <> subCommand <> intersectCommand)) O.<**> O.helper)+    (O.fullDesc <> O.progDesc "Use -h for help. Available commands: cat, sub, int.")   )