transient 0.1.0.8 → 0.1.0.9
raw patch · 7 files changed
+298/−123 lines, 7 filesdep +SHAdep ~process
Dependencies added: SHA
Dependency ranges changed: process
Files
- src/Transient/Base.hs +13/−9
- src/Transient/DDS.hs +1/−10
- src/Transient/Indeterminism.hs +5/−2
- src/Transient/Logged.hs +18/−8
- src/Transient/Move.hs +101/−72
- src/Transient/Move/Services.hs +156/−0
- transient.cabal +4/−22
src/Transient/Base.hs view
@@ -223,14 +223,14 @@ -- x <- g -- return $ k <|> x -data RemoteStatus= WasRemote | NoRemote deriving (Typeable, Eq) +data RemoteStatus= WasRemote | NoRemote deriving (Typeable, Eq, Show) instance MonadPlus TransientIO where mzero= empty mplus x y= Transient $ do mx <- runTrans x -- !> "RUNTRANS11111" was <- getSessionData `onNothing` return NoRemote - if was== WasRemote + if was== WasRemote !> ("WASREMOTE= "++ show was) then return Nothing else case mx of Nothing -> runTrans y -- !> "RUNTRANS22222" @@ -456,14 +456,14 @@ SMore r <- parallel (SMore <$> io) return r --- | variant of + parallel` that execute the IO computation once, and kill the previous child threads +-- | variant of `parallel` that execute the IO computation once, and kill the previous child threads async :: IO b -> TransientIO b async io= do SLast r <- parallel (SLast <$>io) killChilds return r --- | variant that spawn free threads. Since there is not thread control, this is faster +-- | variant that spawn free threads. Since there is no thread control, this is faster spawn :: IO b -> TransientIO b spawn io= freeThreads $ do @@ -474,16 +474,20 @@ --- | return empty to the current thread and launch the IO action in a new thread and attaches the continuation after it. --- if the result of the action is `Right` the process is repeated. if not, it finish. +-- | return empty to the current thread, in new thread, execute the IO action, +-- this IO action modify an internal buffer. then, executes the closure where `parallel` is located +-- In this new execution, since the buffer is filled, `paralle¤` return the content of this buffer. +-- Then it launch the continuation after it with this new value returned by the closure. -- -- If the maximum number of threads, set with `threads` has been reached `parallel` perform -- the work sequentially, in the current thread. +-- So `parallel` means that 'it can be parallelized if there are thread available' -- --- When `parallel`finish, increase the counter of threads available, if there is a limitation of them. +-- if there is a limitation of threads, when a thread finish, the counter of threads available +-- is increased so another `parallel` can make use of it. -- --- The behaviour of `parallel` depend on `StreamData` if `SMore`, `parallel` will excute again the --- IO action. with `SLast`, `SDone` and `SError`, `parallel` will execute the continuation and will stop. +-- The behaviour of `parallel` depend on `StreamData`; If `SMore`, `parallel` will excute again the +-- IO action. with `SLast`, `SDone` and `SError`, `parallel` will not repeat the IO action anymore. parallel :: IO (StreamData b) -> TransientIO (StreamData b) parallel ioaction= Transient $ do cont <- getCont -- !> "PARALLEL"
src/Transient/DDS.hs view
@@ -22,7 +22,7 @@ import Data.TCache import Data.TCache.Defs ---import Data.TCache.DefaultPersistence + import Data.ByteString.Lazy.Char8 (pack,unpack) import Control.Monad.STM @@ -42,15 +42,6 @@ else defaultReadByKey k >>= return . fmap ( read . unpack) writeResource (s@(Part _ _ save _))= if not save then return () else defaultWrite (defPath s ++ key s) (pack$ show s) ---instance Loggable a => Serializable (Partition a) where --- serialize = pack . show --- deserialize = read . unpack --- setPersist (Part _ _ persist _) = --- Just Persist --- {readByKey= if persist then defaultReadByKey else const $ return Nothing --- ,write = if persist then defaultWrite else const $ const $ return () --- ,delete = defaultDelete} - instance Loggable a => Monoid (DDS a) where mempty= DDS mempty
src/Transient/Indeterminism.hs view
@@ -56,6 +56,7 @@ then stop else liftIO $ atomicModifyIORef v $ \(n,xs) -> ((0,[]),xs) +-- | group result for a time interval, measured with `diffUTCTime groupByTime :: Integer -> TransientIO a -> TransientIO [a] groupByTime time proc = do v <- liftIO $ newIORef (0,[]) @@ -89,8 +90,10 @@ collect :: Int -> TransientIO a -> TransientIO [a] collect n = collect' n 1000 0 --- | search also between two time intervals. If the first interval has passed and there is no result, it stop --- after the second interval, it stop unconditionally. It stops as soon as it has enoug result. +-- | search also between two time intervals. If the first interval has passed and there is no result, +--it stops. +-- After the second interval, it stop unconditionally and return the current results. +-- It also stops as soon as there are enough results. collect' :: Int -> NominalDiffTime -> NominalDiffTime -> TransientIO a -> TransientIO [a] collect' n t1 t2 search= do rv <- liftIO $ atomically $ newTVar (0,[]) !> "NEWMVAR"
src/Transient/Logged.hs view
@@ -21,15 +21,25 @@ import Control.Monad.IO.Class ---newtype Logged m a = Logged {runLogged :: m a} --- +newtype TransLIO a = TransLIO {runLogged :: TransIO a} + --data RLogged= forall a.(Read a, Show a) => RLogged a --- ---instance Monad m => Monad (Logged m) where --- return x= Logged $ return x --- Logged x >>= f = Logged $ do --- r <- x --- runLogged $ f r + +instance Functor TransLIO where +-- fmap f mx= mx >>= \(TransLIO x) -> TransLIO (f x) + +instance Applicative TransLIO where +-- pure= return +-- f <*> g= TransLIO $ do +-- x <- f +-- y <- g +-- return $ x y + +instance Monad TransLIO where + return x= TransLIO $ return x + TransLIO x >>= f = TransLIO $ do + r <- x + runLogged $ f r --data IDynamic= IDyns String | forall a.(Read a, Show a,Typeable a) => IDynamic a
src/Transient/Move.hs view
@@ -13,8 +13,9 @@ {-# LANGUAGE DeriveDataTypeable , ExistentialQuantification ,ScopedTypeVariables, StandaloneDeriving, RecordWildCards #-} module Transient.Move where -import Transient.Base +import Transient.Base hiding (onNothing) import Transient.Logged +import Transient.EVars import Data.Typeable import Control.Applicative import Network @@ -26,8 +27,8 @@ import Control.Exception import Data.Maybe import Unsafe.Coerce -import System.Process -import System.Directory + +--import System.Directory import Control.Monad import Network.Info import System.IO.Unsafe @@ -43,32 +44,14 @@ import qualified Network.BSD as BSD -import qualified Data.ByteString.Char8 as BS +import qualified Data.ByteString.Lazy.Char8 as BS import System.IO - import Control.Concurrent +import Data.TCache +import Data.TCache.DefaultPersistence --- | install in a remote node a haskell package with an executable transient service initialized with `listen` --- the package, the git repository and the main exectable must have the same name -installService (node@(Node _ port _)) servport package= do - beamTo node - liftIO $ do - let packagename= name package - exist <- doesDirectoryExist packagename - when (not exist) $ do - runCommand $ "git clone "++ package - runCommand $ "cd "++ packagename - runCommand "cabal install" - return () - createProcess $ shell $ "./dist/build/"++ packagename++"/"++packagename - ++ " " ++ show port - return() - where - name path= - let x= dropWhile (/= '/') path - in if x== "" then tail path else name $ tail x -- | continue the execution in a new node -- all the previous actions from `listen` to this statement must have been logged @@ -76,21 +59,24 @@ beamTo node = do Log rec log _ <- getSData <|> return (Log False [][]) if rec then return () else do - Connection _ bufSize <- getSData <|> return (Connection Nothing 8192) + Connection{bufferSize=bufSize} + <- getSData + <|> return (Connection beamToErr Nothing 8192 beamToErr) h <- assign bufSize node liftIO $ hPutStrLn h (show $ SMore $ reverse log) >> hFlush h release node h let log'= WaitRemote: log setSData $ Log rec log' log' stop - + where + beamToErr= error "beamTo: some connection param has not been set. Use setMyNode and listen" -- | execute in the remote node a process with the same execution state -- all the previous actions from `listen` to this statement must have been logged forkTo :: Node -> TransientIO () forkTo node= do Log rec log _<- getSData <|> return (Log False [][]) if rec then return () else do - Connection _ bufSize <- getSData <|> return (Connection Nothing 8192) + Connection {bufferSize=bufSize} <- getSData <|> return (Connection undefined Nothing 8192 undefined) h <-assign bufSize node liftIO $ hPutStrLn h (show $ SMore $ reverse log) >> hFlush h release node h @@ -117,7 +103,7 @@ then runTrans $ do rnum <- liftIO $ newMVar (0 :: Int) - Connection (Just (_, h, sock, blocked)) _ <- getSData <|> error "callTo: no hander" + Connection _(Just (ConnectionData _ h sock blocked )) _ _ <- getSData <|> error "callTo: no hander" r <- remoteProc !> "executing remoteProc" !> "CALLTO REMOTE" -- LOg="++ show fulLog n <- liftIO $ do -- modifyMVar_ rnum $ \n -> return (n+1) @@ -131,7 +117,7 @@ stop else do - Connection _ bufSize <- getSessionData `onNothing` return (Connection Nothing 8192) + Connection _ _ bufSize _<- getSessionData `onNothing` return (Connection undefined Nothing 8192 undefined) h <- assign bufSize node liftIO $ hSetBuffering h LineBuffering liftIO $ hPutStrLn h ( show $ SLast $ reverse fulLog) {- >> hFlush h -} !> "CALLTO LOCAL" -- send "++ show log @@ -165,6 +151,7 @@ -- | A connectionless version of callTo for long running remote calls +-- myNode should be set with `setMyNode` callTo' :: (Show a, Read a,Typeable a) => Node -> TransIO a -> TransIO a callTo' node remoteProc= logged $ do mynode <- logged getMyNode @@ -175,21 +162,33 @@ type Blocked= MVar () type BuffSize = Int -data Connection= Connection (Maybe(PortID, Handle, Socket, Blocked)) BuffSize +data ConnectionData= ConnectionData{port :: PortID + ,handle :: Handle + ,socket ::Socket + ,blocked :: Blocked} + + + + +data Connection= Connection{myNode :: DBRef MyNode + ,connData :: (Maybe(ConnectionData)) + ,bufferSize ::BuffSize + ,comEvent :: EVar(Node,Service)} deriving Typeable setBufSize :: Int -> TransIO () setBufSize size= Transient $ do - Connection c _ <- getSessionData `onNothing` return (Connection Nothing size) - setSessionData $ Connection c size + Connection n c _ ev <- getSessionData `onNothing` + return (Connection (errorMyNode "setBufSize") Nothing size (error "accessing network events out of listen")) + setSessionData $ Connection n c size ev return $ Just () - - +getBuffSize= + (do Connection _ _ bufSize _ <- getSData ; return bufSize) <|> return 8192 readHandler h= do - line <- hGetLine h -- {- readIORef leftover <> -} unsafeInterleaveIO (hGetLine h) --- liftIO $ print (line, show h) + line <- hGetLine h + let [(v,left)]= readsPrec 0 line --- length v `seq` writeIORef leftover left + return v `catch` (\(e::SomeException) -> do @@ -219,18 +218,25 @@ -- | Wait for messages and replay the rest of the monadic sequence with the log received. listen :: Node -> TransIO () -listen (Node _ port _) = do +listen (node@(Node _ port _ _)) = do addThreads 1 + setMyNode node setSData $ Log False [] [] - Connection _ bufSize <- getSData <|> return (Connection Nothing 8192) - sock <- liftIO $ withSocketsDo $ listenOn port + + Connection node _ bufSize events <- getSData + + sock <- liftIO $ listenOn port liftIO $ do NS.setSocketOption sock NS.RecvBuffer bufSize NS.setSocketOption sock NS.SendBuffer bufSize - SMore(h,host,port1) <- parallel $ SMore <$> accept sock - `catch` (\(e::SomeException) -> print "socket exception" >> sClose sock >> throw e) + SMore(h,host,port1) <- parallel $ (SMore <$> accept sock) + `catch` (\(e::SomeException) -> do + print "socket exception" + sClose sock + return SDone) - setSData $ Connection (Just (port, h, sock, unsafePerformIO $ newMVar ())) bufSize -- !> "setdata port=" ++ show port + setSData $ Connection node (Just (ConnectionData port h sock (unsafePerformIO $ newMVar ()))) bufSize events -- !> "setdata port=" ++ show port + liftIO $ hSetBuffering h LineBuffering -- !> "LISTEN in "++ show (h,host,port1) mlog <- parallel $ readHandler h @@ -249,11 +255,6 @@ - - - - - -- | init a Transient process in a interactive as well as in a replay mode. -- It is intended for twin processes that interact among them in different nodes. beamInit :: Node -> TransIO a -> IO a @@ -261,9 +262,6 @@ listen node <|> return () program - - - instance Read PortNumber where readsPrec n str= let [(n,s)]= readsPrec n str in [(fromIntegral n,s)] @@ -274,10 +272,18 @@ data Pool= Pool{free :: [Handle], pending :: Int} +type Package= String +type Program= String +type Service= (Package, Program, Int) -data Node= Node{host :: HostName, port :: PortID, connection :: IORef Pool} deriving Typeable +data Node= Node{ nodeHost :: HostName + , nodePort :: PortID + , connection :: IORef Pool + , services :: [Service]} + deriving Typeable -release (Node h p rpool) hand= liftIO $ do + +release (Node h p rpool _) hand= liftIO $ do mhs <- atomicModifyIORef rpool $ \(Pool hs pend) -> if pend==0 @@ -288,7 +294,7 @@ Just hs -> mapM_ hClose hs -assign bufSize (Node h p pool)= liftIO $ do +assign bufSize (Node h p pool _)= liftIO $ do mh <- atomicModifyIORef pool $ \(Pool hs p) -> if null hs then (Pool hs p, Nothing) else (Pool (tail hs) p, Just(head hs)) !> "REUSED" @@ -310,30 +316,55 @@ emptyPool= liftIO $ newIORef $ Pool [] 0 createNode :: HostName -> Integer -> Node -createNode h p= Node h ( PortNumber $ fromInteger p) (unsafePerformIO emptyPool) +createNode h p= Node h ( PortNumber $ fromInteger p) (unsafePerformIO emptyPool) [] instance Eq Node where - Node h p _ ==Node h' p' _= h==h' && p==p' + Node h p _ _ ==Node h' p' _ _= h==h' && p==p' -instance Show Node where show (Node h p _)= show (h,p) +instance Show Node where show (Node h p _ servs)= show (h,p,servs) instance Read Node where readsPrec _ s= - let [((h,p),s')]= readsPrec 0 s - in [(Node h p empty,s')] + let r= readsPrec 0 s + in case r of + [] -> [] + [((h,p,ss),s')] -> [(Node h p empty ss,s')] where empty= unsafePerformIO emptyPool +newtype MyNode= MyNode Node deriving(Read,Show,Typeable) +instance Indexable MyNode where key (MyNode Node{nodePort=port}) = "MyNode "++ show port + +instance Serializable MyNode where + serialize= BS.pack . show + deserialize= read . BS.unpack + nodeList :: TVar [Node] nodeList = unsafePerformIO $ newTVarIO [] deriving instance Ord PortID -myNode= unsafePerformIO $ newIORef Nothing +--myNode :: Int -> DBRef MyNode +--myNode= getDBRef $ key $ MyNode undefined -setMyNode node= liftIO $ writeIORef myNode $ Just node -getMyNode= Transient $ liftIO $ readIORef myNode +errorMyNode f= error $ f ++ ": Node not set. Use setMynode before listen" + +getMyNode :: TransIO Node +getMyNode = do + Connection{myNode=rnode} <- getSData <|> errorMyNode "getMyNode" + MyNode node <- liftIO $ atomically $ readDBRef rnode `onNothing` errorMyNode "getMyNode" + return node + +setMyNode :: Node -> TransIO () +setMyNode node= do + addNodes [node] + events <- newEVar + rnode <- liftIO $ atomically $ newDBRef $ MyNode node + let conn= Connection rnode Nothing 8192 events + setSData conn + return () + getNodes :: MonadIO m => m [Node] getNodes = liftIO $ atomically $ readTVar nodeList @@ -379,13 +410,13 @@ -- > createLocalNode n= createNode "localhost" (PortNumber n) clustered :: Loggable a => TransIO a -> TransIO a clustered proc= logged $ do - nodes <- step getNodes + nodes <- getNodes logged $ foldr (<|>) empty $ map (\node -> callTo node proc) nodes !> "fold" -- | a connectionless version of clustered for long running remote computations. Not tested clustered' proc= logged $ do nodes <- getNodes - logged $ mapM (\node -> callTo' node proc) $ nodes + logged $ mapM (\node -> callTo' node proc) nodes -- A variant of clustered that wait for all the responses and `mappend` them mclustered :: (Monoid a, Loggable a) => TransIO a -> TransIO a @@ -399,22 +430,20 @@ -- all the nodes connected to him. this new connected node will receive the list of nodes -- the local list of nodes then is updated with this list. it can be retrieved with `getNodes` connect :: Node -> Node -> TransientIO () -connect (node@(Node h port _)) remotenode= do +connect node remotenode= do listen node <|> return () logged $ do logged $ do - setMyNode node - addNodes [node] liftIO $ putStrLn $ "connecting to: "++ show remotenode newnode <- logged $ return node -- must pass my node the remote node or else it will use his own - port <- logged $ return port + nodes <- callTo remotenode $ do - clustered $ addNodes [newnode] - r <- getNodes - liftIO $ putStrLn $ "Connected to modes: " ++ show r - return r + mclustered $ addNodes [newnode] + getNodes + liftIO $ putStrLn $ "Connected to nodes: " ++ show nodes logged $ addNodes nodes +
+ src/Transient/Move/Services.hs view
@@ -0,0 +1,156 @@+----------------------------------------------------------------------------- +-- +-- Module : Transient.Move.Services +-- Copyright : +-- License : GPL-3 +-- +-- Maintainer : agocorona@gmail.com +-- Stability : +-- Portability : +-- +-- | +-- +----------------------------------------------------------------------------- + +module Transient.Move.Services where + +import Transient.Base +import Transient.Move +import Transient.Logged +import Transient.EVars +import Control.Monad.IO.Class +import System.Process +import System.IO.Unsafe +import Control.Concurrent.MVar +import Control.Applicative +import Network (PortID(..)) +import GHC.Conc +import System.Directory +import Control.Monad +import Data.List +import Data.TCache hiding(onNothing) + +-- for the example +import System.Environment + +startServices :: TransIO () +startServices= do + node <- getMyNode + liftIO $ print node + mapM_ start $ services node + where + start (package,program,port)= liftIO $ do + let prog= pathExe (name package) program port + liftIO $ print prog + createProcess $ shell prog + + +pathExe package program port= package++"/dist/build/"++package++"/"++program + ++ " " ++ show port + +install :: String -> String -> Int -> TransIO () +install package program port = logged $ do + let packagename = name package + exist <- logged $ liftIO $ doesDirectoryExist packagename + when (not exist) $ logged $ liftIO $ do + callProcess "git" ["clone",package] + liftIO $ print "GIT DONE" + setCurrentDirectory packagename + callProcess "cabal" ["install"] + setCurrentDirectory ".." + return() + let prog= pathExe packagename program port + logged $ liftIO $ do + createProcess $ shell program + return () + + let service= (package, program, port) + + Connection{myNode= rnode} <- getSData <|> error "Mynode not set: use setMyNode" + logged $ liftIO $ do + atomically $ do + MyNode( Node h p c servs) <- readDBRef rnode + `onNothing` error "install: myNode: not set with setMyNode" + writeDBRef rnode $ MyNode $ Node h p c $ service:servs + liftIO syncCache + node <- logged getMyNode + clustered $ notifyService node service + return() + +name url= do + let git= "http://github.com/" + if not $ isPrefixOf git url + then error "install: only github repos are admitted, sorry" + else + let segments = split '/' $ drop (length git) url + segs'= reverse segments + in head segs' + + + where + split c []= [] + split c xs= + let (h,t)= span (/= c) xs + in if null t then [h] else h : split c (tail t) + +rfreePort :: MVar Int +rfreePort = unsafePerformIO $ newMVar 3000 + +freePort :: MonadIO m => m Int +freePort= liftIO $ modifyMVar rfreePort $ \ n -> return (n+1,n) + +initService node package program= logged $ + case find (\(package', program',_) -> package==package' && program== program') $ services node of + Just (_,_,port) -> return port + Nothing -> do + beamTo node + port <- logged freePort + install package program port + stop + <|> do + Connection _ _ _ ev<- getSData + (node', (package', program',port)) <- readEVar ev + if node'== node && package' == package && program'== program + then return port + else stop + +notifyService :: Node -> Service -> TransIO () +notifyService node service= logged $ do + liftIO $ atomically $ do + nodes <- readTVar nodeList + let ([nod], nodes')= span (== node) nodes + let nod' = nod{services=service:services nod} + writeTVar nodeList $ nod' : nodes' + return () + + Connection _ _ _ ev<- getSData + writeEVar ev (node,service) + return () + + +main= do +-- keep $ install "http://github.com/agocorona/transient" "MainStreamFiles" 3000 + let node1= createNode "localhost" 2000 + let node2= createNode "localhost" 2001 + args <-getArgs + let [localNode,remoteNode]= if null args then [node1,node2] else [node2,node1] + + addNodes [localNode, remoteNode] + keep $ do + setMyNode localNode + listen localNode <|> return () + step $ option "start" "start" + + logged startServices + port <-initService remoteNode "http://github.com/agocorona/transient" "MainStreamFiles" + liftIO $ putStrLn $ "installed at" ++ show port +-- nodes <- getNodes +-- liftIO $ print nodes +-- liftIO syncCache +-- option "end" "end" +-- liftIO $ print "END" + + + + +
transient.cabal view
@@ -1,5 +1,5 @@ name: transient -version: 0.1.0.8 +version: 0.1.0.9 cabal-version: >=1.10 build-type: Simple license: GPL-3 @@ -7,7 +7,7 @@ maintainer: agocorona@gmail.com homepage: http://www.fpcomplete.com/user/agocorona bug-reports: https://github.com/agocorona/transient/issues -synopsis: A monad for extensible effects and primitives for unrestricted composability of applications +synopsis: Making composable programs with multithreading, events and distributed computing description: see <http://github.com/agocorona/transient> category: Control author: Alberto G. Corona @@ -21,7 +21,7 @@ build-depends: base >4 && <5 , mtl , random , containers , directory , filepath , stm , HTTP , network , transformers , process , network-info , - bytestring , time , TCache + bytestring , time , TCache, SHA exposed-modules: Transient.DDS Transient.Indeterminism Transient.Base Transient.EVars Transient.Backtrack Transient.Move Transient.Logged Transient.Stream.Resource @@ -29,25 +29,7 @@ buildable: True default-language: Haskell2010 hs-source-dirs: src . + other-modules: Transient.Move.Services ---executable transient --- build-depends: base >4 && <5, mtl , random , --- containers , directory , filepath , stm , --- HTTP , network , transformers , transient , --- bytestring --- main-is: examples/Main.hs --- buildable: True --- default-language: Haskell2010 --- hs-source-dirs: . --- other-modules: --- ---executable distributed --- build-depends: base , mtl , random , containers , --- directory , filepath , stm , HTTP , network , --- transformers , transient , bytestring --- main-is: examples/distributedExamples.hs --- buildable: True --- default-language: Haskell2010 --- hs-source-dirs: .