diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,6 @@ +# Version 0.1.0.0 + +[tag v0.1.0.0](https://github.com/stites/gym-http-api/releases/tag/v0.1.0.0) + +- Initial release. API contains basic environments and Aeson-based functions. + diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2016 OpenAI (http://openai.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# Haskell Binding for the OpenAI gym open-source library + +To run the example agent: + +``` +stack build && stack exec example +``` + +This library provides a servant-based REST client to the gym open-source library. +[openai/gym-http-api][openai] itself provides a [python-based REST server][flask] +to the gym open-source library, allowing development in languages other than python. + +[openai]:https://github.com/openai/gym-http-api +[flask]:https://github.com/openai/gym-http-api/blob/master/gym_http_server.py + diff --git a/Setup.hs b/Setup.hs new file mode 100644 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/TODO.md b/TODO.md new file mode 100644 --- /dev/null +++ b/TODO.md @@ -0,0 +1,6 @@ +## Checklist +- [ ] Add environment variable functionality to obtain the API key +- [ ] Optimization (lagging can be detected while running the example agent) +- [ ] Test suite + + diff --git a/examples/Agent.hs b/examples/Agent.hs new file mode 100644 --- /dev/null +++ b/examples/Agent.hs @@ -0,0 +1,80 @@ +------------------------------------------------------------------------------- +-- | +-- Module : Main +-- License : MIT +-- Stability : experimental +-- Portability: non-portable +-- +-- Example of how to build an agent using OpenAI.Gym.Client +------------------------------------------------------------------------------- +{-# LANGUAGE NamedFieldPuns #-} +module Main where + +import Prelude +import Control.Monad (replicateM_, when) +import Control.Monad.Catch +import Control.Exception.Base + +import OpenAI.Gym +import Servant.Client +import Network.HTTP.Client + + +main :: IO () +main = do + manager <- newManager defaultManagerSettings + out <- runClientM example (ClientEnv manager url) + case out of + Left err -> print err + Right ok -> print ok + + where + url :: BaseUrl + url = BaseUrl Http "localhost" 5000 "" + + +example :: ClientM () +example = do + inst <- envCreate CartPoleV0 + Monitor{directory} <- withMonitor inst $ + replicateM_ episodeCount (agent inst) + + -- Upload to the scoreboard. + -- TODO: Implement environment variable support. + upload (Config directory "algo" "") + + where + episodeCount :: Int + episodeCount = 100 + + +agent :: InstID -> ClientM () +agent inst = do + envReset inst + go 0 False + where + maxSteps :: Int + maxSteps = 200 + + reward :: Int + reward = 0 + + go :: Int -> Bool -> ClientM () + go x done = do + Action a <- envActionSpaceSample inst + Outcome ob reward done _ <- envStep inst (Step a True) + when (not done && x < 200) $ go (x + 1) done + + +withMonitor :: InstID -> ClientM () -> ClientM Monitor +withMonitor inst agent = do + envMonitorStart inst configs + agent + envMonitorClose inst + return configs + where + configs :: Monitor + configs = Monitor "/tmp/random-agent-results" True False False + + + diff --git a/gym-http-api.cabal b/gym-http-api.cabal new file mode 100644 --- /dev/null +++ b/gym-http-api.cabal @@ -0,0 +1,62 @@ +-- This file has been generated from package.yaml by hpack version 0.18.1. +-- +-- see: https://github.com/sol/hpack + +name: gym-http-api +version: 0.1.0.0 +synopsis: REST client to the gym-http-api project +description: This library provides a REST client to the gym open-source library. gym-http-api itself provides a server to the gym open-source library, allowing development in languages other than python. + . + Note that the is a monorepo of all language-clients. This hackage library tracks which is the actively-maintained haskell fork. +category: Web, Learning Environments +homepage: https://github.com/stites/gym-http-api#readme +bug-reports: https://github.com/stites/gym-http-api/issues +author: Daniel Lucsanszky, Sam Stites +maintainer: dl3913@ic.ac.uk, sam@stites.io +license: MIT +license-file: LICENSE +build-type: Simple +cabal-version: >= 1.10 + +extra-source-files: + CHANGELOG.md + README.md + TODO.md + +source-repository head + type: git + location: https://github.com/stites/gym-http-api + subdir: binding-hs + +library + hs-source-dirs: + src + default-extensions: OverloadedStrings + build-depends: + base >= 4.7 && < 5 + , servant-client >= 0.9 + , aeson >= 1.0 + , unordered-containers >= 0.2 + , servant >= 0.9 + , servant-lucid >= 0.7 + , text >= 1.2 + exposed-modules: + OpenAI.Gym + OpenAI.Gym.API + OpenAI.Gym.Data + other-modules: + Paths_gym_http_api + default-language: Haskell2010 + +executable example + main-is: Agent.hs + hs-source-dirs: + examples + default-extensions: OverloadedStrings + build-depends: + base >= 4.7 && < 5 + , servant-client >= 0.9 + , gym-http-api + , http-client >= 0.5 + , exceptions >= 0.8 + default-language: Haskell2010 diff --git a/src/OpenAI/Gym.hs b/src/OpenAI/Gym.hs new file mode 100644 --- /dev/null +++ b/src/OpenAI/Gym.hs @@ -0,0 +1,17 @@ +------------------------------------------------------------------------------- +-- | +-- Module : OpenAI.Gym +-- License : MIT +-- Stability : experimental +-- Portability: non-portable +-- +-- re-exports of @OpenAI.Gym.API@ and @OpenAI.Gym.Data@ +------------------------------------------------------------------------------- +module OpenAI.Gym + ( module OpenAI.Gym.API + , module OpenAI.Gym.Data + ) where + +import OpenAI.Gym.API +import OpenAI.Gym.Data + diff --git a/src/OpenAI/Gym/API.hs b/src/OpenAI/Gym/API.hs new file mode 100644 --- /dev/null +++ b/src/OpenAI/Gym/API.hs @@ -0,0 +1,120 @@ +------------------------------------------------------------------------------- +-- | +-- Module : OpenAI.Gym.API +-- License : MIT +-- Stability : experimental +-- Portability: non-portable +-- +-- Servant-client functions to interact with the flask server from +-- . +------------------------------------------------------------------------------- +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE TypeOperators #-} +{-# OPTIONS_GHC -Wno-orphans #-} --for MimeUnrender HTML () +module OpenAI.Gym.API ( + -- * Environment functions + envCreate + , envListAll + , envReset + , envStep + , envActionSpaceInfo + , envActionSpaceSample + , envActionSpaceContains + , envObservationSpaceInfo + , envMonitorStart + , envMonitorClose + , envClose + -- * Http-server commands + , upload + , shutdownServer + -- * Servant code + , gymAPI + ) where + +import Data.Aeson (Object) +import Data.Proxy (Proxy(..)) +import Servant.API +import Servant.HTML.Lucid (HTML) +import Servant.Client (ClientM, client) + +import OpenAI.Gym.Data + +type GymAPI + = "v1" :> ( "envs" :> ( ReqBody '[JSON] GymEnv :> Post '[JSON] InstID + :<|> Get '[JSON] Environment + :<|> Capture "instance_id" InstID :> "reset" :> Post '[JSON] Observation + :<|> Capture "instance_id" InstID :> "step" :> ReqBody '[JSON] Step :> Post '[JSON] Outcome + :<|> Capture "instance_id" InstID :> "action_space" :> Get '[JSON] Info + :<|> Capture "instance_id" InstID :> "action_space" :> "sample" :> Get '[JSON] Action + :<|> Capture "instance_id" InstID :> "action_space" :> "contains" :> Capture "x" Int :> Get '[JSON] Object + :<|> Capture "instance_id" InstID :> "observation_space" :> Get '[JSON] Info + :<|> Capture "instance_id" InstID :> "monitor" :> "start" :> ReqBody '[JSON] Monitor :> Post '[HTML] () + :<|> Capture "instance_id" InstID :> "monitor" :> "close" :> Post '[HTML] () + :<|> Capture "instance_id" InstID :> "close" :> Post '[HTML] ()) + :<|> "upload" :> ReqBody '[JSON] Config :> Post '[HTML] () + :<|> "shutdown" :> Post '[HTML] ()) + + +-- | Proxy type for the full servant-client representation of the gym http api. +gymAPI :: Proxy GymAPI +gymAPI = Proxy + + +-- | Create an instance of the specified environment (@POST \/v1\/envs\/@) +envCreate :: GymEnv -> ClientM InstID + +-- | List all environments running on the server (@GET \/v1\/envs\/@) +envListAll :: ClientM Environment + +-- | Reset the state of the environment and return an initial observation. (@POST \/v1\/envs\/\/reset\/@) +envReset :: InstID -> ClientM Observation + +-- | Step though an environment using an action. (@POST \/v1\/envs\/\/step\/@) +envStep :: InstID -> Step -> ClientM Outcome + +-- | Get information (name and dimensions\/bounds) of the env's action_space (@GET \/v1\/envs\/\/action_space\/@) +envActionSpaceInfo :: InstID -> ClientM Info + +-- | Sample randomly from the env's action_space (@GET \/v1\/envs\/\/action_space\/sample@) +envActionSpaceSample :: InstID -> ClientM Action + +-- | Check to see if a value is valid in the env's action_space (@GET \/v1\/envs\/\/action_space\/contains\/@) +envActionSpaceContains :: InstID -> Int -> ClientM Object + +-- | Get information (name and dimensions\/bounds) of the env's observation_space (@GET \/v1\/envs\/\/observation_space\/@) +envObservationSpaceInfo :: InstID -> ClientM Info + +-- | Start monitoring (@POST \/v1\/envs\/\/monitor\/start\/@) +envMonitorStart :: InstID -> Monitor -> ClientM () + +-- | Flush all monitor data to disk (@POST \/v1\/envs\/\/monitor\/close\/@) +envMonitorClose :: InstID -> ClientM () + +-- | Stop the environment (@POST \/v1\/envs\/\/close\/@) +envClose :: InstID -> ClientM () + +-- | Upload results to OpenAI's servers (@POST \/v1\/upload\/@) +upload :: Config -> ClientM () + +-- | Request a server shutdown (@POST \/v1\/shutdown\/@) +shutdownServer :: ClientM () + + +(envCreate + :<|> envListAll + :<|> envReset + :<|> envStep + :<|> envActionSpaceInfo + :<|> envActionSpaceSample + :<|> envActionSpaceContains + :<|> envObservationSpaceInfo + :<|> envMonitorStart + :<|> envMonitorClose + :<|> envClose) + :<|> upload + :<|> shutdownServer + = client gymAPI + +instance MimeUnrender HTML () where + mimeUnrender _ _ = return () diff --git a/src/OpenAI/Gym/Data.hs b/src/OpenAI/Gym/Data.hs new file mode 100644 --- /dev/null +++ b/src/OpenAI/Gym/Data.hs @@ -0,0 +1,167 @@ +------------------------------------------------------------------------------- +-- | +-- Module : OpenAI.Gym.Data +-- License : MIT +-- Stability : experimental +-- Portability: non-portable +-- +-- Aeson-based data types to be returned by "OpenAI.Gym.API" +------------------------------------------------------------------------------- +{-# LANGUAGE DeriveGeneric #-} +module OpenAI.Gym.Data + ( GymEnv (..) + , InstID (..) + , Environment (..) + , Observation (..) + , Step (..) + , Outcome (..) + , Info (..) + , Action (..) + , Monitor (..) + , Config (..) + ) where + +import Data.Aeson (ToJSON(..), FromJSON(..), Value(..), Object, (.=), (.:), object) +import Data.Aeson.Types (Parser) +import Data.HashMap.Strict (HashMap) +import Data.Text (Text) +import GHC.Generics (Generic) +import Servant.API (ToHttpApiData(..)) +import qualified Data.Text as T +import qualified Data.Aeson as A + +-- | Classic Control Environments +data GymEnv + = CartPoleV0 -- ^ Balance a pole on a cart (for a short time). + | CartPoleV1 -- ^ Balance a pole on a cart. + | AcrobotV1 -- ^ Swing up a two-link robot. + | MountainCarV0 -- ^ Drive up a big hill. + | MountainCarContinuousV0 -- ^ Drive up a big hill with continuous control. + | PendulumV0 -- ^ Swing up a pendulum. + + -- Toy text games + | FrozenLakeV0 -- ^ Swing up a pendulum. + + -- | Atari Games + | PongRamV0 -- ^ Maximize score in the game Pong, with RAM as input + | PongV0 -- ^ Maximize score in the game Pong + deriving (Eq, Enum, Ord) + + +instance Show GymEnv where + show CartPoleV0 = "CartPole-v0" + show CartPoleV1 = "CartPole-v1" + show AcrobotV1 = "Acrobot-v1" + show MountainCarV0 = "MountainCar-v0" + show MountainCarContinuousV0 = "MountainCarContinuous-v0" + show PendulumV0 = "Pendulum-v0" + show FrozenLakeV0 = "FrozenLake-v0" + show PongRamV0 = "Pong-ram-v0" + show PongV0 = "Pong-v0" + +instance ToJSON GymEnv where + toJSON env = object [ "env_id" .= show env ] + +-- | a short identifier (such as '3c657dbc') for the created environment instance. +-- The instance_id is used in future API calls to identify the environment to be manipulated. +newtype InstID = InstID { getInstID :: Text } + deriving (Eq, Show, Generic) + +instance ToHttpApiData InstID where + toUrlPiece (InstID i) = i + +instance ToJSON InstID where + toJSON (InstID i) = toSingleton "instance_id" i + +instance FromJSON InstID where + parseJSON = parseSingleton InstID "instance_id" + +-- | a mapping of instance_id to env_id (e.g. {'3c657dbc': 'CartPole-v0'}) for every env on the server +newtype Environment = Environment { all_envs :: HashMap Text Text } + deriving (Eq, Show, Generic) + +instance ToJSON Environment +instance FromJSON Environment + +-- | The agent's observation of the current environment +newtype Observation = Observation { getObservation :: Value } + deriving (Eq, Show, Generic) + +instance ToJSON Observation where + toJSON (Observation v) = toSingleton "observation" v + +instance FromJSON Observation where + parseJSON = parseSingleton Observation "observation" + +-- | An action to take in the environment and whether or not to render that change +data Step = Step + { action :: !Value + , render :: !Bool + } deriving (Eq, Generic, Show) + +instance ToJSON Step + +-- | The result of taking a step in an environment +data Outcome = Outcome + { observation :: !Value -- ^ agent's observation of the current environment + , reward :: !Double -- ^ amount of reward returned after previous action + , done :: !Bool -- ^ whether the episode has ended + , info :: !Object -- ^ a dict containing auxiliary diagnostic information + } deriving (Eq, Show, Generic) + +instance ToJSON Outcome +instance FromJSON Outcome + +-- | A dict containing auxiliary diagnostic information +newtype Info = Info { getInfo :: Object } + deriving (Eq, Show, Generic) + +instance ToJSON Info where + toJSON (Info v) = toSingleton "info" v + +instance FromJSON Info where + parseJSON = parseSingleton Info "info" + +-- | An action to take in the environment +newtype Action = Action { getAction :: Value } + deriving (Eq, Show, Generic) + +instance ToJSON Action where + toJSON (Action v) = toSingleton "action" v + +instance FromJSON Action where + parseJSON = parseSingleton Action "action" + +-- | Parameters used to start a monitoring session. +data Monitor = Monitor + { directory :: !Text -- ^ directory to use for monitoring + , force :: !Bool -- ^ Clear out existing training data from this directory (by deleting + -- every file prefixed with "openaigym.") (default=False) + , resume :: !Bool -- ^ Retain the training data already in this directory, which will be + -- merged with our new data. (default=False) + , video_callable :: !Bool -- ^ video_callable parameter from the native env.monitor.start function + } deriving (Generic, Eq, Show) + +instance ToJSON Monitor + +-- | Parameters used to upload a monitored session to OpenAI's servers +data Config = Config + { training_dir :: !Text -- ^ A directory containing the results of a training run. + , algorithm_id :: !Text -- ^ An arbitrary string indicating the paricular version of the + -- algorithm (including choices of parameters) you are running. + -- (default=None) + , api_key :: !Text -- ^ Your OpenAI API key + } deriving (Generic, Eq, Show) + +instance ToJSON Config + + +-- | helper to parse a singleton object from aeson +parseSingleton :: FromJSON a => (a -> b) -> Text -> Value -> Parser b +parseSingleton fn f (Object v) = fn <$> v .: f +parseSingleton fn f _ = mempty + +-- | convert a value into a singleton object +toSingleton :: ToJSON a => Text -> a -> Value +toSingleton f a = object [ f .= toJSON a ] +