diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright Erik Aker (c) 2017 + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of Author name here nor the names of other + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,175 @@ +# servant-py + +This library lets you derive automatically Python functions that let you query each endpoint of a *servant* webservice. + +Currently, the only supported method for generating requests is via the `requests` library, which is the recommended way to generate HTTP requests in the Python world (even among Python core devs). + +## Inspiration + +This library is largely inspired by [servant-js](https://github.com/haskell-servant/servant-js) and by the fantastic work of the Servant team in general. Any good ideas you find in here are from their work (any mistakes are almost entirely mine, however). + +## Example + +There are two different styles of function-return supported here: `DangerMode` and `RawResponse`. + +The latter returns the raw response from issuing the request and the former calls `raise_for_status` and then attempts to return `resp.json()`. You can switch which style you'd like to use by creating a proper `CommonGeneratorOptions` object. + +The default options just chucks it all to the wind and goes for `DangerMode` (because, seriously, we're using Haskell to generate Python here...). + +Following is an example of using the Servant DSL to describe endpoints and then using `servant-py` to create Python clients for those endpoints. + +#### Servant DSL API Description + +``` haskell +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE TypeOperators #-} + +module Main where + +import Data.Aeson +import qualified Data.ByteString.Char8 as B +import Data.Proxy +import qualified Data.Text as T +import GHC.Generics +import Servant +import System.FilePath + +import Servant.PY + +-- * A simple Counter data type +newtype Counter = Counter { value :: Int } + deriving (Generic, Show, Num) +instance ToJSON Counter + +data LoginForm = LoginForm + { username :: !T.Text + , password :: !T.Text + , otherMissing :: Maybe T.Text + } deriving (Eq, Show, Generic) +instance ToJSON LoginForm + +-- * Our API type +type TestApi = "counter-req-header" :> Post '[JSON] Counter + :<|> "counter-queryparam" + :> QueryParam "sortby" T.Text + :> Header "Some-Header" T.Text :> Get '[JSON] Counter + :<|> "login-queryflag" :> QueryFlag "published" :> Get '[JSON] LoginForm + :<|> "login-params-authors-with-reqBody" + :> QueryParams "authors" T.Text + :> ReqBody '[JSON] LoginForm :> Post '[JSON] LoginForm + :<|> "login-with-path-var-and-header" + :> Capture "id" Int + :> Capture "Name" T.Text + :> Capture "hungrig" Bool + :> ReqBody '[JSON] LoginForm + :> Post '[JSON] (Headers '[Header "test-head" B.ByteString] LoginForm) + +testApi :: Proxy TestApi +testApi = Proxy + +-- where our static files reside +result :: FilePath +result = "examples" + +main :: IO () +main = writePythonForAPI testApi requests (result "api.py") +``` + +#### Generated Python Code + +If you build the above and run it, you will get some output that looks like the following: + +```python +from urllib import parse + +import requests + +def post_counterreqheader(): + """ + POST "counter-req-header" + + """ + url = "http://localhost:8000/counter-req-header" + + resp = requests.post(url) + resp.raise_for_status() + return resp.json() + + +def get_counterqueryparam(sortby, headerSomeHeader): + """ + GET "counter-queryparam" + + """ + url = "http://localhost:8000/counter-queryparam" + + headers = {"Some-Header": headerSomeHeader} + params = {"sortby": sortby} + resp = requests.get(url, + headers=headers, + params=params) + + resp.raise_for_status() + return resp.json() + + +def get_loginqueryflag(published): + """ + GET "login-queryflag" + + """ + url = "http://localhost:8000/login-queryflag" + + params = {"published": published} + resp = requests.get(url, + params=params) + + resp.raise_for_status() + return resp.json() + + +def post_loginparamsauthorswithreqBody(authors, data): + """ + POST "login-params-authors-with-reqBody" + + """ + url = "http://localhost:8000/login-params-authors-with-reqBody" + + params = {"authors": authors} + resp = requests.post(url, + params=params, + json=data) + + resp.raise_for_status() + return resp.json() + + +def post_loginwithpathvarandheader_by_id_by_Name_by_hungrig(id, Name, hungrig, data): + """ + POST "login-with-path-var-and-header/{id}/{Name}/{hungrig}" + Args: + id + Name + hungrig + """ + url = "http://localhost:8000/login-with-path-var-and-header/{id}/{Name}/{hungrig}".format( + id=parse.quote(id), + Name=parse.quote(Name), + hungrig=parse.quote(hungrig)) + + resp = requests.post(url, + json=data) + + resp.raise_for_status() + return resp.json() +``` + +If you would like to compile and run this example yourself, you can do that like so: + +``` +$ stack build --flag servant-py:example +$ stack exec servant-py-exe +$ cat examples/api.py +``` diff --git a/Setup.hs b/Setup.hs new file mode 100644 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/examples/Main.hs b/examples/Main.hs new file mode 100644 --- /dev/null +++ b/examples/Main.hs @@ -0,0 +1,77 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE DeriveDataTypeable #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE TypeOperators #-} + +module Main where + +import Data.Aeson +import qualified Data.ByteString.Char8 as B +import Data.Data +import qualified Data.Text as T +import GHC.Generics +import Servant +import Servant.Foreign +import System.FilePath + +import Servant.PY +import Servant.PY.Python + +-- * A simple Counter data type +newtype Counter = Counter { value :: Int } + deriving (Generic, Show, Num, Data, Typeable) +instance ToJSON Counter +-- instance HasForeignType Python T.Text Counter where +-- typeFor _ _ _ = "{\"value\": int}" + +data LoginForm = LoginForm + { username :: !T.Text + , password :: !T.Text + , otherMissing :: Maybe T.Text + } deriving (Eq, Show, Generic, Typeable, Data) +instance ToJSON LoginForm + +-- You could provide a direct instance of your types to represent them as strings +-- instance HasForeignType Python T.Text LoginForm where +-- typeFor _ _ _ = "{\"username\": str, \"password\": str, \"otherMissing\": Maybe Text}" + + +-- Alternately, if you make your instance derive Typeable and Data, and +-- you enable the pragma DeriveDataTypeable, +-- then you can use recordToDict to automatically derive your records +instance HasForeignType Python T.Text LoginForm where + typeFor _ _ _ = recordToDict (undefined :: LoginForm) LoginForm + +instance HasForeignType Python T.Text Counter where + typeFor _ _ _ = recordToDict (undefined :: Counter) Counter + +-- * Our API type +type TestApi = "counter-req-header" :> Post '[JSON] Counter + :<|> "counter-queryparam" + :> QueryParam "sortby" T.Text + :> Header "Some-Header" T.Text :> Get '[JSON] Counter + :<|> "login-queryflag" :> QueryFlag "published" :> Get '[JSON] LoginForm + :<|> "login-params-authors-with-reqBody" + :> QueryParams "authors" T.Text + :> ReqBody '[JSON] LoginForm :> Post '[JSON] LoginForm + :<|> "login-with-path-var-and-header" + :> Capture "id" Int + :> Capture "Name" T.Text + :> Capture "hungrig" Bool + :> ReqBody '[JSON] LoginForm + :> Post '[JSON] (Headers '[Header "test-head" B.ByteString] LoginForm) + +testApi :: Proxy TestApi +testApi = Proxy + +-- where our static files reside +result :: FilePath +result = "examples" + +main :: IO () +main = do + writePythonForAPI testApi requests (result "api.py") + writeTypedPythonForAPI testApi requests (result "api_typed.py") diff --git a/servant-py.cabal b/servant-py.cabal new file mode 100644 --- /dev/null +++ b/servant-py.cabal @@ -0,0 +1,92 @@ +name: servant-py +version: 0.1.0.1 +synopsis: Automatically derive python functions to query servant webservices. +description: + Automatically derive python functions to query servant webservices. + . + Supports deriving functions using Python's requests library. +homepage: https://github.com/pellagic-puffbomb/servant-py#readme +license: BSD3 +license-file: LICENSE +author: Erik Aker +maintainer: eraker@gmail.com +copyright: 2017 Erik Aker +category: Web +build-type: Simple +extra-source-files: README.md +cabal-version: >=1.10 + +flag example + description: Build the example too + manual: True + default: False + +library + hs-source-dirs: src + exposed-modules: Servant.PY + , Servant.PY.Requests + , Servant.PY.Internal + , Servant.PY.Python + build-depends: base >= 4.7 && < 5 + , aeson + , bytestring + , charset + , lens + , servant-foreign + , servant + , text + default-language: Haskell2010 + default-extensions: OverloadedStrings + ghc-options: -Wall + +executable servant-py-exe + ghc-options: -Wall -O2 -threaded -rtsopts -with-rtsopts=-N + hs-source-dirs: examples + + if flag(example) + buildable: True + else + buildable: False + hs-source-dirs: examples + main-is: Main.hs + build-depends: base + , blaze-html + , bytestring + , servant-py + , stm + , aeson + , wai + , servant + , servant-foreign + , servant-server + , servant-blaze + , text + , filepath + , warp + default-language: Haskell2010 + +test-suite servant-py-test + type: exitcode-stdio-1.0 + hs-source-dirs: test + ghc-options: -Wall + main-is: Spec.hs + other-modules: + Servant.PYSpec + Servant.PY.InternalSpec + build-depends: base + , servant-py + , aeson + , base-compat + , bytestring + , hspec + , hspec-expectations + , lens + , QuickCheck + , servant + , servant-foreign + , text + default-language: Haskell2010 + +source-repository head + type: git + location: https://github.com/pellagic-puffbomb/servant-py diff --git a/src/Servant/PY.hs b/src/Servant/PY.hs new file mode 100644 --- /dev/null +++ b/src/Servant/PY.hs @@ -0,0 +1,90 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE TypeOperators #-} + +module Servant.PY ( -- * Generating python code from an API type + PythonGenerator + , python + , pythonTyped + , writePythonForAPI + , pyForAPI + , pyTypedForAPI + , writeTypedPythonForAPI + , -- * Options common to all generators + CommonGeneratorOptions(..) + , defCommonGeneratorOptions + + -- Requests library + , requests + + , -- * Function renamers + concatCase + , snakeCase + , camelCase + + , -- * Misc. + listFromAPI + , NoTypes + , GenerateList(..) + , FunctionName(..) + ) where + +import Data.Proxy +import Data.Text (Text) +import qualified Data.Text as T +import Servant.Foreign + +import Servant.PY.Internal +import Servant.PY.Python +import Servant.PY.Requests + +-- | Generate the data necessary to generate Python code +-- for all the endpoints of an API, as ':<|>'-separated values +-- of type 'PyRequest'. +python :: HasForeign NoTypes NoContent api => Proxy api -> Foreign NoContent api +python p = foreignFor (Proxy :: Proxy NoTypes) (Proxy :: Proxy NoContent) p defReq + +-- | Generate the data necessary to generate Python code +-- for all the endpoints of an API, but try to get as much type-information +-- into Python docstrings, in order to aid discoverability of client functions. +pythonTyped :: HasForeign Python Text api => Proxy api -> Foreign Text api +pythonTyped p = foreignFor (Proxy :: Proxy Python) (Proxy :: Proxy Text) p defReq + +-- | Directly generate all the Python functions for your API +-- from a 'Proxy' for your API type. You can then write it to +-- a file or integrate it in a page, for example. +pyForAPI :: (HasForeign NoTypes NoContent api, GenerateList NoContent (Foreign NoContent api)) + => Proxy api -- ^ proxy for your API type + -> PythonGenerator -- ^ python code generator to use (requests is the only one for now) + -> Text -- ^ a text that you can embed in your pages or write to a file +pyForAPI p gen = gen (UnTypedPythonRequest <$> listFromAPI (Proxy :: Proxy NoTypes) (Proxy :: Proxy NoContent) p) + +-- | Directly generate all the Python functions for your API +-- from a 'Proxy' for your API type using the given generator +-- and write the resulting code to a file at the given path. +writePythonForAPI :: (HasForeign NoTypes NoContent api, GenerateList NoContent (Foreign NoContent api)) + => Proxy api -- ^ proxy for your API type + -> PythonGenerator -- ^ python code generator to use (requests is the only one for now) + -> FilePath -- ^ path to the file you want to write the resulting javascript code into + -> IO () +writePythonForAPI p gen fp = writeFile fp (T.unpack $ pyForAPI p gen) + + +-- | Directly generate all the Python functions for your API +-- from a 'Proxy' for your API type. You can then write it to +-- a file or integrate it in a page, for example. +pyTypedForAPI :: (HasForeign Python T.Text api, GenerateList T.Text (Foreign T.Text api)) + => Proxy api -- ^ proxy for your API type + -> PythonGenerator -- ^ python code generator to use (requests is the only one for now) + -> Text -- ^ a text that you can embed in your pages or write to a file +pyTypedForAPI p gen = gen (TypedPythonRequest <$> listFromAPI (Proxy :: Proxy Python) (Proxy :: Proxy T.Text) p) + + +writeTypedPythonForAPI :: (HasForeign Python T.Text api, GenerateList T.Text (Foreign T.Text api)) + => Proxy api -- ^ proxy for your API type + -> PythonGenerator -- ^ python code generator to use (requests is the only one for now) + -> FilePath -- ^ path to the file you want to write the resulting javascript code into + -> IO () +writeTypedPythonForAPI p gen fp = writeFile fp (T.unpack $ pyTypedForAPI p gen) diff --git a/src/Servant/PY/Internal.hs b/src/Servant/PY/Internal.hs new file mode 100644 --- /dev/null +++ b/src/Servant/PY/Internal.hs @@ -0,0 +1,370 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE KindSignatures #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ScopedTypeVariables #-} + +module Servant.PY.Internal + ( PythonGenerator + , ReturnStyle(..) + , PythonRequest(..) + , PyRequestArgs(..) + , CommonGeneratorOptions(..) + , defCommonGeneratorOptions + , defaultPyIndent + , indent + , Indent + , indenter + , makePyUrl + , makePyUrl' + , segmentToStr + , capturesToFormatArgs + , toValidFunctionName + , functionName + , toPyHeader + , retrieveHeaders + , getHeaderDict + , retrieveHeaderText + , toPyDict + , toPyParams + , getParams + , paramNames + , captures + , getMethod + , hasBody + , withFormattedCaptures + , buildDocString + , buildHeaderDict + , functionArguments + , formatBuilder + , remainingReqCall + -- re-exports + , (:<|>)(..) + , (:>) + , defReq + , reqHeaders + , HasForeign(..) + , HasForeignType(..) + , GenerateList(..) + , NoTypes + , ArgType(..) + , HeaderArg(..) + , QueryArg(..) + , Req(..) + , Segment(..) + , SegmentType(..) + , Url(..) + , Path + , Arg(..) + , FunctionName(..) + , PathSegment(..) + , concatCase + , snakeCase + , camelCase + , ReqBody + , JSON + , FormUrlEncoded + , Post + , Get + , Raw + , Header + ) where + +import Control.Lens hiding (List) +import qualified Data.CharSet as Set +import qualified Data.CharSet.Unicode.Category as Set +import Data.Data +import Data.Maybe (isJust) +import Data.Monoid +import Data.Text (Text) +import qualified Data.Text as T +import Data.Text.Encoding (decodeUtf8) +import GHC.TypeLits +import Servant.Foreign + + +-- A 'PythonGenerator' just takes the data found in the API type +-- for each endpoint and generates Python code as Text. +-- There are `NoContent` requests and Text requests with typing information. +type PythonGenerator = [PythonRequest] -> Text +data PythonRequest = TypedPythonRequest (Req Text) + | UnTypedPythonRequest (Req NoContent) + deriving (Eq, Show) + +-- We'd like to encode at the type-level that indentation +-- is some multiplication of whitespace (sorry: never tabs!) +type Indent = (" " :: Symbol) + +indent :: Proxy Indent +indent = Proxy + +-- The defaultPyIndent function is 4 spaces. +-- You can create a different indentation width by passing a different Int to indenter. +defaultPyIndent :: Proxy Indent -> Text +defaultPyIndent = indenter 4 + +-- But you can create alternatives by specializing the `indenter` function +-- to other Ints. Then, to get your indentation, pass `indent` to the created function +indenter :: Int -> Proxy Indent -> Text +indenter width space = mconcat $ width `replicate` (T.pack . symbolVal) space +{-# INLINE indenter #-} + + +-- Created python Functions can have different return styles +data ReturnStyle = DangerMode -- Throw caution to the wind and return JSON + | RawResponse -- Return response object itself + + +data PyRequestArgs = PyRequestArgs { + hasHeaders :: Bool + , hasParams :: Bool + , hasData :: Bool + } deriving (Show) + +-- | This structure is used by specific implementations to let you +-- customize the output +data CommonGeneratorOptions = CommonGeneratorOptions + { + functionNameBuilder :: FunctionName -> Text + -- ^ function generating function names + , requestBody :: Text + -- ^ name used when a user want to send the request body + -- (to let you redefine it) + , urlPrefix :: Text + -- ^ a prefix we should add to the Url in the codegen + , indentation :: Proxy Indent -> Text + -- ^ indentation to use for Python codeblocks. Create this function by passing an Int to indenter. + , returnMode :: ReturnStyle + -- ^ whether the generated functions return the raw response or content + } + +-- | Default options. +-- +-- @ +-- > defCommonGeneratorOptions = CommonGeneratorOptions +-- > { functionNameBuilder = snakeCase +-- > , requestBody = "body" +-- > , urlPrefix = "" +-- > , indentation = " " -- 4 spaces +-- > , returnMode = DangerMode +-- > } +-- @ +defCommonGeneratorOptions :: CommonGeneratorOptions +defCommonGeneratorOptions = CommonGeneratorOptions + { + functionNameBuilder = snakeCase + , requestBody = "data" + , urlPrefix = "http://localhost:8000" + , indentation = defaultPyIndent + , returnMode = DangerMode + } + +-- | Attempts to reduce the function name provided to that allowed by @'Foreign'@. +-- +-- For valid Python function identifiers see the following: +-- https://docs.python.org/3.2/reference/lexical_analysis.html#identifiers +-- valid start chars: Lu, Ll, Lt, Lm, Lo, Nl, the underscore +-- valid continuation chars: valid start chars <> Mn, Mc, Nd, Pc +toValidFunctionName :: Text -> Text +toValidFunctionName t = + case T.uncons t of + Just (x,xs) -> + setFirstChar x `T.cons` T.filter remainder xs + Nothing -> "_" + where + setFirstChar c = if Set.member c firstLetterOK then c else '_' + remainder c = Set.member c remainderOK + firstLetterOK = filterBmpChars $ mconcat + [ Set.fromDistinctAscList "_" + , Set.lowercaseLetter + , Set.uppercaseLetter + , Set.titlecaseLetter + , Set.modifierLetter + , Set.otherLetter + , Set.letterNumber + ] + remainderOK = firstLetterOK + <> filterBmpChars (mconcat + [ Set.nonSpacingMark + , Set.spacingCombiningMark + , Set.decimalNumber + , Set.connectorPunctuation + ]) + +functionName :: CommonGeneratorOptions -> PythonRequest -> Text +functionName opts (TypedPythonRequest req) = toValidFunctionName (functionNameBuilder opts $ req ^. reqFuncName) +functionName opts (UnTypedPythonRequest req) = toValidFunctionName (functionNameBuilder opts $ req ^. reqFuncName) + +-- Identifiers can only contain codepoints in the Basic Multilingual Plane +-- that is, codepoints that can be encoded in UTF-16 without a surrogate pair (UCS-2) +-- that is, codepoints that can fit in 16-bits, up to 0xffff (65535) +filterBmpChars :: Set.CharSet -> Set.CharSet +filterBmpChars = Set.filter (< '\65536') + +-- This function creates a dict where the keys are string representations of variable +-- names. This is due to the way arguments are passed into the function, and these +-- arguments named params. In other words, [("key", "key")] becomes: {"key": key} +toPyDict :: Text -> [Text] -> Text +toPyDict offset dict + | null dict = "{}" + | otherwise = "{" <> T.intercalate (",\n" <> offset) insides <> "}" + where insides = combiner <$> dict + combiner a = "\"" <> a <> "\": " <> a + +-- Query params are passed into the function that makes the request, so we make +-- a python dict out of them. +getParams :: Text -> PythonRequest -> Text +getParams offset (TypedPythonRequest req) = toPyParams offset $ req ^.. reqUrl.queryStr.traverse +getParams offset (UnTypedPythonRequest req) = toPyParams offset $ req ^.. reqUrl.queryStr.traverse + +toPyParams :: Text -> [QueryArg f] -> Text +toPyParams _ [] = "" +toPyParams offset qargs = toPyDict offset paramList + where paramList = fmap (\qarg -> qarg ^. queryArgName.argName._PathSegment) qargs + +-- We also need to make sure we can retrieve just the param names for function args. +paramNames :: PythonRequest -> [Text] +paramNames (TypedPythonRequest req) = map (view $ queryArgName . argPath) $ req ^.. reqUrl.queryStr.traverse +paramNames (UnTypedPythonRequest req) = map (view $ queryArgName . argPath) $ req ^.. reqUrl.queryStr.traverse + +-- Request headers are also passed into the function that makes the request, so we make +-- a python dict out of them. +toPyHeader :: HeaderArg f -> Text +toPyHeader (HeaderArg n) + = toValidFunctionName ("header" <> n ^. argName . _PathSegment) +toPyHeader (ReplaceHeaderArg n p) + | pn `T.isPrefixOf` p = pv <> " + \"" <> rp <> "\"" + | pn `T.isSuffixOf` p = "\"" <> rp <> "\" + " <> pv + | pn `T.isInfixOf` p = "\"" <> T.replace pn ("\" + " <> pv <> " + \"") p + <> "\"" + | otherwise = p + where + pv = toValidFunctionName ("header" <> n ^. argName . _PathSegment) + pn = "{" <> n ^. argName . _PathSegment <> "}" + rp = T.replace pn "" p + +buildHeaderDict :: [HeaderArg f] -> Text +buildHeaderDict [] = "" +buildHeaderDict hs = "{" <> headers <> "}" + where headers = T.intercalate ", " $ map headerStr hs + headerStr header = "\"" <> header ^. headerArg . argPath <> "\": " + <> toPyHeader header + +getHeaderDict :: PythonRequest -> Text +getHeaderDict (TypedPythonRequest req) = buildHeaderDict $ req ^. reqHeaders +getHeaderDict (UnTypedPythonRequest req) = buildHeaderDict $ req ^. reqHeaders + +retrieveHeaders :: PythonRequest -> [Text] +retrieveHeaders (TypedPythonRequest req) = retrieveHeaderText <$> req ^. reqHeaders +retrieveHeaders (UnTypedPythonRequest req) = retrieveHeaderText <$> req ^. reqHeaders + +retrieveHeaderText :: forall f. HeaderArg f -> Text +retrieveHeaderText header = header ^. headerArg . argPath + + +functionArguments :: forall f. Req f -> Text +functionArguments req = + mconcat [ T.intercalate ", " args] + where + args = captures' req ++ qparam ++ body ++ headers + + qparam = map ((<>) "param_" . view (queryArgName . argPath)) queryParams + + body = if isJust $ req ^. reqBody + then ["data"] + else [] + queryParams = req ^.. reqUrl . queryStr . traverse + headers = map ((<>) "header_" + . view (headerArg . argPath) + ) $ req ^. reqHeaders + +captures :: PythonRequest -> [Text] +captures (TypedPythonRequest req) = captures' req +captures (UnTypedPythonRequest req) = captures' req + + +captures' :: forall f. Req f -> [Text] +captures' req = map (view argPath . captureArg) + . filter isCapture + $ req ^. reqUrl.path + +makePyUrl :: CommonGeneratorOptions -> PythonRequest -> Text -> Text +makePyUrl opts (TypedPythonRequest req) offset = makePyUrl' opts req offset +makePyUrl opts (UnTypedPythonRequest req) offset = makePyUrl' opts req offset + +makePyUrl' :: forall f. CommonGeneratorOptions -> Req f -> Text -> Text +makePyUrl' opts req offset = if url' == "\"" then "\"/\"" else url' + where url' = "\"" <> urlPrefix opts <> "/" + <> getSegments pathParts + <> withFormattedCaptures offset pathParts + pathParts = req ^.. reqUrl.path.traverse + +getSegments :: forall f. [Segment f] -> Text +getSegments segments = if null segments + then "" + else T.intercalate "/" (map segmentToStr segments) <> "\"" + +withFormattedCaptures :: Text -> [Segment f] -> Text +withFormattedCaptures offset segments = formattedCaptures (capturesToFormatArgs segments) + where formattedCaptures [] = "" + formattedCaptures xs = ".format(\n" <> offset + <> T.intercalate (",\n" <> offset) (map formatBuilder xs) + <> ")" + +formatBuilder :: Text -> Text +formatBuilder val = val <> "=parse.quote(str("<> val <> "))" + +segmentToStr :: Segment f -> Text +segmentToStr (Segment (Static s)) = s ^. _PathSegment +segmentToStr (Segment (Cap s)) = "{" <> s ^. argName . _PathSegment <> "}" + +capturesToFormatArgs :: [Segment f] -> [Text] +capturesToFormatArgs segments = map getSegment $ filter isCapture segments + where getSegment (Segment (Cap a)) = getCapture a + getSegment _ = "" + getCapture s = s ^. argName . _PathSegment + +captureArgsWithTypes :: [Segment Text] -> [Text] +captureArgsWithTypes segments = map getSegmentArgType (filter isCapture segments) + where getSegmentArgType (Segment (Cap a)) = pathPart a <> " (" <> a ^. argType <> ")" + getSegmentArgType _ = "" + pathPart s = s ^. argName . _PathSegment + + +buildDocString :: PythonRequest -> CommonGeneratorOptions -> Text -> Text +buildDocString (TypedPythonRequest req) opts returnVal = buildDocString' req opts args returnVal + where args = captureArgsWithTypes $ req ^.. reqUrl.path.traverse +buildDocString (UnTypedPythonRequest req) opts returnVal = buildDocString' req opts args returnVal + where args = capturesToFormatArgs $ req ^.. reqUrl.path.traverse + +buildDocString' :: forall f. Req f -> CommonGeneratorOptions -> [Text] -> Text -> Text +buildDocString' req opts args returnVal = T.toUpper method <> " \"" <> url <> "\n" + <> includeArgs <> "\n\n" + <> indent' <> "Returns: " <> "\n" + <> indent' <> indent' <> returnVal + where method = decodeUtf8 $ req ^. reqMethod + url = getSegments $ req ^.. reqUrl.path.traverse + includeArgs = if null args then "" else argDocs + argDocs = indent' <> "Args: " <> "\n" + <> indent' <> indent' <> T.intercalate ("\n" <> indent' <> indent') args + indent' = indentation opts indent + +getMethod :: PythonRequest -> Text +getMethod (TypedPythonRequest req) = decodeUtf8 $ req ^. reqMethod +getMethod (UnTypedPythonRequest req) = decodeUtf8 $ req ^. reqMethod + +hasBody :: PythonRequest -> Bool +hasBody (TypedPythonRequest req) = isJust (req ^. reqBody) +hasBody (UnTypedPythonRequest req) = isJust (req ^. reqBody) + +remainingReqCall :: PyRequestArgs -> Int -> Text +remainingReqCall reqArgs width + | null argsAsList = ")" + | length argsAsList == 1 = ",\n" <> offset <> head argsAsList <> ")\n" + | otherwise = ",\n" <> offset <> T.intercalate (",\n" <> offset) argsAsList <> ")\n" + where argsAsList = requestArgsToList reqArgs + offset = mconcat $ replicate width " " + +requestArgsToList :: PyRequestArgs -> [Text] +requestArgsToList reqArgs = map snd . filter fst $ zip bools strings + where bools = [hasHeaders reqArgs, hasParams reqArgs, hasData reqArgs] + strings = ["headers=headers", "params=params", "json=data"] diff --git a/src/Servant/PY/Python.hs b/src/Servant/PY/Python.hs new file mode 100644 --- /dev/null +++ b/src/Servant/PY/Python.hs @@ -0,0 +1,79 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE KindSignatures #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeOperators #-} + +module Servant.PY.Python where + +import qualified Data.ByteString.Char8 as B +import qualified Data.ByteString.Lazy as LB +import Data.Data +import Data.Monoid ((<>)) +import Data.Text (Text) +import qualified Data.Text as T +import GHC.TypeLits +import Servant.Foreign + +data Python + +getFieldsForInstance :: forall a. Data a => a -> [Text] +getFieldsForInstance = map T.pack . mconcat . map constrFields . getConstr + where getConstr = dataTypeConstrs . dataTypeOf + +getFieldTypesForType :: forall a. Typeable a => a -> [Text] +getFieldTypesForType = init . T.splitOn " -> " . T.pack . show . typeOf + +recordToDict :: forall a b. (Data a, Typeable b) => a -> b -> T.Text +recordToDict fieldsInstance fieldsType = "{" <> T.intercalate ", " (map printPair fieldPairs) <> "}" + where printPair (c, d) = "\"" <> c <> "\": " <> d + fieldPairs = zip fieldNames fieldTypes + fieldNames = getFieldsForInstance fieldsInstance + fieldTypes = getFieldTypesForType fieldsType + +instance HasForeignType Python Text NoContent where + typeFor _ _ _ = "None" + +instance HasForeignType Python Text Int where + typeFor _ _ _ = "int" + +instance HasForeignType Python Text Bool where + typeFor _ _ _ = "bool" + +instance HasForeignType Python Text String where + typeFor _ _ _ = "str" + +instance HasForeignType Python Text Text where + typeFor _ _ _ = "str" + +instance HasForeignType Python Text LB.ByteString where + typeFor _ _ _ = "str" + +instance HasForeignType Python Text B.ByteString where + typeFor _ _ _ = "str" + +instance HasForeignType Python Text JSON where + typeFor _ _ _ = "dict" + +instance HasForeignType Python T.Text (a :: Symbol) where + typeFor _ _ _ = "str" + +instance HasForeignType Python Text a => HasForeignType Python Text (Header a) where + typeFor lang ftype _ = "dict" <> typeFor lang ftype (Proxy :: Proxy a) + +instance HasForeignType Python Text a => HasForeignType Python Text (Headers '[Header a b] c) where + typeFor lang ftype _ = "dict" <> typeFor lang ftype (Proxy :: Proxy a) + +instance HasForeignType Python Text a => HasForeignType Python Text [Header a b] where + typeFor lang ftype _ = "[dict of " <> typeFor lang ftype (Proxy :: Proxy a) <> "]" + +instance HasForeignType Python Text a => HasForeignType Python Text (Headers a) where + typeFor lang ftype _ = "dict" <> typeFor lang ftype (Proxy :: Proxy a) + +instance HasForeignType Python Text a => HasForeignType Python Text (Maybe a) where + typeFor lang ftype _ = "Optional" <> typeFor lang ftype (Proxy :: Proxy a) + +instance HasForeignType Python Text a => HasForeignType Python Text [a] where + typeFor lang ftype _ = "list of " <> typeFor lang ftype (Proxy :: Proxy a) diff --git a/src/Servant/PY/Requests.hs b/src/Servant/PY/Requests.hs new file mode 100644 --- /dev/null +++ b/src/Servant/PY/Requests.hs @@ -0,0 +1,75 @@ +module Servant.PY.Requests where + +import Data.Monoid +import Data.Proxy +import Data.Text (Text) +import qualified Data.Text as T + +import Servant.PY.Internal + +-- | Generate python functions that use the requests library. +-- Uses 'defCommonGeneratorOptions' for the generator options. +requests :: PythonGenerator +requests reqs = defPyImports <> mconcat (map requestsWithDef reqs) + +-- | Generate python functions that use the requests library. +-- Lets you specify your own 'CommonGeneratorOptions'. +requestsWith :: CommonGeneratorOptions -> [PythonRequest] -> Text +requestsWith opts reqs = mconcat (map (generatePyRequestWith opts) reqs) + +-- | python codegen using requests with default options +requestsWithDef :: PythonRequest -> Text +requestsWithDef = generatePyRequestWith defCommonGeneratorOptions + +defPyImports :: Text +defPyImports = + T.unlines + [ "from urllib import parse" + , "" -- Separate stdlib from 3rd-party imports + , "import requests" + ] + +-- | python codegen with requests +generatePyRequestWith :: CommonGeneratorOptions -> PythonRequest -> Text +generatePyRequestWith opts req = "\n" <> + "def " <> functionName opts req <> "(" <> argsStr <> "):\n" + <> indent' <> docStringMarker + <> indent' <> buildDocString req opts returnVal <> "\n" + <> indent' <> docStringMarker + <> indent' <> "url = " <> makePyUrl opts req (indent' <> indent') <> "\n\n" + <> headerDef + <> paramDef + <> requestBuilder <> "(url" <> remaining (T.length requestBuilder + 1) <> "\n" + <> functionReturn (returnMode opts) (indentation opts) + <> "\n\n" + where argsStr = T.intercalate ", " args + args = captures req + ++ qparams + ++ body + ++ map (toValidFunctionName + . (<>) "header" + ) hs + hs = retrieveHeaders req + qparams = paramNames req + method = T.toLower $ getMethod req + remaining = remainingReqCall $ PyRequestArgs (not . null $ hs) (not . null $ qparams) (hasBody req) + paramDef + | null qparams = "" + | otherwise = indent' <> "params = " <> getParams (indent' <> indent') req <> "\n" + headerDef + | null hs = "" + | otherwise = indent' <> "headers = " <> getHeaderDict req <> "\n" + requestBuilder = indent' <> "resp = requests." <> method + body = [requestBody opts | hasBody req] + indent' = indentation opts indent + docStringMarker = "\"\"\"\n" + returnVal = case returnMode opts of + DangerMode -> "JSON response from the endpoint" + RawResponse -> "response (requests.Response) from issuing the request" + +functionReturn :: ReturnStyle -> (Proxy Indent -> T.Text) -> T.Text +functionReturn DangerMode pyindenter = indent' <> "resp.raise_for_status()\n" + <> indent' <> "return resp.json()" + where indent' = pyindenter indent +functionReturn RawResponse pyindenter = indent' <> "return resp" + where indent' = pyindenter indent diff --git a/test/Servant/PY/InternalSpec.hs b/test/Servant/PY/InternalSpec.hs new file mode 100644 --- /dev/null +++ b/test/Servant/PY/InternalSpec.hs @@ -0,0 +1,143 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE TypeOperators #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Servant.PY.InternalSpec where + +import Control.Lens +import Data.Aeson +import qualified Data.ByteString.Char8 as B +import Data.Monoid +import Data.Proxy +import Data.Text (Text) +import qualified Data.Text as T +import GHC.Generics +import Prelude () +import Prelude.Compat +import Test.Hspec hiding + (shouldContain, + shouldNotContain) +import Test.QuickCheck (Arbitrary (..), + choose, listOf, + property) + +import Servant.API.ContentTypes +import Servant.API.Internal.Test.ComprehensiveAPI +import Servant.Foreign + +import Servant.PY.Internal + + +data SomeJson = SomeJson + { uvalue :: !T.Text + , pvalue :: !T.Text + , otherMissing :: Maybe T.Text + } deriving (Eq, Show, Generic) +instance ToJSON SomeJson + +-- * Our API type +type TestApi = "counter-req-header" :> Post '[JSON] SomeJson + :<|> "counter-queryparam" + :> QueryParam "sortby" T.Text + :> Header "Some-Header" T.Text :> Get '[JSON] SomeJson + :<|> "login-queryflag" :> QueryFlag "published" :> Get '[JSON] SomeJson + :<|> "login-params-authors-with-reqBody" + :> QueryParams "authors" T.Text + :> ReqBody '[JSON] SomeJson :> Post '[JSON] SomeJson + +testApi :: Proxy TestApi +testApi = Proxy + + +type CaptureApi = "login-with-path-var-and-header" + :> Capture "id" Int + :> Capture "Name" T.Text + :> Capture "hungrig" Bool + :> ReqBody '[JSON] SomeJson + :> Post '[JSON] (Headers '[Header "test-head" B.ByteString] SomeJson) +captureApi :: Proxy CaptureApi +captureApi = Proxy + +customOptions :: CommonGeneratorOptions +customOptions = defCommonGeneratorOptions + { urlPrefix = "urlForRequesting:9000" + , returnMode = RawResponse + } + +spec :: Spec +spec = describe "Servant.PY.Internal" internalSpec + +shouldContain :: Text -> Text -> Expectation +a `shouldContain` b = shouldSatisfy a (T.isInfixOf b) + +shouldNotContain :: Text -> Text -> Expectation +a `shouldNotContain` b = shouldNotSatisfy a (T.isInfixOf b) + +newtype ASCII = ASCII {getASCII :: T.Text} deriving (Show) + +instance Arbitrary ASCII where + -- Our arbitrary instance is generating only ASCII, since the language-ecmascript's lexer + -- is currently (October 2016) still a bit naïve + arbitrary = fmap (ASCII . T.pack) $ listOf $ choose (minBound, '\127') + shrink xs = (ASCII . T.pack) <$> shrink (T.unpack $ getASCII xs) + + +internalSpec :: Spec +internalSpec = describe "Internal" $ do + describe "pure text functions" $ do + it "should only indent using whitespace" $ + property $ \n -> indenter n indent == mconcat (replicate n (T.pack " ")) + + it "should generate a valid python identifier when supplied with hyphens, unicode whitespace, non-bmp unicode" $ + toValidFunctionName "a_--a\66352b\6158c\65075" `shouldBe` "a_abc\65075" + + it "should produce PyDicts where the key is a quoted version of the variable name" $ do + let dict = toPyDict " " ["forty", "one", "people"] + dict `shouldBe` "{\"forty\": forty,\n \"one\": one,\n \"people\": people}" + + describe "functions that operate on Req objects" $ do + let captureList = listFromAPI (Proxy :: Proxy NoTypes) (Proxy :: Proxy NoContent) captureApi + it "should correctly find captures" $ do + let captured = captures . head $ captureList + captured `shouldBe` ["id", "Name", "hungrig"] + + let reqList = listFromAPI (Proxy :: Proxy NoTypes) (Proxy :: Proxy NoContent) testApi + it "should not incorrectly find captures" $ do + let captured = captures . head $ reqList + captured `shouldBe` [] + + let req = head captureList + let pathParts = req ^.. reqUrl.path.traverse + it "should correctly find captures as a list" $ + capturesToFormatArgs pathParts `shouldBe` ["id", "Name", "hungrig"] + + it "should correctly format captures" $ + withFormattedCaptures " " pathParts `shouldBe` ".format(\n id=parse.quote(str(id)),\n" + <> " Name=parse.quote(str(Name)),\n " + <> "hungrig=parse.quote(str(hungrig)))" + + it "should build a formatted val with parse.quote and str" $ + property $ \s -> T.isInfixOf "=parse.quote(str(" $ formatBuilder $ T.pack s + it "should build a formatted val that ends with parens" $ + property $ \s -> T.isSuffixOf (T.pack s <> "))") $ formatBuilder $ T.pack s + + it "should build urls properly with / separator" $ do + let pyUrl = makePyUrl customOptions req " " + pyUrl `shouldBe` "\"urlForRequesting:9000/login-with-path-var-and-header/{id}/{Name}/{hungrig}\"" + <> withFormattedCaptures " " pathParts + + it "should do segment-to-str as a plain string for Static" $ + segmentToStr (head pathParts) == "login-with-path-var-and-header" + it "should do segment-to-str in formatting braces for a capture" $ + segmentToStr (last pathParts) == "{hungrig}" + it "should build a doctstring that looks like a regular Python docstring" $ do + let docstring = buildDocString req customOptions + docstring `shouldContain` "POST" + docstring `shouldContain` makePyUrl' pathParts + docstring `shouldContain` "Args:" + docstring `shouldContain` "Returns:" diff --git a/test/Servant/PYSpec.hs b/test/Servant/PYSpec.hs new file mode 100644 --- /dev/null +++ b/test/Servant/PYSpec.hs @@ -0,0 +1,61 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE QuasiQuotes #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE TypeOperators #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Servant.PYSpec where + +import Data.Either (isRight) +import Data.Monoid () +import Data.Monoid.Compat ((<>)) +import Data.Proxy +import Data.Text (Text) +import qualified Data.Text as T +import GHC.TypeLits +import Prelude () +import Prelude.Compat +import Test.Hspec hiding + (shouldContain, + shouldNotContain) +import Test.QuickCheck (Arbitrary (..), + choose, listOf, + property) + +import Servant.API.ContentTypes +import Servant.API.Internal.Test.ComprehensiveAPI + +import Servant.PY.Internal + + +customOptions :: CommonGeneratorOptions +customOptions = defCommonGeneratorOptions + { urlPrefix = "urlForRequesting:9000" + , returnMode = DangerMode + } + +spec :: Spec +spec = describe "Servant.PY.Internal" internalSpec + +shouldContain :: Text -> Text -> Expectation +a `shouldContain` b = shouldSatisfy a (T.isInfixOf b) + +shouldNotContain :: Text -> Text -> Expectation +a `shouldNotContain` b = shouldNotSatisfy a (T.isInfixOf b) + +newtype ASCII = ASCII {getASCII :: T.Text} deriving (Show) + +instance Arbitrary ASCII where + -- Our arbitrary instance is generating only ASCII, since the language-ecmascript's lexer + -- is currently (October 2016) still a bit naïve + arbitrary = fmap (ASCII . T.pack) $ listOf $ choose (minBound, '\127') + shrink xs = (ASCII . T.pack) <$> shrink (T.unpack $ getASCII xs) + + +internalSpec :: Spec +internalSpec = describe "Internal" $ do + it "should only indent using whitespace" $ + property $ \n -> indenter n indent == mconcat (replicate n (T.pack " ")) diff --git a/test/Spec.hs b/test/Spec.hs new file mode 100644 --- /dev/null +++ b/test/Spec.hs @@ -0,0 +1,1 @@ +{-# OPTIONS_GHC -F -pgmF hspec-discover #-}