diff --git a/arxiv.cabal b/arxiv.cabal --- a/arxiv.cabal +++ b/arxiv.cabal @@ -1,15 +1,16 @@ Name: arxiv -Version: 0.0.1 -Cabal-Version: >= 1.8 -Copyright: Copyright (c) Tobias Schoofs, 2014 +Version: 0.0.2 +Cabal-Version: >= 1.10 +Copyright: Copyright (c) Tobias Schoofs, 2014 -- 2020 License: LGPL license-file: license/lgpl-3.0.txt Author: Tobias Schoofs Maintainer: Tobias Schoofs -Homepage: http://github.com/toschoo/libs +Homepage: http://github.com/toschoo/Haskell-Libs Category: Network, API, NetworkAPI, Arxiv Build-Type: Simple Synopsis: A client for the Arxiv API +extra-source-files: changelog.md Description: The arXiv is a repository of electronic preprints, known as e-prints, @@ -42,6 +43,7 @@ tagsoup >= 0.13 hs-source-dirs: src, . + default-language: Haskell98 Exposed-Modules: Network.Api.Arxiv other-modules: diff --git a/changelog.md b/changelog.md new file mode 100644 --- /dev/null +++ b/changelog.md @@ -0,0 +1,5 @@ +__0.0.2__ + Documentation and test review + +__0.0.1__ + Initial release diff --git a/src/Network/Api/Arxiv.hs b/src/Network/Api/Arxiv.hs --- a/src/Network/Api/Arxiv.hs +++ b/src/Network/Api/Arxiv.hs @@ -120,19 +120,21 @@ > module Main > where - > + > > import qualified Network.Api.Arxiv as Ax > import Network.Api.Arxiv (Expression(..), - > Field(..), (/*/), (/+/)) - > import Network (withSocketsDo) - > import Network.HTTP.Conduit + > Field(..), (/*/), (/+/)) + > import Network.Socket (withSocketsDo) + > import Network.HTTP.Simple as HT + > import Network.HTTP.Conduit (parseRequest) > import Network.HTTP.Types.Status > import Data.List (intercalate) > import qualified Data.ByteString as B hiding (unpack) > import qualified Data.ByteString.Char8 as B (unpack) - > import Data.Conduit (($$+-), (=$), ($$)) + > import Data.Conduit ((.|)) > import qualified Data.Conduit as C > import qualified Data.Conduit.List as CL + > import Data.Function ((&)) > import Text.HTML.TagSoup > import Control.Monad.Trans (liftIO) > import Control.Monad.Trans.Resource (MonadResource) @@ -156,61 +158,62 @@ > type Soup = Tag String > > execQuery :: Ax.Query -> IO () - > execQuery q = withManager $ \m -> searchAxv m q $$ outSnk + > execQuery q = C.runConduitRes (searchAxv q .| outSnk) > - > ----------------------------------------------------------------- + > ---------------------------------------------------------------------- > -- Execute query and start a source - > ----------------------------------------------------------------- - > searchAxv :: MonadResource m => - > Manager -> Ax.Query -> C.Source m String - > searchAxv m q = do - > u <- liftIO (parseUrl $ mkQuery q) - > rsp <- http u m - > case responseStatus rsp of - > (Status 200 _) -> getSoup rsp >>= results m q - > st -> error $ "Error:" ++ show st + > ---------------------------------------------------------------------- + > searchAxv :: MonadResource m => Ax.Query -> C.ConduitT () String m () + > searchAxv q = + > let s = Ax.mkQuery q + > in do rsp <- HT.httpBS =<< liftIO (parseRequest s) + > case getResponseStatus rsp of + > (Status 200 _) -> getSoup (getResponseBody rsp) + > >>= results q + > st -> error $ "Error:" ++ show st > - > ----------------------------------------------------------------- + > ---------------------------------------------------------------------- > -- Consume page by page - > ----------------------------------------------------------------- - > getSoup :: MonadResource m => - > Response (C.ResumableSource m B.ByteString) -> m [Soup] - > getSoup rsp = concat <$> (responseBody rsp $$+- - > toSoup =$ CL.consume) + > ---------------------------------------------------------------------- + > getSoup :: MonadResource m => + > B.ByteString -> C.ConduitT () String m [Soup] + > getSoup b = concat <$> (C.yield b .| toSoup .| CL.consume) > - > ----------------------------------------------------------------- + > ---------------------------------------------------------------------- > -- Receive a ByteString and yield Soup - > ----------------------------------------------------------------- - > toSoup :: MonadResource m => C.Conduit B.ByteString m [Soup] + > ---------------------------------------------------------------------- + > toSoup :: MonadResource m => C.ConduitT B.ByteString [Soup] m () > toSoup = C.awaitForever (C.yield . parseTags . B.unpack) > - > ------------------------------------------------------------------ + > ---------------------------------------------------------------------- > -- Yield all entries and fetch next page - > ------------------------------------------------------------------ - > results :: MonadResource m => - > Manager -> Ax.Query -> [Soup] -> C.Source m String - > results m q sp = + > ---------------------------------------------------------------------- + > results :: MonadResource m => + > Ax.Query -> [Soup] -> C.ConduitT () String m () + > results q sp = > if Ax.exhausted sp - > then C.yield ("EOT: " ++ show (Ax.totalResults sp) ++ - > " results") + > then C.yield ("EOT: " ++ show (Ax.totalResults sp) ++ " results") > else Ax.forEachEntryM_ sp (C.yield . mkResult) - > >> searchAxv m (Ax.nextPage q) + > >> searchAxv (Ax.nextPage q) > - > ------------------------------------------------------------------ - > -- Get data and format somehow - > ------------------------------------------------------------------ + > ---------------------------------------------------------------------- + > -- Get data and format + > ---------------------------------------------------------------------- > mkResult :: [Soup] -> String > mkResult sp = let aus = Ax.getAuthorNames sp > y = Ax.getYear sp - > tmp = Ax.getTitle sp + > tmp = Ax.getTitle sp & clean ['\n', '\r', '\t'] > ti = if null tmp then "No title" else tmp > in intercalate ", " aus ++ " (" ++ y ++ "): " ++ ti + > where clean _ [] = [] + > clean d (c:cs) | c `elem` d = clean d cs + > | otherwise = c:clean d cs > - > ------------------------------------------------------------------ + > ---------------------------------------------------------------------- > -- Sink results - > ------------------------------------------------------------------ - > outSnk :: MonadResource m => C.Sink String m () - > outSnk = C.awaitForever (liftIO . putStrLn) + > ---------------------------------------------------------------------- + > outSnk :: MonadResource m => C.ConduitT String C.Void m () + > outSnk = C.awaitForever (liftIO . putStrLn) -} ------------------------------------------------------------------------ @@ -223,7 +226,7 @@ -- | The Arxiv API URL \"export.arxiv.org/api\" ------------------------------------------------------------------------ apiUrl :: String - apiUrl = "http://export.arxiv.org/api/query?" + apiUrl = "https://export.arxiv.org/api/query?" ------------------------------------------------------------------------ -- | The query string (\"search_query=\" or \"id_list=\") @@ -576,9 +579,6 @@ -- * The entry (if any) -- -- * The rest of the tag soup following the first entry. - -- - -- With getEntry, we can build a loop through all entries - -- in the result (which is actually implemented in 'forEachEntry'). ------------------------------------------------------------------------ getEntry :: [Tag String] -> ([Tag String],[Tag String]) getEntry = element "entry" @@ -676,8 +676,12 @@ pureError = drop 1 . dropWhile (/= '#') ------------------------------------------------------------------------ - -- | Gets the contents of the \"update\" field in this entry, i.e. + -- | Gets the contents of the \"updated\" field in this entry, i.e. -- the date when the article was last updated. + -- Be aware that there is another \"updated\" field + -- right below the root node of the result. + -- Make sure your are operating on an entry, + -- not on the root node! ------------------------------------------------------------------------ getUpdated :: [Tag String] -> String getUpdated = getString "updated"