diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Revision history for arxiv-client-cli + +## 0.1.0.0 -- YYYY-mm-dd + +* First version. Released on an unsuspecting world. diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +Copyright (c) 2025, Eiko + + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,73 @@ +## Arxiv Cli + +`arxiv-client-cli` is a command-line interface (CLI) tool for searching and downloading research papers from arXiv.org, it warps the library `arxiv-client`. + +### Help Text + +``` +Arxiv Client CLI + +Usage: arxiv-client-cli [-q|--query TEXT] [-H|--helpQuery] [-p|--downloadPdf] + [-s|--downloadSrc] [-u|--ungzip] [-c|--concise] + [-v|--detail] [-a|--abstract] [-j|--json] + [-d|--downloadDir STRING] [--after yyyy-mm-dd] + [--before yyyy-mm-dd] [-m|--maxResult INT] + [-n|--page INT] + +Available options: + -h,--help Show this help text + -q,--query TEXT Search query string + -H,--helpQuery Display help for constructing query strings + -p,--downloadPdf Download PDF files for each entry + -s,--downloadSrc Download source .tar.gz files for each entry + -u,--ungzip Automatically ungzip source files (by running "tar + -xzf") + -c,--concise Concise output (only titles) + -v,--detail Detailed output + -a,--abstract Include abstracts in the output + -j,--json Output results in JSON format, turn off + human-readable output + -d,--downloadDir STRING Directory path to save downloaded files + (default: "./") + --after yyyy-mm-dd Only include papers published after this date + (YYYY-MM-DD) + --before yyyy-mm-dd Only include papers published before this date + (YYYY-MM-DD) + -m,--maxResult INT Maximum results per page (default: 25) + -n,--page INT Page number to retrieve (starting from 0) +``` +### Constructing Query String +``` +--query parameter is mandatory. +Query string is constructed by using + +* + := title | author | abstract | category | anywhere + (, ) := + is -- exact match + has -- substring match + any [, ...] -- any of the strings (ors) + all [, ...] -- all of the strings (ands) + +* You can write a single string "value" without specifying field and match, + which is equivalent to: anywhere has "value" + +* Logical operators: + && -- and (note that these operators are right associative, use brackets for clarity) + || -- or (note that these operators are right associative, use brackets for clarity) + ands [, ...] -- and multiple terms + ors [, ...] -- or multiple terms + not -- negate term + +Examples: + --query 'title has "quantum" && author is "Albert Einstein"' + --query 'author any ["john doe", "jane smith"]' + --query 'ands [title is "coleman", author has "doe"]' + --query 'ors [category is "math.NT", category is "math.AG"]' +``` + +### Installation + +```bash +cabal install arxiv-client-cli +``` diff --git a/app/Main.hs b/app/Main.hs new file mode 100644 --- /dev/null +++ b/app/Main.hs @@ -0,0 +1,192 @@ +{-# LANGUAGE OverloadedStrings, MultilineStrings, OverloadedRecordDot, DataKinds #-} +module Main where + +import Arxiv.Client +import Arxiv.Download +import Arxiv.Entry +import Arxiv.Filters +import Arxiv.Query +import Arxiv.Query.Algebraic +import Arxiv.Query.Parser +import Control.Monad (when, forM_, unless, guard) +import Data.Aeson +import qualified Data.Aeson.KeyMap as K +import Data.IORef +import Data.Maybe +import Data.Time (UTCTime(..), Day) +import Options.Generic +import System.Directory +import System.FilePath (()) +import System.Process (system) +import System.Exit (ExitCode(..)) +import Text.Megaparsec (errorBundlePretty) +import qualified Data.ByteString.Lazy.Char8 as B +import qualified Data.Text as T +import qualified Data.Text.IO as T + +queryHelp :: [Char] +queryHelp = + """ + --query parameter is mandatory. + Query string is constructed by using + + * + := title | author | abstract | category | anywhere + (, ) := + is -- exact match + has -- substring match + any [, ...] -- any of the strings (ors) + all [, ...] -- all of the strings (ands) + + * You can write a single string "value" without specifying field and match, + which is equivalent to: anywhere has "value" + + * Logical operators: + && -- and (note that these operators are right associative, use brackets for clarity) + || -- or (note that these operators are right associative, use brackets for clarity) + ands [, ...] -- and multiple terms + ors [, ...] -- or multiple terms + not -- negate term + + Examples: + --query 'title has "quantum" && author is "Albert Einstein"' + --query 'author any ["john doe", "jane smith"]' + --query 'ands [title is "coleman", author has "doe"]' + --query 'ors [category is "math.NT", category is "math.AG"]' + """ + +data ArxivCliArgs w = ArxivCliArgs + { query :: w ::: Maybe T.Text <#> "q" "Search query string" + , helpQuery :: w ::: Bool <#> "H" "Display help for constructing query strings" + , downloadPdf :: w ::: Bool <#> "p" "Download PDF files for each entry" + , downloadSrc :: w ::: Bool <#> "s" "Download source .tar.gz files for each entry" + , ungzip :: w ::: Bool <#> "u" "Automatically ungzip source files (by running \"tar -xzf\")" + , concise :: w ::: Bool <#> "c" "Concise output (only titles)" + , detail :: w ::: Bool <#> "v" "Detailed output" + , abstract :: w ::: Bool <#> "a" "Include abstracts in the output" + , json :: w ::: Bool <#> "j" "Output results in JSON format, turn off human-readable output" + , downloadDir :: w ::: Maybe FilePath <#> "d" "Directory path to save downloaded files" "./" + , after :: w ::: Maybe Day "Only include papers published after this date (YYYY-MM-DD)" + , before :: w ::: Maybe Day "Only include papers published before this date (YYYY-MM-DD)" + , maxResult :: w ::: Maybe Int <#> "m" "Maximum results per page" "25" + , page :: w ::: Maybe Int <#> "n" "Page number to retrieve (starting from 0)" + } deriving (Generic) + +instance ParseRecord (ArxivCliArgs Wrapped) + +tshow :: Show a => a -> T.Text +tshow = T.pack . show + +main :: IO () +main = do + args <- unwrap <$> getRecord "Arxiv Client CLI" + let mQuery = do + guard (not args.helpQuery) + args.query + maybe (putStrLn queryHelp) (arxivCli args) mQuery + +arxivCli :: ArxivCliArgs Unwrapped -> Text -> IO () +arxivCli args query = do + let nResultPage = fromMaybe 25 args.maxResult + directory = fromMaybe "./" args.downloadDir + entryFilter = foldr (.) id + [ maybe id (publishedAfter . (`UTCTime` 0)) args.after + , maybe id (publishedBefore . (`UTCTime` 0)) args.before + ] + + arxivQuery <- case parseQueryTerm query of + Left err -> error ("Failed to parse query: " ++ errorBundlePretty err) + Right qt -> pure + $ emptyQuery + & applyQueryTerm qt + & setPaging (maybe 0 (* nResultPage) args.page) nResultPage + & setSort SubmittedDate Desc + + entries0 <- queryArxivIO arxivQuery + let entries = entryFilter entries0 + jsonObj <- newIORef (object ["results" .= entries]) + + unless args.json $ do + putStrLn $ "Total received : " <> show (length entries0) <> ", after filtering : " <> show (length entries) + forM_ entries $ \e -> do + T.putStrLn $ "• " <> title e + when (args.detail || not args.concise) $ do + T.putStrLn $ " Authors : " <> T.intercalate ", " e.authors + T.putStrLn $ " Published : " <> tshow e.published + T.putStrLn $ " Categories : " <> T.intercalate ", " e.categories + T.putStrLn $ " Link : " <> e.pdfUrl + when (args.detail || args.abstract) $ do + T.putStrLn " Abstract : " + T.putStrLn $ " " <> T.replace "\n" "\n " (summary e) + unless args.concise $ putStrLn "" + + if null entries + then putStrLn "No result found." + else putStrLn $ "Total papers found: " <> show (length entries) + + when args.downloadPdf $ do + createDirectoryIfMissing True directory + if args.json + then addDownloadedPdfPaths entries directory jsonObj + else putStrLn "Downloading recent papers (PDF)" + mapM_ (\en -> downloadPdfToFile en (directory defaultFileName ".pdf" en)) entries + + when args.downloadSrc $ do + createDirectoryIfMissing True directory + if args.json + then addDownloadedSourcePaths entries directory jsonObj + else putStrLn "Downloading recent papers (Source .tar.gz)" + mapM_ (\en -> downloadSourceToFile en (directory defaultFileName ".tar.gz" en)) entries + when args.ungzip $ do + unless args.json $ putStrLn "Unzipping downloaded source files" + forM_ entries $ \en -> do + let filePath' = "\"" <> directory defaultFileName ".tar.gz" en <> "\"" + destDir = directory defaultFileName "_src" en + destDir' = "\"" <> destDir <> "\"" + createDirectoryIfMissing True destDir + let cmd = "tar -xzf " ++ filePath' ++ " -C " ++ destDir' ++ " && rm " ++ filePath' + unless args.json $ putStrLn $ "Running: " ++ cmd + code <- system cmd + case code of + ExitSuccess -> addUngzippedSourcePath en destDir jsonObj + ExitFailure c -> unless args.json $ putStrLn $ "Command failed with exit code: " ++ show c + return () + + when args.json $ do + finalObj <- readIORef jsonObj + B.putStrLn $ encode finalObj + where + addDownloadedPdfPaths :: [ArxivEntry] -> FilePath -> IORef Value -> IO () + addDownloadedPdfPaths entries dir jsonRef = do + let paths = entries `zip` map (\en -> dir defaultFileName ".pdf" en) entries + downloads = toJSON $ map (\(en, path) -> object + [ "arxiv_id" .= arxivId en + , "pdf_path" .= path + ]) paths + modifyIORef jsonRef $ \obj -> + case obj of + Object o -> Object $ K.insert "downloadPdf" downloads o + _ -> obj + addDownloadedSourcePaths :: [ArxivEntry] -> FilePath -> IORef Value -> IO () + addDownloadedSourcePaths entries dir jsonRef = do + let paths = entries `zip` map (\en -> dir defaultFileName ".tar.gz" en) entries + downloads = toJSON $ map (\(en, path) -> object + [ "arxiv_id" .= arxivId en + , "source_path" .= path + ]) paths + modifyIORef jsonRef $ \obj -> + case obj of + Object o -> Object $ K.insert "downloadSrc" downloads o + _ -> obj + addUngzippedSourcePath :: ArxivEntry -> FilePath -> IORef Value -> IO () + addUngzippedSourcePath en destDir jsonRef = do + let entryObj = object + [ "arxiv_id" .= arxivId en + , "ungzipped_path" .= destDir + ] + appendUngzipped (Array a1) (Array a2) = Array (a1 <> a2) + appendUngzipped _ _ = error "Expected arrays when appending ungzippedSrc entries" + modifyIORef jsonRef $ \obj -> + case obj of + Object o -> Object $ K.insertWith appendUngzipped "ungzippedSrc" (toJSON [entryObj]) o + _ -> obj diff --git a/arxiv-client-cli.cabal b/arxiv-client-cli.cabal new file mode 100644 --- /dev/null +++ b/arxiv-client-cli.cabal @@ -0,0 +1,43 @@ +cabal-version: 3.0 +name: arxiv-client-cli +version: 0.1.0.0 +synopsis: Command line tool to search and download papers from arXiv.org +description: This is a command line interface (cli) application wrapping the library arxiv-client, which is a Haskell tool to search and download papers from arXiv.org using a simple query DSL. +license: BSD-3-Clause +license-file: LICENSE +author: Eiko +maintainer: eikochanowo@outlook.com +-- copyright: +category: Network, CLI +build-type: Simple +extra-doc-files: CHANGELOG.md + , README.md +tested-with: GHC == 9.12.2 +-- extra-source-files: +source-repository head + type: git + location: https://github.com/Eiko-Tokura/arxiv-client-cli.git + +common warnings + ghc-options: -Wall + +executable arxiv-client-cli + import: warnings + main-is: Main.hs + hs-source-dirs: app + -- ghc-options: -threaded -rtsopts -with-rtsopts=-N + -- ^ we will enable them after we do parallel downloading + build-depends: + base >=4 && <5 + , aeson >=2 && <2.3 + , arxiv-client >=0.1 && < 0.2 + , bytestring >=0.10 && <0.13 + , text >=2.0 && <2.2 + , directory >=1.3 && <1.4 + , time >=1.11 && <1.15 + , optparse-generic >=1.5 && <1.6 + , megaparsec >=9.5 && <9.8 + , filepath >=1.5.2 && <1.5.5 + , process >=1.6 && <1.7 + hs-source-dirs: app + default-language: GHC2021