diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 --- a/CHANGELOG.md +++ /dev/null @@ -1,18 +0,0 @@ -# Changelog - -## sloane 1.8 - -Use optparse-applicative instead of cmdargs for command line parsing. -Deprecate '-u', '-?', '-V', '--limit', and '--keys'. - -## sloane 1.7.1 - April 30, 2014 - -* Clean up the README / man page. - -## sloane 1.7 - April 20, 2014 - -* Enable filtering against a local cache of sequences. - -## sloane 1.6 - March 18, 2014 - -* Properly handle multiple search terms. diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ --- -title: SLOANE(1) Sloane User Manual | Version 1.8.2 -date: July 3, 2014 +title: SLOANE(1) Sloane User Manual | Version 1.9.1 +date: 6 Aug 2014 --- # NAME @@ -10,42 +10,64 @@ # SYNOPSIS -sloane [-a | --all | -k *keys* | --url] [-n *entries*] *search-terms* ... -sloane (--update | --version | --help) -sloane [--invert] +sloane [lookup | grep] + [-a | --all | -k *keys* | --url] [-n *entries*] *terms* ... +sloane filter [--invert] +sloane update +sloane version # DESCRIPTION -The sloane command searches Sloane's On-Line Encyclopedia of Integer -Sequences (OEIS). The search terms are typically the leading term of a -sequence. For example, +The `sloane lookup` command searches Sloane's On-Line Encyclopedia of +Integer Sequences (OEIS). The search terms are typically the leading +term of a sequence. For example, + sloane lookup 1,1,2,5,15,52,203,877,4140 + +returns entry A000110 (Bell numbers), and four more entries. If no +command is given sloane will fall back to the lookup command, so the +above query can more simply be given as + sloane 1,1,2,5,15,52,203,877,4140 -returns entry A000110 (Bell numbers), and four more entries. One can -also search by sequence id (A-number), or even search for arbitrary -words. See the **EXAMPLES** section. +One can also search by sequence id (A-number), or even search for +arbitrary words. See the **EXAMPLES** section. -If no search terms are specified the standard input is read -line-by-line. In this mode the search is done locally against a -downloaded list of known sequences. If the sequence is in OEIS, then -it is returned to the standard output; if not, it is ignored. This way -sloane can quickly filter out the sequences from the input that are in -OEIS. Assuming that *FILE* contains one sequence per line, +Alternatively, using the `sloane grep` command, the search can be done +locally against a downloaded list of known sequences. This mode works by +"grepping" for the query in the sequence field. - sloane +-- License : BSD-3 +-- +module Sloane.Config (Config (..), defaultConfig) where + +import System.Console.Terminal.Size (width, size) +import System.FilePath (()) +import System.Directory + +type URL = String + +data Config = Config + { name :: String + , home :: FilePath + , sloaneDir :: FilePath + , sloaneDB :: FilePath + , oeisHost :: URL + , oeisURL :: URL + , sURL :: URL + , nURL :: URL + , termWidth :: Int + } + +defaultConfig :: IO Config +defaultConfig = do + w <- maybe maxBound width `fmap` size + h <- getHomeDirectory + let dsloane = h ".sloane" + return Config + { name = "sloane 1.9.1" + , home = h + , sloaneDir = dsloane + , sloaneDB = dsloane "sloane.db" + , oeisHost = oeisorg + , oeisURL = oeisorg ++ "search?fmt=text" + , sURL = oeisorg ++ "stripped.gz" + , nURL = oeisorg ++ "names.gz" + , termWidth = w + } + where + oeisorg = "https://oeis.org/" diff --git a/Sloane/DB.hs b/Sloane/DB.hs new file mode 100644 --- /dev/null +++ b/Sloane/DB.hs @@ -0,0 +1,150 @@ +-- | +-- Copyright : Anders Claesson 2014 +-- Maintainer : Anders Claesson +-- License : BSD-3 +-- + +module Sloane.DB + ( DB + , Reply + , initDB + , readDB + , writeDB + , putDB + , null + , insert + , lookup + , grep + , take + , aNumbers + , parseOEISEntries + ) where + +import Prelude hiding (lookup, null, take) +import qualified Prelude as P +import Data.List (intersect) +import Data.ByteString (ByteString) +import qualified Data.ByteString.Lazy as BL +import Data.Map (Map, (!)) +import qualified Data.Map as M +import Data.Serialize +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Text.IO as IO +import Data.Text.Encoding (decodeUtf8, encodeUtf8) +import Control.Monad (forM_, unless) +import qualified Codec.Compression.GZip as GZip +import Network.Curl.Download.Lazy (openLazyURI) +import System.Console.ANSI +import Sloane.Config +import System.Directory + +type ANumber = Text +type Key = Char +type Entry = Text +type Reply = Map Key Entry + +type DB = Map ANumber Reply +type DBRaw = Map ByteString (Map Char ByteString) + +encodeDB :: DB -> DBRaw +encodeDB = M.mapKeys encodeUtf8 . M.map (M.map encodeUtf8) + +decodeDB :: DBRaw -> DB +decodeDB = M.mapKeys decodeUtf8 . M.map (M.map decodeUtf8) + +compress :: ByteString -> BL.ByteString +compress = GZip.compress . BL.fromStrict + +compressDB :: DB -> BL.ByteString +compressDB = compress . encode . encodeDB + +decompress :: BL.ByteString -> ByteString +decompress = BL.toStrict . GZip.decompress + +decompressDB :: BL.ByteString -> Either String DB +decompressDB = fmap decodeDB . decode . decompress + +initDB :: Config -> IO () +initDB cfg = do + createDirectoryIfMissing False (sloaneDir cfg) + putStrLn $ "Downloading " ++ sURL cfg + dbS <- openLazyURI (sURL cfg) >>= either error (return . mkDB 'S') + putStrLn $ "Downloading " ++ nURL cfg + dbN <- openLazyURI (nURL cfg) >>= either error (return . mkDB 'N') + putStrLn "Building database" + writeDB cfg $ unionDB dbS dbN + putStrLn "Done." + where + unionDB = M.unionWith M.union + mkDB key = mkMap key . decodeUtf8 . decompress + mkMap key = M.fromList . map (aNumberAndReply key) . drop 4 . T.lines + mkReply key = M.singleton key . T.dropWhile (==',') . T.drop 8 + aNumberAndReply key line = (T.take 7 line, mkReply key line) + +readDB :: Config -> IO DB +readDB cfg = doesFileExist (sloaneDB cfg) >>= \updated -> + if updated + then BL.readFile (sloaneDB cfg) >>= either error return . decompressDB + else error $ "No local database found. " ++ + "You need to run \"sloane update\" first." + +writeDB :: Config -> DB -> IO () +writeDB cfg = BL.writeFile (sloaneDB cfg) . compressDB + +null :: DB -> Bool +null = M.null + +insert :: ANumber -> Reply -> DB -> DB +insert = M.insert + +lookup :: ANumber -> DB -> Maybe Reply +lookup = M.lookup + +grep :: Text -> DB -> DB +grep q = M.filter $ \reply -> q `T.isInfixOf` (reply ! 'S') + +take :: Int -> DB -> DB +take n = M.fromList . P.take n . M.toList + +aNumbers :: DB -> [ANumber] +aNumbers = M.keys + +unions :: [DB] -> DB +unions = M.unionsWith . M.unionWith $ \s t -> + (s `T.append` T.pack "\n") `T.append` t + +singleton :: ANumber -> Key -> Entry -> DB +singleton aNum key entry = M.singleton aNum $ M.singleton key entry + +parseOEISEntries :: Text -> DB +parseOEISEntries = unions . map parseLine . trim + where + trim = map (T.drop 1) . reverse . drop 2 . reverse . drop 5 . T.lines + parseLine = parseWords . T.words + parseWords (key:aNum:rest) = singleton aNum (T.head key) (T.unwords rest) + parseWords _ = M.empty + +putDB :: Config -> [Key] -> DB -> IO () +putDB cfg keys db = do + unless (null db) $ putStrLn "" + forM_ (M.toList db) $ \(aNum, reply) -> do + forM_ (keys `intersect` M.keys reply) $ \key -> do + let entry = reply ! key + forM_ (T.lines entry) $ \line -> do + setSGR [ SetColor Foreground Dull Green ] + putStr [key] + setSGR [ SetColor Foreground Dull Yellow ] + putStr " " >> IO.putStr aNum + setSGR [] + putStr " " >> IO.putStrLn (crop key (termWidth cfg - 10) line) + putStrLn "" + +crop :: Key -> Int -> Text -> Text +crop key = + let cropText f maxLen s = if maxLen < T.length s then f maxLen s else s + in if key `elem` ['S'..'X'] + then cropText $ \maxLen -> + T.reverse . T.dropWhile (/= ',') . T.reverse . T.take maxLen + else cropText $ \maxLen s -> + T.take (maxLen-2) s `T.append` T.pack ".." diff --git a/sloane.1 b/sloane.1 --- a/sloane.1 +++ b/sloane.1 @@ -1,68 +1,101 @@ -.TH "SLOANE" "1" "July 3, 2014" "Sloane User Manual" "Version 1.8.2" +.TH "SLOANE" "1" "6 Aug 2014" "Sloane User Manual" "Version 1.9.1" .SH NAME .PP sloane \- a command line interface to Sloane\[aq]s On\-Line Encyclopedia of Integer Sequences .SH SYNOPSIS .PP -sloane [\-a | \-\-all | \-k \f[I]keys\f[] | \-\-url] [\-n -\f[I]entries\f[]] \f[I]search\-terms\f[] ... +sloane [lookup | grep] [\-a | \-\-all | \-k \f[I]keys\f[] | \-\-url] +[\-n \f[I]entries\f[]] \f[I]terms\f[] ... .PD 0 .P .PD -sloane (\-\-update | \-\-version | \-\-help) +sloane filter [\-\-invert] .PD 0 .P .PD -sloane [\-\-invert] +sloane update +.PD 0 +.P +.PD +sloane version .SH DESCRIPTION .PP -The sloane command searches Sloane\[aq]s On\-Line Encyclopedia of -Integer Sequences (OEIS). +The \f[C]sloane\ lookup\f[] command searches Sloane\[aq]s On\-Line +Encyclopedia of Integer Sequences (OEIS). The search terms are typically the leading term of a sequence. For example, .IP .nf \f[C] -sloane\ 1,1,2,5,15,52,203,877,4140 +sloane\ lookup\ 1,1,2,5,15,52,203,877,4140 \f[] .fi .PP returns entry A000110 (Bell numbers), and four more entries. +If no command is given sloane will fall back to the lookup command, so +the above query can more simply be given as +.IP +.nf +\f[C] +sloane\ 1,1,2,5,15,52,203,877,4140 +\f[] +.fi +.PP One can also search by sequence id (A\-number), or even search for arbitrary words. See the \f[B]EXAMPLES\f[] section. .PP -If no search terms are specified the standard input is read -line\-by\-line. -In this mode the search is done locally against a downloaded list of -known sequences. -If the sequence is in OEIS, then it is returned to the standard output; -if not, it is ignored. -This way sloane can quickly filter out the sequences from the input that -are in OEIS. -Assuming that \f[I]FILE\f[] contains one sequence per line, +Alternatively, using the \f[C]sloane\ grep\f[] command, the search can +be done locally against a downloaded list of known sequences. +This mode works by "grepping" for the query in the sequence field. +.PP +To check a large number of sequences one can use the +\f[C]sloane\ filter\f[] command. +It reads the standard input line\-by\-line, if the sequence read is in +the local database, then it is returned to the standard output; if not, +it is ignored. +This way onw can quickly filter out the sequences from the input that +are in the local database. +In other words, assuming that \f[I]FILE\f[] contains one sequence per +line, .IP .nf \f[C] -sloane\ . +Version: 1.9.1 +Synopsis: A command line interface to Sloane's On-Line Encyclopedia + of Integer Sequences +Description: A command line interface to Sloane's On-Line Encyclopedia + of Integer Sequences. For usage see + . Homepage: http://github.com/akc/sloane License: BSD3 License-file: LICENSE @@ -10,9 +12,9 @@ Maintainer: anders.claesson@gmail.com Category: Math Build-type: Custom -Extra-Source-Files: README.md sloane.1 CHANGELOG.md +Extra-Source-Files: README.md sloane.1 -Cabal-version: >=1.6 +Cabal-version: >=1.9.2 source-repository head type: git @@ -20,16 +22,20 @@ Executable sloane Main-is: sloane.hs + Other-Modules: Sloane.Config + Sloane.DB ghc-options: -Wall Build-depends: base >=3 && <5, - optparse-applicative >=0.9, + containers >=0.5, + optparse-applicative >=0.8, + download-curl >= 0.1.4, HTTP >=4000.0.9, + cereal >=0.4, network >=2.4, bytestring >=0.9, - text >=1.1, + text >=0.11, ansi-terminal >=0.6, terminal-size >=0.2, filepath >=1.3, directory >=1.2, - zlib >=0.5, - time >=1.4 + zlib >=0.5 diff --git a/sloane.hs b/sloane.hs --- a/sloane.hs +++ b/sloane.hs @@ -3,202 +3,151 @@ -- Maintainer : Anders Claesson -- License : BSD-3 -- -import qualified Codec.Compression.GZip as GZip -import Control.Monad (unless, when, liftM2) -import qualified Data.ByteString as B -import qualified Data.ByteString.Lazy as BL import Data.Text (Text) import qualified Data.Text as T import qualified Data.Text.IO as IO import Data.Text.Encoding (decodeUtf8) -import Data.Maybe (fromJust) -import Data.Time (diffUTCTime, getCurrentTime) -import Network.HTTP -import Network.URI (parseURI) +import Network.HTTP (urlEncodeVars) +import Network.Curl.Download (openURI) import Options.Applicative -import System.Console.ANSI -import System.Console.Terminal.Size (Window (..), size) -import System.Directory -import System.FilePath (()) -import System.IO (stderr) - -type OEISEntries = [String] -type ANumbers = [String] -type Query = String -type Keys = String - -data Args = Args - Bool -- a: Print all fields? - String -- k: Keys of fields to print - Int -- n: Fetch at most this many entries - Bool -- invert: Return sequences NOT in OEIS - Bool -- update: Update the local sequence cache - Bool -- url: Print URLs of found entries - Bool -- version - [String] -- search terms - -name, oeisHost, oeisURL, cacheDir, cacheFile, cacheURL :: String - -name = "sloane 1.8.2" - -oeisHost = "http://oeis.org/" -oeisURL = oeisHost ++ "search?fmt=text" - -cacheDir = ".sloane" -cacheFile = "stripped.gz" -cacheURL = oeisHost ++ cacheFile - -msgDownloadingCache, msgCacheIsUpToDate, msgNoCache, msgOldCache :: String - -msgDownloadingCache = unlines - [ "Downloading " ++ cacheURL - , "This may take a minute or two ..." - ] -msgCacheIsUpToDate = unlines - [ "The sequence cache is now up-to-date" - ] -msgNoCache = unlines - [ "No sequence cache found. You need to run \"sloane --update\"" - ] -msgOldCache = unlines - [ "The sequence cache is more than 100 days old" - , "You may want to run \"sloane --update\"" - ] - -select :: Keys -> OEISEntries -> OEISEntries -select ks = filter (\line -> null line || head line `elem` ks) - -aNumbers :: OEISEntries -> ANumbers -aNumbers es = [ words ids !! 1 | ids@(_:_) <- select "I" es ] - -urls :: OEISEntries -> String -urls = unlines . map (oeisHost ++ ) . aNumbers - -get :: HStream b => String -> IO b -get uri = simpleHTTP (defaultGETRequest_ uri') >>= getResponseBody - where - uri' = fromJust $ parseURI uri - -searchOEIS :: Int -> Query -> IO OEISEntries -searchOEIS n s = trim `fmap` get uri - where - trim = map (drop 1) . reverse . drop 2 . reverse . drop 5 . lines - uri = oeisURL ++ "&" ++ urlEncodeVars [("n", show n), ("q", s)] +import Sloane.Config +import Sloane.DB hiding (null) +import qualified Sloane.DB as DB -cropStr :: (Int -> String -> String) -> Int -> String -> String -cropStr f maxLen s = if maxLen < length s then f maxLen s else s +type URL = String +type Seq = Text -cropSeq :: Int -> String -> String -cropSeq = cropStr $ \maxLen -> - reverse . dropWhile (/= ',') . reverse . take maxLen +data Visibility = Visible | Internal -cropLine :: Int -> String -> String -cropLine = cropStr $ \maxLen s -> take (maxLen-2) s ++ ".." +data Options + = Cmd Command + | IntOpts SearchOpts -- Internal opts for fallback to lookup -getWidth :: IO Int -getWidth = maybe maxBound width `fmap` size +data Command + = Lookup SearchOpts + | Grep SearchOpts + | Filter FilterOpts + | Update + | Version -put :: String -> IO () -put = IO.putStr . T.pack +data SearchOpts = SearchOpts + { full :: Bool -- Print all fields? + , keys :: String -- Keys of fields to print + , limit :: Int -- Fetch at most this many entries + , url :: Bool -- Print URLs of found entries + , terms :: [String] -- Search terms + } -putErr :: String -> IO () -putErr = IO.hPutStr stderr . T.pack +data FilterOpts = FilterOpts + { invert :: Bool -- Return sequences NOT in DB + } -newline :: IO () -newline = IO.putStrLn T.empty +oeisKeys :: String +oeisKeys = "ISTUVWXNDHFYAOEeptoKC" -- Valid OEIS keys -putEntries :: Int -> OEISEntries -> IO () -putEntries ncols = mapM_ $ \line -> do - case words line of - [] -> return () - [w] -> put w -- Should never be reached - (key:aNum:rest) -> do - setSGR [ SetColor Foreground Dull Green ] - put key - setSGR [ SetColor Foreground Dull Yellow ] - put $ ' ' : aNum - setSGR [] - let crop = if key == "S" then cropSeq else cropLine - put $ ' ' : crop ncols (unwords rest) - newline +oeisUrls :: Config -> DB -> [URL] +oeisUrls cfg = map ((oeisHost cfg ++) . T.unpack) . aNumbers -updateCache :: FilePath -> IO () -updateCache home = do - createDirectoryIfMissing False dir - put msgDownloadingCache - get cacheURL >>= BL.writeFile (dir cacheFile) - put msgCacheIsUpToDate +oeisLookup :: SearchOpts -> Config -> IO DB +oeisLookup opts cfg = + (parseOEISEntries . decodeUtf8 . either error id) <$> + openURI (oeisURL cfg ++ "&" ++ urlEncodeVars [("n", show n), ("q", q)]) where - dir = home cacheDir + n = limit opts + q = unwords $ terms opts -readCache :: FilePath -> IO Text -readCache home = do - updated <- doesFileExist fname - if updated - then do - age <- liftM2 diffUTCTime getCurrentTime (getModificationTime fname) - when (age > 100*day) $ putErr msgOldCache - (dropPreamble . decompress) `fmap` BL.readFile fname - else - error msgNoCache +grepDB :: SearchOpts -> DB -> DB +grepDB opts = DB.take n . DB.grep (T.pack q) where - day = 60*60*24 - fname = home cacheDir cacheFile - decompress = decodeUtf8 . B.concat . BL.toChunks . GZip.decompress - dropPreamble = T.unlines . drop 4 . T.lines + n = limit opts + q = unwords $ terms opts -seqs :: Text -> [Text] -seqs = filter (not . T.null) . map mkSeq . T.lines +filterDB :: FilterOpts -> DB -> IO [Seq] +filterDB opts db = filter match . parseSeqs <$> IO.getContents where + match q = (if invert opts then id else not) (DB.null $ DB.grep q db) + parseSeqs = filter (not . T.null) . map mkSeq . T.lines mkSeq = normalize . dropComment dropComment = T.takeWhile (/= '#') - normalize = T.intercalate (T.pack ",") . T.words . clean . T.map tr - tr c = if c `elem` ";," then ' ' else c + normalize = T.intercalate (T.pack ",") . T.words . clean . T.map tr + tr c = if c `elem` ";," then ' ' else c clean = T.filter (`elem` " 0123456789-") -filterSeqs :: Bool -> FilePath -> IO () -filterSeqs invert home = do - cache <- readCache home - let f q = (if invert then not else id) (q `T.isInfixOf` cache) - IO.getContents >>= mapM_ IO.putStrLn . filter f . seqs - -args :: Parser Args -args = Args - <$> switch (short 'a' <> long "all" <> help "Print all fields") +searchOptionsParser :: Visibility -> Parser SearchOpts +searchOptionsParser visibility = hiddenHelp <*> (SearchOpts + <$> switch + ( short 'a' + <> long "all" + <> help "Print all fields" + <> f ) <*> strOption ( short 'k' <> metavar "KEYS" <> value "SN" - <> help "Keys of fields to print [default: SN]" ) + <> help "Keys of fields to print [default: SN]" + <> f ) <*> option ( short 'n' <> metavar "N" <> value 5 - <> help "Fetch at most this many entries [default: 5]" ) + <> help "Fetch at most this many entries [default: 5]" + <> f ) <*> switch - ( long "invert" - <> help "When used as a filter, return sequences NOT in OEIS" ) - <*> switch (long "update" <> help "Update the local sequence cache") - <*> switch (long "url" <> help "Print URLs of found entries") - <*> switch (hidden <> long "version") - <*> many (argument str (metavar "TERMS...")) + ( long "url" + <> help "Print URLs of found entries" + <> f ) + <*> some (argument str (metavar "TERMS..."))) + where + f = case visibility of {Visible -> idm; Internal -> internal} -sloane :: Args -> IO () -sloane (Args _ _ _ _ _ _ True _ ) = put name >> newline -sloane (Args _ _ _ _ True _ _ _ ) = getHomeDirectory >>= updateCache -sloane (Args _ _ _ inv _ _ _ []) = getHomeDirectory >>= filterSeqs inv -sloane (Args a keys n _ _ url _ ts) = do - ncols <- getWidth - hits <- searchOEIS n (unwords ts) - let pick = if a then id else select keys - unless (null hits) $ do - newline - if url - then put (urls hits) - else putEntries (ncols - 10) (pick hits) - newline +filterOptionsParser :: Parser FilterOpts +filterOptionsParser = FilterOpts + <$> switch (long "invert" <> help "Return sequences NOT in the database") +commandParser :: Parser Command +commandParser = subparser + ( command "lookup" (info (Lookup <$> searchOptionsParser Visible) + ( progDesc "Lookup a sequence, or other search term, in OEIS" )) + <> command "grep" (info (Grep <$> searchOptionsParser Visible) + ( progDesc "Grep for a sequence in the local database" )) + <> command "filter" (info (Filter <$> filterOptionsParser) + ( progDesc ("Read sequences from stdin and " + ++ "return those that are in the local database"))) + <> command "update" (info (pure Update) + ( progDesc "Update the local database" )) + <> command "version" (info (pure Version) + ( progDesc "Show version info" )) + ) + +optionsParser :: Parser Options +optionsParser = + (Cmd <$> commandParser) <|> (IntOpts <$> searchOptionsParser Internal) + +runSearch :: (SearchOpts -> Config -> IO DB) -> SearchOpts -> Config -> IO () +runSearch f opts cfg = f opts cfg >>= + if url opts + then putStr . unlines . oeisUrls cfg + else putDB cfg (if full opts then oeisKeys else keys opts) + +runCmd :: Command -> Config -> IO () +runCmd (Lookup opts) = runSearch oeisLookup opts +runCmd (Grep opts) = runSearch (\o cfg -> grepDB o <$> readDB cfg) opts +runCmd (Filter opts) = \c -> readDB c >>= filterDB opts >>= mapM_ IO.putStrLn +runCmd Update = initDB +runCmd Version = putStrLn . name + +hiddenHelp :: Parser (a -> a) +hiddenHelp = abortOption ShowHelpText $ hidden <> short 'h' <> long "help" + main :: IO () -main = execParser (info (h <*> args) (fullDesc <> header name)) >>= sloane +main = do + conf <- defaultConfig + opts <- customExecParser preferences (info parser description) + case opts of + (Cmd cmd) -> runCmd cmd conf + (IntOpts o) -> runCmd (Lookup o) conf -- Fallback to 'lookup' where - h = abortOption ShowHelpText $ hidden <> long "help" + parser = hiddenHelp <*> optionsParser + preferences = prefs showHelpOnError + description = fullDesc <> footer + "Run 'sloane COMMAND --help' for help on a specific command."