wikicfp-scraper (empty) → 0.1.0.0
raw patch · 10 files changed
+491/−0 lines, 10 filesdep +attoparsecdep +basedep +bytestringsetup-changed
Dependencies added: attoparsec, base, bytestring, filepath, hspec, scalpel, text, time, wikicfp-scraper
Files
- ChangeLog.md +5/−0
- LICENSE +30/−0
- README.md +10/−0
- Setup.hs +2/−0
- src/Web/WikiCFP/Scraper.hs +74/−0
- src/Web/WikiCFP/Scraper/Scalpel.hs +160/−0
- src/Web/WikiCFP/Scraper/Type.hs +36/−0
- test/Spec.hs +1/−0
- test/Web/WikiCFP/ScraperSpec.hs +117/−0
- wikicfp-scraper.cabal +56/−0
+ ChangeLog.md view
@@ -0,0 +1,5 @@+# Revision history for wikicfp-scraper++## 0.1.0.0 -- 2016-05-21++* First version. Released on an unsuspecting world.
+ LICENSE view
@@ -0,0 +1,30 @@+Copyright (c) 2016, Toshio Ito++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.++ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following+ disclaimer in the documentation and/or other materials provided+ with the distribution.++ * Neither the name of Toshio Ito nor the names of other+ contributors may be used to endorse or promote products derived+ from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ README.md view
@@ -0,0 +1,10 @@+# wikicfp-scraper++This module scrapes WikiCFP pages (http://wikicfp.com/) for call-for-papers.++See http://hackage.haskell.org/package/wikicfp-scraper or module document of `Web.WikiCFP.Scraper`.+++## Author++Toshio Ito <debug.ito@gmail.com>
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ src/Web/WikiCFP/Scraper.hs view
@@ -0,0 +1,74 @@+{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-}+-- |+-- Module: Web.WikiCFP.Scraper+-- Description: Scrape WikiCFP web site+-- Maintainer: Toshio Ito <debug.ito@gmail.com>+-- +-- Synopsis:+--+-- > import qualified Network.HTTP as H+-- > import Web.WikiCFP.Scraper (scrapeSearchEvents)+-- > +-- > main :: IO ()+-- > main = do+-- > res <- H.getResponseBody =<< H.simpleHTTP (H.getRequest "http://wikicfp.com/cfp/servlet/tool.search?q=japan&year=t")+-- > print $ scrapeSearchEvents res+--+-- This module scrapes WikiCFP pages (<http://wikicfp.com/>) for+-- call-for-papers. It helps you stay up to date with deadlines of+-- academic paper submissions.+module Web.WikiCFP.Scraper+ ( -- * Scraper routines+ scrapeConfEvents,+ scrapeSearchEvents,+ -- * Types+ ErrorMsg,+ HTML(..),+ When(..),+ Event(..)+ ) where++import qualified Data.ByteString as SB+import qualified Data.ByteString.Lazy as LB+import Data.Text (Text, pack)+import Data.Text.Encoding (decodeUtf8')+import qualified Data.Text.Lazy as LT+import Text.HTML.Scalpel (scrapeStringLike)++import Web.WikiCFP.Scraper.Type (When(..), Event(..))+import Web.WikiCFP.Scraper.Scalpel (ErrorMsg, Scraper', confRoot, searchRoot)+++-- | Types of input HTML data to scrape.+class HTML a where+ decodeToText :: a -> Either ErrorMsg Text++instance HTML Text where+ decodeToText = Right++instance HTML LT.Text where+ decodeToText = Right . LT.toStrict++-- | It just assumes UTF-8 encoding.+instance HTML SB.ByteString where+ decodeToText = either (\e -> Left $ "UTF-8 decoding error: " ++ show e) Right . decodeUtf8'++instance HTML LB.ByteString where+ decodeToText = decodeToText . LB.toStrict++instance HTML String where+ decodeToText = Right . pack++runScraper :: Scraper' (Either ErrorMsg a) -> Text -> Either ErrorMsg a+runScraper s input = maybe (Left "Scraping error") id $ scrapeStringLike input s++-- | Scrape a page of a conference, for example,+-- <http://wikicfp.com/cfp/program?id=2671>+scrapeConfEvents :: HTML input => input -> Either ErrorMsg [Event]+scrapeConfEvents t = runScraper confRoot =<< decodeToText t++-- | Scrape a page of search results, for example,+-- <http://wikicfp.com/cfp/servlet/tool.search?q=cloud&year=t>+scrapeSearchEvents :: HTML input => input -> Either ErrorMsg [Event]+scrapeSearchEvents t = runScraper searchRoot =<< decodeToText t+
+ src/Web/WikiCFP/Scraper/Scalpel.hs view
@@ -0,0 +1,160 @@+-- |+-- Module: Web.WikiCFP.Scraper.Scalpel+-- Description: Scraper implementation with Scalpel+-- Maintainer: Toshio Ito <debug.ito@gmail.com>+--+-- +module Web.WikiCFP.Scraper.Scalpel+ ( ErrorMsg,+ Scraper',+ confRoot,+ searchRoot+ ) where++import Control.Applicative ((<$>), (<*>), (<|>), (<*), (*>), optional, pure)+import Control.Monad (guard, forM_, mzero)+import Data.List (sort)+import Data.Maybe (catMaybes)+import Data.Monoid ((<>))+import Data.Text (Text, pack)+import Data.Time (Day, fromGregorian)+import Data.Attoparsec.Text (Parser, parseOnly, skipSpace, string, endOfInput, decimal, takeText, char)+import Text.HTML.Scalpel+ ( Scraper,+ (@:), (@=), (//), chroot, chroots, text, texts, attr, hasClass+ )++import Web.WikiCFP.Scraper.Type (Event(..), When(..))++type ErrorMsg = String++type Scraper' = Scraper Text++-- | Root scraper for conference Events.+confRoot :: Scraper' (Either ErrorMsg [Event])+confRoot = concatSuccess $ chroots ("div" @: [hasClass "contsec"] // "table") $ eventsTable++-- | Root scraper for searched Events.+searchRoot :: Scraper' (Either ErrorMsg [Event])+searchRoot = confRoot++concatSuccess :: (Functor m, Monad m) => m [Either e [a]] -> m (Either e [a])+concatSuccess = fmap $ fmap concat . sequence++-- | Scrape events from a table. Use with the root at @\<table\>@ tag.+eventsTable :: Scraper' (Either ErrorMsg [Event])+eventsTable = do+ rows <- chroots "tr" (eventRow1 <|> eventRow2 <|> eventRowHeader)+ case rows of+ (EventRowHeader : rest) -> return $ rowsToEvents_noHeader rest+ _ -> mzero+ ++-- | Intermediate result of parsing under events \<table\>.+data EventRow = EventRowHeader+ | EventRow1 Text Text Text -- ^ shortName, URL and longName+ | EventRow2 Text Text Text -- ^ when, where, deadlines+ deriving (Eq,Ord,Show)++-- | Scrape header row. Use with the root at @\<tr\>@ tag.+eventRowHeader :: Scraper' EventRow+eventRowHeader = do+ tds <- texts "td"+ guard $ length tds == 4+ -- We cannot use OverloadedStrings with scalpel (as of 0.3.0.1),+ -- because it requires explicit (:: String) declarations everywhere!+ let expected_labels = map pack ["Event", "When", "Where", "Deadline"]+ forM_ [0..(length expected_labels - 1)] $ \i -> guard $ parsable (spacedText $ expected_labels !! i) $ tds !! i+ return EventRowHeader++-- | Scrape shortName, URL and longName. Use with the root at @\<tr\>@ tag.+eventRow1 :: Scraper' EventRow+eventRow1 = do+ (sname, url) <- chroot ("td" @: ["rowspan" @= "2"]) $ ((,) <$> text "a" <*> attr "href" "a")+ lname <- text ("td" @: ["colspan" @= "3"])+ return $ EventRow1 sname url lname++-- | Scrape when, where, deadlines in Texts. Use the the root at @\<tr\>@ tag.+eventRow2 :: Scraper' EventRow+eventRow2 = do+ tds <- texts "td"+ guard $ length tds == 3+ return $ EventRow2 (tds !! 0) (tds !! 1) (tds !! 2)++rowsToEvents_noHeader :: [EventRow] -> Either ErrorMsg [Event]+rowsToEvents_noHeader [] = return []+rowsToEvents_noHeader ((EventRow1 sn url ln) : (EventRow2 when wher dl) : rest) =+ (:) <$> createEvent sn url ln when wher dl <*> rowsToEvents_noHeader rest+rowsToEvents_noHeader rows = Left ("Error while parsing rows: " ++ show rows)++-- | TODO: make it configurable.+urlBase :: Text+urlBase = pack "http://wikicfp.com"++createEvent :: Text -> Text -> Text -> Text -> Text -> Text -> Either ErrorMsg Event+createEvent sname url lname when wher dl = do+ when' <- parseWhen when+ wher' <- parseWhere wher+ dl' <- parseDeadlines dl+ return Event { eventShortName = sname,+ eventURL = urlBase <> url,+ eventLongName = lname,+ eventWhen = when',+ eventWhere = wher',+ eventDeadlines = dl'+ }++-- * Parsers++parsable :: Parser a -> Text -> Bool+parsable p t = either (const False) (const True) $ parseOnly p t++spacedText :: Text -> Parser Text+spacedText expected = skipSpace *> string expected <* skipSpace <* endOfInput++spacedText' :: String -> Parser Text+spacedText' = spacedText . pack++string' :: String -> Parser Text+string' = string . pack++parseWhen :: Text -> Either ErrorMsg (Maybe When)+parseWhen = parseOnly (parserWhen <* endOfInput) where+ parserWhen = (spacedText' "N/A" *> pure Nothing)+ <|> (Just <$> parserJustWhen)+ parserJustWhen = When+ <$> (parserDay <* skipSpace <* (char '-') <* skipSpace)+ <*> (parserDay <* skipSpace)++parseWhere :: Text -> Either ErrorMsg (Maybe Text)+parseWhere = parseOnly (parserWhere <* endOfInput) where+ parserWhere = (spacedText' "N/A" *> pure Nothing) <|> (Just <$> takeText)++parseDeadlines :: Text -> Either ErrorMsg [Day]+parseDeadlines input = sort <$> parseOnly (parserDeadlines <* endOfInput) input where+ parserDeadlines = do+ primary <- parserDay <* skipSpace+ msecondary <- optional $ (char '(' *> skipSpace *> parserDay <* skipSpace <* char ')')+ return $ maybe [primary] (: [primary]) msecondary++parserDay :: Parser Day+parserDay = impl where+ impl = do+ m <- parserMonth <* skipSpace+ d <- decimal <* (optional $ char ',') <* skipSpace+ y <- decimal+ return $ fromGregorian y m d+ parserMonth = (string' "Jan" *> pure 1)+ <|> (string' "Feb" *> pure 2)+ <|> (string' "Mar" *> pure 3)+ <|> (string' "Apr" *> pure 4)+ <|> (string' "May" *> pure 5)+ <|> (string' "Jun" *> pure 6)+ <|> (string' "Jul" *> pure 7)+ <|> (string' "Aug" *> pure 8)+ <|> (string' "Sep" *> pure 9)+ <|> (string' "Oct" *> pure 10)+ <|> (string' "Nov" *> pure 11)+ <|> (string' "Dec" *> pure 12)+ +
+ src/Web/WikiCFP/Scraper/Type.hs view
@@ -0,0 +1,36 @@+-- |+-- Module: Web.WikiCFP.Scraper.Type+-- Description: data types for WikiCFP scraper+-- Maintainer: Toshio Ito <debug.ito@gmail.com>+--+-- +module Web.WikiCFP.Scraper.Type+ ( When(..),+ Event(..),+ ) where++import Data.Text (Text)+import Data.Time (Day)++-- | Period of dates (inclusive).+data When = When { whenFrom :: !Day,+ whenTo :: !Day+ } deriving (Eq,Ord,Show)++-- | A conference event posted to WikiCFP site. It corresponds to a+-- row in the table you see conference pages etc, for example,+-- <http://wikicfp.com/cfp/program?id=1172>+data Event = Event { eventShortName :: !Text,+ + eventURL :: !Text,+ -- ^ URL to the WikiCFP page of this event.+ + eventLongName :: !Text, + eventWhen :: !(Maybe When),+ eventWhere :: !(Maybe Text),+ + eventDeadlines :: ![Day]+ -- ^ deadlines are in an ascending order, i.e.,+ -- the earliest deadline is the head.+ + } deriving (Eq,Ord,Show)
+ test/Spec.hs view
@@ -0,0 +1,1 @@+{-# OPTIONS_GHC -F -pgmF hspec-discover #-}
+ test/Web/WikiCFP/ScraperSpec.hs view
@@ -0,0 +1,117 @@+module Web.WikiCFP.ScraperSpec (main, spec) where++import Data.ByteString (ByteString)+import qualified Data.ByteString as BS+import Data.Time (Day, fromGregorian)+import System.FilePath (FilePath, joinPath)+import Test.Hspec++import Web.WikiCFP.Scraper+ ( scrapeConfEvents, scrapeSearchEvents,+ Event(..), When(..)+ )++main :: IO ()+main = hspec spec++spec :: Spec+spec = do+ describe "scrapeConfEvents" $ do+ forFile "conf_sigmod20160505.html" $ \raw_html -> do+ let ret = scrapeConfEvents raw_html+ fmap (take 5) ret `shouldBe`+ Right [ Event { eventShortName = "SIGMOD 2017",+ eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=53445©ownerid=70798",+ eventLongName = "2017 ACM SIGMOD Conference",+ eventWhen = Just $ newWhen (2017, 5, 14) (2017, 5, 19),+ eventWhere = Just "Raleigh",+ eventDeadlines = fmap newDay [(2016, 7, 15), (2016, 7, 22)]+ },+ Event { eventShortName = "SIGMOD 2016",+ eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=44695©ownerid=76071",+ eventLongName = "ACM International Conference on Management of Data ",+ eventWhen = Just $ newWhen (2016, 6, 26) (2016, 7, 1),+ eventWhere = Just "San Francisco, USA",+ eventDeadlines = fmap newDay [(2015, 7, 9), (2015, 7, 16)]+ },+ Event { eventShortName = "SIGMOD 2015",+ eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=37140©ownerid=43021",+ eventLongName = "International Conference on Management of Data",+ eventWhen = Just $ newWhen (2015, 5, 31) (2015, 6, 4),+ eventWhere = Just "Melbourne, VIC, Australia",+ eventDeadlines = fmap newDay [(2014, 10, 30), (2014, 11, 6)]+ },+ Event { eventShortName = "SIGMOD 2014",+ eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=32160©ownerid=52931",+ eventLongName = "ACM Conference on Management of Data",+ eventWhen = Just $ newWhen (2014, 6, 22) (2014, 6, 27),+ eventWhere = Just "Snowbird, Utah, USA",+ eventDeadlines = fmap newDay [(2013, 9, 9), (2013, 9, 16)]+ },+ Event { eventShortName = "SIGMOD 2013",+ eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=24911©ownerid=5823",+ eventLongName = "ACM SIGMOD International Conference on Management of Data",+ eventWhen = Just $ newWhen (2013, 6, 23) (2013, 6, 28),+ eventWhere = Just "New York",+ eventDeadlines = fmap newDay [(2012, 11, 13)]+ }+ ]+ fmap length ret `shouldBe` Right 10++ describe "scrapeSearchEvents" $ do+ forFile "search_cloud20160505.html" $ \raw_html -> do+ let ret = scrapeSearchEvents raw_html+ fmap (take 5) ret `shouldBe`+ Right [ Event { eventShortName = "CloudCom 2016",+ eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=52874©ownerid=9221",+ eventLongName = "The 8th IEEE International Conference on Cloud Computing Technology and Science",+ eventWhen = Just $ newWhen (2016, 12, 12) (2016, 12, 15),+ eventWhere = Just "Luxembourg City, Luxembourg",+ eventDeadlines = fmap newDay [(2016, 6, 8), (2016, 6, 15)]+ },+ Event { eventShortName = "UCC 2016",+ eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=52465©ownerid=49250",+ eventLongName = "Utility and Cloud Computing",+ eventWhen = Just $ newWhen (2016, 12, 6) (2016, 12, 9),+ eventWhere = Just "Shanghai, China",+ eventDeadlines = fmap newDay [(2016, 7, 3)]+ },+ Event { eventShortName = "SI-Cloud-2 2016",+ eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=50034©ownerid=53027",+ eventLongName = "International Journal of Services Technology and Management (EI) - Special Issue on Big Data Management in the Cloud",+ eventWhen = Nothing,+ eventWhere = Nothing,+ eventDeadlines = fmap newDay [(2016, 8, 30), (2016, 9, 30)]+ },+ Event { eventShortName = "IEEE SC 2016",+ eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=52039©ownerid=85546",+ eventLongName = "The 6th IEEE International Symposium on Cloud and Service Computing",+ eventWhen = Just $ newWhen (2016, 12, 8) (2016, 12, 10),+ eventWhere = Just "Fiji",+ eventDeadlines = fmap newDay [(2016, 8, 10)]+ },+ Event { eventShortName = "ISDSA 2016",+ eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=51832©ownerid=35379",+ eventLongName = "First International Symposium on Data Science and Applications",+ eventWhen = Just $ newWhen (2016, 10, 10) (2016, 10, 11),+ eventWhere = Just "Milan, Italy",+ eventDeadlines = fmap newDay [(2016, 7, 25)]+ }+ ]+ fmap length ret `shouldBe` Right 30++ forFile "search_noresult20160516.html" $ \raw_html -> do+ scrapeSearchEvents raw_html `shouldBe` Right []+++newDay :: (Integer, Int, Int) -> Day+newDay (y, m, d) = fromGregorian y m d++newWhen :: (Integer, Int, Int) -> (Integer, Int, Int) -> When+newWhen from to = When (newDay from) (newDay to)++testFile :: FilePath -> FilePath+testFile filename = joinPath ["test", "data", filename]++forFile :: FilePath -> (ByteString -> IO ()) -> SpecWith ()+forFile filename action = specify filename $ BS.readFile (testFile filename) >>= action
+ wikicfp-scraper.cabal view
@@ -0,0 +1,56 @@+name: wikicfp-scraper+version: 0.1.0.0+author: Toshio Ito <debug.ito@gmail.com>+maintainer: Toshio Ito <debug.ito@gmail.com>+license: BSD3+license-file: LICENSE+synopsis: Scrape WikiCFP web site+description: Scrape WikiCFP web site. See 'Web.WikiCFP.Scraper'.+category: Web+cabal-version: >= 1.10+build-type: Simple+extra-source-files: README.md, ChangeLog.md+homepage: https://github.com/debug-ito/wikicfp-scraper+bug-reports: https://github.com/debug-ito/wikicfp-scraper/issues++library+ default-language: Haskell2010+ hs-source-dirs: src+ ghc-options: -Wall -fno-warn-unused-imports+ -- default-extensions: + exposed-modules: Web.WikiCFP.Scraper+ + other-modules: Web.WikiCFP.Scraper.Type,+ Web.WikiCFP.Scraper.Scalpel+ build-depends: base >=4.6.0 && <4.9,+ bytestring >=0.10.0 && <0.11,+ text >=0.11.3.1 && <1.3,+ scalpel >=0.2.1 && <0.4,+ time >=1.4.0 && <1.6,+ attoparsec >=0.10.4 && <0.14++-- executable wikicfp-scraper+-- default-language: Haskell2010+-- hs-source-dirs: app+-- main-is: Main.hs+-- ghc-options: -Wall -fno-warn-unused-imports+-- -- other-modules: +-- -- other-extensions: +-- build-depends: base++test-suite spec+ type: exitcode-stdio-1.0+ default-language: Haskell2010+ hs-source-dirs: test+ ghc-options: -Wall -fno-warn-unused-imports "-with-rtsopts=-M512m"+ main-is: Spec.hs+ default-extensions: OverloadedStrings+ other-modules: Web.WikiCFP.ScraperSpec+ build-depends: base, wikicfp-scraper,+ bytestring, time,+ hspec >=2.1.5 && <2.3,+ filepath >=1.3.0 && <1.5++source-repository head+ type: git+ location: https://github.com/debug-ito/wikicfp-scraper.git