diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,5 @@ +# Revision history for wikicfp-scraper + +## 0.1.0.0 -- 2016-05-21 + +* First version. Released on an unsuspecting world. diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright (c) 2016, Toshio Ito + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of Toshio Ito nor the names of other + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +# wikicfp-scraper + +This module scrapes WikiCFP pages (http://wikicfp.com/) for call-for-papers. + +See http://hackage.haskell.org/package/wikicfp-scraper or module document of `Web.WikiCFP.Scraper`. + + +## Author + +Toshio Ito diff --git a/Setup.hs b/Setup.hs new file mode 100644 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/src/Web/WikiCFP/Scraper.hs b/src/Web/WikiCFP/Scraper.hs new file mode 100644 --- /dev/null +++ b/src/Web/WikiCFP/Scraper.hs @@ -0,0 +1,74 @@ +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} +-- | +-- Module: Web.WikiCFP.Scraper +-- Description: Scrape WikiCFP web site +-- Maintainer: Toshio Ito +-- +-- Synopsis: +-- +-- > import qualified Network.HTTP as H +-- > import Web.WikiCFP.Scraper (scrapeSearchEvents) +-- > +-- > main :: IO () +-- > main = do +-- > res <- H.getResponseBody =<< H.simpleHTTP (H.getRequest "http://wikicfp.com/cfp/servlet/tool.search?q=japan&year=t") +-- > print $ scrapeSearchEvents res +-- +-- This module scrapes WikiCFP pages () for +-- call-for-papers. It helps you stay up to date with deadlines of +-- academic paper submissions. +module Web.WikiCFP.Scraper + ( -- * Scraper routines + scrapeConfEvents, + scrapeSearchEvents, + -- * Types + ErrorMsg, + HTML(..), + When(..), + Event(..) + ) where + +import qualified Data.ByteString as SB +import qualified Data.ByteString.Lazy as LB +import Data.Text (Text, pack) +import Data.Text.Encoding (decodeUtf8') +import qualified Data.Text.Lazy as LT +import Text.HTML.Scalpel (scrapeStringLike) + +import Web.WikiCFP.Scraper.Type (When(..), Event(..)) +import Web.WikiCFP.Scraper.Scalpel (ErrorMsg, Scraper', confRoot, searchRoot) + + +-- | Types of input HTML data to scrape. +class HTML a where + decodeToText :: a -> Either ErrorMsg Text + +instance HTML Text where + decodeToText = Right + +instance HTML LT.Text where + decodeToText = Right . LT.toStrict + +-- | It just assumes UTF-8 encoding. +instance HTML SB.ByteString where + decodeToText = either (\e -> Left $ "UTF-8 decoding error: " ++ show e) Right . decodeUtf8' + +instance HTML LB.ByteString where + decodeToText = decodeToText . LB.toStrict + +instance HTML String where + decodeToText = Right . pack + +runScraper :: Scraper' (Either ErrorMsg a) -> Text -> Either ErrorMsg a +runScraper s input = maybe (Left "Scraping error") id $ scrapeStringLike input s + +-- | Scrape a page of a conference, for example, +-- +scrapeConfEvents :: HTML input => input -> Either ErrorMsg [Event] +scrapeConfEvents t = runScraper confRoot =<< decodeToText t + +-- | Scrape a page of search results, for example, +-- +scrapeSearchEvents :: HTML input => input -> Either ErrorMsg [Event] +scrapeSearchEvents t = runScraper searchRoot =<< decodeToText t + diff --git a/src/Web/WikiCFP/Scraper/Scalpel.hs b/src/Web/WikiCFP/Scraper/Scalpel.hs new file mode 100644 --- /dev/null +++ b/src/Web/WikiCFP/Scraper/Scalpel.hs @@ -0,0 +1,160 @@ +-- | +-- Module: Web.WikiCFP.Scraper.Scalpel +-- Description: Scraper implementation with Scalpel +-- Maintainer: Toshio Ito +-- +-- +module Web.WikiCFP.Scraper.Scalpel + ( ErrorMsg, + Scraper', + confRoot, + searchRoot + ) where + +import Control.Applicative ((<$>), (<*>), (<|>), (<*), (*>), optional, pure) +import Control.Monad (guard, forM_, mzero) +import Data.List (sort) +import Data.Maybe (catMaybes) +import Data.Monoid ((<>)) +import Data.Text (Text, pack) +import Data.Time (Day, fromGregorian) +import Data.Attoparsec.Text (Parser, parseOnly, skipSpace, string, endOfInput, decimal, takeText, char) +import Text.HTML.Scalpel + ( Scraper, + (@:), (@=), (//), chroot, chroots, text, texts, attr, hasClass + ) + +import Web.WikiCFP.Scraper.Type (Event(..), When(..)) + +type ErrorMsg = String + +type Scraper' = Scraper Text + +-- | Root scraper for conference Events. +confRoot :: Scraper' (Either ErrorMsg [Event]) +confRoot = concatSuccess $ chroots ("div" @: [hasClass "contsec"] // "table") $ eventsTable + +-- | Root scraper for searched Events. +searchRoot :: Scraper' (Either ErrorMsg [Event]) +searchRoot = confRoot + +concatSuccess :: (Functor m, Monad m) => m [Either e [a]] -> m (Either e [a]) +concatSuccess = fmap $ fmap concat . sequence + +-- | Scrape events from a table. Use with the root at @\@ tag. +eventsTable :: Scraper' (Either ErrorMsg [Event]) +eventsTable = do + rows <- chroots "tr" (eventRow1 <|> eventRow2 <|> eventRowHeader) + case rows of + (EventRowHeader : rest) -> return $ rowsToEvents_noHeader rest + _ -> mzero + + +-- | Intermediate result of parsing under events \. +data EventRow = EventRowHeader + | EventRow1 Text Text Text -- ^ shortName, URL and longName + | EventRow2 Text Text Text -- ^ when, where, deadlines + deriving (Eq,Ord,Show) + +-- | Scrape header row. Use with the root at @\@ tag. +eventRowHeader :: Scraper' EventRow +eventRowHeader = do + tds <- texts "td" + guard $ length tds == 4 + -- We cannot use OverloadedStrings with scalpel (as of 0.3.0.1), + -- because it requires explicit (:: String) declarations everywhere! + let expected_labels = map pack ["Event", "When", "Where", "Deadline"] + forM_ [0..(length expected_labels - 1)] $ \i -> guard $ parsable (spacedText $ expected_labels !! i) $ tds !! i + return EventRowHeader + +-- | Scrape shortName, URL and longName. Use with the root at @\@ tag. +eventRow1 :: Scraper' EventRow +eventRow1 = do + (sname, url) <- chroot ("td" @: ["rowspan" @= "2"]) $ ((,) <$> text "a" <*> attr "href" "a") + lname <- text ("td" @: ["colspan" @= "3"]) + return $ EventRow1 sname url lname + +-- | Scrape when, where, deadlines in Texts. Use the the root at @\@ tag. +eventRow2 :: Scraper' EventRow +eventRow2 = do + tds <- texts "td" + guard $ length tds == 3 + return $ EventRow2 (tds !! 0) (tds !! 1) (tds !! 2) + +rowsToEvents_noHeader :: [EventRow] -> Either ErrorMsg [Event] +rowsToEvents_noHeader [] = return [] +rowsToEvents_noHeader ((EventRow1 sn url ln) : (EventRow2 when wher dl) : rest) = + (:) <$> createEvent sn url ln when wher dl <*> rowsToEvents_noHeader rest +rowsToEvents_noHeader rows = Left ("Error while parsing rows: " ++ show rows) + +-- | TODO: make it configurable. +urlBase :: Text +urlBase = pack "http://wikicfp.com" + +createEvent :: Text -> Text -> Text -> Text -> Text -> Text -> Either ErrorMsg Event +createEvent sname url lname when wher dl = do + when' <- parseWhen when + wher' <- parseWhere wher + dl' <- parseDeadlines dl + return Event { eventShortName = sname, + eventURL = urlBase <> url, + eventLongName = lname, + eventWhen = when', + eventWhere = wher', + eventDeadlines = dl' + } + +-- * Parsers + +parsable :: Parser a -> Text -> Bool +parsable p t = either (const False) (const True) $ parseOnly p t + +spacedText :: Text -> Parser Text +spacedText expected = skipSpace *> string expected <* skipSpace <* endOfInput + +spacedText' :: String -> Parser Text +spacedText' = spacedText . pack + +string' :: String -> Parser Text +string' = string . pack + +parseWhen :: Text -> Either ErrorMsg (Maybe When) +parseWhen = parseOnly (parserWhen <* endOfInput) where + parserWhen = (spacedText' "N/A" *> pure Nothing) + <|> (Just <$> parserJustWhen) + parserJustWhen = When + <$> (parserDay <* skipSpace <* (char '-') <* skipSpace) + <*> (parserDay <* skipSpace) + +parseWhere :: Text -> Either ErrorMsg (Maybe Text) +parseWhere = parseOnly (parserWhere <* endOfInput) where + parserWhere = (spacedText' "N/A" *> pure Nothing) <|> (Just <$> takeText) + +parseDeadlines :: Text -> Either ErrorMsg [Day] +parseDeadlines input = sort <$> parseOnly (parserDeadlines <* endOfInput) input where + parserDeadlines = do + primary <- parserDay <* skipSpace + msecondary <- optional $ (char '(' *> skipSpace *> parserDay <* skipSpace <* char ')') + return $ maybe [primary] (: [primary]) msecondary + +parserDay :: Parser Day +parserDay = impl where + impl = do + m <- parserMonth <* skipSpace + d <- decimal <* (optional $ char ',') <* skipSpace + y <- decimal + return $ fromGregorian y m d + parserMonth = (string' "Jan" *> pure 1) + <|> (string' "Feb" *> pure 2) + <|> (string' "Mar" *> pure 3) + <|> (string' "Apr" *> pure 4) + <|> (string' "May" *> pure 5) + <|> (string' "Jun" *> pure 6) + <|> (string' "Jul" *> pure 7) + <|> (string' "Aug" *> pure 8) + <|> (string' "Sep" *> pure 9) + <|> (string' "Oct" *> pure 10) + <|> (string' "Nov" *> pure 11) + <|> (string' "Dec" *> pure 12) + + diff --git a/src/Web/WikiCFP/Scraper/Type.hs b/src/Web/WikiCFP/Scraper/Type.hs new file mode 100644 --- /dev/null +++ b/src/Web/WikiCFP/Scraper/Type.hs @@ -0,0 +1,36 @@ +-- | +-- Module: Web.WikiCFP.Scraper.Type +-- Description: data types for WikiCFP scraper +-- Maintainer: Toshio Ito +-- +-- +module Web.WikiCFP.Scraper.Type + ( When(..), + Event(..), + ) where + +import Data.Text (Text) +import Data.Time (Day) + +-- | Period of dates (inclusive). +data When = When { whenFrom :: !Day, + whenTo :: !Day + } deriving (Eq,Ord,Show) + +-- | A conference event posted to WikiCFP site. It corresponds to a +-- row in the table you see conference pages etc, for example, +-- +data Event = Event { eventShortName :: !Text, + + eventURL :: !Text, + -- ^ URL to the WikiCFP page of this event. + + eventLongName :: !Text, + eventWhen :: !(Maybe When), + eventWhere :: !(Maybe Text), + + eventDeadlines :: ![Day] + -- ^ deadlines are in an ascending order, i.e., + -- the earliest deadline is the head. + + } deriving (Eq,Ord,Show) diff --git a/test/Spec.hs b/test/Spec.hs new file mode 100644 --- /dev/null +++ b/test/Spec.hs @@ -0,0 +1,1 @@ +{-# OPTIONS_GHC -F -pgmF hspec-discover #-} diff --git a/test/Web/WikiCFP/ScraperSpec.hs b/test/Web/WikiCFP/ScraperSpec.hs new file mode 100644 --- /dev/null +++ b/test/Web/WikiCFP/ScraperSpec.hs @@ -0,0 +1,117 @@ +module Web.WikiCFP.ScraperSpec (main, spec) where + +import Data.ByteString (ByteString) +import qualified Data.ByteString as BS +import Data.Time (Day, fromGregorian) +import System.FilePath (FilePath, joinPath) +import Test.Hspec + +import Web.WikiCFP.Scraper + ( scrapeConfEvents, scrapeSearchEvents, + Event(..), When(..) + ) + +main :: IO () +main = hspec spec + +spec :: Spec +spec = do + describe "scrapeConfEvents" $ do + forFile "conf_sigmod20160505.html" $ \raw_html -> do + let ret = scrapeConfEvents raw_html + fmap (take 5) ret `shouldBe` + Right [ Event { eventShortName = "SIGMOD 2017", + eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=53445©ownerid=70798", + eventLongName = "2017 ACM SIGMOD Conference", + eventWhen = Just $ newWhen (2017, 5, 14) (2017, 5, 19), + eventWhere = Just "Raleigh", + eventDeadlines = fmap newDay [(2016, 7, 15), (2016, 7, 22)] + }, + Event { eventShortName = "SIGMOD 2016", + eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=44695©ownerid=76071", + eventLongName = "ACM International Conference on Management of Data ", + eventWhen = Just $ newWhen (2016, 6, 26) (2016, 7, 1), + eventWhere = Just "San Francisco, USA", + eventDeadlines = fmap newDay [(2015, 7, 9), (2015, 7, 16)] + }, + Event { eventShortName = "SIGMOD 2015", + eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=37140©ownerid=43021", + eventLongName = "International Conference on Management of Data", + eventWhen = Just $ newWhen (2015, 5, 31) (2015, 6, 4), + eventWhere = Just "Melbourne, VIC, Australia", + eventDeadlines = fmap newDay [(2014, 10, 30), (2014, 11, 6)] + }, + Event { eventShortName = "SIGMOD 2014", + eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=32160©ownerid=52931", + eventLongName = "ACM Conference on Management of Data", + eventWhen = Just $ newWhen (2014, 6, 22) (2014, 6, 27), + eventWhere = Just "Snowbird, Utah, USA", + eventDeadlines = fmap newDay [(2013, 9, 9), (2013, 9, 16)] + }, + Event { eventShortName = "SIGMOD 2013", + eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=24911©ownerid=5823", + eventLongName = "ACM SIGMOD International Conference on Management of Data", + eventWhen = Just $ newWhen (2013, 6, 23) (2013, 6, 28), + eventWhere = Just "New York", + eventDeadlines = fmap newDay [(2012, 11, 13)] + } + ] + fmap length ret `shouldBe` Right 10 + + describe "scrapeSearchEvents" $ do + forFile "search_cloud20160505.html" $ \raw_html -> do + let ret = scrapeSearchEvents raw_html + fmap (take 5) ret `shouldBe` + Right [ Event { eventShortName = "CloudCom 2016", + eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=52874©ownerid=9221", + eventLongName = "The 8th IEEE International Conference on Cloud Computing Technology and Science", + eventWhen = Just $ newWhen (2016, 12, 12) (2016, 12, 15), + eventWhere = Just "Luxembourg City, Luxembourg", + eventDeadlines = fmap newDay [(2016, 6, 8), (2016, 6, 15)] + }, + Event { eventShortName = "UCC 2016", + eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=52465©ownerid=49250", + eventLongName = "Utility and Cloud Computing", + eventWhen = Just $ newWhen (2016, 12, 6) (2016, 12, 9), + eventWhere = Just "Shanghai, China", + eventDeadlines = fmap newDay [(2016, 7, 3)] + }, + Event { eventShortName = "SI-Cloud-2 2016", + eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=50034©ownerid=53027", + eventLongName = "International Journal of Services Technology and Management (EI) - Special Issue on Big Data Management in the Cloud", + eventWhen = Nothing, + eventWhere = Nothing, + eventDeadlines = fmap newDay [(2016, 8, 30), (2016, 9, 30)] + }, + Event { eventShortName = "IEEE SC 2016", + eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=52039©ownerid=85546", + eventLongName = "The 6th IEEE International Symposium on Cloud and Service Computing", + eventWhen = Just $ newWhen (2016, 12, 8) (2016, 12, 10), + eventWhere = Just "Fiji", + eventDeadlines = fmap newDay [(2016, 8, 10)] + }, + Event { eventShortName = "ISDSA 2016", + eventURL = "http://wikicfp.com/cfp/servlet/event.showcfp?eventid=51832©ownerid=35379", + eventLongName = "First International Symposium on Data Science and Applications", + eventWhen = Just $ newWhen (2016, 10, 10) (2016, 10, 11), + eventWhere = Just "Milan, Italy", + eventDeadlines = fmap newDay [(2016, 7, 25)] + } + ] + fmap length ret `shouldBe` Right 30 + + forFile "search_noresult20160516.html" $ \raw_html -> do + scrapeSearchEvents raw_html `shouldBe` Right [] + + +newDay :: (Integer, Int, Int) -> Day +newDay (y, m, d) = fromGregorian y m d + +newWhen :: (Integer, Int, Int) -> (Integer, Int, Int) -> When +newWhen from to = When (newDay from) (newDay to) + +testFile :: FilePath -> FilePath +testFile filename = joinPath ["test", "data", filename] + +forFile :: FilePath -> (ByteString -> IO ()) -> SpecWith () +forFile filename action = specify filename $ BS.readFile (testFile filename) >>= action diff --git a/wikicfp-scraper.cabal b/wikicfp-scraper.cabal new file mode 100644 --- /dev/null +++ b/wikicfp-scraper.cabal @@ -0,0 +1,56 @@ +name: wikicfp-scraper +version: 0.1.0.0 +author: Toshio Ito +maintainer: Toshio Ito +license: BSD3 +license-file: LICENSE +synopsis: Scrape WikiCFP web site +description: Scrape WikiCFP web site. See 'Web.WikiCFP.Scraper'. +category: Web +cabal-version: >= 1.10 +build-type: Simple +extra-source-files: README.md, ChangeLog.md +homepage: https://github.com/debug-ito/wikicfp-scraper +bug-reports: https://github.com/debug-ito/wikicfp-scraper/issues + +library + default-language: Haskell2010 + hs-source-dirs: src + ghc-options: -Wall -fno-warn-unused-imports + -- default-extensions: + exposed-modules: Web.WikiCFP.Scraper + + other-modules: Web.WikiCFP.Scraper.Type, + Web.WikiCFP.Scraper.Scalpel + build-depends: base >=4.6.0 && <4.9, + bytestring >=0.10.0 && <0.11, + text >=0.11.3.1 && <1.3, + scalpel >=0.2.1 && <0.4, + time >=1.4.0 && <1.6, + attoparsec >=0.10.4 && <0.14 + +-- executable wikicfp-scraper +-- default-language: Haskell2010 +-- hs-source-dirs: app +-- main-is: Main.hs +-- ghc-options: -Wall -fno-warn-unused-imports +-- -- other-modules: +-- -- other-extensions: +-- build-depends: base + +test-suite spec + type: exitcode-stdio-1.0 + default-language: Haskell2010 + hs-source-dirs: test + ghc-options: -Wall -fno-warn-unused-imports "-with-rtsopts=-M512m" + main-is: Spec.hs + default-extensions: OverloadedStrings + other-modules: Web.WikiCFP.ScraperSpec + build-depends: base, wikicfp-scraper, + bytestring, time, + hspec >=2.1.5 && <2.3, + filepath >=1.3.0 && <1.5 + +source-repository head + type: git + location: https://github.com/debug-ito/wikicfp-scraper.git