wikicfp-scraper 0.1.0.12 → 0.1.0.13
raw patch · 6 files changed
+93/−85 lines, 6 filesdep ~bytestringdep ~timePVP ok
version bump matches the API change (PVP)
Dependency ranges changed: bytestring, time
API changes (from Hackage documentation)
Files
- ChangeLog.md +4/−0
- src/Web/WikiCFP/Scraper.hs +21/−20
- src/Web/WikiCFP/Scraper/Scalpel.hs +29/−28
- src/Web/WikiCFP/Scraper/Type.hs +25/−23
- test/Web/WikiCFP/ScraperSpec.hs +11/−11
- wikicfp-scraper.cabal +3/−3
ChangeLog.md view
@@ -1,5 +1,9 @@ # Revision history for wikicfp-scraper +## 0.1.0.13 -- 2022-11-25++* Confirmed test with ghc-9.2.5.+ ## 0.1.0.12 -- 2021-03-23 * Confirmed test with `attoparsec-0.14.1`.
src/Web/WikiCFP/Scraper.hs view
@@ -1,14 +1,15 @@-{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-}+{-# LANGUAGE FlexibleInstances #-}+{-# LANGUAGE TypeSynonymInstances #-} -- | -- Module: Web.WikiCFP.Scraper -- Description: Scrape WikiCFP web site -- Maintainer: Toshio Ito <debug.ito@gmail.com>--- +-- -- Synopsis: -- -- > import qualified Network.HTTP as H -- > import Web.WikiCFP.Scraper (scrapeSearchEvents)--- > +-- > -- > main :: IO () -- > main = do -- > res <- H.getResponseBody =<< H.simpleHTTP (H.getRequest "http://wikicfp.com/cfp/servlet/tool.search?q=japan&year=t")@@ -18,25 +19,25 @@ -- call-for-papers. It helps you stay up to date with deadlines of -- academic paper submissions. module Web.WikiCFP.Scraper- ( -- * Scraper routines- scrapeConfEvents,- scrapeSearchEvents,- -- * Types- ErrorMsg,- HTML(..),- When(..),- Event(..)- ) where+ ( -- * Scraper routines+ scrapeConfEvents+ , scrapeSearchEvents+ -- * Types+ , ErrorMsg+ , HTML (..)+ , When (..)+ , Event (..)+ ) where -import qualified Data.ByteString as SB-import qualified Data.ByteString.Lazy as LB-import Data.Text (Text, pack)-import Data.Text.Encoding (decodeUtf8')-import qualified Data.Text.Lazy as LT-import Text.HTML.Scalpel.Core (scrapeStringLike)+import qualified Data.ByteString as SB+import qualified Data.ByteString.Lazy as LB+import Data.Text (Text, pack)+import Data.Text.Encoding (decodeUtf8')+import qualified Data.Text.Lazy as LT+import Text.HTML.Scalpel.Core (scrapeStringLike) -import Web.WikiCFP.Scraper.Type (When(..), Event(..))-import Web.WikiCFP.Scraper.Scalpel (ErrorMsg, Scraper', confRoot, searchRoot)+import Web.WikiCFP.Scraper.Scalpel (ErrorMsg, Scraper', confRoot, searchRoot)+import Web.WikiCFP.Scraper.Type (Event (..), When (..)) -- | Types of input HTML data to scrape.
src/Web/WikiCFP/Scraper/Scalpel.hs view
@@ -3,30 +3,30 @@ -- Description: Scraper implementation with Scalpel -- Maintainer: Toshio Ito <debug.ito@gmail.com> ----- -{-# LANGUAGE CPP, OverloadedStrings #-}+--+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-} module Web.WikiCFP.Scraper.Scalpel- ( ErrorMsg,- Scraper',- confRoot,- searchRoot- ) where+ ( ErrorMsg+ , Scraper'+ , confRoot+ , searchRoot+ ) where -import Control.Applicative ((<$>), (<*>), (<|>), (<*), (*>), optional, pure)-import Control.Monad (guard, forM_, mzero)-import Data.List (sort)-import Data.Maybe (catMaybes)-import Data.Monoid ((<>))-import Data.Text (Text, pack)-import Data.Time (Day, fromGregorian)-import Data.Attoparsec.Text (Parser, parseOnly, skipSpace, string, endOfInput, decimal, takeText, char)-import Text.HTML.Scalpel.Core- ( Scraper,- (@:), (@=), (//), chroot, chroots, text, texts, attr, hasClass- )+import Control.Applicative (optional, pure, (*>), (<$>), (<*), (<*>), (<|>))+import Control.Monad (forM_, guard, mzero)+import Data.Attoparsec.Text (Parser, char, decimal, endOfInput, parseOnly, skipSpace,+ string, takeText)+import Data.List (sort)+import Data.Maybe (catMaybes)+import Data.Monoid ((<>))+import Data.Text (Text, pack)+import Data.Time (Day, fromGregorian)+import Text.HTML.Scalpel.Core (Scraper, attr, chroot, chroots, hasClass, text, texts,+ (//), (@:), (@=)) -import Web.WikiCFP.Scraper.Type (Event(..), When(..))+import Web.WikiCFP.Scraper.Type (Event (..), When (..)) type ErrorMsg = String @@ -49,14 +49,15 @@ rows <- chroots "tr" (eventRow1 <|> eventRow2 <|> eventRowHeader) case rows of (EventRowHeader : rest) -> return $ rowsToEvents_noHeader rest- _ -> mzero- + _ -> mzero + -- | Intermediate result of parsing under events \<table\>.-data EventRow = EventRowHeader- | EventRow1 Text Text Text -- ^ shortName, URL and longName- | EventRow2 Text Text Text -- ^ when, where, deadlines- deriving (Eq,Ord,Show)+data EventRow+ = EventRowHeader+ | EventRow1 Text Text Text -- ^ shortName, URL and longName+ | EventRow2 Text Text Text -- ^ when, where, deadlines+ deriving (Eq, Ord, Show) -- | Scrape header row. Use with the root at @\<tr\>@ tag. eventRowHeader :: Scraper' EventRow@@ -159,5 +160,5 @@ <|> (string' "Oct" *> pure 10) <|> (string' "Nov" *> pure 11) <|> (string' "Dec" *> pure 12)- - ++
src/Web/WikiCFP/Scraper/Type.hs view
@@ -3,34 +3,36 @@ -- Description: data types for WikiCFP scraper -- Maintainer: Toshio Ito <debug.ito@gmail.com> ----- +-- module Web.WikiCFP.Scraper.Type- ( When(..),- Event(..),- ) where+ ( When (..)+ , Event (..)+ ) where -import Data.Text (Text)-import Data.Time (Day)+import Data.Text (Text)+import Data.Time (Day) -- | Period of dates (inclusive).-data When = When { whenFrom :: !Day,- whenTo :: !Day- } deriving (Eq,Ord,Show)+data When+ = When+ { whenFrom :: !Day+ , whenTo :: !Day+ }+ deriving (Eq, Ord, Show) -- | A conference event posted to WikiCFP site. It corresponds to a -- row in the table you see conference pages etc, for example, -- <http://wikicfp.com/cfp/program?id=1172>-data Event = Event { eventShortName :: !Text,- - eventURL :: !Text,- -- ^ URL to the WikiCFP page of this event.- - eventLongName :: !Text, - eventWhen :: !(Maybe When),- eventWhere :: !(Maybe Text),- - eventDeadlines :: ![Day]- -- ^ deadlines are in an ascending order, i.e.,- -- the earliest deadline is the head.- - } deriving (Eq,Ord,Show)+data Event+ = Event+ { eventShortName :: !Text+ , eventURL :: !Text+ -- ^ URL to the WikiCFP page of this event.+ , eventLongName :: !Text+ , eventWhen :: !(Maybe When)+ , eventWhere :: !(Maybe Text)+ , eventDeadlines :: ![Day]+ -- ^ deadlines are in an ascending order, i.e.,+ -- the earliest deadline is the head.+ }+ deriving (Eq, Ord, Show)
test/Web/WikiCFP/ScraperSpec.hs view
@@ -1,15 +1,15 @@-module Web.WikiCFP.ScraperSpec (main, spec) where+module Web.WikiCFP.ScraperSpec+ ( main+ , spec+ ) where -import Data.ByteString (ByteString)-import qualified Data.ByteString as BS-import Data.Time (Day, fromGregorian)-import System.FilePath (FilePath, joinPath)-import Test.Hspec+import Data.ByteString (ByteString)+import qualified Data.ByteString as BS+import Data.Time (Day, fromGregorian)+import System.FilePath (FilePath, joinPath)+import Test.Hspec -import Web.WikiCFP.Scraper- ( scrapeConfEvents, scrapeSearchEvents,- Event(..), When(..)- )+import Web.WikiCFP.Scraper (Event (..), When (..), scrapeConfEvents, scrapeSearchEvents) main :: IO () main = hspec spec@@ -125,7 +125,7 @@ forFile "search_noresult20160516.html" $ \raw_html -> do scrapeSearchEvents raw_html `shouldBe` Right [] - + newDay :: (Integer, Int, Int) -> Day
wikicfp-scraper.cabal view
@@ -1,5 +1,5 @@ name: wikicfp-scraper-version: 0.1.0.12+version: 0.1.0.13 author: Toshio Ito <debug.ito@gmail.com> maintainer: Toshio Ito <debug.ito@gmail.com> license: BSD3@@ -24,10 +24,10 @@ other-modules: Web.WikiCFP.Scraper.Type, Web.WikiCFP.Scraper.Scalpel build-depends: base >=4.6.0 && <5.0,- bytestring >=0.10.0 && <0.11,+ bytestring >=0.10.0 && <0.12, text >=0.11.3.1 && <1.3, scalpel-core >=0.5.0 && <0.7,- time >=1.4.0 && <1.10,+ time >=1.4.0 && <1.12, attoparsec >=0.10.4 && <0.15 -- executable wikicfp-scraper