reddit-scrape (empty) → 0.0.0

raw patch · 3 files changed

+198/−0 lines, 3 filesdep +basedep +hspecdep +http-client

Dependencies added: base, hspec, http-client, http-client-tls, reddit-scrape, rio, scalpel

Files

• reddit-scrape.cabal +135/−0
• src/ScrapeReddit.hs +62/−0
• test/Spec.hs +1/−0

@@ -0,0 +1,135 @@+cabal-version: 1.12++-- This file has been generated from package.yaml by hpack version 0.34.4.+--+-- see: https://github.com/sol/hpack++name:           reddit-scrape+version:        0.0.0+description:    Library for getting links from a sub-reddit+category:       Web Scraping,Library+maintainer:     Rickard Andersson <gonz@severnatazvezda.com>+license:        BSD2+build-type:     Simple++library+  exposed-modules:+      ScrapeReddit+  other-modules:+      Paths_reddit_scrape+  hs-source-dirs:+      src+  default-extensions:+      ApplicativeDo+      BangPatterns+      BinaryLiterals+      ConstraintKinds+      DataKinds+      DefaultSignatures+      DeriveDataTypeable+      DeriveFoldable+      DeriveFunctor+      DeriveGeneric+      DeriveTraversable+      DeriveLift+      DerivingStrategies+      DoAndIfThenElse+      DuplicateRecordFields+      EmptyDataDecls+      EmptyCase+      ExistentialQuantification+      FlexibleContexts+      FlexibleInstances+      FunctionalDependencies+      GADTs+      GeneralizedNewtypeDeriving+      InstanceSigs+      KindSignatures+      LambdaCase+      MultiParamTypeClasses+      MultiWayIf+      NamedFieldPuns+      NoImplicitPrelude+      OverloadedStrings+      PartialTypeSignatures+      PatternGuards+      PolyKinds+      RankNTypes+      RecordWildCards+      ScopedTypeVariables+      StandaloneDeriving+      TupleSections+      TypeFamilies+      TypeSynonymInstances+      ViewPatterns+      TypeApplications+  ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wpartial-fields -Wredundant-constraints+  build-depends:+      base >=4.9.1.0 && <5+    , http-client+    , http-client-tls+    , rio+    , scalpel+  default-language: Haskell2010++test-suite reddit-scrape-test+  type: exitcode-stdio-1.0+  main-is: Spec.hs+  other-modules:+      Paths_reddit_scrape+  hs-source-dirs:+      test+  default-extensions:+      ApplicativeDo+      BangPatterns+      BinaryLiterals+      ConstraintKinds+      DataKinds+      DefaultSignatures+      DeriveDataTypeable+      DeriveFoldable+      DeriveFunctor+      DeriveGeneric+      DeriveTraversable+      DeriveLift+      DerivingStrategies+      DoAndIfThenElse+      DuplicateRecordFields+      EmptyDataDecls+      EmptyCase+      ExistentialQuantification+      FlexibleContexts+      FlexibleInstances+      FunctionalDependencies+      GADTs+      GeneralizedNewtypeDeriving+      InstanceSigs+      KindSignatures+      LambdaCase+      MultiParamTypeClasses+      MultiWayIf+      NamedFieldPuns+      NoImplicitPrelude+      OverloadedStrings+      PartialTypeSignatures+      PatternGuards+      PolyKinds+      RankNTypes+      RecordWildCards+      ScopedTypeVariables+      StandaloneDeriving+      TupleSections+      TypeFamilies+      TypeSynonymInstances+      ViewPatterns+      TypeApplications+  ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wpartial-fields -Wredundant-constraints -Wall+  build-depends:+      base >=4.9.1.0 && <5+    , hspec >=2.0.0+    , http-client+    , http-client-tls+    , reddit-scrape+    , rio+    , scalpel+  default-language: Haskell2010

@@ -0,0 +1,62 @@+module ScrapeReddit where++import Network.HTTP.Client (Manager)+import RIO+import qualified RIO.Text as Text+import RIO.Time (UTCTime, defaultTimeLocale, iso8601DateFormat, parseTimeM)+import Text.HTML.Scalpel+  ( Config (..),+    Scraper,+    anySelector,+    attr,+    chroot,+    chroots,+    defaultDecoder,+    hasClass,+    scrapeURLWithConfig,+    text,+    (@:),+  )++data Link = Link+  { title :: Text,+    href :: String,+    currentScore :: Maybe Int,+    comments :: Maybe Int,+    date :: Maybe UTCTime+  }+  deriving (Eq, Show, Generic)++scrapeSubReddit :: Manager -> String -> IO (Maybe [Link])+scrapeSubReddit manager subReddit =+  scrapeURLWithConfig+    (Config {decoder = defaultDecoder, manager = Just manager})+    (mconcat ["https://old.reddit.com/r/", subReddit, "/"])+    links++links :: Scraper Text [Link]+links = chroots ("div" @: [hasClass "thing", hasClass "link"]) link'++link' :: Scraper Text Link+link' = do+  title <- text $ "a" @: [hasClass "title"]+  href <- Text.unpack <$> attr "href" ("a" @: [hasClass "title"])+  currentScore <- (Text.unpack >>> readMaybe) <$> attr "title" ("div" @: [hasClass "score"])+  comments <- chroot ("a" @: [hasClass "comments"]) commentNumber+  date <- chroot ("time" @: [hasClass "live-timestamp"]) dateTimeFromTime+  pure $ Link {title, href, currentScore, comments, date}++commentNumber :: Scraper Text (Maybe Int)+commentNumber = do+  commentLinkText <- Text.unpack <$> text anySelector+  case words commentLinkText of+    [numberText, _comments] -> pure $ readMaybe numberText+    _anythingElse -> pure Nothing++dateTimeFromTime :: Scraper Text (Maybe UTCTime)+dateTimeFromTime = do+  timeString <- Text.unpack <$> attr "datetime" anySelector+  pure $ parseTimeM True defaultTimeLocale dateFormat timeString++dateFormat :: String+dateFormat = iso8601DateFormat $ Just "%H:%M:%S+00:00"

@@ -0,0 +1,1 @@+{-# OPTIONS_GHC -F -pgmF hspec-discover #-}