chez-grater 0.0.1 → 0.0.2

raw patch · 6 files changed

+54/−21 lines, 6 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• chez-grater.cabal +8/−3
• src/Chez/Grater.hs +3/−3
• src/Chez/Grater/Scraper.hs +9/−10
• src/Chez/Grater/Scraper/Site.hs +23/−3
• src/Chez/Grater/Scraper/Types.hs +7/−0
• test/Chez/GraterSpec.hs +4/−2

@@ -1,11 +1,16 @@ cabal-version: 3.0 name:          chez-grater-version:       0.0.1+version:       0.0.2+author:        Dan Fithian maintainer:    Dan Fithian copyright:     2022 Dan Fithian build-type:    Simple-description:   Parse and scrape recipe blogs license:       MIT+category:      Web+homepage:      https://github.com/dfithian/chez-grater+bug-reports:   https://github.com/dfithian/chez-grater/issues+synopsis:      Parse and scrape recipe blogs+description:   Remove the life story! Parse and scrape recipe blogs. Powers https://app.mo-nomz.com/. extra-source-files:     test/fixtures/banana-bread-rachelmansfield.txt     test/fixtures/chicken-pot-pie-allrecipes.txt@@ -48,7 +53,7 @@   default-language: Haskell2010  common ghc-options-  ghc-options: -Wall -Wunused-packages -fwarn-tabs -fwarn-redundant-constraints -Wincomplete-uni-patterns -eventlog -O0+  ghc-options: -Wall -Wunused-packages -fwarn-tabs -fwarn-redundant-constraints -Wincomplete-uni-patterns -eventlog  library   import: options, ghc-options

@@ -5,7 +5,7 @@ import Chez.Grater.Parser (parseScrapedIngredients, parseScrapedSteps) import Chez.Grater.Scraper (scrape) import Chez.Grater.Scraper.Types-  ( ScrapedRecipeName(..), ScrapeMetaWrapper, ScrapedIngredient, ScrapedStep+  ( ScrapedRecipeName(..), ScrapeMetaWrapper, ScrapedIngredient, ScrapedStep, Scrapers   ) import Chez.Grater.Types (RecipeName(..), Ingredient, Step) import Control.Monad ((>=>))@@ -13,11 +13,11 @@ import Network.URI (URI)  -- |Scrape a URL without parsing it.-scrapeUrl :: Manager -> URI -> IO (ScrapedRecipeName, [ScrapedIngredient], [ScrapedStep], ScrapeMetaWrapper)+scrapeUrl :: Scrapers -> Manager -> URI -> IO (ScrapedRecipeName, [ScrapedIngredient], [ScrapedStep], ScrapeMetaWrapper) scrapeUrl = scrape id Right Right  -- |Scrape a URL and also parse it.-scrapeAndParseUrl :: Manager -> URI -> IO (RecipeName, [Ingredient], [Step], ScrapeMetaWrapper)+scrapeAndParseUrl :: Scrapers -> Manager -> URI -> IO (RecipeName, [Ingredient], [Step], ScrapeMetaWrapper) scrapeAndParseUrl = scrape   (RecipeName . unScrapedRecipeName)   (nonempty "ingredients" parseScrapedIngredients)

@@ -2,12 +2,9 @@  import Chez.Grater.Internal.Prelude -import Chez.Grater.Scraper.Site-  ( allIngredientScrapers, allStepScrapers, ingredientScrapers, stepScrapers-  ) import Chez.Grater.Scraper.Types   ( IngredientScraper(..), ScrapeError(..), ScrapeMetaWrapper(..), ScrapedRecipeName(..)-  , SiteName(..), StepScraper(..), ScrapedIngredient, ScrapedStep, title+  , Scrapers(..), SiteName(..), StepScraper(..), ScrapedIngredient, ScrapedStep, title   ) import Network.HTTP.Client (Manager) import Network.URI (URI, uriAuthority, uriRegName)@@ -19,8 +16,8 @@   :: (ScrapedRecipeName -> a)   -> ([ScrapedIngredient] -> Either Text [b])   -> ([ScrapedStep] -> Either Text [c])-  -> Manager -> URI -> IO (a, [b], [c], ScrapeMetaWrapper)-scrape mkName runIngredientParser runStepParser manager uri = do+  -> Scrapers -> Manager -> URI -> IO (a, [b], [c], ScrapeMetaWrapper)+scrape mkName runIngredientParser runStepParser scrapers manager uri = do   let cfg = Scalpel.Config Scalpel.defaultDecoder (Just manager)   tags <- Scalpel.fetchTagsWithConfig cfg (show uri)   let domainMay = SiteName . Text.replace "www." "" . Text.pack . uriRegName <$> uriAuthority uri@@ -36,21 +33,23 @@         Just True -> (,stepScraperMeta) <$> runScraper runStepParser stepScraperRun         _ -> Nothing -  (ingredients, ingredientMeta) <- case flip HashMap.lookup ingredientScrapers =<< domainMay of+  (ingredients, ingredientMeta) <- case flip HashMap.lookup (scrapersIngredientBySite scrapers) =<< domainMay of     Just IngredientScraper {..} -> maybe (throwIO $ ScrapeError "Failed to scrape known URL") (pure . (,ingredientScraperMeta)) $       runScraper runIngredientParser ingredientScraperRun     Nothing -> maybe (throwIO $ ScrapeError "Failed to scrape URL from defaults") pure      . lastMay      . sortOn (length . fst)      . mapMaybe goIngredient-     $ allIngredientScrapers-  stepsMay <- case flip HashMap.lookup stepScrapers =<< domainMay of+     . scrapersIngredients+     $ scrapers+  stepsMay <- case flip HashMap.lookup (scrapersStepBySite scrapers) =<< domainMay of     Just StepScraper {..} -> pure . fmap (,stepScraperMeta) . runScraper runStepParser $ stepScraperRun     Nothing -> pure       . lastMay       . sortOn (length . fst)       . mapMaybe goStep-      $ allStepScrapers+      . scrapersSites+      $ scrapers   case stepsMay of     Just (steps, stepMeta) | not (null steps) -> pure (mkName name, ingredients, steps, ScrapeMetaWrapperIngredientAndStep ingredientMeta stepMeta)     _ -> pure (mkName name, ingredients, [], ScrapeMetaWrapperIngredient ingredientMeta)

@@ -1,11 +1,13 @@ -- |Description: Ingredient and step scrapers for all the websites we know about.-module Chez.Grater.Scraper.Site where+module Chez.Grater.Scraper.Site+  ( allScrapers, debugI, debugS+  ) where  import Chez.Grater.Internal.Prelude  import Chez.Grater.Scraper.Types   ( IngredientScraper(..), ScrapeMeta(..), ScrapeName(..), ScrapeVersion(..), ScrapedIngredient(..)-  , ScrapedStep(..), SiteName(..), StepScraper(..), inception+  , ScrapedStep(..), Scrapers(..), SiteName(..), StepScraper(..), inception   ) import Data.Function (on) import Text.HTML.Scalpel ((//), (@:), (@=), Scraper, Selector)@@ -13,6 +15,9 @@ import qualified Data.Text as Text import qualified Text.HTML.Scalpel as Scalpel +allScrapers :: Scrapers+allScrapers = Scrapers ingredientScrapers allIngredientScrapers stepScrapers allStepScrapers+ ingredientScrapers :: HashMap SiteName IngredientScraper ingredientScrapers = HashMap.fromList   [ ("allrecipes.com", allrecipesI)@@ -113,6 +118,8 @@   , ("thekitchn.com", thekitchnI)    , ("eatwell101.com", eatwell101I)++  , ("bbcgoodfood.com", bbcGoodFoodI)   ]  stepScrapers :: HashMap SiteName StepScraper@@ -209,11 +216,14 @@   , ("food52.com", food52S)    , ("thekitchn.com", thekitchnS)++  , ("bbcgoodfood.com", bbcGoodFoodS)   ]  -- |Get all ingredient scrapers, ordered by most popular first. allIngredientScrapers :: [IngredientScraper]-allIngredientScrapers = fmap head . reverse . sortOn length  . groupBy ((==) `on` (scrapeMetaName . ingredientScraperMeta)) . sortOn (scrapeMetaName . ingredientScraperMeta) . HashMap.elems $ ingredientScrapers+allIngredientScrapers = fmap head . reverse . sortOn length  . groupBy ((==) `on` (scrapeMetaName . ingredientScraperMeta)) . sortOn (scrapeMetaName . ingredientScraperMeta) $+   HashMap.elems ingredientScrapers <> [ingredientLi4, ingredientLi12]  -- |Get all step scrapers, ordered by most popular first. allStepScrapers :: [StepScraper]@@ -608,3 +618,13 @@ eatwell101I = simpleIngredientScraper "eatwell101"   denyAll   ("div" @: [Scalpel.hasClass "pf-content"] // "li")++bbcGoodFoodI :: IngredientScraper+bbcGoodFoodI = simpleIngredientScraper "bbcgoodfood"+  denyAll+  ("section" @: [Scalpel.hasClass "recipe__ingredients"] // "li")++bbcGoodFoodS :: StepScraper+bbcGoodFoodS = simpleStepScraper "bbcgoodfood"+  denyAll+  ("section" @: [Scalpel.hasClass "recipe__method-steps"] // "li")

@@ -63,6 +63,13 @@ newtype SiteName = SiteName { unSiteName :: Text }   deriving (Eq, Ord, Show, IsString, Hashable) +data Scrapers = Scrapers+  { scrapersIngredientBySite :: HashMap SiteName IngredientScraper+  , scrapersIngredients      :: [IngredientScraper]+  , scrapersStepBySite       :: HashMap SiteName StepScraper+  , scrapersSites            :: [StepScraper]+  }+ data ScrapeError = ScrapeError Text   deriving (Eq, Show) 

@@ -9,6 +9,7 @@   , pillsburySteps, rachelMansfieldIngredients, rachelMansfieldSteps, sallysBakingIngredients   , sallysBakingSteps, tasteOfHomeIngredients, tasteOfHomeSteps   )+import Chez.Grater.Scraper.Site (allScrapers) import Chez.Grater.Scraper.Types () import Chez.Grater.TestEnv (Env(..)) import Chez.Grater.Types@@ -48,7 +49,7 @@ scrapeAndParse :: Env -> String -> String -> ([Ingredient], [Step]) -> Expectation scrapeAndParse Env {..} url expectedName (expectedIngredients, expectedSteps) = do   uri <- maybe (fail "Invalid URL") pure $ parseURI url-  (name, ingredients, steps, _) <- scrapeAndParseUrl envManager uri+  (name, ingredients, steps, _) <- scrapeAndParseUrl allScrapers envManager uri   name `shouldBe` RecipeName (Text.pack expectedName)   ingredients `shouldMatchList` expectedIngredients   steps `shouldMatchList` expectedSteps@@ -57,7 +58,7 @@ scrapeAndParseConfig TestCfg {..} url = do   let Env {..} = env   uri <- maybe (fail "Invalid URL") pure $ parseURI url-  (name, ingredients, steps, _) <- scrapeAndParseUrl envManager uri+  (name, ingredients, steps, _) <- scrapeAndParseUrl allScrapers envManager uri   unRecipeName name `shouldSatisfy` not . Text.null   ingredients `shouldSatisfy` (\xs -> length xs >= requiredIngredients)   ingredients `shouldSatisfy` any hasQuantityAndUnit@@ -221,6 +222,7 @@     it "handles chefspencil" $ scrapeAndParseConfig (defCfg { requiredSteps = 0 }) "https://www.chefspencil.com/recipe/carrot-tarte-tatin/"     it "handles sweetandsavorymeals" $ scrapeAndParseConfig defCfg "https://sweetandsavorymeals.com/air-fryer-eggplant/"     it "handles eatwell101" $ scrapeAndParseConfig (defCfg { requiredSteps = 0 }) "https://www.eatwell101.com/garlic-butter-chicken-bites-asparagus-recipe"+    it "handles bbcgoodfood" $ scrapeAndParseConfig defCfg "https://www.bbcgoodfood.com/recipes/challah/"    describe "Implicit" $ do     describe "WPRM" $ do