Holumbus-MapReduce-0.0.1: Examples/MapReduce/Crawler/Config.hs
module Examples.MapReduce.Crawler.Config
(
getConfig
, MRCrawlerConfig(..)
)
where
import Text.XML.HXT.Arrow
import Holumbus.Build.Config
import Holumbus.Index.Documents
import Holumbus.Build.Crawl
import Holumbus.Build.Index
import Holumbus.Index.Documents
import Holumbus.Utility
import Data.Maybe
import Data.List
data MRCrawlerConfig d a = MRCrawlerConfig {
cc_CrawlerState :: CrawlerState d a
, cc_TraceLevel :: Int
, cc_DocsPerCrawl :: Int
, cc_IndexerConfig :: IndexerConfig
}
getConfig = MRCrawlerConfig
(initialCrawlerState idxConfig emptyDocuments customFunction)
traceLevel
docsPerCrawl
ic_test
where
idxConfig = ic_test
traceLevel = 0
docsPerCrawl = 10
customFunction :: ArrowXml a => a XmlTree (Maybe Int)
customFunction = constA Nothing
ic_test :: IndexerConfig
ic_test = ic_fhw {ic_startPages = ["http://april"], ic_fCrawlFilter = const True}
ic_fhw :: IndexerConfig
ic_fhw
= IndexerConfig
{ ic_startPages = [ "http://www.fh-wedel.de/"
, "http://www.fh-wedel.de/sonstiges/sitemap/"
, "http://www.fh-wedel.de/wir-ueber-uns/mitarbeiter-innen/?no_cache=1"
]
, ic_tempPath = Just "/tmp/"
, ic_indexPath = "/tmp/fhw"
, ic_indexerTimeOut = 10 * 60 * 1000000
, ic_contextConfigs = ccs_fhw
, ic_readAttributes = standardReadDocumentAttributes
, ic_fCrawlFilter = simpleCrawlFilter -- [ "^http://www\\.fh-wedel\\.de"] --
["^http://[a-z]*\\.?fh-wedel\\.de" ] -- allow
(["tx_fhwunternehmensforum_pi3" -- deny
, "http://asta.fh-wedel.de" -- slow
, "http://biblserv.fh-wedel.de" -- slow
, "http://darcs.fh-wedel.de" -- hackers only
, "http://stud.fh-wedel.de" -- boring
, "http://holumbus.fh-wedel.de/branches"
, "http://holumbus.fh-wedel.de/cgi-bin"
, "/HXmlToolbox/hdoc", "si/doc/javadoc/docs"
, "~herbert/html", "/java/jdk1.1.1/docs"
-- , "/~", "/%7E", "http://www.fh-wedel.de/mitarbeiter/"
, "\\?L=0", "\\&L=0"
, ".pdf$", ".jpg$", ".gif$", ".png$", ".tar.gz$"
, ".ppt$", ".exe$", ".txt$", ".zip$", ".doc$"
, ".dot$", ".png$", ".ps$", ".ps.gz$", ".nb$"
, ".swf$", ".JPG$", ".tex$", ".rss$", ".mpg$"
, ".mp3$", ".m3u$", ".java$", ".tgz$", ".svg", ".mdb$"
, ".PDF$", ".xls$", ".dta$", ".lst$", ".rar", ".avi$", ".mp4$"
, "%7Edi", "/~di"
, "ws99/Ausarbeitung/mico/Beispiel"
, "/rundgang/id=", "/vorlesungsplan/id="
, "/vorlesungsplan/sem=", "/tv-infosystem/", "/~splan/"
, "http://www\\.fh-wedel\\.de/index\\.php\\?eID=tx_cms_showpic"
, "http://www.fh-wedel.de/fileadmin/mitarbeiter/ne/CG/opengl_man"
, "http://www.fh-wedel.de/%7Esi/vorlesungen/c/beispiele"
, "http://www.fh-wedel.de/~si/vorlesungen/c/beispiele"
, "http://www.fh-wedel.de/~wol/fhtml" -- very slow and boring pages
, "http://www.fh-wedel.de/%7Esi/vorlesungen/internet/TclTk/program1.html" -- slow
] ++ list404)
}
list404 :: [String]
list404 =
[ "http://www.fh-wedel.de/work/medienlabor-der-fh-wedel/raeumlichkeiten/rechenzentrum/"
, "http://www.fh-wedel.de/typo3"
, "http://www.fh-wedel.de/~ue"
, "http://www.fh-wedel.de/~wk"
, "http://www.fh-wedel.de/work/medienlabor-der-fh-wedel/raeumlichkeiten/videostudio/"
, "http://www.fh-wedel.de/studierende/campus-infos/fh-stream/team.php"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/research/index.html"
, "http://www.fh-wedel.de/studierende/campus-infos/fh-stream/streams.php"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/vrlab/index.html"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/refs/index.html"
, "http://www.fh-wedel.de/studierende/campus-infos/fh-stream/specials.php"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/virtual-reality-labor/index.html"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/index.html"
, "http://www.fh-wedel.de/studierende/campus-infos/fh-stream/kontakt.php"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/virtual-reality-labor/cobench/index.html"
, "http://www.fh-wedel.de/studierende/campus-infos/online-tutorials/vlc-usage.htm"
, "http://www.fh-wedel.de/studierende/campus-infos/fh-stream/index.php"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/courses/index.html"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/virtual-reality-labor/cave/index.html"
, "http://www.fh-wedel.de/studierende/campus-infos/online-tutorials/toc_index.htm"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/vrlab/cobench/index.html"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/news/index.html"
, "http://www.fh-wedel.de/wir-ueber-uns/labore-fe/informatik/contact/index.html"
, "http://www.fh-wedel.de/studierende/campus-infos/fh-stream/faq.php"
, "http://www.fh-wedel.de/internationaloffice/Deutsch/"
, "http://www.fh-wedel.de/%7Ege/Seminar.htm"
, "http://www.fh-wedel.de/HXmlToolbox/index.html"
, "http://www.fh-wedel.de/Literatur_08.html"
, "http://www.fh-wedel.de/Seminar_Algorithmische_Graphentheorie.html"
, "http://www.fh-wedel.de/Seminar_Kombinatorik.html"
, "http://www.fh-wedel.de/absolventen/members/absolventensuche.html"
, "http://www.fh-wedel.de/analysis-im-Netz.htm"
, "http://www.fh-wedel.de/bba/index.html"
, "http://www.fh-wedel.de/buecher.html"
, "http://www.fh-wedel.de/buero_n15.html"
, "http://www.fh-wedel.de/c/index.html"
, "http://www.fh-wedel.de/cg1/index.html"
, "http://www.fh-wedel.de/cg2/index.html"
, "http://www.fh-wedel.de/cprog.html"
, "http://www.fh-wedel.de/ctrl.html"
, "http://www.fh-wedel.de/dbprog.html"
, "http://www.fh-wedel.de/diplom.html"
, "http://www.fh-wedel.de/download.htm"
, "http://www.fh-wedel.de/downloads.html"
, "http://www.fh-wedel.de/dozent.html"
, "http://www.fh-wedel.de/faelle.html"
, "http://www.fh-wedel.de/gesetze.html"
, "http://www.fh-wedel.de/gisela.html"
, "http://www.fh-wedel.de/handout.html"
, "http://www.fh-wedel.de/index.html"
, "http://www.fh-wedel.de/index/index.html"
, "http://www.fh-wedel.de/infos/mitarbeiter"
, "http://www.fh-wedel.de/infos/mitarbeiter/index.html"
, "http://www.fh-wedel.de/infos/sprechzeiten/"
, "http://www.fh-wedel.de/klausuren.html"
, "http://www.fh-wedel.de/~si/seminare/ws05/Ausarbeitung/7.concurrent/index.html"
, "http://www.fh-wedel.de/cis/semesterinfo/richtlinien/doku-richtl.html"
, "http://www.fh-wedel.de/~si/seminare/ws06/Ausarbeitung/04.UserModeLinux/index.html"
, "http://www.fh-wedel.de/cis/semesterinfo/studienordnung/index.html"
, "http://www.fh-wedel.de/~ge/Datenbanken.htm"
, "http://www.fh-wedel.de/~si/seminare/ws06/Ausarbeitung/14.Cocoon/cocoon0.htm"
, "http://www.fh-wedel.de/~si/seminare/ws00/Ausarbeitung/index.html"
, "http://www.fh-wedel.de/~si/seminare/ws01/Ausarbeitung/index.html"
, "http://www.fh-wedel.de/%7Efi/odb/index.html"
, "http://www.fh-wedel.de/~si/seminare/ws02/Ausarbeitung/index.html"
, "http://www.fh-wedel.de/%7Ekap/"
, "http://www.fh-wedel.de/~si/seminare/ws99/Ausarbeitung/index.html"
, "http://www.fh-wedel.de/%7Eps2/programmierstil/delphi_progstil.html"
, "http://www.fh-wedel.de/%7Eps2/programmierstil/inline_doku.html"
, "http://www.fh-wedel.de/%7Eps2/programmierstil/pascal_progstil.html"
, "http://www.fh-wedel.de/~mo/lectures/seminar-sose03.html"
, "http://www.fh-wedel.de/~si/seminare/ws03/Ausarbeitung/index.html"
, "http://www.fh-wedel.de/~si/HXmlToolbox/hdoc/index.htm"
, "http://www.fh-wedel.de/archiv/iw/Lehrveranstaltungen/WS2006/GdT.html"
, "http://www.fh-wedel.de/~si/seminare/ws04/Ausarbeitung/index.html"
, "http://www.fh-wedel.de/~si/seminare/ws04/Termine/www.wikipedia.org"
, "http://www.fh-wedel.de/~ki/ia/sem3/seminar1/code.htm"
, "http://www.fh-wedel.de/klausuren/Compilerbau.html"
, "http://www.fh-wedel.de/klausuren/CundOOP.html"
, "http://www.fh-wedel.de/klausuren/SD.html"
, "http://www.fh-wedel.de/klausuren/Unix.html"
, "http://www.fh-wedel.de/klausuren/index.html"
, "http://www.fh-wedel.de/klr2.html"
, "http://www.fh-wedel.de/kontakt.html"
, "http://www.fh-wedel.de/lebenslauf.html"
, "http://www.fh-wedel.de/links.htm"
, "http://www.fh-wedel.de/links.html"
, "http://www.fh-wedel.de/material.html"
, "http://www.fh-wedel.de/mec_MInf.html"
, "http://www.fh-wedel.de/mitarbeiter/iw/lehrveranstaltungen-im-naechsten-semester/informatik-seminar-algorithmen/"
, "http://www.fh-wedel.de/oop/index.html"
, "http://www.fh-wedel.de/or.html"
, "http://www.fh-wedel.de/personal.html"
, "http://www.fh-wedel.de/phys1_TInf.html"
, "http://www.fh-wedel.de/phys1_WIng.html"
, "http://www.fh-wedel.de/planspiel.html"
, "http://www.fh-wedel.de/planspiel2.html"
, "http://www.fh-wedel.de/pmod.html"
, "http://www.fh-wedel.de/praktika/MultimediaProjekte/SVG/SVG_Tutorial_mi3794/index.htm"
, "http://www.fh-wedel.de/praktika/MultimediaProjekte/index.html"
, "http://www.fh-wedel.de/praktika/SoftwarePraktikum/index.html"
, "http://www.fh-wedel.de/projekt.html"
, "http://www.fh-wedel.de/projekte/aktuell.htm"
, "http://www.fh-wedel.de/pse.html"
, "http://www.fh-wedel.de/rechnernetze/nwslab.htm"
, "http://www.fh-wedel.de/seite2.htm"
, "http://www.fh-wedel.de/seminare.html"
, "http://www.fh-wedel.de/seminare/index.html"
, "http://www.fh-wedel.de/seminare/ss03/Termine/Themen.html"
, "http://www.fh-wedel.de/seminare/ss04/Termine/Themen.html"
, "http://www.fh-wedel.de/seminare/ws03/Termine/Themen.html"
, "http://www.fh-wedel.de/seminare/ws04/Termine/Themen.html"
, "http://www.fh-wedel.de/seminare/ws05/Termine/Themen.html"
, "http://www.fh-wedel.de/seminare/ws06/Termine/Themen.html"
, "http://www.fh-wedel.de/sprechstunde.htm"
, "http://www.fh-wedel.de/suche/"
, "http://www.fh-wedel.de/termine.htm"
, "http://www.fh-wedel.de/termine/Klausureinsicht.html"
, "http://www.fh-wedel.de/termine/Sprechstunde.html"
, "http://www.fh-wedel.de/termine/index.html"
, "http://www.fh-wedel.de/toc_index.htm"
, "http://www.fh-wedel.de/uebungen/hinweise/hinweise.html"
, "http://www.fh-wedel.de/uebungen/oop/aufgaben/aufg04.html"
, "http://www.fh-wedel.de/uebungen/oop/aufgaben/aufg05.html"
, "http://www.fh-wedel.de/uebungen/oop/index.html"
, "http://www.fh-wedel.de/uebungen/oop/richtlinien.html"
, "http://www.fh-wedel.de/uebungen/unix/index.html"
, "http://www.fh-wedel.de/veranstaltungen.htm"
, "http://www.fh-wedel.de/vl.html"
, "http://www.fh-wedel.de/vlc-usage.htm"
, "http://www.fh-wedel.de/vorlesungen/c/c.html"
, "http://www.fh-wedel.de/vorlesungen/cb/cb.html"
, "http://www.fh-wedel.de/vorlesungen/fp/fp.html"
, "http://www.fh-wedel.de/vorlesungen/index.html"
, "http://www.fh-wedel.de/vorlesungen/internet/Literatur/Buecher.html"
, "http://www.fh-wedel.de/vorlesungen/internet/internet.html"
, "http://www.fh-wedel.de/vorlesungen/java/Literatur/Buecher.html"
, "http://www.fh-wedel.de/vorlesungen/java/java.html"
, "http://www.fh-wedel.de/vorlesungen/softwaredesign/Literatur/Buecher.html"
, "http://www.fh-wedel.de/vorlesungen/softwaredesign/design.html"
, "http://www.fh-wedel.de/vr/index.html"
, "http://www.fh-wedel.de/xml2html.2/xml2html/xml/index.xml"
, "http://www.fh-wedel.de/zettelkasten/etc/archiv.html"
, "http://www.fh-wedel.de/zettelkasten/etc/index.html"
, "http://www.fh-wedel.de/zettelkasten/etc/schwob.html"
, "http://www.fh-wedel.de/zettelkasten/fortune/fhspruch.html"
, "http://www.fh-wedel.de/zettelkasten/fortune/index.html"
, "http://www.fh-wedel.de/zettelkasten/fortune/index.html?KEY=1147779735"
]
-- | The list of context configurations for the Fh Wedel pages
ccs_fhw :: [ContextConfig]
ccs_fhw = []
++ [cc_title]
++ [cc_meta]
++ [cc_content]
++ [cc_raw]
-- | Context for title-tags
cc_title :: ContextConfig
cc_title
= ContextConfig
{ cc_name = "title"
, cc_preFilter = this
, cc_fExtract = getXPathTrees "/html/head/title"
, cc_fTokenize = map (stripWith (=='.')) . (parseWords isWordChar)
, cc_fIsStopWord = const False -- (\s -> length s < 2)
, cc_addToCache = False
}
-- | Context for meta information. Description and keywords will be indexed
cc_meta :: ContextConfig
cc_meta
= ContextConfig
{ cc_name = "meta"
, cc_preFilter = this
, cc_fExtract = getXPathTrees "/html/head/meta[@name='description' or @name='keywords']/@content"
, cc_fTokenize = map (stripWith (=='.')) . (parseWords isWordChar)
, cc_fIsStopWord = (\s -> length s < 2)
, cc_addToCache = False
}
-- | Context for normal page content. This indexes everything that is inside a div element with id
-- "col2_content" as it is defined in the central fhw template. Pages that do not use the normal
-- template will only be indexed in the raw context
cc_content :: ContextConfig
cc_content
= ContextConfig
{ cc_name = "content"
, cc_preFilter = this
, cc_fExtract = getXPathTrees "//div[@id='col2_content']"
, cc_fTokenize = map (stripWith (=='.')) . (parseWords isWordChar)
, cc_fIsStopWord = (\s -> length s < 2)
, cc_addToCache = True
}
-- | Raw context. This indexes all the text that is listed on the page without check if it is in
-- a menu or if it is real text. This context will have very low importance for search results
cc_raw :: ContextConfig
cc_raw
= ContextConfig
{ cc_name = "raw"
, cc_preFilter = this
, cc_fExtract = getXPathTrees "//body"
, cc_fTokenize = map (stripWith (=='.')) . (parseWords isWordChar)
, cc_fIsStopWord = (\s -> length s < 2)
, cc_addToCache = False
}