packages feed

htsn-import 0.1.3 → 0.2.0

raw patch · 11 files changed

+93/−70 lines, 11 files

Files

+ doc/CHANGES.database view
@@ -0,0 +1,16 @@+Any database changes requiring manual intervention will be mentioned+here. These should always be accompanied by a medium-version bump.++0.1.3 -> 0.2.0+--------------++The newsxml teams now have an "id" attribute, which means that we can+link each news entry to the common "teams" table rather than using a+custom "news_teams".++The migration will work automatically, creating a news__teams join+table, but the two old tables will still be left behind: news_teams+and news__news_teams. These can safely be deleted:++  DROP TABLE news_teams;+  DROP TABLE news__news_teams;
doc/dbschema/newsxml.png view

binary file changed (78490 → 84667 bytes)

htsn-import.cabal view
@@ -1,5 +1,5 @@ name:           htsn-import-version:        0.1.3+version:        0.2.0 cabal-version:  >= 1.8 author:         Michael Orlitzky maintainer:	Michael Orlitzky <michael@orlitzky.com>@@ -11,6 +11,7 @@   doc/dbschema/*.png   doc/htsn-importrc.example   doc/man1/htsn-import.1+  doc/CHANGES.database   doc/README.development   doc/TODO   makefile
schema/newsxml.dtd view
@@ -18,3 +18,6 @@ <!ELEMENT Editor (#PCDATA)>  <!ATTLIST msg_id EventId CDATA #REQUIRED>++<!-- The team id used to be nonexistent, now it's required. -->+<!ATTLIST team id CDATA #REQUIRED>
+ schemagen/newsxml/22041367.xml view
@@ -0,0 +1,1 @@+<?xml version="1.0" standalone="no" ?>
<!DOCTYPE message PUBLIC "-//TSN//DTD News 1.0/EN" "newsxml.dtd">
<message>
<XML_File_ID>22041367</XML_File_ID>
<heading>ADN;PREVIEW-WAS-CAL</heading>
<msg_id EventId="">4733420</msg_id>
<category>News</category>
<sport>NHL</sport>
<url>/nhl/news/ADN4733420.htm</url>
<team id="123">CAL</team>
<team id="140">WAS</team>
<location>
<city>Calgary</city>
<state>ALBERTA</state>
<country>CANADA</country>
</location>
<location>
<city>Washington</city>
<state>DC</state>
<country>USA</country>
</location>
<SMS>Caps seek to rebound against Flames</SMS>
<text>
 (SportsNetwork.com) - The Washington Capitals will aim to bounce back from
 their first regulation loss of the season on Saturday night as they visit the
 Calgary Flames.
</text>
<continue>
<P>
 The Capitals had won two in a row and put together a 3-0-2 record ahead of a
 3-2 loss to the Edmonton Oilers on Wednesday. Washington got goals from
 defenseman Mike Green and John Carlson, while Nicklas Backstrom had an assist
 to record his 500th career point.
</P>
<P>
 Braden Holtby made 17 saves in defeat.
</P>
<P>
 "We played good, we just didn't execute," Capitals center Marcus Johansson
 said.
</P>
<P>
 Washington also is set to visit Vancouver on Sunday, so it is unknown if
 Holtby or backup Justin Peters will get the start tonight.
</P>
<P>
 Holtby has allowed six goals on 25 shots in two prior meetings with the Flames
 for a 12.34 goals against average and .760 save percentage, while Peters took
 a post-regulation setback in his only previous meeting with the Flames. He
 gave up two goals on 30 shots.
</P>
<P>
 The Flames play the third contest of a five-game homestand and are 4-1-1 in
 their last six. They lost in overtime to Tampa Bay on Tuesday before blanking
 the Carolina Hurricanes 5-0 on Thursday.
</P>
<P>
 Jonas Hiller recorded his first shutout with Calgary and the 22nd of his
 career after making 16 saves, while defenseman Dennis Wideman scored his
 fourth goal of the season. All four have come in his past five games.
</P>
<P>
 Sean Monahan scored twice, while Matt Stajan and Lance Bouma also tallied.
</P>
<P>
 "To score that many goals is great," Calgary head coach Bob Hartley said. "I
 challenged the players before the game to win the three periods. They did it,
 and they did it in a great fashion."
</P>
<P>
 Despite Hiller's shutout, Hartley will continue his goaltender rotation and
 give Karri Ramo the start tonight. Ramo is 1-3-2 with a 3.11 GAA and .906 save
 percentage in his career versus the Capitals.
</P>
<P>
 The Caps are 8-3-0 with a tie in their last 12 versus the Flames, but lost 5-2
 in their most recent trip to Calgary.
</P>
</continue>
<time_stamp> October 25, 2014, at 10:59 AM ET </time_stamp>
</message>
src/TSN/XML/News.hs view
@@ -17,9 +17,8 @@   news_tests,   -- * WARNING: these are private but exported to silence warnings   News_LocationConstructor(..),-  News_NewsTeamConstructor(..),-  NewsConstructor(..),-  NewsTeamConstructor(..) )+  News_TeamConstructor(..),+  NewsConstructor(..) ) where  -- System imports.@@ -39,7 +38,6 @@ import Database.Groundhog.Generic ( runDbConn ) import Database.Groundhog.Sqlite ( withSqliteConn ) import Database.Groundhog.TH (-  defaultCodegenConfig,   groundhog,   mkPersist ) import Test.Tasty ( TestTree, testGroup )@@ -72,8 +70,9 @@   tsn_db_field_namer ) -- Used in a test import TSN.Database ( insert_or_select ) import TSN.DbImport ( DbImport(..), ImportResult(..), run_dbmigrate )-import TSN.Picklers ( xp_time_stamp ) import TSN.Location ( Location(..), pickle_location )+import TSN.Picklers ( xp_time_stamp )+import TSN.Team ( Team(..) ) import TSN.XmlImport ( XmlImport(..) ) import Xml (   FromXml(..),@@ -118,7 +117,7 @@     xml_category :: String,     xml_sport :: String,     xml_url :: Maybe String,-    xml_teams :: [NewsTeam],+    xml_teams :: [NewsTeamXml],     xml_locations :: [Location],     xml_sms :: Maybe String,     xml_editor :: Maybe String,@@ -173,32 +172,43 @@ instance XmlImport Message  --- * NewsTeam+-- * NewsTeamXml --- | The database/XML type for teams as they show up in the news. We---   can't reuse the representation from "TSN.Team" because they---   require a team id. We wouldn't want to make the team ID optional---   and then insert a team with no id, only to find the same team---   later with an id and be unable to update the record. (We could---   add the update logic, but it would be more trouble than it's---   worth.)+-- | The XML type for teams as they show up in the news. We can't+--   reuse the representation from "TSN.Team" because our name doesn't+--   appear optional and we have no abbreviation. ---data NewsTeam =-  NewsTeam { team_name :: String }+data NewsTeamXml =+  NewsTeamXml { xml_team_id :: String,+                xml_team_name :: String }   deriving (Eq, Show) +instance ToDb NewsTeamXml where+  -- | The database representation of 'NewsTeamXml' is 'Team'.+  type Db NewsTeamXml = Team +-- | Convert the XML representation 'NewsTeamXml' to the database+--   representation 'Team'.+--+instance FromXml NewsTeamXml where+  from_xml NewsTeamXml{..} =+    Team { team_id = xml_team_id,+           abbreviation = Nothing,+           name = Just xml_team_name } --- * News_NewsTeam+-- | Allow us to import 'NewsTeamXml' directly.+--+instance XmlImport NewsTeamXml --- | Mapping between News records and NewsTeam records in the---   database. We don't name the fields because we don't use the names---   explicitly; that means we have to give them nice database names---   via groundhog.++-- * News_Team++-- | Mapping between News records and Team records in the database. We+--   don't name the fields because we don't use the names explicitly;+--   that means we have to give them nice database names via+--   groundhog. ---data News_NewsTeam = News_NewsTeam-                       (DefaultKey News)-                       (DefaultKey NewsTeam)+data News_Team = News_Team (DefaultKey News) (DefaultKey Team)   -- * News_Location@@ -248,7 +258,7 @@ --  -- | Define 'dbmigrate' and 'dbimport' for 'Message's. The import is---   slightly non-generic because of our 'News_NewsTeam' and+--   slightly non-generic because of our 'News_Team' and --   'News_Location' join tables. -- instance DbImport Message where@@ -256,23 +266,23 @@     run_dbmigrate $ do       migrate (undefined :: Location)       migrate (undefined :: News)-      migrate (undefined :: NewsTeam)-      migrate (undefined :: News_NewsTeam)+      migrate (undefined :: Team)+      migrate (undefined :: News_Team)       migrate (undefined :: News_Location)    dbimport message = do     -- Insert the message and acquire its primary key (unique ID)     news_id <- insert_xml message -    -- Now insert the teams. We use insert_or_select because we know-    -- that most teams will already exist, and we want to get back the-    -- id for the existing team when there's a collision.-    nt_ids <- mapM insert_or_select (xml_teams message)+    -- Now insert the teams. We use insert_xml_or_select because we+    -- know that most teams will already exist, and we want to get+    -- back the id for the existing team when there's a collision.+    team_ids <- mapM insert_xml_or_select (xml_teams message)      -- Now that the teams have been inserted, create-    -- news__news_team records mapping beween the two.-    let news_news_teams = map (News_NewsTeam news_id) nt_ids-    mapM_ insert_ news_news_teams+    -- news__team records mapping beween the two.+    let news_teams = map (News_Team news_id) team_ids+    mapM_ insert_ news_teams      -- Do all of that over again for the Locations.     loc_ids <- mapM insert_or_select (xml_locations message)@@ -282,22 +292,7 @@     return ImportSucceeded  --- These types don't have special XML representations or field name--- collisions so we use the defaultCodegenConfig and give their--- fields nice simple names.-mkPersist defaultCodegenConfig [groundhog|-- entity: NewsTeam-  dbName: news_teams-  constructors:-    - name: NewsTeam-      uniques:-        - name: unique_news_teams-          type: constraint-          fields: [team_name] -|]-- -- These types have fields with e.g. db_ and xml_ prefixes, so we -- use our own codegen to peel those off before naming the columns. mkPersist tsn_codegen_config [groundhog|@@ -322,17 +317,17 @@     - name: db_event_id       dbName: event_id -- entity: News_NewsTeam-  dbName: news__news_teams+- entity: News_Team+  dbName: news__teams   constructors:-    - name: News_NewsTeam+    - name: News_Team       fields:-        - name: news_NewsTeam0 # Default created by mkNormalFieldName+        - name: news_Team0 # Default created by mkNormalFieldName           dbName: news_id           reference:             onDelete: cascade-        - name: news_NewsTeam1 # Default created by mkNormalFieldName-          dbName: news_teams_id+        - name: news_Team1 # Default created by mkNormalFieldName+          dbName: teams_id           reference:             onDelete: cascade @@ -356,18 +351,20 @@ -- XML Picklers -- --- | Convert a 'NewsTeam' to/from XML.+-- | Convert a 'NewsTeamXml' to/from XML. ---pickle_news_team :: PU NewsTeam+pickle_news_team :: PU NewsTeamXml pickle_news_team =   xpElem "team" $-    xpWrap (from_string, to_string) xpText+    xpWrap (from_pair, to_pair) $+      xpPair (xpAttr "id" xpText)+             xpText -- team name   where-    to_string :: NewsTeam -> String-    to_string = team_name+    from_pair :: (String, String) -> NewsTeamXml+    from_pair = uncurry NewsTeamXml -    from_string :: String -> NewsTeam-    from_string = NewsTeam+    to_pair :: NewsTeamXml -> (String, String)+    to_pair (NewsTeamXml x y) = (x,y)   -- | Convert a 'MsgId' to/from XML.@@ -539,8 +536,8 @@       news <- unsafe_unpickle path pickle_message       let a = undefined :: Location       let b = undefined :: News-      let c = undefined :: NewsTeam-      let d = undefined :: News_NewsTeam+      let c = undefined :: Team+      let d = undefined :: News_Team       let e = undefined :: News_Location       actual <- withSqliteConn ":memory:" $ runDbConn $ do                   runMigration silentMigrationLogger $ do
test/shell/import-duplicates.test view
@@ -20,8 +20,10 @@ >>>= 0  # Run the imports again; we should get complaints about the duplicate-# xml_file_ids. There are 2 errors for each violation, so we expect 2*28-# occurrences of the string 'ERROR'.+# xml_file_ids. There are 2 errors for each violation, so we expect+# (twice the number of files) occurrences of the string 'ERROR'. Oh+# and we subtract 2 because 2 of the files are supposed to fail. I+# know, sorry. ./dist/build/htsn-import/htsn-import -c 'shelltest.sqlite3' test/xml/*.xml 2>&1 | grep ERROR | wc -l >>> 68
test/xml/newsxml-empty-sms.xml view
@@ -1,1 +1,1 @@-<?xml version="1.0" standalone="no" ?>
<!DOCTYPE message PUBLIC "-//TSN//DTD News 1.0/EN" "newsxml.dtd">
<message>
<XML_File_ID>21904559</XML_File_ID>
<heading>AFN;UTAH-UCLA</heading>
<msg_id EventId="">0</msg_id>
<category>News</category>
<sport>CFOOT</sport>
<url></url>
<team>UCLA</team>
<team>UTAH</team>
<location>
<city>U-C-L-A</city>
<state>CA</state>
<country>USA</country>
</location>
<location>
<city>Utah</city>
<state>UT</state>
<country>USA</country>
</location>
<SMS></SMS>
<text>
 Pasadena, CA (SportsNetwork.com) -
</text>
<continue>
</continue>
<time_stamp> October 1, 2014, at 11:11 AM ET </time_stamp>
</message>
+<?xml version="1.0" standalone="no" ?>
<!DOCTYPE message PUBLIC "-//TSN//DTD News 1.0/EN" "newsxml.dtd">
<message>
<XML_File_ID>21904559</XML_File_ID>
<heading>AFN;UTAH-UCLA</heading>
<msg_id EventId="">0</msg_id>
<category>News</category>
<sport>CFOOT</sport>
<url></url>
<team id="foo">UCLA</team>
<team id="90210">UTAH</team>
<location>
<city>U-C-L-A</city>
<state>CA</state>
<country>USA</country>
</location>
<location>
<city>Utah</city>
<state>UT</state>
<country>USA</country>
</location>
<SMS></SMS>
<text>
 Pasadena, CA (SportsNetwork.com) -
</text>
<continue>
</continue>
<time_stamp> October 1, 2014, at 11:11 AM ET </time_stamp>
</message>
test/xml/newsxml-with-editor.xml view
@@ -1,1 +1,1 @@-<?xml version="1.0" standalone="no" ?>
<!DOCTYPE message PUBLIC "-//TSN//DTD News 1.0/EN" "newsxml.dtd">
<message>
<!-- This file was munged by hand; make up an ID -->
<XML_File_ID>12345</XML_File_ID>
<heading>ACN;RECAP-HOU-SAN</heading>
<msg_id EventId="">4650108</msg_id>
<category>News</category>
<sport>NBA</sport>
<url>/nba/news/ACN4650108.htm</url>
<team>HOU</team>
<team>SAN</team>
<location>
<city>Houston</city>
<state>TX</state>
<country>USA</country>
</location>
<location>
<city>San Antonio</city>
<state>TX</state>
<country>USA</country>
</location>
<SMS>Harden powers Rockets past Spurs</SMS>
<Editor>Washington Irving</Editor>
<text>
 San  Antonio,  TX (SportsNetwork.com) -  James Harden returned from a two-game
 absence and led all five Houston starters in double figures with 28 points, as
 the Rockets topped the San Antonio Spurs, 111-98, at AT&amp;T Center.
</text>
<continue>
<P>
 Harden  sat out games against Detroit and Dallas due to a sprained left ankle,
 but  the veteran  shooting guard  was a  one-man wrecking  crew in  the fourth
 quarter  against  the Spurs. Harden had  11 straight Houston points during the
 final period and scored 16 in the frame.
</P>
<P>
 Dwight  Howard contributed  15 points and 20 rebounds for the Rockets, who are
 within  3  1/2 games of  the Spurs for first  place in the Southwest Division.
 Terrence Jones had 21 points and 14 boards, while Chandler Parsons also scored
 21 to go with six rebounds and six assists. Jeremy Lin had 13 points and eight
 helpers.
</P>
<P>
 The Rockets, who didn't trail the entire game, made 12-of-24 shots from beyond
 the arc.
</P>
<P>
 Manu Ginobili had 22 points for the Spurs, who suffered only their fourth home
 loss  all season.  Two of  those defeats  have come  against the  Rockets. Tim
 Duncan finished with 11 points, 14 rebounds, six blocks and five assists.
</P>
<P>
 Kawhi  Leonard scored 13 points for the Spurs, who lost for the second time in
 three contests.
</P>
</continue>
<time_stamp> December 25, 2013, at 10:38 PM ET </time_stamp>
</message>
+<?xml version="1.0" standalone="no" ?>
<!DOCTYPE message PUBLIC "-//TSN//DTD News 1.0/EN" "newsxml.dtd">
<message>
<!-- This file was munged by hand; make up an ID -->
<XML_File_ID>12345</XML_File_ID>
<heading>ACN;RECAP-HOU-SAN</heading>
<msg_id EventId="">4650108</msg_id>
<category>News</category>
<sport>NBA</sport>
<url>/nba/news/ACN4650108.htm</url>
<team id="foo">HOU</team>
<team id="bar">SAN</team>
<location>
<city>Houston</city>
<state>TX</state>
<country>USA</country>
</location>
<location>
<city>San Antonio</city>
<state>TX</state>
<country>USA</country>
</location>
<SMS>Harden powers Rockets past Spurs</SMS>
<Editor>Washington Irving</Editor>
<text>
 San  Antonio,  TX (SportsNetwork.com) -  James Harden returned from a two-game
 absence and led all five Houston starters in double figures with 28 points, as
 the Rockets topped the San Antonio Spurs, 111-98, at AT&amp;T Center.
</text>
<continue>
<P>
 Harden  sat out games against Detroit and Dallas due to a sprained left ankle,
 but  the veteran  shooting guard  was a  one-man wrecking  crew in  the fourth
 quarter  against  the Spurs. Harden had  11 straight Houston points during the
 final period and scored 16 in the frame.
</P>
<P>
 Dwight  Howard contributed  15 points and 20 rebounds for the Rockets, who are
 within  3  1/2 games of  the Spurs for first  place in the Southwest Division.
 Terrence Jones had 21 points and 14 boards, while Chandler Parsons also scored
 21 to go with six rebounds and six assists. Jeremy Lin had 13 points and eight
 helpers.
</P>
<P>
 The Rockets, who didn't trail the entire game, made 12-of-24 shots from beyond
 the arc.
</P>
<P>
 Manu Ginobili had 22 points for the Spurs, who suffered only their fourth home
 loss  all season.  Two of  those defeats  have come  against the  Rockets. Tim
 Duncan finished with 11 points, 14 rebounds, six blocks and five assists.
</P>
<P>
 Kawhi  Leonard scored 13 points for the Spurs, who lost for the second time in
 three contests.
</P>
</continue>
<time_stamp> December 25, 2013, at 10:38 PM ET </time_stamp>
</message>
test/xml/newsxml.dtd view
@@ -18,3 +18,6 @@ <!ELEMENT Editor (#PCDATA)>  <!ATTLIST msg_id EventId CDATA #REQUIRED>++<!-- The team id used to be nonexistent, now it's required. -->+<!ATTLIST team id CDATA #REQUIRED>
test/xml/newsxml.xml view
@@ -1,1 +1,1 @@-<?xml version="1.0" standalone="no" ?>
<!DOCTYPE message PUBLIC "-//TSN//DTD News 1.0/EN" "newsxml.dtd">
<message>
<XML_File_ID>19956478</XML_File_ID>
<heading>ACN;RECAP-HOU-SAN</heading>
<msg_id EventId="">4650108</msg_id>
<category>News</category>
<sport>NBA</sport>
<url>/nba/news/ACN4650108.htm</url>
<team>HOU</team>
<team>SAN</team>
<location>
<city>Houston</city>
<state>TX</state>
<country>USA</country>
</location>
<location>
<city>San Antonio</city>
<state>TX</state>
<country>USA</country>
</location>
<SMS>Harden powers Rockets past Spurs</SMS>
<text>
 San  Antonio,  TX (SportsNetwork.com) -  James Harden returned from a two-game
 absence and led all five Houston starters in double figures with 28 points, as
 the Rockets topped the San Antonio Spurs, 111-98, at AT&amp;T Center.
</text>
<continue>
<P>
 Harden  sat out games against Detroit and Dallas due to a sprained left ankle,
 but  the veteran  shooting guard  was a  one-man wrecking  crew in  the fourth
 quarter  against  the Spurs. Harden had  11 straight Houston points during the
 final period and scored 16 in the frame.
</P>
<P>
 Dwight  Howard contributed  15 points and 20 rebounds for the Rockets, who are
 within  3  1/2 games of  the Spurs for first  place in the Southwest Division.
 Terrence Jones had 21 points and 14 boards, while Chandler Parsons also scored
 21 to go with six rebounds and six assists. Jeremy Lin had 13 points and eight
 helpers.
</P>
<P>
 The Rockets, who didn't trail the entire game, made 12-of-24 shots from beyond
 the arc.
</P>
<P>
 Manu Ginobili had 22 points for the Spurs, who suffered only their fourth home
 loss  all season.  Two of  those defeats  have come  against the  Rockets. Tim
 Duncan finished with 11 points, 14 rebounds, six blocks and five assists.
</P>
<P>
 Kawhi  Leonard scored 13 points for the Spurs, who lost for the second time in
 three contests.
</P>
</continue>
<time_stamp> December 25, 2013, at 10:38 PM ET </time_stamp>
</message>
+<?xml version="1.0" standalone="no" ?>
<!DOCTYPE message PUBLIC "-//TSN//DTD News 1.0/EN" "newsxml.dtd">
<message>
<XML_File_ID>19956478</XML_File_ID>
<heading>ACN;RECAP-HOU-SAN</heading>
<msg_id EventId="">4650108</msg_id>
<category>News</category>
<sport>NBA</sport>
<url>/nba/news/ACN4650108.htm</url>
<team id="foo">HOU</team>
<team id="6">SAN</team>
<location>
<city>Houston</city>
<state>TX</state>
<country>USA</country>
</location>
<location>
<city>San Antonio</city>
<state>TX</state>
<country>USA</country>
</location>
<SMS>Harden powers Rockets past Spurs</SMS>
<text>
 San  Antonio,  TX (SportsNetwork.com) -  James Harden returned from a two-game
 absence and led all five Houston starters in double figures with 28 points, as
 the Rockets topped the San Antonio Spurs, 111-98, at AT&amp;T Center.
</text>
<continue>
<P>
 Harden  sat out games against Detroit and Dallas due to a sprained left ankle,
 but  the veteran  shooting guard  was a  one-man wrecking  crew in  the fourth
 quarter  against  the Spurs. Harden had  11 straight Houston points during the
 final period and scored 16 in the frame.
</P>
<P>
 Dwight  Howard contributed  15 points and 20 rebounds for the Rockets, who are
 within  3  1/2 games of  the Spurs for first  place in the Southwest Division.
 Terrence Jones had 21 points and 14 boards, while Chandler Parsons also scored
 21 to go with six rebounds and six assists. Jeremy Lin had 13 points and eight
 helpers.
</P>
<P>
 The Rockets, who didn't trail the entire game, made 12-of-24 shots from beyond
 the arc.
</P>
<P>
 Manu Ginobili had 22 points for the Spurs, who suffered only their fourth home
 loss  all season.  Two of  those defeats  have come  against the  Rockets. Tim
 Duncan finished with 11 points, 14 rebounds, six blocks and five assists.
</P>
<P>
 Kawhi  Leonard scored 13 points for the Spurs, who lost for the second time in
 three contests.
</P>
</continue>
<time_stamp> December 25, 2013, at 10:38 PM ET </time_stamp>
</message>