concrete-haskell 0.1.0.15 → 0.1.0.16
raw patch · 24 files changed
+1278/−3411 lines, 24 filesdep +bzlib-conduitdep +conduitdep +conduit-combinatorsdep ~megaparsecnew-component:exe:apply_annotatorsnew-component:exe:build_graphs
Dependencies added: bzlib-conduit, conduit, conduit-combinators, conduit-extra, cryptohash-conduit, deepseq, lens, tar-conduit, zip-conduit
Dependency ranges changed: megaparsec
Files
- concrete-haskell.cabal +215/−16
- src/Data/Concrete/Internal/Lens.hs +178/−0
- src/Data/Concrete/Parsers.hs +39/−37
- src/Data/Concrete/Parsers/CONLL.hs +28/−46
- src/Data/Concrete/Parsers/CSV.hs +29/−21
- src/Data/Concrete/Parsers/Email.hs +74/−12
- src/Data/Concrete/Parsers/HTML.hs +76/−14
- src/Data/Concrete/Parsers/JSON.hs +28/−21
- src/Data/Concrete/Parsers/PTB.hs +60/−24
- src/Data/Concrete/Parsers/RFC2822.lhs +0/−2889
- src/Data/Concrete/Parsers/Types.hs +12/−7
- src/Data/Concrete/Parsers/Utils.hs +100/−22
- src/Data/Concrete/Parsers/XML.hs +74/−134
- src/Data/Concrete/Prelude.hs +51/−0
- src/Data/Concrete/Services/Annotate.hs +2/−0
- src/Data/Concrete/Services/Fetch.hs +133/−47
- src/Data/Concrete/Services/Store.hs +9/−2
- src/Data/Concrete/Utils.hs +2/−2
- tests/TestIngesters.hs +12/−59
- tests/TestSerialization.hs +22/−0
- utils/ApplyAnnotators.hs +38/−0
- utils/BuildGraphs.hs +44/−0
- utils/IngestCommunications.hs +37/−40
- utils/InspectCommunications.hs +15/−18
concrete-haskell.cabal view
@@ -1,9 +1,11 @@--- This file has been generated from package.yaml by hpack version 0.17.1.+-- This file has been generated from package.yaml by hpack version 0.20.0. -- -- see: https://github.com/sol/hpack+--+-- hash: 6713e725c67f25da63035bde809aeb0804583fde977b95d1b7d338ce35eaf71e name: concrete-haskell-version: 0.1.0.15+version: 0.1.0.16 synopsis: Library for the Concrete data format. description: Concrete is a Thrift-based data specification designed for Natural Language Processing (NLP) applications. This library provides a Haskell interface to code generated from the latest release of Concrete (the concrete-haskell-autogen package). It also has an ingest utility for converting various formats (JSON, CSV, XML, etc) to Concrete Communication objects. category: Data@@ -26,12 +28,19 @@ , binary ==0.8.3.0 , bytestring ==0.10.8.1 , bzlib ==0.5.0.5+ , bzlib-conduit ==0.2.1.4 , concrete-haskell-autogen ==0.0.0.1+ , conduit ==1.2.12+ , conduit-combinators ==1.1.1+ , conduit-extra ==1.1.17 , containers ==0.5.7.1+ , cryptohash-conduit ==0.1.1+ , deepseq ==1.4.3.0 , directory ==1.3.0.0 , filepath ==1.4.1.1 , hashable ==1.2.6.1- , megaparsec ==5.3.1+ , lens ==4.15.4+ , megaparsec >=6.1 , monad-extras ==0.6.0 , mtl ==2.2.1 , network ==2.6.3.2@@ -42,6 +51,7 @@ , scientific ==0.3.5.1 , stm ==2.4.4.1 , tar ==0.5.0.3+ , tar-conduit ==0.1.1 , text ==1.2.2.2 , thrift ==0.10.0 , time ==1.6.0.1@@ -49,8 +59,10 @@ , uuid ==1.3.13 , vector ==0.10.12.2 , zip ==0.1.11+ , zip-conduit ==0.2.2.2 , zlib ==0.6.1.2 exposed-modules:+ Data.Concrete.Prelude Data.Concrete.Utils Data.Concrete.Services Data.Concrete.Services.Fetch@@ -59,18 +71,111 @@ Data.Concrete.Parsers.Types Data.Concrete.Parsers.Utils Data.Concrete.Parsers.JSON- Data.Concrete.Parsers.CONLL- Data.Concrete.Parsers.CSV- Data.Concrete.Parsers.PTB other-modules: Data.Concrete+ Data.Concrete.Internal.Lens+ Data.Concrete.Parsers.CONLL+ Data.Concrete.Parsers.CSV Data.Concrete.Parsers.Email Data.Concrete.Parsers.HTML- Data.Concrete.Parsers.RFC2822+ Data.Concrete.Parsers.PTB Data.Concrete.Parsers.XML+ Data.Concrete.Services.Annotate Paths_concrete_haskell default-language: Haskell2010 +executable apply_annotators+ main-is: ApplyAnnotators.hs+ hs-source-dirs:+ utils+ build-depends:+ QuickCheck ==2.9.2+ , base >=4.6 && <5+ , binary ==0.8.3.0+ , bytestring ==0.10.8.1+ , bzlib ==0.5.0.5+ , bzlib-conduit ==0.2.1.4+ , concrete-haskell+ , concrete-haskell-autogen ==0.0.0.1+ , conduit ==1.2.12+ , conduit-combinators ==1.1.1+ , conduit-extra ==1.1.17+ , containers ==0.5.7.1+ , cryptohash-conduit ==0.1.1+ , deepseq ==1.4.3.0+ , directory ==1.3.0.0+ , filepath ==1.4.1.1+ , hashable ==1.2.6.1+ , lens ==4.15.4+ , megaparsec >=6.1+ , monad-extras ==0.6.0+ , mtl ==2.2.1+ , network ==2.6.3.2+ , optparse-generic ==1.2.2+ , path ==0.5.13+ , path-io ==1.2.2+ , process ==1.4.3.0+ , scientific ==0.3.5.1+ , stm ==2.4.4.1+ , tar ==0.5.0.3+ , tar-conduit ==0.1.1+ , text ==1.2.2.2+ , thrift ==0.10.0+ , time ==1.6.0.1+ , unordered-containers ==0.2.8.0+ , uuid ==1.3.13+ , vector ==0.10.12.2+ , zip ==0.1.11+ , zip-conduit ==0.2.2.2+ , zlib ==0.6.1.2+ default-language: Haskell2010++executable build_graphs+ main-is: BuildGraphs.hs+ hs-source-dirs:+ utils+ build-depends:+ QuickCheck ==2.9.2+ , base >=4.6 && <5+ , binary ==0.8.3.0+ , bytestring ==0.10.8.1+ , bzlib ==0.5.0.5+ , bzlib-conduit ==0.2.1.4+ , concrete-haskell+ , concrete-haskell-autogen ==0.0.0.1+ , conduit ==1.2.12+ , conduit-combinators ==1.1.1+ , conduit-extra ==1.1.17+ , containers ==0.5.7.1+ , cryptohash-conduit ==0.1.1+ , deepseq ==1.4.3.0+ , directory ==1.3.0.0+ , filepath ==1.4.1.1+ , hashable ==1.2.6.1+ , lens ==4.15.4+ , megaparsec >=6.1+ , monad-extras ==0.6.0+ , mtl ==2.2.1+ , network ==2.6.3.2+ , optparse-generic ==1.2.2+ , path ==0.5.13+ , path-io ==1.2.2+ , process ==1.4.3.0+ , scientific ==0.3.5.1+ , stm ==2.4.4.1+ , tar ==0.5.0.3+ , tar-conduit ==0.1.1+ , text ==1.2.2.2+ , thrift ==0.10.0+ , time ==1.6.0.1+ , unordered-containers ==0.2.8.0+ , uuid ==1.3.13+ , vector ==0.10.12.2+ , zip ==0.1.11+ , zip-conduit ==0.2.2.2+ , zlib ==0.6.1.2+ default-language: Haskell2010+ executable fetch_service main-is: FetchService.hs hs-source-dirs:@@ -81,12 +186,20 @@ , binary ==0.8.3.0 , bytestring ==0.10.8.1 , bzlib ==0.5.0.5+ , bzlib-conduit ==0.2.1.4+ , concrete-haskell , concrete-haskell-autogen ==0.0.0.1+ , conduit ==1.2.12+ , conduit-combinators ==1.1.1+ , conduit-extra ==1.1.17 , containers ==0.5.7.1+ , cryptohash-conduit ==0.1.1+ , deepseq ==1.4.3.0 , directory ==1.3.0.0 , filepath ==1.4.1.1 , hashable ==1.2.6.1- , megaparsec ==5.3.1+ , lens ==4.15.4+ , megaparsec >=6.1 , monad-extras ==0.6.0 , mtl ==2.2.1 , network ==2.6.3.2@@ -97,6 +210,7 @@ , scientific ==0.3.5.1 , stm ==2.4.4.1 , tar ==0.5.0.3+ , tar-conduit ==0.1.1 , text ==1.2.2.2 , thrift ==0.10.0 , time ==1.6.0.1@@ -104,8 +218,8 @@ , uuid ==1.3.13 , vector ==0.10.12.2 , zip ==0.1.11+ , zip-conduit ==0.2.2.2 , zlib ==0.6.1.2- , concrete-haskell default-language: Haskell2010 executable ingest_communications@@ -118,12 +232,20 @@ , binary ==0.8.3.0 , bytestring ==0.10.8.1 , bzlib ==0.5.0.5+ , bzlib-conduit ==0.2.1.4+ , concrete-haskell , concrete-haskell-autogen ==0.0.0.1+ , conduit ==1.2.12+ , conduit-combinators ==1.1.1+ , conduit-extra ==1.1.17 , containers ==0.5.7.1+ , cryptohash-conduit ==0.1.1+ , deepseq ==1.4.3.0 , directory ==1.3.0.0 , filepath ==1.4.1.1 , hashable ==1.2.6.1- , megaparsec ==5.3.1+ , lens ==4.15.4+ , megaparsec >=6.1 , monad-extras ==0.6.0 , mtl ==2.2.1 , network ==2.6.3.2@@ -134,6 +256,7 @@ , scientific ==0.3.5.1 , stm ==2.4.4.1 , tar ==0.5.0.3+ , tar-conduit ==0.1.1 , text ==1.2.2.2 , thrift ==0.10.0 , time ==1.6.0.1@@ -141,8 +264,8 @@ , uuid ==1.3.13 , vector ==0.10.12.2 , zip ==0.1.11+ , zip-conduit ==0.2.2.2 , zlib ==0.6.1.2- , concrete-haskell default-language: Haskell2010 executable inspect_communications@@ -156,12 +279,20 @@ , binary ==0.8.3.0 , bytestring ==0.10.8.1 , bzlib ==0.5.0.5+ , bzlib-conduit ==0.2.1.4+ , concrete-haskell , concrete-haskell-autogen ==0.0.0.1+ , conduit ==1.2.12+ , conduit-combinators ==1.1.1+ , conduit-extra ==1.1.17 , containers ==0.5.7.1+ , cryptohash-conduit ==0.1.1+ , deepseq ==1.4.3.0 , directory ==1.3.0.0 , filepath ==1.4.1.1 , hashable ==1.2.6.1- , megaparsec ==5.3.1+ , lens ==4.15.4+ , megaparsec >=6.1 , monad-extras ==0.6.0 , mtl ==2.2.1 , network ==2.6.3.2@@ -172,6 +303,7 @@ , scientific ==0.3.5.1 , stm ==2.4.4.1 , tar ==0.5.0.3+ , tar-conduit ==0.1.1 , text ==1.2.2.2 , thrift ==0.10.0 , time ==1.6.0.1@@ -179,8 +311,8 @@ , uuid ==1.3.13 , vector ==0.10.12.2 , zip ==0.1.11+ , zip-conduit ==0.2.2.2 , zlib ==0.6.1.2- , concrete-haskell default-language: Haskell2010 executable store_service@@ -193,12 +325,20 @@ , binary ==0.8.3.0 , bytestring ==0.10.8.1 , bzlib ==0.5.0.5+ , bzlib-conduit ==0.2.1.4+ , concrete-haskell , concrete-haskell-autogen ==0.0.0.1+ , conduit ==1.2.12+ , conduit-combinators ==1.1.1+ , conduit-extra ==1.1.17 , containers ==0.5.7.1+ , cryptohash-conduit ==0.1.1+ , deepseq ==1.4.3.0 , directory ==1.3.0.0 , filepath ==1.4.1.1 , hashable ==1.2.6.1- , megaparsec ==5.3.1+ , lens ==4.15.4+ , megaparsec >=6.1 , monad-extras ==0.6.0 , mtl ==2.2.1 , network ==2.6.3.2@@ -209,6 +349,7 @@ , scientific ==0.3.5.1 , stm ==2.4.4.1 , tar ==0.5.0.3+ , tar-conduit ==0.1.1 , text ==1.2.2.2 , thrift ==0.10.0 , time ==1.6.0.1@@ -216,8 +357,8 @@ , uuid ==1.3.13 , vector ==0.10.12.2 , zip ==0.1.11+ , zip-conduit ==0.2.2.2 , zlib ==0.6.1.2- , concrete-haskell default-language: Haskell2010 test-suite ingesters@@ -229,12 +370,20 @@ , binary ==0.8.3.0 , bytestring ==0.10.8.1 , bzlib ==0.5.0.5+ , bzlib-conduit ==0.2.1.4+ , concrete-haskell , concrete-haskell-autogen ==0.0.0.1+ , conduit ==1.2.12+ , conduit-combinators ==1.1.1+ , conduit-extra ==1.1.17 , containers ==0.5.7.1+ , cryptohash-conduit ==0.1.1+ , deepseq ==1.4.3.0 , directory ==1.3.0.0 , filepath ==1.4.1.1 , hashable ==1.2.6.1- , megaparsec ==5.3.1+ , lens ==4.15.4+ , megaparsec >=6.1 , monad-extras ==0.6.0 , mtl ==2.2.1 , network ==2.6.3.2@@ -245,6 +394,7 @@ , scientific ==0.3.5.1 , stm ==2.4.4.1 , tar ==0.5.0.3+ , tar-conduit ==0.1.1 , text ==1.2.2.2 , thrift ==0.10.0 , time ==1.6.0.1@@ -252,6 +402,55 @@ , uuid ==1.3.13 , vector ==0.10.12.2 , zip ==0.1.11+ , zip-conduit ==0.2.2.2 , zlib ==0.6.1.2+ other-modules:+ Paths_concrete_haskell+ default-language: Haskell2010++test-suite serialization+ type: exitcode-stdio-1.0+ main-is: tests/TestSerialization.hs+ build-depends:+ QuickCheck ==2.9.2+ , base >=4.6 && <5+ , binary ==0.8.3.0+ , bytestring ==0.10.8.1+ , bzlib ==0.5.0.5+ , bzlib-conduit ==0.2.1.4 , concrete-haskell+ , concrete-haskell-autogen ==0.0.0.1+ , conduit ==1.2.12+ , conduit-combinators ==1.1.1+ , conduit-extra ==1.1.17+ , containers ==0.5.7.1+ , cryptohash-conduit ==0.1.1+ , deepseq ==1.4.3.0+ , directory ==1.3.0.0+ , filepath ==1.4.1.1+ , hashable ==1.2.6.1+ , lens ==4.15.4+ , megaparsec >=6.1+ , monad-extras ==0.6.0+ , mtl ==2.2.1+ , network ==2.6.3.2+ , optparse-generic ==1.2.2+ , path ==0.5.13+ , path-io ==1.2.2+ , process ==1.4.3.0+ , scientific ==0.3.5.1+ , stm ==2.4.4.1+ , tar ==0.5.0.3+ , tar-conduit ==0.1.1+ , text ==1.2.2.2+ , thrift ==0.10.0+ , time ==1.6.0.1+ , unordered-containers ==0.2.8.0+ , uuid ==1.3.13+ , vector ==0.10.12.2+ , zip ==0.1.11+ , zip-conduit ==0.2.2.2+ , zlib ==0.6.1.2+ other-modules:+ Paths_concrete_haskell default-language: Haskell2010
+ src/Data/Concrete/Internal/Lens.hs view
@@ -0,0 +1,178 @@+{-# LANGUAGE TemplateHaskell #-}++module Data.Concrete.Internal.Lens ( module Data.Concrete.Autogen.Lens.Annotation+ , module Data.Concrete.Autogen.Lens.AnnotationMetadata+ , module Data.Concrete.Autogen.Lens.AnnotationTask+ , module Data.Concrete.Autogen.Lens.AnnotationTaskType+ , module Data.Concrete.Autogen.Lens.AnnotationUnitIdentifier+ , module Data.Concrete.Autogen.Lens.AnnotationUnitType+ , module Data.Concrete.Autogen.Lens.Arc+ , module Data.Concrete.Autogen.Lens.Argument+ , module Data.Concrete.Autogen.Lens.AsyncContactInfo+ , module Data.Concrete.Autogen.Lens.AudioSpan+ , module Data.Concrete.Autogen.Lens.BoundingBox+ , module Data.Concrete.Autogen.Lens.Cluster+ , module Data.Concrete.Autogen.Lens.Clustering+ , module Data.Concrete.Autogen.Lens.ClusterMember+ , module Data.Concrete.Autogen.Lens.Communication+ , module Data.Concrete.Autogen.Lens.CommunicationMetadata+ , module Data.Concrete.Autogen.Lens.CommunicationSet+ , module Data.Concrete.Autogen.Lens.CommunicationTagging+ , module Data.Concrete.Autogen.Lens.ConcreteThriftException+ , module Data.Concrete.Autogen.Lens.Constituent+ , module Data.Concrete.Autogen.Lens.ConstituentRef+ , module Data.Concrete.Autogen.Lens.Dependency+ , module Data.Concrete.Autogen.Lens.DependencyParse+ , module Data.Concrete.Autogen.Lens.DependencyParseStructure+ , module Data.Concrete.Autogen.Lens.Digest+ , module Data.Concrete.Autogen.Lens.EmailAddress+ , module Data.Concrete.Autogen.Lens.EmailCommunicationInfo+ , module Data.Concrete.Autogen.Lens.Entity+ , module Data.Concrete.Autogen.Lens.EntityMention+ , module Data.Concrete.Autogen.Lens.EntityMentionSet+ , module Data.Concrete.Autogen.Lens.EntitySet+ , module Data.Concrete.Autogen.Lens.FetchRequest+ , module Data.Concrete.Autogen.Lens.FetchResult+ , module Data.Concrete.Autogen.Lens.HashTag+ , module Data.Concrete.Autogen.Lens.Justification+ , module Data.Concrete.Autogen.Lens.LanguageIdentification+ , module Data.Concrete.Autogen.Lens.LatticePath+ , module Data.Concrete.Autogen.Lens.Link+ , module Data.Concrete.Autogen.Lens.Linking+ , module Data.Concrete.Autogen.Lens.LinkTarget+ , module Data.Concrete.Autogen.Lens.MentionArgument+ , module Data.Concrete.Autogen.Lens.NITFInfo+ , module Data.Concrete.Autogen.Lens.NotImplementedException+ , module Data.Concrete.Autogen.Lens.Parse+ , module Data.Concrete.Autogen.Lens.PlaceAttributes+ , module Data.Concrete.Autogen.Lens.Property+ , module Data.Concrete.Autogen.Lens.SearchCapability+ , module Data.Concrete.Autogen.Lens.SearchFeedback+ , module Data.Concrete.Autogen.Lens.SearchQuery+ , module Data.Concrete.Autogen.Lens.SearchResult+ , module Data.Concrete.Autogen.Lens.SearchResultItem+ , module Data.Concrete.Autogen.Lens.SearchType+ , module Data.Concrete.Autogen.Lens.Section+ , module Data.Concrete.Autogen.Lens.Sentence+ , module Data.Concrete.Autogen.Lens.ServiceInfo+ , module Data.Concrete.Autogen.Lens.ServicesException+ , module Data.Concrete.Autogen.Lens.Situation+ , module Data.Concrete.Autogen.Lens.SituationMention+ , module Data.Concrete.Autogen.Lens.SituationMentionSet+ , module Data.Concrete.Autogen.Lens.SituationSet+ , module Data.Concrete.Autogen.Lens.Sound+ , module Data.Concrete.Autogen.Lens.SpanLink+ , module Data.Concrete.Autogen.Lens.SummarizationCapability+ , module Data.Concrete.Autogen.Lens.SummarizationRequest+ , module Data.Concrete.Autogen.Lens.SummaryConcept+ , module Data.Concrete.Autogen.Lens.Summary+ , module Data.Concrete.Autogen.Lens.SummarySourceType+ , module Data.Concrete.Autogen.Lens.TaggedToken+ , module Data.Concrete.Autogen.Lens.TextSpan+ , module Data.Concrete.Autogen.Lens.TheoryDependencies+ , module Data.Concrete.Autogen.Lens.TimeML+ , module Data.Concrete.Autogen.Lens.Token+ , module Data.Concrete.Autogen.Lens.Tokenization+ , module Data.Concrete.Autogen.Lens.TokenizationKind+ , module Data.Concrete.Autogen.Lens.TokenLattice+ , module Data.Concrete.Autogen.Lens.TokenList+ , module Data.Concrete.Autogen.Lens.TokenRefSequence+ , module Data.Concrete.Autogen.Lens.TokenTagging+ , module Data.Concrete.Autogen.Lens.TweetInfo+ , module Data.Concrete.Autogen.Lens.TwitterCoordinates+ , module Data.Concrete.Autogen.Lens.TwitterEntities+ , module Data.Concrete.Autogen.Lens.TwitterLatLong+ , module Data.Concrete.Autogen.Lens.TwitterPlace+ , module Data.Concrete.Autogen.Lens.TwitterUser+ , module Data.Concrete.Autogen.Lens.URL+ , module Data.Concrete.Autogen.Lens.UserMention+ , module Data.Concrete.Autogen.Lens.UUID+ ) where++import Data.Concrete.Autogen.Lens.Annotation+import Data.Concrete.Autogen.Lens.AnnotationMetadata+import Data.Concrete.Autogen.Lens.AnnotationTask+import Data.Concrete.Autogen.Lens.AnnotationTaskType+import Data.Concrete.Autogen.Lens.AnnotationUnitIdentifier+import Data.Concrete.Autogen.Lens.AnnotationUnitType+import Data.Concrete.Autogen.Lens.Arc+import Data.Concrete.Autogen.Lens.Argument+import Data.Concrete.Autogen.Lens.AsyncContactInfo+import Data.Concrete.Autogen.Lens.AudioSpan+import Data.Concrete.Autogen.Lens.BoundingBox+import Data.Concrete.Autogen.Lens.Cluster+import Data.Concrete.Autogen.Lens.Clustering+import Data.Concrete.Autogen.Lens.ClusterMember+import Data.Concrete.Autogen.Lens.Communication+import Data.Concrete.Autogen.Lens.CommunicationMetadata+import Data.Concrete.Autogen.Lens.CommunicationSet+import Data.Concrete.Autogen.Lens.CommunicationTagging+import Data.Concrete.Autogen.Lens.ConcreteThriftException+import Data.Concrete.Autogen.Lens.Constituent+import Data.Concrete.Autogen.Lens.ConstituentRef+import Data.Concrete.Autogen.Lens.Dependency+import Data.Concrete.Autogen.Lens.DependencyParse+import Data.Concrete.Autogen.Lens.DependencyParseStructure+import Data.Concrete.Autogen.Lens.Digest+import Data.Concrete.Autogen.Lens.EmailAddress+import Data.Concrete.Autogen.Lens.EmailCommunicationInfo+import Data.Concrete.Autogen.Lens.Entity+import Data.Concrete.Autogen.Lens.EntityMention+import Data.Concrete.Autogen.Lens.EntityMentionSet+import Data.Concrete.Autogen.Lens.EntitySet+import Data.Concrete.Autogen.Lens.FetchRequest+import Data.Concrete.Autogen.Lens.FetchResult+import Data.Concrete.Autogen.Lens.HashTag+import Data.Concrete.Autogen.Lens.Justification+import Data.Concrete.Autogen.Lens.LanguageIdentification+import Data.Concrete.Autogen.Lens.LatticePath+import Data.Concrete.Autogen.Lens.Link+import Data.Concrete.Autogen.Lens.Linking+import Data.Concrete.Autogen.Lens.LinkTarget+import Data.Concrete.Autogen.Lens.MentionArgument+import Data.Concrete.Autogen.Lens.NITFInfo+import Data.Concrete.Autogen.Lens.NotImplementedException+import Data.Concrete.Autogen.Lens.Parse+import Data.Concrete.Autogen.Lens.PlaceAttributes+import Data.Concrete.Autogen.Lens.Property+import Data.Concrete.Autogen.Lens.SearchCapability+import Data.Concrete.Autogen.Lens.SearchFeedback+import Data.Concrete.Autogen.Lens.SearchQuery+import Data.Concrete.Autogen.Lens.SearchResult+import Data.Concrete.Autogen.Lens.SearchResultItem+import Data.Concrete.Autogen.Lens.SearchType+import Data.Concrete.Autogen.Lens.Section+import Data.Concrete.Autogen.Lens.Sentence+import Data.Concrete.Autogen.Lens.ServiceInfo+import Data.Concrete.Autogen.Lens.ServicesException+import Data.Concrete.Autogen.Lens.Situation+import Data.Concrete.Autogen.Lens.SituationMention+import Data.Concrete.Autogen.Lens.SituationMentionSet+import Data.Concrete.Autogen.Lens.SituationSet+import Data.Concrete.Autogen.Lens.Sound+import Data.Concrete.Autogen.Lens.SpanLink+import Data.Concrete.Autogen.Lens.SummarizationCapability+import Data.Concrete.Autogen.Lens.SummarizationRequest+import Data.Concrete.Autogen.Lens.SummaryConcept+import Data.Concrete.Autogen.Lens.Summary+import Data.Concrete.Autogen.Lens.SummarySourceType+import Data.Concrete.Autogen.Lens.TaggedToken+import Data.Concrete.Autogen.Lens.TextSpan+import Data.Concrete.Autogen.Lens.TheoryDependencies+import Data.Concrete.Autogen.Lens.TimeML+import Data.Concrete.Autogen.Lens.Token+import Data.Concrete.Autogen.Lens.Tokenization+import Data.Concrete.Autogen.Lens.TokenizationKind+import Data.Concrete.Autogen.Lens.TokenLattice+import Data.Concrete.Autogen.Lens.TokenList+import Data.Concrete.Autogen.Lens.TokenRefSequence+import Data.Concrete.Autogen.Lens.TokenTagging+import Data.Concrete.Autogen.Lens.TweetInfo+import Data.Concrete.Autogen.Lens.TwitterCoordinates+import Data.Concrete.Autogen.Lens.TwitterEntities+import Data.Concrete.Autogen.Lens.TwitterLatLong+import Data.Concrete.Autogen.Lens.TwitterPlace+import Data.Concrete.Autogen.Lens.TwitterUser+import Data.Concrete.Autogen.Lens.URL+import Data.Concrete.Autogen.Lens.UserMention+import Data.Concrete.Autogen.Lens.UUID
src/Data/Concrete/Parsers.hs view
@@ -22,81 +22,91 @@ import System.IO (stdin, stdout, stderr, openFile, Handle, IOMode(..), hPutStrLn) import Control.Monad.State (runStateT) import Data.ByteString.Lazy (ByteString)-import Data.Text.Lazy (Text, pack)+import Data.Text.Lazy (Text, pack, unpack) import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..)) import Data.Concrete.Parsers.Types import Control.Monad.IO.Class (liftIO)-import Text.Megaparsec (runParserT', initialPos, State(..), unsafePos, parseErrorPretty, eof, space)+import Text.Megaparsec.Char (space)+import Text.Megaparsec (runParserT', initialPos, State(..), mkPos, parseErrorPretty, eof, ParsecT) import qualified Data.List.NonEmpty as NE+import Data.Void (Void) import Data.Vector (Vector, fromList, snoc, empty)+import Data.Concrete.Parsers.Utils (finalizeCommunication) import qualified Data.Concrete.Parsers.JSON as JSON+import Control.Monad.Identity (Identity(..))+import Data.Conduit.List (unfold)+import Conduit++import Text.Printf (printf) import qualified Data.Concrete.Parsers.CONLL as CONLL+import qualified Data.Concrete.Parsers.CSV as CSV+import qualified Data.Concrete.Parsers.PTB as PTB import qualified Data.Concrete.Parsers.HTML as HTML import qualified Data.Concrete.Parsers.XML as XML-import qualified Data.Concrete.Parsers.CSV as CSV import qualified Data.Concrete.Parsers.Email as Email-import qualified Data.Concrete.Parsers.PTB as PTB+import Control.Monad (void, join, liftM)+import Data.Text.Lazy.Encoding (decodeUtf8) -- | List of ingest configurations and default parameters-communicationParsers = [( "JSON"- , ( "JSON array of arbitrary objects"- , JSON.parser+communicationParsers = [( "JSON-ARRAY"+ , ( "JSON array of objects"+ , JSON.arraySource , [ "catchphrase" , "relatives.0.name" ] , "json_${name}" ) )- , ( "JSON-LINES"+ , ( "JSON-SEQUENCE" , ( "One JSON object per line"- , JSON.lineParser+ , JSON.sequenceSource , [ "author" , "subreddit" ] , "json-lines_${name}" ) )- , ( "CSV"- , ( "CSV format (with header, commas)"- , CSV.parser Nothing ','- , [ "technology"- , "Bush"- , "Gore"- ]- , "csv_${county}"- )- ) , ("CONLL-U" , ( "CONLL-U format"- , CONLL.parser CONLL.conllufields+ , CONLL.sequenceSource CONLL.ufields , ["sentence"] , "conll_${}" ) ) , ( "PTB" , ( "PENN Treebank format"- , PTB.parser- , ["sentence"]- , "ptb_${}"+ , PTB.sequenceSource+ , ["sentence"] :: [Text]+ , "ptb_${}" :: Text ) )+ -- , ( "CSV"+ -- , ( "CSV format (with header, commas)"+ -- , CSV.sequenceSource True ','+ -- , [ "technology"+ -- , "Bush"+ -- , "Gore"+ -- ]+ -- , "csv_${county}"+ -- )+ -- ) -- , ("HTML" -- , ("HTML format"- -- , HTML.parser+ -- , HTML.sequenceSource -- , [] -- , "id_${}" -- ) -- ) -- , ("XML" -- , ("XML format"- -- , XML.parser+ -- , XML.sequenceSource -- , [] -- , "id_${}" -- ) -- ) -- , ("Email" -- , ("Email format"- -- , Email.parser+ -- , Email.sequenceSource -- , [] -- , "id_${}" -- )@@ -104,14 +114,6 @@ ] -- | Run CommunicationAction on each entry created during the ingest process-ingest :: CommunicationAction -> CommunicationParser a -> Text -> [String] -> String -> String -> IO ()-ingest a p t cs i ct = do- let s = State { stateInput=t- , statePos=NE.fromList $ [initialPos "Text File"]- , stateTokensProcessed=0- , stateTabWidth=unsafePos 8- }- ((_, e), _) <- runStateT (runParserT' (space >> p >> space >> eof) s) (Bookkeeper (default_Communication { communication_sectionList=Just empty }) Map.empty [] [] [] [] a cs (pack i) ct 0 0)- case e of- Left x -> putStrLn $ parseErrorPretty x- _ -> return ()+ingest :: (Text -> ConduitM () Communication IO ()) -> (Communication -> IO ()) -> (Communication -> Bool) -> Text -> [Text] -> Text -> Text -> IO ()+ingest src cb filt txt cs idStr ct = do+ runConduit $ src txt .| mergeSource (yieldMany [1..]) .| mapMC (finalizeCommunication idStr []) .| mapMC cb .| sinkNull
src/Data/Concrete/Parsers/CONLL.hs view
@@ -1,10 +1,8 @@ {-# LANGUAGE DeriveGeneric, OverloadedStrings, ApplicativeDo #-} module Data.Concrete.Parsers.CONLL- ( parser- , singleParser- , conllxfields- , conllufields- , conll2009fields+ ( sequenceSource+ , xfields+ , ufields ) where import Data.List (intercalate)@@ -16,19 +14,14 @@ import qualified Data.Map as Map import Data.Map (Map) import Data.List.NonEmpty (fromList)-import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+import Text.Megaparsec.Char.Lexer (symbol, lexeme, signed, scientific) import Text.Megaparsec.Pos (initialPos, defaultTabWidth)-import Text.Megaparsec.Error (Dec) import Text.Megaparsec ( parseErrorPretty , (<|>)- , space- , hexDigitChar , count , manyTill- , anyChar , runParser , some- , char , choice , sepBy , between@@ -37,49 +30,36 @@ , runParserT' , State(..) , getParserState- , eol- , tab- , newline , sepBy1 , many- , noneOf , eof- , separatorChar , someTill )--import Text.Megaparsec.Text.Lazy (Parser)---import Data.Concrete (default_Communication, Communication(..))+import Text.Megaparsec.Char ( eol+ , noneOf+ , separatorChar+ , tab+ , newline+ , char+ , anyChar+ , space+ , hexDigitChar+ ) import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..)) import qualified Control.Monad.State as S import qualified Control.Monad.Identity as I---import Data.Concrete.Types-import Data.Concrete.Parsers.Utils (communicationRule, sectionRule, sentenceRule, tokenRule, pushPathComponent, popPathComponent)---- | Field names for the CONLL-X format-conllxfields = ["ID", "FORM", "LEMMA", "PLEMMA", "POS", "PPOS", "FEAT", "PFEAT", "HEAD", "PHEAD"] :: [Text]+import Data.Concrete.Parsers.Utils (communicationRule, sectionRule, pushPathComponent, popPathComponent)+import Data.Concrete.Parsers.Utils (unfoldParseNewline) --- | Field names for the CONLL-U format-conllufields = ["ID", "FORM", "LEMMA", "UPOSTAG", "XPOSTAG", "FEATS", "HEAD", "DEPREL", "DEPS", "MISC"] :: [Text]+sequenceSource fs = unfoldParseNewline (communicationRule id (sentence fs)) --- | Field names for the CONLL2009 format-conll2009fields = ["ID", "FORM", "LEMMA", "UPOSTAG", "XPOSTAG", "FEATS", "HEAD", "DEPREL", "DEPS", "MISC", "DEPREL", "PDEPREL", "FILLPRED", "PRED"] ++ (map (\x -> pack $ "APRED" ++ (show x)) [1..16]) :: [Text]+ufields = ["ID", "FORM", "LEMMA", "UPOSTAG", "XPOSTAG", "FEATS", "HEAD", "DEPREL", "DEPS", "MISC"] :: [Text]+xfields = ["ID", "FORM", "LEMMA", "PLEMMA", "POS", "PPOS", "FEAT", "PFEAT", "HEAD", "PHEAD", "DEPREL", "PDEPREL"] :: [Text] --- | Parses different flavors of CONLL depending on which fields are passed as arguments--- Each sentence is made into its own Communication-parser :: [Text] -> CommunicationParser ()-parser fs = (communicationRule id (sentence fs)) `sepBy1` sentenceBreak >> return ()+--parserSentence :: [Text] -> CommunicationParser ()+--parser fs = (communicationRule id (sentence fs)) `sepBy1` sentenceBreak >> return () --- | Parses different flavors of CONLL depending on which fields are passed as arguments--- All input sentences are placed in a single Communication-singleParser :: [Text] -> CommunicationParser ()-singleParser fs = (communicationRule id (sentence fs)) `sepBy1` sentenceBreak >> return ()- -sentence fs = do- pushPathComponent "sentence"- (sectionRule id . sentenceRule id) $ (some (commentLine <|> wordLine fs))- popPathComponent- return ()+sentence fs = (some (commentLine <|> wordLine fs)) commentLine = (char '#') >> (manyTill anyChar newline) @@ -91,8 +71,10 @@ newline return () --- | Parse a CONLL cell, which if it's "FORM" should be treated as a Token-namedEntry :: Text -> ParsecT Dec Text (S.StateT Bookkeeper IO) Text-namedEntry f = if f == "FORM" then tokenRule id $ pack <$> (some (noneOf ['\t', '\n'])) else pack <$> (some (noneOf ['\t', '\n']))+namedEntry f = do+ pushPathComponent (unpack f)+ t <- sectionRule id $ pack <$> (some (noneOf ['\t', '\n']))+ popPathComponent+ return t -sentenceBreak = newline+--sentenceBreak = newline
src/Data/Concrete/Parsers/CSV.hs view
@@ -1,6 +1,6 @@ {-# LANGUAGE DeriveGeneric, OverloadedStrings, ApplicativeDo #-} module Data.Concrete.Parsers.CSV- ( parser+ ( sequenceSource ) where import Data.List (intercalate)@@ -10,54 +10,62 @@ import Data.Text.Lazy (pack, unpack, Text) import qualified Data.Text.Lazy as T import Data.Functor (($>))-import Data.Maybe (fromJust) import qualified Data.Map as Map import Data.Map (Map) import Data.List.NonEmpty (fromList)-import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+import Text.Megaparsec.Char.Lexer (symbol, lexeme, signed, scientific) import Text.Megaparsec.Pos (initialPos, defaultTabWidth)-import Text.Megaparsec.Error (Dec)+import Text.Megaparsec.Char ( eol+ , noneOf+ , newline+ , char+ , anyChar+ , space+ , hexDigitChar+ ) import Text.Megaparsec ( parseErrorPretty , (<|>)- , space- , hexDigitChar , count , manyTill- , anyChar , runParser , some- , char , choice , sepBy , sepBy1 , sepEndBy , between , match- , newline , ParsecT , runParserT' , State(..) , getParserState- , eol- , noneOf , eof , many ) import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..))-import Text.Megaparsec.Text.Lazy (Parser)+--import Text.Megaparsec.Text.Lazy (Parser)+--import Data.Concrete (default_Communication, Communication(..)) import qualified Control.Monad.State as S import qualified Control.Monad.Identity as I+--import Data.Concrete.Types+import Data.Concrete.Parsers.Utils (unfoldParse, unfoldParseArray) import Data.Concrete.Parsers.Utils (communicationRule, sectionRule, pushPathComponent, popPathComponent)+import Conduit --- | Parser for CSV files-parser :: Maybe [Text] -> Char -> CommunicationParser ()-parser h d = do- fs <- if h == Nothing then header d else return $ fromJust h- space- withFields fs d- space- eof- return ()+--sequenceSource = undefined+sequenceSource :: Bool -> Char -> [String] -> Text -> Text -> Text -> ConduitM () Communication IO ()+sequenceSource h d = undefined -- do+-- fs <- if h == True then header d else return []+-- unfoldParse (communicationRule id (row d fs))++-- parser :: Bool -> Char -> CommunicationParser ()+-- parser h d = do+-- fs <- if h == True then header d else return []+-- space+-- withFields fs d+-- space+-- eof+-- return () withFields :: [Text] -> Char -> CommunicationParser () withFields fs d = (communicationRule id (row d fs)) `sepEndBy` (newline) >> return ()
src/Data/Concrete/Parsers/Email.hs view
@@ -1,8 +1,9 @@ {-# LANGUAGE DeriveGeneric, OverloadedStrings, ApplicativeDo #-} module Data.Concrete.Parsers.Email- ( parser+ ( sequenceSource ) where +import Data.Char (isSpace) import Data.List (intercalate) import Data.Concrete.Parsers.Types (Bookkeeper(..), CommunicationParser) import Data.Concrete.Parsers.Utils (communicationRule)@@ -12,19 +13,28 @@ import qualified Data.Map as Map import Data.Map (Map) import Data.List.NonEmpty (fromList)-import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+--import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+ import Text.Megaparsec.Pos (initialPos, defaultTabWidth)-import Text.Megaparsec.Error (Dec)+--import Text.Megaparsec.Error (Dec)+import Text.Megaparsec.Char.Lexer (symbol, lexeme, signed, scientific)+import Text.Megaparsec.Char ( eol+ , noneOf+ , newline+ , char+ , anyChar+ , space+ , hexDigitChar+ , tab+ , separatorChar+ , satisfy+ ) import Text.Megaparsec ( parseErrorPretty , (<|>)- , space- , hexDigitChar , count , manyTill- , anyChar , runParser , some- , char , choice , sepBy , between@@ -33,19 +43,71 @@ , runParserT' , State(..) , getParserState+ , sepBy1 , many+ , eof+ , someTill )+-- import Text.Megaparsec ( parseErrorPretty+-- , (<|>)+-- , satisfy+-- , space+-- , hexDigitChar+-- , count+-- , manyTill+-- , anyChar+-- , runParser+-- , some+-- , char+-- , choice+-- , sepBy+-- , between+-- , match+-- , ParsecT+-- , runParserT'+-- , State(..)+-- , getParserState+-- , spaceChar+-- , eof+-- , noneOf+-- , try+-- ) -import Text.Megaparsec.Text.Lazy (Parser)+import Control.Monad.IO.Class (liftIO) +--import Text.Megaparsec.Text.Lazy (Parser) import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..))---import Data.Concrete (default_Communication, Communication(..)) import qualified Control.Monad.State as S import qualified Control.Monad.Identity as I --import Data.Concrete.Types import Data.Concrete.Parsers.Utils (communicationRule, sectionRule)-import qualified Data.Concrete.Parsers.RFC2822 as R +sequenceSource = undefined+ parser :: CommunicationParser ()-parser = email >> return ()+parser = do+ space+ some document+ space+ eof+ return () -email = many anyChar+-- type CS = CommunicationParser String+-- type CSS = CommunicationParser [String]+-- type CC = CommunicationParser Char++document :: CommunicationParser ()+document = lexeme' $ communicationRule id (parens (some sentence)) >> return ()++sentence = lexeme' $ between (symbol' "(S") (symbol' ")") (some phrase)++phrase = lexeme' $ parens (tag >> some (tag <|> phrase)) >> return []++tag = lexicalItem++lexicalItem = lexeme' $ some notSpaceOrParen++notSpaceOrParen = satisfy (\c -> and [(not . isSpace) c, ('(' /= c), (')' /= c)])++lexeme' = lexeme space+symbol' = symbol space+parens = between (symbol' "(") (symbol' ")")
src/Data/Concrete/Parsers/HTML.hs view
@@ -1,31 +1,40 @@-{-# LANGUAGE DeriveGeneric, OverloadedStrings #-}+{-# LANGUAGE DeriveGeneric, OverloadedStrings, ApplicativeDo #-} module Data.Concrete.Parsers.HTML- ( parser+ ( sequenceSource ) where -import Data.Maybe (fromJust)+import Data.Char (isSpace) import Data.List (intercalate) import Data.Concrete.Parsers.Types (Bookkeeper(..), CommunicationParser)-import Data.Concrete.Parsers.Utils (communicationRule, Located(..))+import Data.Concrete.Parsers.Utils (communicationRule) import Data.Scientific (scientific, Scientific(..)) import Data.Text.Lazy (pack, Text) import Data.Functor (($>)) import qualified Data.Map as Map import Data.Map (Map) import Data.List.NonEmpty (fromList)-import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+--import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+ import Text.Megaparsec.Pos (initialPos, defaultTabWidth)-import Text.Megaparsec.Error (Dec)+--import Text.Megaparsec.Error (Dec)+import Text.Megaparsec.Char.Lexer (symbol, lexeme, signed, scientific)+import Text.Megaparsec.Char ( eol+ , noneOf+ , newline+ , char+ , anyChar+ , space+ , hexDigitChar+ , tab+ , separatorChar+ , satisfy+ ) import Text.Megaparsec ( parseErrorPretty , (<|>)- , space- , hexDigitChar , count , manyTill- , anyChar , runParser , some- , char , choice , sepBy , between@@ -34,18 +43,71 @@ , runParserT' , State(..) , getParserState+ , sepBy1 , many+ , eof+ , someTill )+-- import Text.Megaparsec ( parseErrorPretty+-- , (<|>)+-- , satisfy+-- , space+-- , hexDigitChar+-- , count+-- , manyTill+-- , anyChar+-- , runParser+-- , some+-- , char+-- , choice+-- , sepBy+-- , between+-- , match+-- , ParsecT+-- , runParserT'+-- , State(..)+-- , getParserState+-- , spaceChar+-- , eof+-- , noneOf+-- , try+-- ) -import Text.Megaparsec.Text.Lazy (Parser)+import Control.Monad.IO.Class (liftIO) +--import Text.Megaparsec.Text.Lazy (Parser) import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..))---import Data.Concrete (default_Communication, Communication(..), Section(..), TextSpan(..)) import qualified Control.Monad.State as S import qualified Control.Monad.Identity as I --import Data.Concrete.Types import Data.Concrete.Parsers.Utils (communicationRule, sectionRule) +sequenceSource = undefined+ parser :: CommunicationParser ()-parser = html >> return ()+parser = do+ space+ some document+ space+ eof+ return () -html = many anyChar+-- type CS = CommunicationParser String+-- type CSS = CommunicationParser [String]+-- type CC = CommunicationParser Char++document :: CommunicationParser ()+document = lexeme' $ communicationRule id (parens (some sentence)) >> return ()++sentence = lexeme' $ between (symbol' "(S") (symbol' ")") (some phrase)++phrase = lexeme' $ parens (tag >> some (tag <|> phrase)) >> return []++tag = lexicalItem++lexicalItem = lexeme' $ some notSpaceOrParen++notSpaceOrParen = satisfy (\c -> and [(not . isSpace) c, ('(' /= c), (')' /= c)])++lexeme' = lexeme space+symbol' = symbol space+parens = between (symbol' "(") (symbol' ")")
src/Data/Concrete/Parsers/JSON.hs view
@@ -1,33 +1,32 @@ {-# LANGUAGE DeriveGeneric, OverloadedStrings #-} module Data.Concrete.Parsers.JSON- ( parser- , lineParser+ ( parseCommunication+ , sequenceSource+ , arraySource ) where import Control.Monad.State (State, get, put, modify, modify') import Data.Maybe (fromJust) import Data.List (intercalate)-import Data.Scientific (scientific, Scientific(..)) import Data.Text.Lazy (pack, Text) import Data.Functor (($>)) import qualified Data.Map as Map import Data.Map (Map) import Data.List.NonEmpty (fromList)-import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+import Text.Megaparsec.Char.Lexer (symbol, lexeme, signed, scientific) import Text.Megaparsec.Pos (initialPos, defaultTabWidth)-import Text.Megaparsec.Error (Dec)+import Text.Megaparsec.Char ( char+ , space+ , anyChar+ , hexDigitChar+ , satisfy+ ) import Text.Megaparsec ( parseErrorPretty , (<|>)- , eol- , space- , hexDigitChar , count , manyTill- , anyChar , runParser- , try , some- , char , choice , sepBy , between@@ -36,15 +35,16 @@ , runParserT' , State(..) , getParserState- , eof , many+ , eof+ , try+ , mkPos ) import Control.Monad.IO.Class (liftIO)-import Text.Megaparsec.Text.Lazy (Parser) import qualified Control.Monad.State as S import qualified Control.Monad.Identity as I import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..))-import Data.Concrete.Parsers.Types (Bookkeeper(..), CommunicationParser)+import Data.Concrete.Parsers.Types (Bookkeeper(..), CommunicationParser, IngestStream) import Data.Concrete.Parsers.Utils ( communicationRule , sectionRule , pathArrayRule@@ -57,22 +57,29 @@ , modifyPathComponent , incrementPathComponent )+import Data.Concrete.Parsers.Utils (unfoldParse, unfoldParseArray)+import Conduit+import qualified Data.List.NonEmpty as NE --- | Parses an array of JSON objects, turning each into a Communication-parser :: CommunicationParser ()-parser = brackets ((communicationRule id objectP) `sepBy` comma) >> return ()+-- | Parses a sequence of JSON objects into a stream+sequenceSource :: Text -> ConduitM () Communication IO ()+sequenceSource = unfoldParse parseCommunication --- | Parses a sequence of JSON objects (i.e. not a valid JSON object overall), like one object per line-lineParser :: CommunicationParser ()-lineParser = (many (communicationRule id objectP)) >> return ()+-- | Parses an array of JSON objects into a stream+arraySource :: Text -> ConduitM () Communication IO ()+arraySource = unfoldParseArray parseCommunication +-- | Parser for turning a single JSON object into a Communication+parseCommunication :: CommunicationParser Communication+parseCommunication = communicationRule id objectP -- >> return default_Communication+ jsonP = lexeme' $ choice [nullP, numberP, stringP, boolP, objectP, arrayP] nullP = sectionRule id $ symbol' "null" >> return () boolP = sectionRule id $ (symbol' "true" <|> symbol' "false") >> return () -numberP = sectionRule id $ signed space number >> return ()+numberP = sectionRule id $ signed space scientific >> return () stringP = stringPLiteral >> return ()
src/Data/Concrete/Parsers/PTB.hs view
@@ -1,6 +1,6 @@ {-# LANGUAGE DeriveGeneric, OverloadedStrings, ApplicativeDo #-} module Data.Concrete.Parsers.PTB- ( parser+ ( sequenceSource ) where import Data.Char (isSpace)@@ -13,21 +13,28 @@ import qualified Data.Map as Map import Data.Map (Map) import Data.List.NonEmpty (fromList)-import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+--import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+ import Text.Megaparsec.Pos (initialPos, defaultTabWidth)-import Text.Megaparsec.Error (Dec)-import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+--import Text.Megaparsec.Error (Dec)+import Text.Megaparsec.Char.Lexer (symbol, lexeme, signed, scientific)+import Text.Megaparsec.Char ( eol+ , noneOf+ , newline+ , char+ , anyChar+ , space+ , hexDigitChar+ , tab+ , separatorChar+ , satisfy+ ) import Text.Megaparsec ( parseErrorPretty , (<|>)- , satisfy- , space- , hexDigitChar , count , manyTill- , anyChar , runParser , some- , char , choice , sepBy , between@@ -36,21 +43,51 @@ , runParserT' , State(..) , getParserState- , spaceChar+ , sepBy1+ , many , eof- , noneOf- , try+ , someTill )+-- import Text.Megaparsec ( parseErrorPretty+-- , (<|>)+-- , satisfy+-- , space+-- , hexDigitChar+-- , count+-- , manyTill+-- , anyChar+-- , runParser+-- , some+-- , char+-- , choice+-- , sepBy+-- , between+-- , match+-- , ParsecT+-- , runParserT'+-- , State(..)+-- , getParserState+-- , spaceChar+-- , eof+-- , noneOf+-- , try+-- ) import Control.Monad.IO.Class (liftIO) -import Text.Megaparsec.Text.Lazy (Parser)+--import Text.Megaparsec.Text.Lazy (Parser) import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..)) import qualified Control.Monad.State as S import qualified Control.Monad.Identity as I-import Data.Concrete.Parsers.Utils (communicationRule, sectionRule, sentenceRule, tokenRule, pushPathComponent, popPathComponent)+--import Data.Concrete.Types+import Data.Concrete.Parsers.Utils (communicationRule, sectionRule)+import Data.Concrete.Parsers.Utils (unfoldParse, unfoldParseArray)+import Conduit --- | Parser for PENN Treebank format--- NOTE: currently, doesn't capture tags/parses+--sequenceSource = undefined+-- | Parses a sequence of JSON objects into a stream+sequenceSource :: Text -> ConduitM () Communication IO ()+sequenceSource = unfoldParse (communicationRule id (parens (some sentence)))+ parser :: CommunicationParser () parser = do space@@ -59,19 +96,18 @@ eof return () +-- type CS = CommunicationParser String+-- type CSS = CommunicationParser [String]+-- type CC = CommunicationParser Char+ document :: CommunicationParser () document = lexeme' $ communicationRule id (parens (some sentence)) >> return () -sentence = do- pushPathComponent "sentence"- (sectionRule id . sentenceRule id) $ lexeme' $ between (symbol' "(S") (symbol' ")") (some phrase)- popPathComponent - -phrase = lexeme' $ parens (tag >> some (token <|> phrase)) >> return []+sentence = lexeme' $ between (symbol' "(S") (symbol' ")") (some phrase) -tag = lexicalItem+phrase = lexeme' $ parens (tag >> some (tag <|> phrase)) >> return [] -token = tokenRule id lexicalItem+tag = lexicalItem lexicalItem = lexeme' $ some notSpaceOrParen
− src/Data/Concrete/Parsers/RFC2822.lhs
@@ -1,2889 +0,0 @@-{-# LANGUAGE DeriveGeneric, OverloadedStrings, ApplicativeDo #-}--> module Data.Concrete.Parsers.RFC2822-> (-> ) where--import Data.List (intercalate)-import Data.Concrete.Parsers.Types (Bookkeeper(..), CommunicationParser)-import Data.Concrete.Parsers.Utils (communicationRule)-import Data.Scientific (scientific, Scientific(..))-import Data.Text.Lazy (pack, Text)-import Data.Functor (($>))-import qualified Data.Map as Map-import Data.Map (Map)-import Data.List.NonEmpty (fromList)-import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)-import Text.Megaparsec.Pos (initialPos, defaultTabWidth)-import Text.Megaparsec.Error (Dec)-import Text.Megaparsec ( parseErrorPretty- , (<|>)- , space- , hexDigitChar- , count- , manyTill- , anyChar- , runParser- , some- , char- , choice- , sepBy- , between- , match- , ParsecT- , runParserT'- , State(..)- , getParserState- , many- )--import Text.Megaparsec.Text.Lazy (Parser)-import Data.Concrete (default_Communication, Communication(..))-import qualified Control.Monad.State as S-import qualified Control.Monad.Identity as I-import Data.Concrete.Types-import Data.Concrete.Parsers.Utils (communicationRule, sectionRule)--parser :: CommunicationParser ()-parser = email >> return ()--email = many anyChar-----Network Working Group P. Resnick, Editor-Request for Comments: 2822 QUALCOMM Incorporated-Obsoletes: 822 April 2001-Category: Standards Track--- Internet Message Format--Status of this Memo-- This document specifies an Internet standards track protocol for the- Internet community, and requests discussion and suggestions for- improvements. Please refer to the current edition of the "Internet- Official Protocol Standards" (STD 1) for the standardization state- and status of this protocol. Distribution of this memo is unlimited.--Copyright Notice-- Copyright (C) The Internet Society (2001). All Rights Reserved.--Abstract-- This standard specifies a syntax for text messages that are sent- between computer users, within the framework of "electronic mail"- messages. This standard supersedes the one specified in Request For- Comments (RFC) 822, "Standard for the Format of ARPA Internet Text- Messages", updating it to reflect current practice and incorporating- incremental changes that were specified in other RFCs.--Table of Contents-- 1. Introduction ............................................... 3- 1.1. Scope .................................................... 3- 1.2. Notational conventions ................................... 4- 1.2.1. Requirements notation .................................. 4- 1.2.2. Syntactic notation ..................................... 4- 1.3. Structure of this document ............................... 4- 2. Lexical Analysis of Messages ............................... 5- 2.1. General Description ...................................... 5- 2.1.1. Line Length Limits ..................................... 6- 2.2. Header Fields ............................................ 7- 2.2.1. Unstructured Header Field Bodies ....................... 7- 2.2.2. Structured Header Field Bodies ......................... 7- 2.2.3. Long Header Fields ..................................... 7- 2.3. Body ..................................................... 8- 3. Syntax ..................................................... 9- 3.1. Introduction ............................................. 9- 3.2. Lexical Tokens ........................................... 9----Resnick Standards Track [Page 1]--RFC 2822 Internet Message Format April 2001--- 3.2.1. Primitive Tokens ....................................... 9- 3.2.2. Quoted characters ......................................10- 3.2.3. Folding white space and comments .......................11- 3.2.4. Atom ...................................................12- 3.2.5. Quoted strings .........................................13- 3.2.6. Miscellaneous tokens ...................................13- 3.3. Date and Time Specification ..............................14- 3.4. Address Specification ....................................15- 3.4.1. Addr-spec specification ................................16- 3.5 Overall message syntax ....................................17- 3.6. Field definitions ........................................18- 3.6.1. The origination date field .............................20- 3.6.2. Originator fields ......................................21- 3.6.3. Destination address fields .............................22- 3.6.4. Identification fields ..................................23- 3.6.5. Informational fields ...................................26- 3.6.6. Resent fields ..........................................26- 3.6.7. Trace fields ...........................................28- 3.6.8. Optional fields ........................................29- 4. Obsolete Syntax ............................................29- 4.1. Miscellaneous obsolete tokens ............................30- 4.2. Obsolete folding white space .............................31- 4.3. Obsolete Date and Time ...................................31- 4.4. Obsolete Addressing ......................................33- 4.5. Obsolete header fields ...................................33- 4.5.1. Obsolete origination date field ........................34- 4.5.2. Obsolete originator fields .............................34- 4.5.3. Obsolete destination address fields ....................34- 4.5.4. Obsolete identification fields .........................35- 4.5.5. Obsolete informational fields ..........................35- 4.5.6. Obsolete resent fields .................................35- 4.5.7. Obsolete trace fields ..................................36- 4.5.8. Obsolete optional fields ...............................36- 5. Security Considerations ....................................36- 6. Bibliography ...............................................37- 7. Editor's Address ...........................................38- 8. Acknowledgements ...........................................39- Appendix A. Example messages ..................................41- A.1. Addressing examples ......................................41- A.1.1. A message from one person to another with simple- addressing .............................................41- A.1.2. Different types of mailboxes ...........................42- A.1.3. Group addresses ........................................43- A.2. Reply messages ...........................................43- A.3. Resent messages ..........................................44- A.4. Messages with trace fields ...............................46- A.5. White space, comments, and other oddities ................47- A.6. Obsoleted forms ..........................................47----Resnick Standards Track [Page 2]--RFC 2822 Internet Message Format April 2001--- A.6.1. Obsolete addressing ....................................48- A.6.2. Obsolete dates .........................................48- A.6.3. Obsolete white space and comments ......................48- Appendix B. Differences from earlier standards ................49- Appendix C. Notices ...........................................50- Full Copyright Statement ......................................51--1. Introduction--1.1. Scope-- This standard specifies a syntax for text messages that are sent- between computer users, within the framework of "electronic mail"- messages. This standard supersedes the one specified in Request For- Comments (RFC) 822, "Standard for the Format of ARPA Internet Text- Messages" [RFC822], updating it to reflect current practice and- incorporating incremental changes that were specified in other RFCs- [STD3].-- This standard specifies a syntax only for text messages. In- particular, it makes no provision for the transmission of images,- audio, or other sorts of structured data in electronic mail messages.- There are several extensions published, such as the MIME document- series [RFC2045, RFC2046, RFC2049], which describe mechanisms for the- transmission of such data through electronic mail, either by- extending the syntax provided here or by structuring such messages to- conform to this syntax. Those mechanisms are outside of the scope of- this standard.-- In the context of electronic mail, messages are viewed as having an- envelope and contents. The envelope contains whatever information is- needed to accomplish transmission and delivery. (See [RFC2821] for a- discussion of the envelope.) The contents comprise the object to be- delivered to the recipient. This standard applies only to the format- and some of the semantics of message contents. It contains no- specification of the information in the envelope.-- However, some message systems may use information from the contents- to create the envelope. It is intended that this standard facilitate- the acquisition of such information by programs.-- This specification is intended as a definition of what message- content format is to be passed between systems. Though some message- systems locally store messages in this format (which eliminates the- need for translation between formats) and others use formats that- differ from the one specified in this standard, local storage is- outside of the scope of this standard.-----Resnick Standards Track [Page 3]--RFC 2822 Internet Message Format April 2001--- Note: This standard is not intended to dictate the internal formats- used by sites, the specific message system features that they are- expected to support, or any of the characteristics of user interface- programs that create or read messages. In addition, this standard- does not specify an encoding of the characters for either transport- or storage; that is, it does not specify the number of bits used or- how those bits are specifically transferred over the wire or stored- on disk.--1.2. Notational conventions--1.2.1. Requirements notation-- This document occasionally uses terms that appear in capital letters.- When the terms "MUST", "SHOULD", "RECOMMENDED", "MUST NOT", "SHOULD- NOT", and "MAY" appear capitalized, they are being used to indicate- particular requirements of this specification. A discussion of the- meanings of these terms appears in [RFC2119].--1.2.2. Syntactic notation-- This standard uses the Augmented Backus-Naur Form (ABNF) notation- specified in [RFC2234] for the formal definitions of the syntax of- messages. Characters will be specified either by a decimal value- (e.g., the value %d65 for uppercase A and %d97 for lowercase A) or by- a case-insensitive literal value enclosed in quotation marks (e.g.,- "A" for either uppercase or lowercase A). See [RFC2234] for the full- description of the notation.--1.3. Structure of this document-- This document is divided into several sections.-- This section, section 1, is a short introduction to the document.-- Section 2 lays out the general description of a message and its- constituent parts. This is an overview to help the reader understand- some of the general principles used in the later portions of this- document. Any examples in this section MUST NOT be taken as- specification of the formal syntax of any part of a message.-- Section 3 specifies formal ABNF rules for the structure of each part- of a message (the syntax) and describes the relationship between- those parts and their meaning in the context of a message (the- semantics). That is, it describes the actual rules for the structure- of each part of a message (the syntax) as well as a description of- the parts and instructions on how they ought to be interpreted (the- semantics). This includes analysis of the syntax and semantics of----Resnick Standards Track [Page 4]--RFC 2822 Internet Message Format April 2001--- subparts of messages that have specific structure. The syntax- included in section 3 represents messages as they MUST be created.- There are also notes in section 3 to indicate if any of the options- specified in the syntax SHOULD be used over any of the others.-- Both sections 2 and 3 describe messages that are legal to generate- for purposes of this standard.-- Section 4 of this document specifies an "obsolete" syntax. There are- references in section 3 to these obsolete syntactic elements. The- rules of the obsolete syntax are elements that have appeared in- earlier revisions of this standard or have previously been widely- used in Internet messages. As such, these elements MUST be- interpreted by parsers of messages in order to be conformant to this- standard. However, since items in this syntax have been determined- to be non-interoperable or to cause significant problems for- recipients of messages, they MUST NOT be generated by creators of- conformant messages.-- Section 5 details security considerations to take into account when- implementing this standard.-- Section 6 is a bibliography of references in this document.-- Section 7 contains the editor's address.-- Section 8 contains acknowledgements.-- Appendix A lists examples of different sorts of messages. These- examples are not exhaustive of the types of messages that appear on- the Internet, but give a broad overview of certain syntactic forms.-- Appendix B lists the differences between this standard and earlier- standards for Internet messages.-- Appendix C has copyright and intellectual property notices.--2. Lexical Analysis of Messages--2.1. General Description-- At the most basic level, a message is a series of characters. A- message that is conformant with this standard is comprised of- characters with values in the range 1 through 127 and interpreted as- US-ASCII characters [ASCII]. For brevity, this document sometimes- refers to this range of characters as simply "US-ASCII characters".------Resnick Standards Track [Page 5]--RFC 2822 Internet Message Format April 2001--- Note: This standard specifies that messages are made up of characters- in the US-ASCII range of 1 through 127. There are other documents,- specifically the MIME document series [RFC2045, RFC2046, RFC2047,- RFC2048, RFC2049], that extend this standard to allow for values- outside of that range. Discussion of those mechanisms is not within- the scope of this standard.-- Messages are divided into lines of characters. A line is a series of- characters that is delimited with the two characters carriage-return- and line-feed; that is, the carriage return (CR) character (ASCII- value 13) followed immediately by the line feed (LF) character (ASCII- value 10). (The carriage-return/line-feed pair is usually written in- this document as "CRLF".)-- A message consists of header fields (collectively called "the header- of the message") followed, optionally, by a body. The header is a- sequence of lines of characters with special syntax as defined in- this standard. The body is simply a sequence of characters that- follows the header and is separated from the header by an empty line- (i.e., a line with nothing preceding the CRLF).--2.1.1. Line Length Limits-- There are two limits that this standard places on the number of- characters in a line. Each line of characters MUST be no more than- 998 characters, and SHOULD be no more than 78 characters, excluding- the CRLF.-- The 998 character limit is due to limitations in many implementations- which send, receive, or store Internet Message Format messages that- simply cannot handle more than 998 characters on a line. Receiving- implementations would do well to handle an arbitrarily large number- of characters in a line for robustness sake. However, there are so- many implementations which (in compliance with the transport- requirements of [RFC2821]) do not accept messages containing more- than 1000 character including the CR and LF per line, it is important- for implementations not to create such messages.-- The more conservative 78 character recommendation is to accommodate- the many implementations of user interfaces that display these- messages which may truncate, or disastrously wrap, the display of- more than 78 characters per line, in spite of the fact that such- implementations are non-conformant to the intent of this- specification (and that of [RFC2821] if they actually cause- information to be lost). Again, even though this limitation is put on- messages, it is encumbant upon implementations which display messages------Resnick Standards Track [Page 6]--RFC 2822 Internet Message Format April 2001--- to handle an arbitrarily large number of characters in a line- (certainly at least up to the 998 character limit) for the sake of- robustness.--2.2. Header Fields-- Header fields are lines composed of a field name, followed by a colon- (":"), followed by a field body, and terminated by CRLF. A field- name MUST be composed of printable US-ASCII characters (i.e.,- characters that have values between 33 and 126, inclusive), except- colon. A field body may be composed of any US-ASCII characters,- except for CR and LF. However, a field body may contain CRLF when- used in header "folding" and "unfolding" as described in section- 2.2.3. All field bodies MUST conform to the syntax described in- sections 3 and 4 of this standard.--2.2.1. Unstructured Header Field Bodies-- Some field bodies in this standard are defined simply as- "unstructured" (which is specified below as any US-ASCII characters,- except for CR and LF) with no further restrictions. These are- referred to as unstructured field bodies. Semantically, unstructured- field bodies are simply to be treated as a single line of characters- with no further processing (except for header "folding" and- "unfolding" as described in section 2.2.3).--2.2.2. Structured Header Field Bodies-- Some field bodies in this standard have specific syntactical- structure more restrictive than the unstructured field bodies- described above. These are referred to as "structured" field bodies.- Structured field bodies are sequences of specific lexical tokens as- described in sections 3 and 4 of this standard. Many of these tokens- are allowed (according to their syntax) to be introduced or end with- comments (as described in section 3.2.3) as well as the space (SP,- ASCII value 32) and horizontal tab (HTAB, ASCII value 9) characters- (together known as the white space characters, WSP), and those WSP- characters are subject to header "folding" and "unfolding" as- described in section 2.2.3. Semantic analysis of structured field- bodies is given along with their syntax.--2.2.3. Long Header Fields-- Each header field is logically a single line of characters comprising- the field name, the colon, and the field body. For convenience- however, and to deal with the 998/78 character limitations per line,- the field body portion of a header field can be split into a multiple- line representation; this is called "folding". The general rule is----Resnick Standards Track [Page 7]--RFC 2822 Internet Message Format April 2001--- that wherever this standard allows for folding white space (not- simply WSP characters), a CRLF may be inserted before any WSP. For- example, the header field:-- Subject: This is a test-- can be represented as:-- Subject: This- is a test-- Note: Though structured field bodies are defined in such a way that- folding can take place between many of the lexical tokens (and even- within some of the lexical tokens), folding SHOULD be limited to- placing the CRLF at higher-level syntactic breaks. For instance, if- a field body is defined as comma-separated values, it is recommended- that folding occur after the comma separating the structured items in- preference to other places where the field could be folded, even if- it is allowed elsewhere.-- The process of moving from this folded multiple-line representation- of a header field to its single line representation is called- "unfolding". Unfolding is accomplished by simply removing any CRLF- that is immediately followed by WSP. Each header field should be- treated in its unfolded form for further syntactic and semantic- evaluation.--2.3. Body-- The body of a message is simply lines of US-ASCII characters. The- only two limitations on the body are as follows:-- - CR and LF MUST only occur together as CRLF; they MUST NOT appear- independently in the body.-- - Lines of characters in the body MUST be limited to 998 characters,- and SHOULD be limited to 78 characters, excluding the CRLF.-- Note: As was stated earlier, there are other standards documents,- specifically the MIME documents [RFC2045, RFC2046, RFC2048, RFC2049]- that extend this standard to allow for different sorts of message- bodies. Again, these mechanisms are beyond the scope of this- document.---------Resnick Standards Track [Page 8]--RFC 2822 Internet Message Format April 2001---3. Syntax--3.1. Introduction-- The syntax as given in this section defines the legal syntax of- Internet messages. Messages that are conformant to this standard- MUST conform to the syntax in this section. If there are options in- this section where one option SHOULD be generated, that is indicated- either in the prose or in a comment next to the syntax.-- For the defined expressions, a short description of the syntax and- use is given, followed by the syntax in ABNF, followed by a semantic- analysis. Primitive tokens that are used but otherwise unspecified- come from [RFC2234].-- In some of the definitions, there will be nonterminals whose names- start with "obs-". These "obs-" elements refer to tokens defined in- the obsolete syntax in section 4. In all cases, these productions- are to be ignored for the purposes of generating legal Internet- messages and MUST NOT be used as part of such a message. However,- when interpreting messages, these tokens MUST be honored as part of- the legal syntax. In this sense, section 3 defines a grammar for- generation of messages, with "obs-" elements that are to be ignored,- while section 4 adds grammar for interpretation of messages.--3.2. Lexical Tokens-- The following rules are used to define an underlying lexical- analyzer, which feeds tokens to the higher-level parsers. This- section defines the tokens used in structured header field bodies.-- Note: Readers of this standard need to pay special attention to how- these lexical tokens are used in both the lower-level and- higher-level syntax later in the document. Particularly, the white- space tokens and the comment tokens defined in section 3.2.3 get used- in the lower-level tokens defined here, and those lower-level tokens- are in turn used as parts of the higher-level tokens defined later.- Therefore, the white space and comments may be allowed in the- higher-level tokens even though they may not explicitly appear in a- particular definition.--3.2.1. Primitive Tokens-- The following are primitive tokens referred to elsewhere in this- standard, but not otherwise defined in [RFC2234]. Some of them will- not appear anywhere else in the syntax, but they are convenient to- refer to in other parts of this document.-----Resnick Standards Track [Page 9]--RFC 2822 Internet Message Format April 2001--- Note: The "specials" below are just such an example. Though the- specials token does not appear anywhere else in this standard, it is- useful for implementers who use tools that lexically analyze- messages. Each of the characters in specials can be used to indicate- a tokenization point in lexical analysis.--NO-WS-CTL = %d1-8 / ; US-ASCII control characters- %d11 / ; that do not include the- %d12 / ; carriage return, line feed,- %d14-31 / ; and white space characters- %d127--text = %d1-9 / ; Characters excluding CR and LF- %d11 /- %d12 /- %d14-127 /- obs-text--specials = "(" / ")" / ; Special characters used in- "<" / ">" / ; other parts of the syntax- "[" / "]" /- ":" / ";" /- "@" / "\" /- "," / "." /- DQUOTE-- No special semantics are attached to these tokens. They are simply- single characters.--3.2.2. Quoted characters-- Some characters are reserved for special interpretation, such as- delimiting lexical tokens. To permit use of these characters as- uninterpreted data, a quoting mechanism is provided.--quoted-pair = ("\" text) / obs-qp-- Where any quoted-pair appears, it is to be interpreted as the text- character alone. That is to say, the "\" character that appears as- part of a quoted-pair is semantically "invisible".-- Note: The "\" character may appear in a message where it is not part- of a quoted-pair. A "\" character that does not appear in a- quoted-pair is not semantically invisible. The only places in this- standard where quoted-pair currently appears are ccontent, qcontent,- dcontent, no-fold-quote, and no-fold-literal.------Resnick Standards Track [Page 10]--RFC 2822 Internet Message Format April 2001---3.2.3. Folding white space and comments-- White space characters, including white space used in folding- (described in section 2.2.3), may appear between many elements in- header field bodies. Also, strings of characters that are treated as- comments may be included in structured field bodies as characters- enclosed in parentheses. The following defines the folding white- space (FWS) and comment constructs.-- Strings of characters enclosed in parentheses are considered comments- so long as they do not appear within a "quoted-string", as defined in- section 3.2.5. Comments may nest.-- There are several places in this standard where comments and FWS may- be freely inserted. To accommodate that syntax, an additional token- for "CFWS" is defined for places where comments and/or FWS can occur.- However, where CFWS occurs in this standard, it MUST NOT be inserted- in such a way that any line of a folded header field is made up- entirely of WSP characters and nothing else.--FWS = ([*WSP CRLF] 1*WSP) / ; Folding white space- obs-FWS--ctext = NO-WS-CTL / ; Non white space controls-- %d33-39 / ; The rest of the US-ASCII- %d42-91 / ; characters not including "(",- %d93-126 ; ")", or "\"--ccontent = ctext / quoted-pair / comment--comment = "(" *([FWS] ccontent) [FWS] ")"--CFWS = *([FWS] comment) (([FWS] comment) / FWS)-- Throughout this standard, where FWS (the folding white space token)- appears, it indicates a place where header folding, as discussed in- section 2.2.3, may take place. Wherever header folding appears in a- message (that is, a header field body containing a CRLF followed by- any WSP), header unfolding (removal of the CRLF) is performed before- any further lexical analysis is performed on that header field- according to this standard. That is to say, any CRLF that appears in- FWS is semantically "invisible."-- A comment is normally used in a structured field body to provide some- human readable informational text. Since a comment is allowed to- contain FWS, folding is permitted within the comment. Also note that- since quoted-pair is allowed in a comment, the parentheses and----Resnick Standards Track [Page 11]--RFC 2822 Internet Message Format April 2001--- backslash characters may appear in a comment so long as they appear- as a quoted-pair. Semantically, the enclosing parentheses are not- part of the comment; the comment is what is contained between the two- parentheses. As stated earlier, the "\" in any quoted-pair and the- CRLF in any FWS that appears within the comment are semantically- "invisible" and therefore not part of the comment either.-- Runs of FWS, comment or CFWS that occur between lexical tokens in a- structured field header are semantically interpreted as a single- space character.--3.2.4. Atom-- Several productions in structured header field bodies are simply- strings of certain basic characters. Such productions are called- atoms.-- Some of the structured header field bodies also allow the period- character (".", ASCII value 46) within runs of atext. An additional- "dot-atom" token is defined for those purposes.--atext = ALPHA / DIGIT / ; Any character except controls,- "!" / "#" / ; SP, and specials.- "$" / "%" / ; Used for atoms- "&" / "'" /- "*" / "+" /- "-" / "/" /- "=" / "?" /- "^" / "_" /- "`" / "{" /- "|" / "}" /- "~"--atom = [CFWS] 1*atext [CFWS]--dot-atom = [CFWS] dot-atom-text [CFWS]--dot-atom-text = 1*atext *("." 1*atext)-- Both atom and dot-atom are interpreted as a single unit, comprised of- the string of characters that make it up. Semantically, the optional- comments and FWS surrounding the rest of the characters are not part- of the atom; the atom is only the run of atext characters in an atom,- or the atext and "." characters in a dot-atom.--------Resnick Standards Track [Page 12]--RFC 2822 Internet Message Format April 2001---3.2.5. Quoted strings-- Strings of characters that include characters other than those- allowed in atoms may be represented in a quoted string format, where- the characters are surrounded by quote (DQUOTE, ASCII value 34)- characters.--qtext = NO-WS-CTL / ; Non white space controls-- %d33 / ; The rest of the US-ASCII- %d35-91 / ; characters not including "\"- %d93-126 ; or the quote character--qcontent = qtext / quoted-pair--quoted-string = [CFWS]- DQUOTE *([FWS] qcontent) [FWS] DQUOTE- [CFWS]-- A quoted-string is treated as a unit. That is, quoted-string is- identical to atom, semantically. Since a quoted-string is allowed to- contain FWS, folding is permitted. Also note that since quoted-pair- is allowed in a quoted-string, the quote and backslash characters may- appear in a quoted-string so long as they appear as a quoted-pair.-- Semantically, neither the optional CFWS outside of the quote- characters nor the quote characters themselves are part of the- quoted-string; the quoted-string is what is contained between the two- quote characters. As stated earlier, the "\" in any quoted-pair and- the CRLF in any FWS/CFWS that appears within the quoted-string are- semantically "invisible" and therefore not part of the quoted-string- either.--3.2.6. Miscellaneous tokens-- Three additional tokens are defined, word and phrase for combinations- of atoms and/or quoted-strings, and unstructured for use in- unstructured header fields and in some places within structured- header fields.--word = atom / quoted-string--phrase = 1*word / obs-phrase---------Resnick Standards Track [Page 13]--RFC 2822 Internet Message Format April 2001---utext = NO-WS-CTL / ; Non white space controls- %d33-126 / ; The rest of US-ASCII- obs-utext--unstructured = *([FWS] utext) [FWS]--3.3. Date and Time Specification-- Date and time occur in several header fields. This section specifies- the syntax for a full date and time specification. Though folding- white space is permitted throughout the date-time specification, it- is RECOMMENDED that a single space be used in each place that FWS- appears (whether it is required or optional); some older- implementations may not interpret other occurrences of folding white- space correctly.--date-time = [ day-of-week "," ] date FWS time [CFWS]--day-of-week = ([FWS] day-name) / obs-day-of-week--day-name = "Mon" / "Tue" / "Wed" / "Thu" /- "Fri" / "Sat" / "Sun"--date = day month year--year = 4*DIGIT / obs-year--month = (FWS month-name FWS) / obs-month--month-name = "Jan" / "Feb" / "Mar" / "Apr" /- "May" / "Jun" / "Jul" / "Aug" /- "Sep" / "Oct" / "Nov" / "Dec"--day = ([FWS] 1*2DIGIT) / obs-day--time = time-of-day FWS zone--time-of-day = hour ":" minute [ ":" second ]--hour = 2DIGIT / obs-hour--minute = 2DIGIT / obs-minute--second = 2DIGIT / obs-second--zone = (( "+" / "-" ) 4DIGIT) / obs-zone------Resnick Standards Track [Page 14]--RFC 2822 Internet Message Format April 2001--- The day is the numeric day of the month. The year is any numeric- year 1900 or later.-- The time-of-day specifies the number of hours, minutes, and- optionally seconds since midnight of the date indicated.-- The date and time-of-day SHOULD express local time.-- The zone specifies the offset from Coordinated Universal Time (UTC,- formerly referred to as "Greenwich Mean Time") that the date and- time-of-day represent. The "+" or "-" indicates whether the- time-of-day is ahead of (i.e., east of) or behind (i.e., west of)- Universal Time. The first two digits indicate the number of hours- difference from Universal Time, and the last two digits indicate the- number of minutes difference from Universal Time. (Hence, +hhmm- means +(hh * 60 + mm) minutes, and -hhmm means -(hh * 60 + mm)- minutes). The form "+0000" SHOULD be used to indicate a time zone at- Universal Time. Though "-0000" also indicates Universal Time, it is- used to indicate that the time was generated on a system that may be- in a local time zone other than Universal Time and therefore- indicates that the date-time contains no information about the local- time zone.-- A date-time specification MUST be semantically valid. That is, the- day-of-the-week (if included) MUST be the day implied by the date,- the numeric day-of-month MUST be between 1 and the number of days- allowed for the specified month (in the specified year), the- time-of-day MUST be in the range 00:00:00 through 23:59:60 (the- number of seconds allowing for a leap second; see [STD12]), and the- zone MUST be within the range -9959 through +9959.--3.4. Address Specification-- Addresses occur in several message header fields to indicate senders- and recipients of messages. An address may either be an individual- mailbox, or a group of mailboxes.--address = mailbox / group--mailbox = name-addr / addr-spec--name-addr = [display-name] angle-addr--angle-addr = [CFWS] "<" addr-spec ">" [CFWS] / obs-angle-addr--group = display-name ":" [mailbox-list / CFWS] ";"- [CFWS]-----Resnick Standards Track [Page 15]--RFC 2822 Internet Message Format April 2001---display-name = phrase--mailbox-list = (mailbox *("," mailbox)) / obs-mbox-list--address-list = (address *("," address)) / obs-addr-list-- A mailbox receives mail. It is a conceptual entity which does not- necessarily pertain to file storage. For example, some sites may- choose to print mail on a printer and deliver the output to the- addressee's desk. Normally, a mailbox is comprised of two parts: (1)- an optional display name that indicates the name of the recipient- (which could be a person or a system) that could be displayed to the- user of a mail application, and (2) an addr-spec address enclosed in- angle brackets ("<" and ">"). There is also an alternate simple form- of a mailbox where the addr-spec address appears alone, without the- recipient's name or the angle brackets. The Internet addr-spec- address is described in section 3.4.1.-- Note: Some legacy implementations used the simple form where the- addr-spec appears without the angle brackets, but included the name- of the recipient in parentheses as a comment following the addr-spec.- Since the meaning of the information in a comment is unspecified,- implementations SHOULD use the full name-addr form of the mailbox,- instead of the legacy form, to specify the display name associated- with a mailbox. Also, because some legacy implementations interpret- the comment, comments generally SHOULD NOT be used in address fields- to avoid confusing such implementations.-- When it is desirable to treat several mailboxes as a single unit- (i.e., in a distribution list), the group construct can be used. The- group construct allows the sender to indicate a named group of- recipients. This is done by giving a display name for the group,- followed by a colon, followed by a comma separated list of any number- of mailboxes (including zero and one), and ending with a semicolon.- Because the list of mailboxes can be empty, using the group construct- is also a simple way to communicate to recipients that the message- was sent to one or more named sets of recipients, without actually- providing the individual mailbox address for each of those- recipients.--3.4.1. Addr-spec specification-- An addr-spec is a specific Internet identifier that contains a- locally interpreted string followed by the at-sign character ("@",- ASCII value 64) followed by an Internet domain. The locally- interpreted string is either a quoted-string or a dot-atom. If the- string can be represented as a dot-atom (that is, it contains no- characters other than atext characters or "." surrounded by atext----Resnick Standards Track [Page 16]--RFC 2822 Internet Message Format April 2001--- characters), then the dot-atom form SHOULD be used and the- quoted-string form SHOULD NOT be used. Comments and folding white- space SHOULD NOT be used around the "@" in the addr-spec.--addr-spec = local-part "@" domain--local-part = dot-atom / quoted-string / obs-local-part--domain = dot-atom / domain-literal / obs-domain--domain-literal = [CFWS] "[" *([FWS] dcontent) [FWS] "]" [CFWS]--dcontent = dtext / quoted-pair--dtext = NO-WS-CTL / ; Non white space controls-- %d33-90 / ; The rest of the US-ASCII- %d94-126 ; characters not including "[",- ; "]", or "\"-- The domain portion identifies the point to which the mail is- delivered. In the dot-atom form, this is interpreted as an Internet- domain name (either a host name or a mail exchanger name) as- described in [STD3, STD13, STD14]. In the domain-literal form, the- domain is interpreted as the literal Internet address of the- particular host. In both cases, how addressing is used and how- messages are transported to a particular host is covered in the mail- transport document [RFC2821]. These mechanisms are outside of the- scope of this document.-- The local-part portion is a domain dependent string. In addresses,- it is simply interpreted on the particular host as a name of a- particular mailbox.--3.5 Overall message syntax-- A message consists of header fields, optionally followed by a message- body. Lines in a message MUST be a maximum of 998 characters- excluding the CRLF, but it is RECOMMENDED that lines be limited to 78- characters excluding the CRLF. (See section 2.1.1 for explanation.)- In a message body, though all of the characters listed in the text- rule MAY be used, the use of US-ASCII control characters (values 1- through 8, 11, 12, and 14 through 31) is discouraged since their- interpretation by receivers for display is not guaranteed.--------Resnick Standards Track [Page 17]--RFC 2822 Internet Message Format April 2001---message = (fields / obs-fields)- [CRLF body]--body = *(*998text CRLF) *998text-- The header fields carry most of the semantic information and are- defined in section 3.6. The body is simply a series of lines of text- which are uninterpreted for the purposes of this standard.--3.6. Field definitions-- The header fields of a message are defined here. All header fields- have the same general syntactic structure: A field name, followed by- a colon, followed by the field body. The specific syntax for each- header field is defined in the subsequent sections.-- Note: In the ABNF syntax for each field in subsequent sections, each- field name is followed by the required colon. However, for brevity- sometimes the colon is not referred to in the textual description of- the syntax. It is, nonetheless, required.-- It is important to note that the header fields are not guaranteed to- be in a particular order. They may appear in any order, and they- have been known to be reordered occasionally when transported over- the Internet. However, for the purposes of this standard, header- fields SHOULD NOT be reordered when a message is transported or- transformed. More importantly, the trace header fields and resent- header fields MUST NOT be reordered, and SHOULD be kept in blocks- prepended to the message. See sections 3.6.6 and 3.6.7 for more- information.-- The only required header fields are the origination date field and- the originator address field(s). All other header fields are- syntactically optional. More information is contained in the table- following this definition.--fields = *(trace- *(resent-date /- resent-from /- resent-sender /- resent-to /- resent-cc /- resent-bcc /- resent-msg-id))- *(orig-date /- from /- sender /- reply-to /----Resnick Standards Track [Page 18]--RFC 2822 Internet Message Format April 2001--- to /- cc /- bcc /- message-id /- in-reply-to /- references /- subject /- comments /- keywords /- optional-field)-- The following table indicates limits on the number of times each- field may occur in a message header as well as any special- limitations on the use of those fields. An asterisk next to a value- in the minimum or maximum column indicates that a special restriction- appears in the Notes column.--Field Min number Max number Notes--trace 0 unlimited Block prepended - see- 3.6.7--resent-date 0* unlimited* One per block, required- if other resent fields- present - see 3.6.6--resent-from 0 unlimited* One per block - see- 3.6.6--resent-sender 0* unlimited* One per block, MUST- occur with multi-address- resent-from - see 3.6.6--resent-to 0 unlimited* One per block - see- 3.6.6--resent-cc 0 unlimited* One per block - see- 3.6.6--resent-bcc 0 unlimited* One per block - see- 3.6.6--resent-msg-id 0 unlimited* One per block - see- 3.6.6--orig-date 1 1--from 1 1 See sender and 3.6.2----Resnick Standards Track [Page 19]--RFC 2822 Internet Message Format April 2001---sender 0* 1 MUST occur with multi-- address from - see 3.6.2--reply-to 0 1--to 0 1--cc 0 1--bcc 0 1--message-id 0* 1 SHOULD be present - see- 3.6.4--in-reply-to 0* 1 SHOULD occur in some- replies - see 3.6.4--references 0* 1 SHOULD occur in some- replies - see 3.6.4--subject 0 1--comments 0 unlimited--keywords 0 unlimited--optional-field 0 unlimited-- The exact interpretation of each field is described in subsequent- sections.--3.6.1. The origination date field-- The origination date field consists of the field name "Date" followed- by a date-time specification.--orig-date = "Date:" date-time CRLF-- The origination date specifies the date and time at which the creator- of the message indicated that the message was complete and ready to- enter the mail delivery system. For instance, this might be the time- that a user pushes the "send" or "submit" button in an application- program. In any case, it is specifically not intended to convey the- time that the message is actually transported, but rather the time at- which the human or other creator of the message has put the message- into its final form, ready for transport. (For example, a portable- computer user who is not connected to a network might queue a message-----Resnick Standards Track [Page 20]--RFC 2822 Internet Message Format April 2001--- for delivery. The origination date is intended to contain the date- and time that the user queued the message, not the time when the user- connected to the network to send the message.)--3.6.2. Originator fields-- The originator fields of a message consist of the from field, the- sender field (when applicable), and optionally the reply-to field.- The from field consists of the field name "From" and a- comma-separated list of one or more mailbox specifications. If the- from field contains more than one mailbox specification in the- mailbox-list, then the sender field, containing the field name- "Sender" and a single mailbox specification, MUST appear in the- message. In either case, an optional reply-to field MAY also be- included, which contains the field name "Reply-To" and a- comma-separated list of one or more addresses.--from = "From:" mailbox-list CRLF--sender = "Sender:" mailbox CRLF--reply-to = "Reply-To:" address-list CRLF-- The originator fields indicate the mailbox(es) of the source of the- message. The "From:" field specifies the author(s) of the message,- that is, the mailbox(es) of the person(s) or system(s) responsible- for the writing of the message. The "Sender:" field specifies the- mailbox of the agent responsible for the actual transmission of the- message. For example, if a secretary were to send a message for- another person, the mailbox of the secretary would appear in the- "Sender:" field and the mailbox of the actual author would appear in- the "From:" field. If the originator of the message can be indicated- by a single mailbox and the author and transmitter are identical, the- "Sender:" field SHOULD NOT be used. Otherwise, both fields SHOULD- appear.-- The originator fields also provide the information required when- replying to a message. When the "Reply-To:" field is present, it- indicates the mailbox(es) to which the author of the message suggests- that replies be sent. In the absence of the "Reply-To:" field,- replies SHOULD by default be sent to the mailbox(es) specified in the- "From:" field unless otherwise specified by the person composing the- reply.-- In all cases, the "From:" field SHOULD NOT contain any mailbox that- does not belong to the author(s) of the message. See also section- 3.6.3 for more information on forming the destination addresses for a- reply.----Resnick Standards Track [Page 21]--RFC 2822 Internet Message Format April 2001---3.6.3. Destination address fields-- The destination fields of a message consist of three possible fields,- each of the same form: The field name, which is either "To", "Cc", or- "Bcc", followed by a comma-separated list of one or more addresses- (either mailbox or group syntax).--to = "To:" address-list CRLF--cc = "Cc:" address-list CRLF--bcc = "Bcc:" (address-list / [CFWS]) CRLF-- The destination fields specify the recipients of the message. Each- destination field may have one or more addresses, and each of the- addresses indicate the intended recipients of the message. The only- difference between the three fields is how each is used.-- The "To:" field contains the address(es) of the primary recipient(s)- of the message.-- The "Cc:" field (where the "Cc" means "Carbon Copy" in the sense of- making a copy on a typewriter using carbon paper) contains the- addresses of others who are to receive the message, though the- content of the message may not be directed at them.-- The "Bcc:" field (where the "Bcc" means "Blind Carbon Copy") contains- addresses of recipients of the message whose addresses are not to be- revealed to other recipients of the message. There are three ways in- which the "Bcc:" field is used. In the first case, when a message- containing a "Bcc:" field is prepared to be sent, the "Bcc:" line is- removed even though all of the recipients (including those specified- in the "Bcc:" field) are sent a copy of the message. In the second- case, recipients specified in the "To:" and "Cc:" lines each are sent- a copy of the message with the "Bcc:" line removed as above, but the- recipients on the "Bcc:" line get a separate copy of the message- containing a "Bcc:" line. (When there are multiple recipient- addresses in the "Bcc:" field, some implementations actually send a- separate copy of the message to each recipient with a "Bcc:"- containing only the address of that particular recipient.) Finally,- since a "Bcc:" field may contain no addresses, a "Bcc:" field can be- sent without any addresses indicating to the recipients that blind- copies were sent to someone. Which method to use with "Bcc:" fields- is implementation dependent, but refer to the "Security- Considerations" section of this document for a discussion of each.-------Resnick Standards Track [Page 22]--RFC 2822 Internet Message Format April 2001--- When a message is a reply to another message, the mailboxes of the- authors of the original message (the mailboxes in the "From:" field)- or mailboxes specified in the "Reply-To:" field (if it exists) MAY- appear in the "To:" field of the reply since these would normally be- the primary recipients of the reply. If a reply is sent to a message- that has destination fields, it is often desirable to send a copy of- the reply to all of the recipients of the message, in addition to the- author. When such a reply is formed, addresses in the "To:" and- "Cc:" fields of the original message MAY appear in the "Cc:" field of- the reply, since these are normally secondary recipients of the- reply. If a "Bcc:" field is present in the original message,- addresses in that field MAY appear in the "Bcc:" field of the reply,- but SHOULD NOT appear in the "To:" or "Cc:" fields.-- Note: Some mail applications have automatic reply commands that- include the destination addresses of the original message in the- destination addresses of the reply. How those reply commands behave- is implementation dependent and is beyond the scope of this document.- In particular, whether or not to include the original destination- addresses when the original message had a "Reply-To:" field is not- addressed here.--3.6.4. Identification fields-- Though optional, every message SHOULD have a "Message-ID:" field.- Furthermore, reply messages SHOULD have "In-Reply-To:" and- "References:" fields as appropriate, as described below.-- The "Message-ID:" field contains a single unique message identifier.- The "References:" and "In-Reply-To:" field each contain one or more- unique message identifiers, optionally separated by CFWS.-- The message identifier (msg-id) is similar in syntax to an angle-addr- construct without the internal CFWS.--message-id = "Message-ID:" msg-id CRLF--in-reply-to = "In-Reply-To:" 1*msg-id CRLF--references = "References:" 1*msg-id CRLF--msg-id = [CFWS] "<" id-left "@" id-right ">" [CFWS]--id-left = dot-atom-text / no-fold-quote / obs-id-left--id-right = dot-atom-text / no-fold-literal / obs-id-right--no-fold-quote = DQUOTE *(qtext / quoted-pair) DQUOTE----Resnick Standards Track [Page 23]--RFC 2822 Internet Message Format April 2001---no-fold-literal = "[" *(dtext / quoted-pair) "]"-- The "Message-ID:" field provides a unique message identifier that- refers to a particular version of a particular message. The- uniqueness of the message identifier is guaranteed by the host that- generates it (see below). This message identifier is intended to be- machine readable and not necessarily meaningful to humans. A message- identifier pertains to exactly one instantiation of a particular- message; subsequent revisions to the message each receive new message- identifiers.-- Note: There are many instances when messages are "changed", but those- changes do not constitute a new instantiation of that message, and- therefore the message would not get a new message identifier. For- example, when messages are introduced into the transport system, they- are often prepended with additional header fields such as trace- fields (described in section 3.6.7) and resent fields (described in- section 3.6.6). The addition of such header fields does not change- the identity of the message and therefore the original "Message-ID:"- field is retained. In all cases, it is the meaning that the sender- of the message wishes to convey (i.e., whether this is the same- message or a different message) that determines whether or not the- "Message-ID:" field changes, not any particular syntactic difference- that appears (or does not appear) in the message.-- The "In-Reply-To:" and "References:" fields are used when creating a- reply to a message. They hold the message identifier of the original- message and the message identifiers of other messages (for example,- in the case of a reply to a message which was itself a reply). The- "In-Reply-To:" field may be used to identify the message (or- messages) to which the new message is a reply, while the- "References:" field may be used to identify a "thread" of- conversation.-- When creating a reply to a message, the "In-Reply-To:" and- "References:" fields of the resultant message are constructed as- follows:-- The "In-Reply-To:" field will contain the contents of the "Message-- ID:" field of the message to which this one is a reply (the "parent- message"). If there is more than one parent message, then the "In-- Reply-To:" field will contain the contents of all of the parents'- "Message-ID:" fields. If there is no "Message-ID:" field in any of- the parent messages, then the new message will have no "In-Reply-To:"- field.-------Resnick Standards Track [Page 24]--RFC 2822 Internet Message Format April 2001--- The "References:" field will contain the contents of the parent's- "References:" field (if any) followed by the contents of the parent's- "Message-ID:" field (if any). If the parent message does not contain- a "References:" field but does have an "In-Reply-To:" field- containing a single message identifier, then the "References:" field- will contain the contents of the parent's "In-Reply-To:" field- followed by the contents of the parent's "Message-ID:" field (if- any). If the parent has none of the "References:", "In-Reply-To:",- or "Message-ID:" fields, then the new message will have no- "References:" field.-- Note: Some implementations parse the "References:" field to display- the "thread of the discussion". These implementations assume that- each new message is a reply to a single parent and hence that they- can walk backwards through the "References:" field to find the parent- of each message listed there. Therefore, trying to form a- "References:" field for a reply that has multiple parents is- discouraged and how to do so is not defined in this document.-- The message identifier (msg-id) itself MUST be a globally unique- identifier for a message. The generator of the message identifier- MUST guarantee that the msg-id is unique. There are several- algorithms that can be used to accomplish this. Since the msg-id has- a similar syntax to angle-addr (identical except that comments and- folding white space are not allowed), a good method is to put the- domain name (or a domain literal IP address) of the host on which the- message identifier was created on the right hand side of the "@", and- put a combination of the current absolute date and time along with- some other currently unique (perhaps sequential) identifier available- on the system (for example, a process id number) on the left hand- side. Using a date on the left hand side and a domain name or domain- literal on the right hand side makes it possible to guarantee- uniqueness since no two hosts use the same domain name or IP address- at the same time. Though other algorithms will work, it is- RECOMMENDED that the right hand side contain some domain identifier- (either of the host itself or otherwise) such that the generator of- the message identifier can guarantee the uniqueness of the left hand- side within the scope of that domain.-- Semantically, the angle bracket characters are not part of the- msg-id; the msg-id is what is contained between the two angle bracket- characters.----------Resnick Standards Track [Page 25]--RFC 2822 Internet Message Format April 2001---3.6.5. Informational fields-- The informational fields are all optional. The "Keywords:" field- contains a comma-separated list of one or more words or- quoted-strings. The "Subject:" and "Comments:" fields are- unstructured fields as defined in section 2.2.1, and therefore may- contain text or folding white space.--subject = "Subject:" unstructured CRLF--comments = "Comments:" unstructured CRLF--keywords = "Keywords:" phrase *("," phrase) CRLF-- These three fields are intended to have only human-readable content- with information about the message. The "Subject:" field is the most- common and contains a short string identifying the topic of the- message. When used in a reply, the field body MAY start with the- string "Re: " (from the Latin "res", in the matter of) followed by- the contents of the "Subject:" field body of the original message.- If this is done, only one instance of the literal string "Re: " ought- to be used since use of other strings or more than one instance can- lead to undesirable consequences. The "Comments:" field contains any- additional comments on the text of the body of the message. The- "Keywords:" field contains a comma-separated list of important words- and phrases that might be useful for the recipient.--3.6.6. Resent fields-- Resent fields SHOULD be added to any message that is reintroduced by- a user into the transport system. A separate set of resent fields- SHOULD be added each time this is done. All of the resent fields- corresponding to a particular resending of the message SHOULD be- together. Each new set of resent fields is prepended to the message;- that is, the most recent set of resent fields appear earlier in the- message. No other fields in the message are changed when resent- fields are added.-- Each of the resent fields corresponds to a particular field elsewhere- in the syntax. For instance, the "Resent-Date:" field corresponds to- the "Date:" field and the "Resent-To:" field corresponds to the "To:"- field. In each case, the syntax for the field body is identical to- the syntax given previously for the corresponding field.-- When resent fields are used, the "Resent-From:" and "Resent-Date:"- fields MUST be sent. The "Resent-Message-ID:" field SHOULD be sent.- "Resent-Sender:" SHOULD NOT be used if "Resent-Sender:" would be- identical to "Resent-From:".----Resnick Standards Track [Page 26]--RFC 2822 Internet Message Format April 2001---resent-date = "Resent-Date:" date-time CRLF--resent-from = "Resent-From:" mailbox-list CRLF--resent-sender = "Resent-Sender:" mailbox CRLF--resent-to = "Resent-To:" address-list CRLF--resent-cc = "Resent-Cc:" address-list CRLF--resent-bcc = "Resent-Bcc:" (address-list / [CFWS]) CRLF--resent-msg-id = "Resent-Message-ID:" msg-id CRLF-- Resent fields are used to identify a message as having been- reintroduced into the transport system by a user. The purpose of- using resent fields is to have the message appear to the final- recipient as if it were sent directly by the original sender, with- all of the original fields remaining the same. Each set of resent- fields correspond to a particular resending event. That is, if a- message is resent multiple times, each set of resent fields gives- identifying information for each individual time. Resent fields are- strictly informational. They MUST NOT be used in the normal- processing of replies or other such automatic actions on messages.-- Note: Reintroducing a message into the transport system and using- resent fields is a different operation from "forwarding".- "Forwarding" has two meanings: One sense of forwarding is that a mail- reading program can be told by a user to forward a copy of a message- to another person, making the forwarded message the body of the new- message. A forwarded message in this sense does not appear to have- come from the original sender, but is an entirely new message from- the forwarder of the message. On the other hand, forwarding is also- used to mean when a mail transport program gets a message and- forwards it on to a different destination for final delivery. Resent- header fields are not intended for use with either type of- forwarding.-- The resent originator fields indicate the mailbox of the person(s) or- system(s) that resent the message. As with the regular originator- fields, there are two forms: a simple "Resent-From:" form which- contains the mailbox of the individual doing the resending, and the- more complex form, when one individual (identified in the- "Resent-Sender:" field) resends a message on behalf of one or more- others (identified in the "Resent-From:" field).-- Note: When replying to a resent message, replies behave just as they- would with any other message, using the original "From:",----Resnick Standards Track [Page 27]--RFC 2822 Internet Message Format April 2001--- "Reply-To:", "Message-ID:", and other fields. The resent fields are- only informational and MUST NOT be used in the normal processing of- replies.-- The "Resent-Date:" indicates the date and time at which the resent- message is dispatched by the resender of the message. Like the- "Date:" field, it is not the date and time that the message was- actually transported.-- The "Resent-To:", "Resent-Cc:", and "Resent-Bcc:" fields function- identically to the "To:", "Cc:", and "Bcc:" fields respectively,- except that they indicate the recipients of the resent message, not- the recipients of the original message.-- The "Resent-Message-ID:" field provides a unique identifier for the- resent message.--3.6.7. Trace fields-- The trace fields are a group of header fields consisting of an- optional "Return-Path:" field, and one or more "Received:" fields.- The "Return-Path:" header field contains a pair of angle brackets- that enclose an optional addr-spec. The "Received:" field contains a- (possibly empty) list of name/value pairs followed by a semicolon and- a date-time specification. The first item of the name/value pair is- defined by item-name, and the second item is either an addr-spec, an- atom, a domain, or a msg-id. Further restrictions may be applied to- the syntax of the trace fields by standards that provide for their- use, such as [RFC2821].--trace = [return]- 1*received--return = "Return-Path:" path CRLF--path = ([CFWS] "<" ([CFWS] / addr-spec) ">" [CFWS]) /- obs-path--received = "Received:" name-val-list ";" date-time CRLF--name-val-list = [CFWS] [name-val-pair *(CFWS name-val-pair)]--name-val-pair = item-name CFWS item-value--item-name = ALPHA *(["-"] (ALPHA / DIGIT))--item-value = 1*angle-addr / addr-spec /- atom / domain / msg-id----Resnick Standards Track [Page 28]--RFC 2822 Internet Message Format April 2001--- A full discussion of the Internet mail use of trace fields is- contained in [RFC2821]. For the purposes of this standard, the trace- fields are strictly informational, and any formal interpretation of- them is outside of the scope of this document.--3.6.8. Optional fields-- Fields may appear in messages that are otherwise unspecified in this- standard. They MUST conform to the syntax of an optional-field.- This is a field name, made up of the printable US-ASCII characters- except SP and colon, followed by a colon, followed by any text which- conforms to unstructured.-- The field names of any optional-field MUST NOT be identical to any- field name specified elsewhere in this standard.--optional-field = field-name ":" unstructured CRLF--field-name = 1*ftext--ftext = %d33-57 / ; Any character except- %d59-126 ; controls, SP, and- ; ":".-- For the purposes of this standard, any optional field is- uninterpreted.--4. Obsolete Syntax-- Earlier versions of this standard allowed for different (usually more- liberal) syntax than is allowed in this version. Also, there have- been syntactic elements used in messages on the Internet whose- interpretation have never been documented. Though some of these- syntactic forms MUST NOT be generated according to the grammar in- section 3, they MUST be accepted and parsed by a conformant receiver.- This section documents many of these syntactic elements. Taking the- grammar in section 3 and adding the definitions presented in this- section will result in the grammar to use for interpretation of- messages.-- Note: This section identifies syntactic forms that any implementation- MUST reasonably interpret. However, there are certainly Internet- messages which do not conform to even the additional syntax given in- this section. The fact that a particular form does not appear in any- section of this document is not justification for computer programs- to crash or for malformed data to be irretrievably lost by any- implementation. To repeat an example, though this document requires- lines in messages to be no longer than 998 characters, silently----Resnick Standards Track [Page 29]--RFC 2822 Internet Message Format April 2001--- discarding the 999th and subsequent characters in a line without- warning would still be bad behavior for an implementation. It is up- to the implementation to deal with messages robustly.-- One important difference between the obsolete (interpreting) and the- current (generating) syntax is that in structured header field bodies- (i.e., between the colon and the CRLF of any structured header- field), white space characters, including folding white space, and- comments can be freely inserted between any syntactic tokens. This- allows many complex forms that have proven difficult for some- implementations to parse.-- Another key difference between the obsolete and the current syntax is- that the rule in section 3.2.3 regarding lines composed entirely of- white space in comments and folding white space does not apply. See- the discussion of folding white space in section 4.2 below.-- Finally, certain characters that were formerly allowed in messages- appear in this section. The NUL character (ASCII value 0) was once- allowed, but is no longer for compatibility reasons. CR and LF were- allowed to appear in messages other than as CRLF; this use is also- shown here.-- Other differences in syntax and semantics are noted in the following- sections.--4.1. Miscellaneous obsolete tokens-- These syntactic elements are used elsewhere in the obsolete syntax or- in the main syntax. The obs-char and obs-qp elements each add ASCII- value 0. Bare CR and bare LF are added to obs-text and obs-utext.- The period character is added to obs-phrase. The obs-phrase-list- provides for "empty" elements in a comma-separated list of phrases.-- Note: The "period" (or "full stop") character (".") in obs-phrase is- not a form that was allowed in earlier versions of this or any other- standard. Period (nor any other character from specials) was not- allowed in phrase because it introduced a parsing difficulty- distinguishing between phrases and portions of an addr-spec (see- section 4.4). It appears here because the period character is- currently used in many messages in the display-name portion of- addresses, especially for initials in names, and therefore must be- interpreted properly. In the future, period may appear in the- regular syntax of phrase.--obs-qp = "\" (%d0-127)--obs-text = *LF *CR *(obs-char *LF *CR)----Resnick Standards Track [Page 30]--RFC 2822 Internet Message Format April 2001---obs-char = %d0-9 / %d11 / ; %d0-127 except CR and- %d12 / %d14-127 ; LF--obs-utext = obs-text--obs-phrase = word *(word / "." / CFWS)--obs-phrase-list = phrase / 1*([phrase] [CFWS] "," [CFWS]) [phrase]-- Bare CR and bare LF appear in messages with two different meanings.- In many cases, bare CR or bare LF are used improperly instead of CRLF- to indicate line separators. In other cases, bare CR and bare LF are- used simply as ASCII control characters with their traditional ASCII- meanings.--4.2. Obsolete folding white space-- In the obsolete syntax, any amount of folding white space MAY be- inserted where the obs-FWS rule is allowed. This creates the- possibility of having two consecutive "folds" in a line, and- therefore the possibility that a line which makes up a folded header- field could be composed entirely of white space.-- obs-FWS = 1*WSP *(CRLF 1*WSP)--4.3. Obsolete Date and Time-- The syntax for the obsolete date format allows a 2 digit year in the- date field and allows for a list of alphabetic time zone- specifications that were used in earlier versions of this standard.- It also permits comments and folding white space between many of the- tokens.--obs-day-of-week = [CFWS] day-name [CFWS]--obs-year = [CFWS] 2*DIGIT [CFWS]--obs-month = CFWS month-name CFWS--obs-day = [CFWS] 1*2DIGIT [CFWS]--obs-hour = [CFWS] 2DIGIT [CFWS]--obs-minute = [CFWS] 2DIGIT [CFWS]--obs-second = [CFWS] 2DIGIT [CFWS]--obs-zone = "UT" / "GMT" / ; Universal Time----Resnick Standards Track [Page 31]--RFC 2822 Internet Message Format April 2001--- ; North American UT- ; offsets- "EST" / "EDT" / ; Eastern: - 5/ - 4- "CST" / "CDT" / ; Central: - 6/ - 5- "MST" / "MDT" / ; Mountain: - 7/ - 6- "PST" / "PDT" / ; Pacific: - 8/ - 7-- %d65-73 / ; Military zones - "A"- %d75-90 / ; through "I" and "K"- %d97-105 / ; through "Z", both- %d107-122 ; upper and lower case-- Where a two or three digit year occurs in a date, the year is to be- interpreted as follows: If a two digit year is encountered whose- value is between 00 and 49, the year is interpreted by adding 2000,- ending up with a value between 2000 and 2049. If a two digit year is- encountered with a value between 50 and 99, or any three digit year- is encountered, the year is interpreted by adding 1900.-- In the obsolete time zone, "UT" and "GMT" are indications of- "Universal Time" and "Greenwich Mean Time" respectively and are both- semantically identical to "+0000".-- The remaining three character zones are the US time zones. The first- letter, "E", "C", "M", or "P" stands for "Eastern", "Central",- "Mountain" and "Pacific". The second letter is either "S" for- "Standard" time, or "D" for "Daylight" (or summer) time. Their- interpretations are as follows:-- EDT is semantically equivalent to -0400- EST is semantically equivalent to -0500- CDT is semantically equivalent to -0500- CST is semantically equivalent to -0600- MDT is semantically equivalent to -0600- MST is semantically equivalent to -0700- PDT is semantically equivalent to -0700- PST is semantically equivalent to -0800-- The 1 character military time zones were defined in a non-standard- way in [RFC822] and are therefore unpredictable in their meaning.- The original definitions of the military zones "A" through "I" are- equivalent to "+0100" through "+0900" respectively; "K", "L", and "M"- are equivalent to "+1000", "+1100", and "+1200" respectively; "N"- through "Y" are equivalent to "-0100" through "-1200" respectively;- and "Z" is equivalent to "+0000". However, because of the error in- [RFC822], they SHOULD all be considered equivalent to "-0000" unless- there is out-of-band information confirming their meaning.-----Resnick Standards Track [Page 32]--RFC 2822 Internet Message Format April 2001--- Other multi-character (usually between 3 and 5) alphabetic time zones- have been used in Internet messages. Any such time zone whose- meaning is not known SHOULD be considered equivalent to "-0000"- unless there is out-of-band information confirming their meaning.--4.4. Obsolete Addressing-- There are three primary differences in addressing. First, mailbox- addresses were allowed to have a route portion before the addr-spec- when enclosed in "<" and ">". The route is simply a comma-separated- list of domain names, each preceded by "@", and the list terminated- by a colon. Second, CFWS were allowed between the period-separated- elements of local-part and domain (i.e., dot-atom was not used). In- addition, local-part is allowed to contain quoted-string in addition- to just atom. Finally, mailbox-list and address-list were allowed to- have "null" members. That is, there could be two or more commas in- such a list with nothing in between them.--obs-angle-addr = [CFWS] "<" [obs-route] addr-spec ">" [CFWS]--obs-route = [CFWS] obs-domain-list ":" [CFWS]--obs-domain-list = "@" domain *(*(CFWS / "," ) [CFWS] "@" domain)--obs-local-part = word *("." word)--obs-domain = atom *("." atom)--obs-mbox-list = 1*([mailbox] [CFWS] "," [CFWS]) [mailbox]--obs-addr-list = 1*([address] [CFWS] "," [CFWS]) [address]-- When interpreting addresses, the route portion SHOULD be ignored.--4.5. Obsolete header fields-- Syntactically, the primary difference in the obsolete field syntax is- that it allows multiple occurrences of any of the fields and they may- occur in any order. Also, any amount of white space is allowed- before the ":" at the end of the field name.--obs-fields = *(obs-return /- obs-received /- obs-orig-date /- obs-from /- obs-sender /- obs-reply-to /- obs-to /----Resnick Standards Track [Page 33]--RFC 2822 Internet Message Format April 2001--- obs-cc /- obs-bcc /- obs-message-id /- obs-in-reply-to /- obs-references /- obs-subject /- obs-comments /- obs-keywords /- obs-resent-date /- obs-resent-from /- obs-resent-send /- obs-resent-rply /- obs-resent-to /- obs-resent-cc /- obs-resent-bcc /- obs-resent-mid /- obs-optional)-- Except for destination address fields (described in section 4.5.3),- the interpretation of multiple occurrences of fields is unspecified.- Also, the interpretation of trace fields and resent fields which do- not occur in blocks prepended to the message is unspecified as well.- Unless otherwise noted in the following sections, interpretation of- other fields is identical to the interpretation of their non-obsolete- counterparts in section 3.--4.5.1. Obsolete origination date field--obs-orig-date = "Date" *WSP ":" date-time CRLF--4.5.2. Obsolete originator fields--obs-from = "From" *WSP ":" mailbox-list CRLF--obs-sender = "Sender" *WSP ":" mailbox CRLF--obs-reply-to = "Reply-To" *WSP ":" mailbox-list CRLF--4.5.3. Obsolete destination address fields--obs-to = "To" *WSP ":" address-list CRLF--obs-cc = "Cc" *WSP ":" address-list CRLF--obs-bcc = "Bcc" *WSP ":" (address-list / [CFWS]) CRLF-------Resnick Standards Track [Page 34]--RFC 2822 Internet Message Format April 2001--- When multiple occurrences of destination address fields occur in a- message, they SHOULD be treated as if the address-list in the first- occurrence of the field is combined with the address lists of the- subsequent occurrences by adding a comma and concatenating.--4.5.4. Obsolete identification fields-- The obsolete "In-Reply-To:" and "References:" fields differ from the- current syntax in that they allow phrase (words or quoted strings) to- appear. The obsolete forms of the left and right sides of msg-id- allow interspersed CFWS, making them syntactically identical to- local-part and domain respectively.--obs-message-id = "Message-ID" *WSP ":" msg-id CRLF--obs-in-reply-to = "In-Reply-To" *WSP ":" *(phrase / msg-id) CRLF--obs-references = "References" *WSP ":" *(phrase / msg-id) CRLF--obs-id-left = local-part--obs-id-right = domain-- For purposes of interpretation, the phrases in the "In-Reply-To:" and- "References:" fields are ignored.-- Semantically, none of the optional CFWS surrounding the local-part- and the domain are part of the obs-id-left and obs-id-right- respectively.--4.5.5. Obsolete informational fields--obs-subject = "Subject" *WSP ":" unstructured CRLF--obs-comments = "Comments" *WSP ":" unstructured CRLF--obs-keywords = "Keywords" *WSP ":" obs-phrase-list CRLF--4.5.6. Obsolete resent fields-- The obsolete syntax adds a "Resent-Reply-To:" field, which consists- of the field name, the optional comments and folding white space, the- colon, and a comma separated list of addresses.--obs-resent-from = "Resent-From" *WSP ":" mailbox-list CRLF--obs-resent-send = "Resent-Sender" *WSP ":" mailbox CRLF-----Resnick Standards Track [Page 35]--RFC 2822 Internet Message Format April 2001---obs-resent-date = "Resent-Date" *WSP ":" date-time CRLF--obs-resent-to = "Resent-To" *WSP ":" address-list CRLF--obs-resent-cc = "Resent-Cc" *WSP ":" address-list CRLF--obs-resent-bcc = "Resent-Bcc" *WSP ":"- (address-list / [CFWS]) CRLF--obs-resent-mid = "Resent-Message-ID" *WSP ":" msg-id CRLF--obs-resent-rply = "Resent-Reply-To" *WSP ":" address-list CRLF-- As with other resent fields, the "Resent-Reply-To:" field is to be- treated as trace information only.--4.5.7. Obsolete trace fields-- The obs-return and obs-received are again given here as template- definitions, just as return and received are in section 3. Their- full syntax is given in [RFC2821].--obs-return = "Return-Path" *WSP ":" path CRLF--obs-received = "Received" *WSP ":" name-val-list CRLF--obs-path = obs-angle-addr--4.5.8. Obsolete optional fields--obs-optional = field-name *WSP ":" unstructured CRLF--5. Security Considerations-- Care needs to be taken when displaying messages on a terminal or- terminal emulator. Powerful terminals may act on escape sequences- and other combinations of ASCII control characters with a variety of- consequences. They can remap the keyboard or permit other- modifications to the terminal which could lead to denial of service- or even damaged data. They can trigger (sometimes programmable)- answerback messages which can allow a message to cause commands to be- issued on the recipient's behalf. They can also effect the operation- of terminal attached devices such as printers. Message viewers may- wish to strip potentially dangerous terminal escape sequences from- the message prior to display. However, other escape sequences appear- in messages for useful purposes (cf. [RFC2045, RFC2046, RFC2047,- RFC2048, RFC2049, ISO2022]) and therefore should not be stripped- indiscriminately.----Resnick Standards Track [Page 36]--RFC 2822 Internet Message Format April 2001--- Transmission of non-text objects in messages raises additional- security issues. These issues are discussed in [RFC2045, RFC2046,- RFC2047, RFC2048, RFC2049].-- Many implementations use the "Bcc:" (blind carbon copy) field- described in section 3.6.3 to facilitate sending messages to- recipients without revealing the addresses of one or more of the- addressees to the other recipients. Mishandling this use of "Bcc:"- has implications for confidential information that might be revealed,- which could eventually lead to security problems through knowledge of- even the existence of a particular mail address. For example, if- using the first method described in section 3.6.3, where the "Bcc:"- line is removed from the message, blind recipients have no explicit- indication that they have been sent a blind copy, except insofar as- their address does not appear in the message header. Because of- this, one of the blind addressees could potentially send a reply to- all of the shown recipients and accidentally reveal that the message- went to the blind recipient. When the second method from section- 3.6.3 is used, the blind recipient's address appears in the "Bcc:"- field of a separate copy of the message. If the "Bcc:" field sent- contains all of the blind addressees, all of the "Bcc:" recipients- will be seen by each "Bcc:" recipient. Even if a separate message is- sent to each "Bcc:" recipient with only the individual's address,- implementations still need to be careful to process replies to the- message as per section 3.6.3 so as not to accidentally reveal the- blind recipient to other recipients.--6. Bibliography-- [ASCII] American National Standards Institute (ANSI), Coded- Character Set - 7-Bit American National Standard Code for- Information Interchange, ANSI X3.4, 1986.-- [ISO2022] International Organization for Standardization (ISO),- Information processing - ISO 7-bit and 8-bit coded- character sets - Code extension techniques, Third edition- - 1986-05-01, ISO 2022, 1986.-- [RFC822] Crocker, D., "Standard for the Format of ARPA Internet- Text Messages", RFC 822, August 1982.-- [RFC2045] Freed, N. and N. Borenstein, "Multipurpose Internet Mail- Extensions (MIME) Part One: Format of Internet Message- Bodies", RFC 2045, November 1996.-- [RFC2046] Freed, N. and N. Borenstein, "Multipurpose Internet Mail- Extensions (MIME) Part Two: Media Types", RFC 2046,- November 1996.----Resnick Standards Track [Page 37]--RFC 2822 Internet Message Format April 2001--- [RFC2047] Moore, K., "Multipurpose Internet Mail Extensions (MIME)- Part Three: Message Header Extensions for Non-ASCII Text",- RFC 2047, November 1996.-- [RFC2048] Freed, N., Klensin, J. and J. Postel, "Multipurpose- Internet Mail Extensions (MIME) Part Four: Format of- Internet Message Bodies", RFC 2048, November 1996.-- [RFC2049] Freed, N. and N. Borenstein, "Multipurpose Internet Mail- Extensions (MIME) Part Five: Conformance Criteria and- Examples", RFC 2049, November 1996.-- [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate- Requirement Levels", BCP 14, RFC 2119, March 1997.-- [RFC2234] Crocker, D., Editor, and P. Overell, "Augmented BNF for- Syntax Specifications: ABNF", RFC 2234, November 1997.-- [RFC2821] Klensin, J., Editor, "Simple Mail Transfer Protocol", RFC- 2821, March 2001.-- [STD3] Braden, R., "Host Requirements", STD 3, RFC 1122 and RFC- 1123, October 1989.-- [STD12] Mills, D., "Network Time Protocol", STD 12, RFC 1119,- September 1989.-- [STD13] Mockapetris, P., "Domain Name System", STD 13, RFC 1034- and RFC 1035, November 1987.-- [STD14] Partridge, C., "Mail Routing and the Domain System", STD- 14, RFC 974, January 1986.--7. Editor's Address-- Peter W. Resnick- QUALCOMM Incorporated- 5775 Morehouse Drive- San Diego, CA 92121-1714- USA-- Phone: +1 858 651 4478- Fax: +1 858 651 1102- EMail: presnick@qualcomm.com--------Resnick Standards Track [Page 38]--RFC 2822 Internet Message Format April 2001---8. Acknowledgements-- Many people contributed to this document. They included folks who- participated in the Detailed Revision and Update of Messaging- Standards (DRUMS) Working Group of the Internet Engineering Task- Force (IETF), the chair of DRUMS, the Area Directors of the IETF, and- people who simply sent their comments in via e-mail. The editor is- deeply indebted to them all and thanks them sincerely. The below- list includes everyone who sent e-mail concerning this document.- Hopefully, everyone who contributed is named here:-- Matti Aarnio Barry Finkel Larry Masinter- Tanaka Akira Erik Forsberg Denis McKeon- Russ Allbery Chuck Foster William P McQuillan- Eric Allman Paul Fox Alexey Melnikov- Harald Tveit Alvestrand Klaus M. Frank Perry E. Metzger- Ran Atkinson Ned Freed Steven Miller- Jos Backus Jochen Friedrich Keith Moore- Bruce Balden Randall C. Gellens John Gardiner Myers- Dave Barr Sukvinder Singh Gill Chris Newman- Alan Barrett Tim Goodwin John W. Noerenberg- John Beck Philip Guenther Eric Norman- J. Robert von Behren Tony Hansen Mike O'Dell- Jos den Bekker John Hawkinson Larry Osterman- D. J. Bernstein Philip Hazel Paul Overell- James Berriman Kai Henningsen Jacob Palme- Norbert Bollow Robert Herriot Michael A. Patton- Raj Bose Paul Hethmon Uzi Paz- Antony Bowesman Jim Hill Michael A. Quinlan- Scott Bradner Paul E. Hoffman Eric S. Raymond- Randy Bush Steve Hole Sam Roberts- Tom Byrer Kari Hurtta Hugh Sasse- Bruce Campbell Marco S. Hyman Bart Schaefer- Larry Campbell Ofer Inbar Tom Scola- W. J. Carpenter Olle Jarnefors Wolfgang Segmuller- Michael Chapman Kevin Johnson Nick Shelness- Richard Clayton Sudish Joseph John Stanley- Maurizio Codogno Maynard Kang Einar Stefferud- Jim Conklin Prabhat Keni Jeff Stephenson- R. Kelley Cook John C. Klensin Bernard Stern- Steve Coya Graham Klyne Peter Sylvester- Mark Crispin Brad Knowles Mark Symons- Dave Crocker Shuhei Kobayashi Eric Thomas- Matt Curtin Peter Koch Lee Thompson- Michael D'Errico Dan Kohn Karel De Vriendt- Cyrus Daboo Christian Kuhtz Matthew Wall- Jutta Degener Anand Kumria Rolf Weber- Mark Delany Steen Larsen Brent B. Welch----Resnick Standards Track [Page 39]--RFC 2822 Internet Message Format April 2001--- Steve Dorner Eliot Lear Dan Wing- Harold A. Driscoll Barry Leiba Jack De Winter- Michael Elkins Jay Levitt Gregory J. Woodhouse- Robert Elz Lars-Johan Liman Greg A. Woods- Johnny Eriksson Charles Lindsey Kazu Yamamoto- Erik E. Fair Pete Loshin Alain Zahm- Roger Fajman Simon Lyall Jamie Zawinski- Patrik Faltstrom Bill Manning Timothy S. Zurcher- Claus Andre Farber John Martin-------------------------------------------Resnick Standards Track [Page 40]--RFC 2822 Internet Message Format April 2001---Appendix A. Example messages-- This section presents a selection of messages. These are intended to- assist in the implementation of this standard, but should not be- taken as normative; that is to say, although the examples in this- section were carefully reviewed, if there happens to be a conflict- between these examples and the syntax described in sections 3 and 4- of this document, the syntax in those sections is to be taken as- correct.-- Messages are delimited in this section between lines of "----". The- "----" lines are not part of the message itself.--A.1. Addressing examples-- The following are examples of messages that might be sent between two- individuals.--A.1.1. A message from one person to another with simple addressing-- This could be called a canonical message. It has a single author,- John Doe, a single recipient, Mary Smith, a subject, the date, a- message identifier, and a textual message in the body.-------From: John Doe <jdoe@machine.example>-To: Mary Smith <mary@example.net>-Subject: Saying Hello-Date: Fri, 21 Nov 1997 09:55:06 -0600-Message-ID: <1234@local.machine.example>--This is a message just to say hello.-So, "Hello".-----------------------Resnick Standards Track [Page 41]--RFC 2822 Internet Message Format April 2001--- If John's secretary Michael actually sent the message, though John- was the author and replies to this message should go back to him, the- sender field would be used:-------From: John Doe <jdoe@machine.example>-Sender: Michael Jones <mjones@machine.example>-To: Mary Smith <mary@example.net>-Subject: Saying Hello-Date: Fri, 21 Nov 1997 09:55:06 -0600-Message-ID: <1234@local.machine.example>--This is a message just to say hello.-So, "Hello".-------A.1.2. Different types of mailboxes-- This message includes multiple addresses in the destination fields- and also uses several different forms of addresses.-------From: "Joe Q. Public" <john.q.public@example.com>-To: Mary Smith <mary@x.test>, jdoe@example.org, Who? <one@y.test>-Cc: <boss@nil.test>, "Giant; \"Big\" Box" <sysservices@example.net>-Date: Tue, 1 Jul 2003 10:52:37 +0200-Message-ID: <5678.21-Nov-1997@example.com>--Hi everyone.------- Note that the display names for Joe Q. Public and Giant; "Big" Box- needed to be enclosed in double-quotes because the former contains- the period and the latter contains both semicolon and double-quote- characters (the double-quote characters appearing as quoted-pair- construct). Conversely, the display name for Who? could appear- without them because the question mark is legal in an atom. Notice- also that jdoe@example.org and boss@nil.test have no display names- associated with them at all, and jdoe@example.org uses the simpler- address form without the angle brackets.------------Resnick Standards Track [Page 42]--RFC 2822 Internet Message Format April 2001---A.1.3. Group addresses-------From: Pete <pete@silly.example>-To: A Group:Chris Jones <c@a.test>,joe@where.test,John <jdoe@one.test>;-Cc: Undisclosed recipients:;-Date: Thu, 13 Feb 1969 23:32:54 -0330-Message-ID: <testabcd.1234@silly.example>--Testing.------- In this message, the "To:" field has a single group recipient named A- Group which contains 3 addresses, and a "Cc:" field with an empty- group recipient named Undisclosed recipients.--A.2. Reply messages-- The following is a series of three messages that make up a- conversation thread between John and Mary. John firsts sends a- message to Mary, Mary then replies to John's message, and then John- replies to Mary's reply message.-- Note especially the "Message-ID:", "References:", and "In-Reply-To:"- fields in each message.-------From: John Doe <jdoe@machine.example>-To: Mary Smith <mary@example.net>-Subject: Saying Hello-Date: Fri, 21 Nov 1997 09:55:06 -0600-Message-ID: <1234@local.machine.example>--This is a message just to say hello.-So, "Hello".---------------------Resnick Standards Track [Page 43]--RFC 2822 Internet Message Format April 2001--- When sending replies, the Subject field is often retained, though- prepended with "Re: " as described in section 3.6.5.-------From: Mary Smith <mary@example.net>-To: John Doe <jdoe@machine.example>-Reply-To: "Mary Smith: Personal Account" <smith@home.example>-Subject: Re: Saying Hello-Date: Fri, 21 Nov 1997 10:01:10 -0600-Message-ID: <3456@example.net>-In-Reply-To: <1234@local.machine.example>-References: <1234@local.machine.example>--This is a reply to your hello.------- Note the "Reply-To:" field in the above message. When John replies- to Mary's message above, the reply should go to the address in the- "Reply-To:" field instead of the address in the "From:" field.-------To: "Mary Smith: Personal Account" <smith@home.example>-From: John Doe <jdoe@machine.example>-Subject: Re: Saying Hello-Date: Fri, 21 Nov 1997 11:00:00 -0600-Message-ID: <abcd.1234@local.machine.tld>-In-Reply-To: <3456@example.net>-References: <1234@local.machine.example> <3456@example.net>--This is a reply to your reply.-------A.3. Resent messages-- Start with the message that has been used as an example several- times:-------From: John Doe <jdoe@machine.example>-To: Mary Smith <mary@example.net>-Subject: Saying Hello-Date: Fri, 21 Nov 1997 09:55:06 -0600-Message-ID: <1234@local.machine.example>--This is a message just to say hello.-So, "Hello".----------Resnick Standards Track [Page 44]--RFC 2822 Internet Message Format April 2001--- Say that Mary, upon receiving this message, wishes to send a copy of- the message to Jane such that (a) the message would appear to have- come straight from John; (b) if Jane replies to the message, the- reply should go back to John; and (c) all of the original- information, like the date the message was originally sent to Mary,- the message identifier, and the original addressee, is preserved. In- this case, resent fields are prepended to the message:-------Resent-From: Mary Smith <mary@example.net>-Resent-To: Jane Brown <j-brown@other.example>-Resent-Date: Mon, 24 Nov 1997 14:22:01 -0800-Resent-Message-ID: <78910@example.net>-From: John Doe <jdoe@machine.example>-To: Mary Smith <mary@example.net>-Subject: Saying Hello-Date: Fri, 21 Nov 1997 09:55:06 -0600-Message-ID: <1234@local.machine.example>--This is a message just to say hello.-So, "Hello".------- If Jane, in turn, wished to resend this message to another person,- she would prepend her own set of resent header fields to the above- and send that.--------------------------Resnick Standards Track [Page 45]--RFC 2822 Internet Message Format April 2001---A.4. Messages with trace fields-- As messages are sent through the transport system as described in- [RFC2821], trace fields are prepended to the message. The following- is an example of what those trace fields might look like. Note that- there is some folding white space in the first one since these lines- can be long.-------Received: from x.y.test- by example.net- via TCP- with ESMTP- id ABC12345- for <mary@example.net>; 21 Nov 1997 10:05:43 -0600-Received: from machine.example by x.y.test; 21 Nov 1997 10:01:22 -0600-From: John Doe <jdoe@machine.example>-To: Mary Smith <mary@example.net>-Subject: Saying Hello-Date: Fri, 21 Nov 1997 09:55:06 -0600-Message-ID: <1234@local.machine.example>--This is a message just to say hello.-So, "Hello".--------------------------------Resnick Standards Track [Page 46]--RFC 2822 Internet Message Format April 2001---A.5. White space, comments, and other oddities-- White space, including folding white space, and comments can be- inserted between many of the tokens of fields. Taking the example- from A.1.3, white space and comments can be inserted into all of the- fields.-------From: Pete(A wonderful \) chap) <pete(his account)@silly.test(his host)>-To:A Group(Some people)- :Chris Jones <c@(Chris's host.)public.example>,- joe@example.org,- John <jdoe@one.test> (my dear friend); (the end of the group)-Cc:(Empty list)(start)Undisclosed recipients :(nobody(that I know)) ;-Date: Thu,- 13- Feb- 1969- 23:32- -0330 (Newfoundland Time)-Message-ID: <testabcd.1234@silly.test>--Testing.------- The above example is aesthetically displeasing, but perfectly legal.- Note particularly (1) the comments in the "From:" field (including- one that has a ")" character appearing as part of a quoted-pair); (2)- the white space absent after the ":" in the "To:" field as well as- the comment and folding white space after the group name, the special- character (".") in the comment in Chris Jones's address, and the- folding white space before and after "joe@example.org,"; (3) the- multiple and nested comments in the "Cc:" field as well as the- comment immediately following the ":" after "Cc"; (4) the folding- white space (but no comments except at the end) and the missing- seconds in the time of the date field; and (5) the white space before- (but not within) the identifier in the "Message-ID:" field.--A.6. Obsoleted forms-- The following are examples of obsolete (that is, the "MUST NOT- generate") syntactic elements described in section 4 of this- document.---------Resnick Standards Track [Page 47]--RFC 2822 Internet Message Format April 2001---A.6.1. Obsolete addressing-- Note in the below example the lack of quotes around Joe Q. Public,- the route that appears in the address for Mary Smith, the two commas- that appear in the "To:" field, and the spaces that appear around the- "." in the jdoe address.-------From: Joe Q. Public <john.q.public@example.com>-To: Mary Smith <@machine.tld:mary@example.net>, , jdoe@test . example-Date: Tue, 1 Jul 2003 10:52:37 +0200-Message-ID: <5678.21-Nov-1997@example.com>--Hi everyone.-------A.6.2. Obsolete dates-- The following message uses an obsolete date format, including a non-- numeric time zone and a two digit year. Note that although the- day-of-week is missing, that is not specific to the obsolete syntax;- it is optional in the current syntax as well.-------From: John Doe <jdoe@machine.example>-To: Mary Smith <mary@example.net>-Subject: Saying Hello-Date: 21 Nov 97 09:55:06 GMT-Message-ID: <1234@local.machine.example>--This is a message just to say hello.-So, "Hello".-------A.6.3. Obsolete white space and comments-- White space and comments can appear between many more elements than- in the current syntax. Also, folding lines that are made up entirely- of white space are legal.-------------Resnick Standards Track [Page 48]--RFC 2822 Internet Message Format April 2001--------From : John Doe <jdoe@machine(comment). example>-To : Mary Smith-__- <mary@example.net>-Subject : Saying Hello-Date : Fri, 21 Nov 1997 09(comment): 55 : 06 -0600-Message-ID : <1234 @ local(blah) .machine .example>--This is a message just to say hello.-So, "Hello".------- Note especially the second line of the "To:" field. It starts with- two space characters. (Note that "__" represent blank spaces.)- Therefore, it is considered part of the folding as described in- section 4.2. Also, the comments and white space throughout- addresses, dates, and message identifiers are all part of the- obsolete syntax.--Appendix B. Differences from earlier standards-- This appendix contains a list of changes that have been made in the- Internet Message Format from earlier standards, specifically [RFC822]- and [STD3]. Items marked with an asterisk (*) below are items which- appear in section 4 of this document and therefore can no longer be- generated.-- 1. Period allowed in obsolete form of phrase.- 2. ABNF moved out of document to [RFC2234].- 3. Four or more digits allowed for year.- 4. Header field ordering (and lack thereof) made explicit.- 5. Encrypted header field removed.- 6. Received syntax loosened to allow any token/value pair.- 7. Specifically allow and give meaning to "-0000" time zone.- 8. Folding white space is not allowed between every token.- 9. Requirement for destinations removed.- 10. Forwarding and resending redefined.- 11. Extension header fields no longer specifically called out.- 12. ASCII 0 (null) removed.*- 13. Folding continuation lines cannot contain only white space.*- 14. Free insertion of comments not allowed in date.*- 15. Non-numeric time zones not allowed.*- 16. Two digit years not allowed.*- 17. Three digit years interpreted, but not allowed for generation.- 18. Routes in addresses not allowed.*- 19. CFWS within local-parts and domains not allowed.*- 20. Empty members of address lists not allowed.*----Resnick Standards Track [Page 49]--RFC 2822 Internet Message Format April 2001--- 21. Folding white space between field name and colon not allowed.*- 22. Comments between field name and colon not allowed.- 23. Tightened syntax of in-reply-to and references.*- 24. CFWS within msg-id not allowed.*- 25. Tightened semantics of resent fields as informational only.- 26. Resent-Reply-To not allowed.*- 27. No multiple occurrences of fields (except resent and received).*- 28. Free CR and LF not allowed.*- 29. Routes in return path not allowed.*- 30. Line length limits specified.- 31. Bcc more clearly specified.--Appendix C. Notices-- Intellectual Property-- The IETF takes no position regarding the validity or scope of any- intellectual property or other rights that might be claimed to- pertain to the implementation or use of the technology described in- this document or the extent to which any license under such rights- might or might not be available; neither does it represent that it- has made any effort to identify any such rights. Information on the- IETF's procedures with respect to rights in standards-track and- standards-related documentation can be found in BCP-11. Copies of- claims of rights made available for publication and any assurances of- licenses to be made available, or the result of an attempt made to- obtain a general license or permission for the use of such- proprietary rights by implementors or users of this specification can- be obtained from the IETF Secretariat.-----------------------Resnick Standards Track [Page 50]--RFC 2822 Internet Message Format April 2001---Full Copyright Statement-- Copyright (C) The Internet Society (2001). All Rights Reserved.-- This document and translations of it may be copied and furnished to- others, and derivative works that comment on or otherwise explain it- or assist in its implementation may be prepared, copied, published- and distributed, in whole or in part, without restriction of any- kind, provided that the above copyright notice and this paragraph are- included on all such copies and derivative works. However, this- document itself may not be modified in any way, such as by removing- the copyright notice or references to the Internet Society or other- Internet organizations, except as needed for the purpose of- developing Internet standards in which case the procedures for- copyrights defined in the Internet Standards process must be- followed, or as required to translate it into languages other than- English.-- The limited permissions granted above are perpetual and will not be- revoked by the Internet Society or its successors or assigns.-- This document and the information contained herein is provided on an- "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING- TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING- BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION- HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF- MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.--Acknowledgement-- Funding for the RFC Editor function is currently provided by the- Internet Society.---
src/Data/Concrete/Parsers/Types.hs view
@@ -4,17 +4,22 @@ , CommunicationAction , PathComponent(..) , Path+ , IngestStream ) where import Data.Text.Lazy (Text) import Data.Concrete.Autogen.Communication_Types (Communication) import Data.Concrete.Autogen.Structure_Types (Section, Sentence, Token) import Text.Megaparsec (ParsecT)-import Text.Megaparsec.Error (Dec) import Data.Map (Map) import Control.Monad.State (StateT) import GHC.Int+import Data.Void (Void)+import Conduit+import Control.Monad.Identity (Identity) +type IngestStream = ConduitM Text Communication (ResourceT IO) ()+ -- | A 'CommunicationAction' gets called on each Communication -- as parsing proceeds type CommunicationAction = Communication -> IO ()@@ -60,18 +65,18 @@ , sections :: [Section] -- | List of Sections accumulated for the Communication currently being parsed , sentences :: [Sentence] -- | List of Sections accumulated for the Communication currently being parsed , tokens :: [Token] -- | List of Sections accumulated for the Communication currently being parsed- , action :: CommunicationAction- , contentSections :: [String]- , commId :: Text- , commType :: String- , commNum :: Int+ --, contentSections :: [String]+ --, commId :: Text+ --, commType :: Text+ --, commNum :: Int , offset :: GHC.Int.Int32 } -- | A StatefulParser is just a Megaparsec Parser that carries -- a State, and has access to the IO monad.-type StatefulParser s a = ParsecT Dec Text (StateT s IO) a+type StatefulParser s a = ParsecT Void Text (StateT s Identity) a -- | A 'CommunicationParser' is a stateful Megaparsec parser that, as it -- processes a Text stream, builds a list of Concrete Communications.+--type CommunicationParser a = ParsecT Void Text Identity a type CommunicationParser a = StatefulParser Bookkeeper a
src/Data/Concrete/Parsers/Utils.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE DeriveGeneric, OverloadedStrings #-}+{-# LANGUAGE DeriveGeneric, OverloadedStrings, RecordWildCards, FlexibleContexts #-} module Data.Concrete.Parsers.Utils ( communicationRule , sectionRule , sentenceRule@@ -12,26 +12,41 @@ , modifyPathComponent , incrementPathComponent , Located(..)+ , unfoldParse+ , unfoldParseArray+ , unfoldParseNewline+ , finalizeCommunication ) where import Data.Text.Lazy (Text, pack, unpack, replace) import qualified Data.Text.Lazy as T import Data.List (intercalate) import Data.Concrete.Parsers.Types (Bookkeeper(..), CommunicationParser, CommunicationAction)-import Text.Megaparsec (ParsecT, getParserState, stateTokensProcessed, match)-import Text.Megaparsec.Error (Dec)+import Text.Megaparsec (ParsecT, getParserState, stateTokensProcessed, match, State(..), mkPos, initialPos, runParserT', parseErrorPretty)+import Text.Megaparsec.Char (char, oneOf, space, newline)+import qualified Data.List.NonEmpty as NE import Data.Map (Map) import qualified Data.Map as Map import Control.Monad.State (State, get, put, modify, modify')+--import Data.Concrete.Prelude import Data.Concrete.Autogen.Communication_Types (Communication(..), default_Communication) import Data.Concrete.Autogen.Structure_Types (Section(..), default_Section, Token(..), default_Token, Sentence(..), default_Sentence, TokenizationKind(..), Tokenization(..), default_Tokenization, TokenList(..), default_TokenList) import Data.Concrete.Autogen.Spans_Types (TextSpan(..), default_TextSpan, AudioSpan(..), default_AudioSpan)+import Data.Concrete.Autogen.Metadata_Types (default_AnnotationMetadata)+import Data.Concrete.Autogen.Uuid_Types (default_UUID) import Data.Concrete.Utils (getUUID, createAnnotationMetadata, incrementUUID) import Control.Monad.IO.Class (liftIO) import Data.Vector (Vector, fromList, snoc, empty, cons, toList) import qualified Data.Vector as V-import Data.Maybe (fromJust)+import Data.Maybe (fromJust, catMaybes) import Text.Printf (printf)+import Conduit+import Data.Conduit.List (unfold, unfoldM)+import Control.Monad.State (runStateT)+import Control.Lens hiding (cons)+--import Data.Concrete.Autogen.Lens.Communication hiding (communication)+--import Data.Concrete.Autogen.Lens.Section+import Data.Concrete.Prelude hiding (communication) -- | Wraps a rule corresponding to a Token tokenRule :: (Token -> Token) -> CommunicationParser a -> CommunicationParser a@@ -54,8 +69,10 @@ v <- p e <- (fromIntegral . stateTokensProcessed) <$> getParserState bs@(Bookkeeper {..}) <- get- u <- liftIO getUUID- m <- liftIO $ createAnnotationMetadata "concrete-haskell ingester"+ let u = default_UUID+ m = default_AnnotationMetadata+ --u <- liftIO getUUID+ --m <- liftIO $ createAnnotationMetadata "concrete-haskell ingester" let tokenList = default_TokenList { tokenList_tokenList=V.fromList (reverse tokens) } tokenization = default_Tokenization { tokenization_tokenList=Just tokenList@@ -96,8 +113,9 @@ } return v + -- | Wraps a rule that corresponds to a Communication-communicationRule :: (Communication -> Communication) -> CommunicationParser a -> CommunicationParser a+communicationRule :: (Communication -> Communication) -> CommunicationParser a -> CommunicationParser Communication communicationRule tr p = do offset <- (fromIntegral . stateTokensProcessed) <$> getParserState bs' <- get@@ -105,37 +123,34 @@ (t, o) <- match p bs@(Bookkeeper {..}) <- get let sections = (toList . fromJust) (communication_sectionList communication)- u <- liftIO getUUID+ u = default_UUID+ --u <- liftIO getUUID let us = iterate incrementUUID u- m <- liftIO $ createAnnotationMetadata "concrete-haskell ingester"+ m = default_AnnotationMetadata+ --m <- liftIO $ createAnnotationMetadata "concrete-haskell ingester" let sections' = [s { section_uuid=u'- , section_kind=if elem ((unpack . fromJust) section_label) contentSections then "content" else "metadata"+ , section_kind="" -- if elem ((unpack . fromJust) section_label) contentSections then "content" else "metadata" , section_textSpan=(\ (Just (TextSpan{..})) -> Just $ TextSpan (textSpan_start - offset) (textSpan_ending - offset)) section_textSpan } | (u', s@(Section{..})) <- zip us sections] - sectionVals = [(fromJust section_label, pack $ substr (pack t) ((fromIntegral . textSpan_start . fromJust) section_textSpan) ((fromIntegral . textSpan_ending . fromJust) section_textSpan)) | Section{..} <- sections']+ sectionVals = [(fromJust section_label, substr t ((fromIntegral . textSpan_start . fromJust) section_textSpan) ((fromIntegral . textSpan_ending . fromJust) section_textSpan)) | Section{..} <- sections'] c = communication { communication_metadata=m- , communication_text=Just $ pack t+ , communication_text=Just t , communication_uuid=u- , communication_id=makeId sectionVals commId commNum+ -- , communication_id=makeId sectionVals commId commNum , communication_sectionList=Just $ fromList sections' }- put $ bs { communication=default_Communication { communication_sectionList=Just empty }, valueMap=Map.fromList [], sections=[], commNum=commNum + 1 }- liftIO $ action (tr c)+ put $ bs { communication=default_Communication { communication_sectionList=Just empty }, valueMap=Map.fromList [], sections=[] } clearState- return o+ return c -- | Extracts a sub-string from a text object-substr :: T.Text -> Int -> Int -> String-substr t s e = T.unpack res+substr :: T.Text -> Int -> Int -> Text+substr t s e = res where (_, start) = T.splitAt (fromIntegral s) t res = T.take (fromIntegral $ e - s) start --- | Performs variable substitution on an ID string-makeId :: [(Text, Text)] -> Text -> Int -> Text-makeId ss i n = foldr (\ (a, b) x -> T.replace (T.concat ["${", a, "}"]) b x) i (("", (pack . show) n):ss)- -- | Resets the "Communication-building" state inside the parser clearState :: CommunicationParser () clearState = do@@ -213,6 +228,66 @@ modify (\ bs -> bs { path=(show p'):(tail path) }) return p' +-- | Performs variable substitution on an ID string+makeId :: Communication -> Text -> Int -> Text+makeId c idStr n = foldr (\ (a, b) x -> T.replace (T.concat ["${", a, "}"]) b x) idStr (("", (pack . show) n):ss'')+ where+ Just ss' = V.toList <$> c ^. _communication_sectionList + Just t = communication_text c+ ss'' = map (\s -> ((fromJust . section_label) s, (spanText t . fromJust . section_textSpan) s)) ss'++ --k = map (\s -> (section_label s, spanText t s)) (catMaybes (map section_textSpan ss'))++finalizeCommunication :: Text -> [Text] -> (Int, Communication) -> IO Communication+finalizeCommunication idStr cs (i, c) = return $ c & _communication_id .~ cid+ where+ cid = makeId c idStr i+ +oneParse b p s = case runStateT (runParserT' p s) b of+ Identity ((_, Left e), _) -> Nothing -- error $ parseErrorPretty e+ Identity ((s', Right c), _) -> Just (c, s')++unfoldParse :: Monad m => CommunicationParser Communication -> Text -> ConduitM () Communication m ()+unfoldParse p t = unfoldC (oneParse b p) s+ where+ s = State { stateInput=t+ , statePos=NE.fromList $ [initialPos "Text File"]+ , stateTokensProcessed=0+ , stateTabWidth=mkPos 8+ }+ b = Bookkeeper (default_Communication { communication_sectionList=Just empty }) Map.empty [] [] [] [] 0++unfoldParseArray :: Monad m => CommunicationParser Communication -> Text -> ConduitM () Communication m ()+unfoldParseArray p t = unfoldC (oneParse b p') s+ where+ t' = T.dropWhile (\c -> c /= '{') t+ s = State { stateInput=t'+ , statePos=NE.fromList $ [initialPos "Text File"]+ , stateTokensProcessed=0+ , stateTabWidth=mkPos 8+ }+ b = Bookkeeper (default_Communication { communication_sectionList=Just empty }) Map.empty [] [] [] [] 0 + p' = do+ c <- p+ space+ oneOf [',', ']']+ space+ return c++unfoldParseNewline :: Monad m => CommunicationParser Communication -> Text -> ConduitM () Communication m ()+unfoldParseNewline p t = unfoldC (oneParse b p') s+ where+ s = State { stateInput=t+ , statePos=NE.fromList $ [initialPos "Text File"]+ , stateTokensProcessed=0+ , stateTabWidth=mkPos 8+ }+ b = Bookkeeper (default_Communication { communication_sectionList=Just empty }) Map.empty [] [] [] [] 0 + p' = do+ c <- p+ newline+ return c+ -- | A data structure that is positioned inside a document and whose boundaries can be adjusted class Located a where getTextSpan :: a -> TextSpan@@ -235,3 +310,6 @@ instance Located Token where getTextSpan s = (fromJust . token_textSpan) s setTextSpan ts s = s { token_textSpan=Just ts }++spanText :: Text -> TextSpan -> Text+spanText t ts = substr t (fromIntegral $ textSpan_start ts) (fromIntegral $ textSpan_ending ts)
src/Data/Concrete/Parsers/XML.hs view
@@ -1,9 +1,9 @@ {-# LANGUAGE DeriveGeneric, OverloadedStrings, ApplicativeDo #-} module Data.Concrete.Parsers.XML- ( parser+ ( sequenceSource ) where--- https://www.w3.org/TR/REC-xml/ +import Data.Char (isSpace) import Data.List (intercalate) import Data.Concrete.Parsers.Types (Bookkeeper(..), CommunicationParser) import Data.Concrete.Parsers.Utils (communicationRule)@@ -13,19 +13,28 @@ import qualified Data.Map as Map import Data.Map (Map) import Data.List.NonEmpty (fromList)-import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+--import Text.Megaparsec.Lexer (symbol, lexeme, signed, number)+ import Text.Megaparsec.Pos (initialPos, defaultTabWidth)-import Text.Megaparsec.Error (Dec)+--import Text.Megaparsec.Error (Dec)+import Text.Megaparsec.Char.Lexer (symbol, lexeme, signed, scientific)+import Text.Megaparsec.Char ( eol+ , noneOf+ , newline+ , char+ , anyChar+ , space+ , hexDigitChar+ , tab+ , separatorChar+ , satisfy+ ) import Text.Megaparsec ( parseErrorPretty , (<|>)- , space- , hexDigitChar , count , manyTill- , anyChar , runParser , some- , char , choice , sepBy , between@@ -34,140 +43,71 @@ , runParserT' , State(..) , getParserState+ , sepBy1 , many+ , eof+ , someTill )+-- import Text.Megaparsec ( parseErrorPretty+-- , (<|>)+-- , satisfy+-- , space+-- , hexDigitChar+-- , count+-- , manyTill+-- , anyChar+-- , runParser+-- , some+-- , char+-- , choice+-- , sepBy+-- , between+-- , match+-- , ParsecT+-- , runParserT'+-- , State(..)+-- , getParserState+-- , spaceChar+-- , eof+-- , noneOf+-- , try+-- ) -import Text.Megaparsec.Text.Lazy (Parser)+import Control.Monad.IO.Class (liftIO) +--import Text.Megaparsec.Text.Lazy (Parser) import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..)) import qualified Control.Monad.State as S import qualified Control.Monad.Identity as I+--import Data.Concrete.Types import Data.Concrete.Parsers.Utils (communicationRule, sectionRule) +sequenceSource = undefined+ parser :: CommunicationParser ()-parser = document >> return ()+parser = do+ space+ some document+ space+ eof+ return () -document = many anyChar- --prolog element (many misc)+-- type CS = CommunicationParser String+-- type CSS = CommunicationParser [String]+-- type CC = CommunicationParser Char --- Char = #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]--- White Space--- [3] S ::= (#x20 | #x9 | #xD | #xA)+--- Names and Tokens--- [4] NameChar ::= Letter | Digit | '.' | '-' | '_' | ':' | CombiningChar | Extender--- [5] Name ::= (Letter | '_' | ':') (NameChar)*--- [6] Names ::= Name (S Name)*--- [7] Nmtoken ::= (NameChar)+--- [8] Nmtokens ::= Nmtoken (S Nmtoken)*--- Literals--- [9] EntityValue ::= '"' ([^%&"] | PEReference | Reference)* '"' | "'" ([^%&'] | PEReference | Reference)* "'"--- [10] AttValue ::= '"' ([^<&"] | Reference)* '"' | "'" ([^<&'] | Reference)* "'"--- [11] SystemLiteral ::= ('"' [^"]* '"') | ("'" [^']* "'")--- [12] PubidLiteral ::= '"' PubidChar* '"' | "'" (PubidChar - "'")* "'"--- [13] PubidChar ::= #x20 | #xD | #xA | [a-zA-Z0-9] | [-'()+,./:=?;!*#@$_%]--- Character Data--- [14] CharData ::= [^<&]* - ([^<&]* ']]>' [^<&]*)--- Comments--- [15] Comment ::= '<!--' ((Char - '-') | ('-' (Char - '-')))* '-->'--- Processing Instructions--- [16] PI ::= '<?' PITarget (S (Char* - (Char* '?>' Char*)))? '?>'--- [17] PITarget ::= Name - (('X' | 'x') ('M' | 'm') ('L' | 'l'))--- CDATA Sections--- [18] CDSect ::= CDStart CData CDEnd--- [19] CDStart ::= '<![CDATA['--- [20] CData ::= (Char* - (Char* ']]>' Char*))--- [21] CDEnd ::= ']]>'--- Prolog--- [22] prolog ::= XMLDecl? Misc* (doctypedecl Misc*)?--- [23] XMLDecl ::= '<?xml' VersionInfo EncodingDecl? SDDecl? S? '?>'--- [24] VersionInfo ::= S 'version' Eq (' VersionNum ' | " VersionNum ")--- [25] Eq ::= S? '=' S?--- [26] VersionNum ::= ([a-zA-Z0-9_.:] | '-')+--- [27] Misc ::= Comment | PI | S--- Document Type Definition--- [28] doctypedecl ::= '<!DOCTYPE' S Name (S ExternalID)? S? ('[' (markupdecl | PEReference | S)* ']' S?)? '>'--- [29] markupdecl ::= elementdecl | AttlistDecl | EntityDecl | NotationDecl | PI | Comment--- External Subset--- [30] extSubset ::= TextDecl? extSubsetDecl--- [31] extSubsetDecl ::= ( markupdecl | conditionalSect | PEReference | S )*--- Standalone Document Declaration--- [32] SDDecl ::= S 'standalone' Eq (("'" ('yes' | 'no') "'") | ('"' ('yes' | 'no') '"'))--- Language Identification--- [33] LanguageID ::= Langcode ('-' Subcode)*--- [34] Langcode ::= ISO639Code | IanaCode | UserCode--- [35] ISO639Code ::= ([a-z] | [A-Z]) ([a-z] | [A-Z])--- [36] IanaCode ::= ('i' | 'I') '-' ([a-z] | [A-Z])+--- [37] UserCode ::= ('x' | 'X') '-' ([a-z] | [A-Z])+--- [38] Subcode ::= ([a-z] | [A-Z])+--- Element--- [39] element ::= EmptyElemTag | STag content ETag--- Start-tag--- [40] STag ::= '<' Name (S Attribute)* S? '>'--- [41] Attribute ::= Name Eq AttValue--- End-tag--- [42] ETag ::= '</' Name S? '>'--- Content of Elements--- [43] content ::= (element | CharData | Reference | CDSect | PI | Comment)*--- Tags for Empty Elements--- [44] EmptyElemTag ::= '<' Name (S Attribute)* S? '/>'--- Element Type Declaration--- [45] elementdecl ::= '<!ELEMENT' S Name S contentspec S? '>'--- [46] contentspec ::= 'EMPTY' | 'ANY' | Mixed | children--- Element-content Models--- [47] children ::= (choice | seq) ('?' | '*' | '+')?--- [48] cp ::= (Name | choice | seq) ('?' | '*' | '+')?--- [49] choice ::= '(' S? cp ( S? '|' S? cp )* S? ')'--- [50] seq ::= '(' S? cp ( S? ',' S? cp )* S? ')'--- Mixed-content Declaration--- [51] Mixed ::= '(' S? '#PCDATA' (S? '|' S? Name)* S? ')*' | '(' S? '#PCDATA' S? ')'--- Attribute-list Declaration--- [52] AttlistDecl ::= '<!ATTLIST' S Name AttDef* S? '>'--- [53] AttDef ::= S Name S AttType S DefaultDecl--- Attribute Types--- [54] AttType ::= StringType | TokenizedType | EnumeratedType--- [55] StringType ::= 'CDATA'--- [56] TokenizedType ::= 'ID' | 'IDREF' | 'IDREFS' | 'ENTITY' | 'ENTITIES' | 'NMTOKEN' | 'NMTOKENS'--- Enumerated Attribute Types--- [57] EnumeratedType ::= NotationType | Enumeration--- [58] NotationType ::= 'NOTATION' S '(' S? Name (S? '|' S? Name)* S? ')'--- [59] Enumeration ::= '(' S? Nmtoken (S? '|' S? Nmtoken)* S? ')'--- Attribute Defaults--- [60] DefaultDecl ::= '#REQUIRED' | '#IMPLIED' | (('#FIXED' S)? AttValue)--- Conditional Section--- [61] conditionalSect ::= includeSect | ignoreSect--- [62] includeSect ::= '<![' S? 'INCLUDE' S? '[' extSubsetDecl ']]>'--- [63] ignoreSect ::= '<![' S? 'IGNORE' S? '[' ignoreSectContents* ']]>'--- [64] ignoreSectContents ::= Ignore ('<![' ignoreSectContents ']]>' Ignore)*--- [65] Ignore ::= Char* - (Char* ('<![' | ']]>') Char*)--- Character Reference--- [66] CharRef ::= '&#' [0-9]+ ';' | '&#x' [0-9a-fA-F]+ ';'--- Entity Reference--- [67] Reference ::= EntityRef | CharRef--- [68] EntityRef ::= '&' Name ';'--- [69] PEReference ::= '%' Name ';'--- Entity Declaration--- [70] EntityDecl ::= GEDecl | PEDecl--- [71] GEDecl ::= '<!ENTITY' S Name S EntityDef S? '>'--- [72] PEDecl ::= '<!ENTITY' S '%' S Name S PEDef S? '>'--- [73] EntityDef ::= EntityValue | (ExternalID NDataDecl?)--- [74] PEDef ::= EntityValue | ExternalID--- External Entity Declaration--- [75] ExternalID ::= 'SYSTEM' S SystemLiteral | 'PUBLIC' S PubidLiteral S SystemLiteral--- [76] NDataDecl ::= S 'NDATA' S Name--- Text Declaration--- [77] TextDecl ::= '<?xml' VersionInfo? EncodingDecl S? '?>'--- Well-Formed External Parsed Entity--- [78] extParsedEnt ::= TextDecl? content--- [79] extPE ::= TextDecl? extSubsetDecl--- Encoding Declaration--- [80] EncodingDecl ::= S 'encoding' Eq ('"' EncName '"' | "'" EncName "'" ) --- [81] EncName ::= [A-Za-z] ([A-Za-z0-9._] | '-')* /* Encoding name contains only Latin characters */--- Notation Declarations--- [82] NotationDecl ::= '<!NOTATION' S Name S (ExternalID | PublicID) S? '>'--- [83] PublicID ::= 'PUBLIC' S PubidLiteral--- Characters--- [84] Letter ::= BaseChar | Ideographic--- [85] BaseChar ::= [#x0041-#x005A] | [#x0061-#x007A] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x00FF] | [#x0100-#x0131] | [#x0134-#x013E] | [#x0141-#x0148] | [#x014A-#x017E] | [#x0180-#x01C3] | [#x01CD-#x01F0] | [#x01F4-#x01F5] | [#x01FA-#x0217] | [#x0250-#x02A8] | [#x02BB-#x02C1] | #x0386 | [#x0388-#x038A] | #x038C | [#x038E-#x03A1] | [#x03A3-#x03CE] | [#x03D0-#x03D6] | #x03DA | #x03DC | #x03DE | #x03E0 | [#x03E2-#x03F3] | [#x0401-#x040C] | [#x040E-#x044F] | [#x0451-#x045C] | [#x045E-#x0481] | [#x0490-#x04C4] | [#x04C7-#x04C8] | [#x04CB-#x04CC] | [#x04D0-#x04EB] | [#x04EE-#x04F5] | [#x04F8-#x04F9] | [#x0531-#x0556] | #x0559 | [#x0561-#x0586] | [#x05D0-#x05EA] | [#x05F0-#x05F2] | [#x0621-#x063A] | [#x0641-#x064A] | [#x0671-#x06B7] | [#x06BA-#x06BE] | [#x06C0-#x06CE] | [#x06D0-#x06D3] | #x06D5 | [#x06E5-#x06E6] | [#x0905-#x0939] | #x093D | [#x0958-#x0961] | [#x0985-#x098C] | [#x098F-#x0990] | [#x0993-#x09A8] | [#x09AA-#x09B0] | #x09B2 | [#x09B6-#x09B9] | [#x09DC-#x09DD] | [#x09DF-#x09E1] | [#x09F0-#x09F1] | [#x0A05-#x0A0A] | [#x0A0F-#x0A10] | [#x0A13-#x0A28] | [#x0A2A-#x0A30] | [#x0A32-#x0A33] | [#x0A35-#x0A36] | [#x0A38-#x0A39] | [#x0A59-#x0A5C] | #x0A5E | [#x0A72-#x0A74] | [#x0A85-#x0A8B] | #x0A8D | [#x0A8F-#x0A91] | [#x0A93-#x0AA8] | [#x0AAA-#x0AB0] | [#x0AB2-#x0AB3] | [#x0AB5-#x0AB9] | #x0ABD | #x0AE0 | [#x0B05-#x0B0C] | [#x0B0F-#x0B10] | [#x0B13-#x0B28] | [#x0B2A-#x0B30] | [#x0B32-#x0B33] | [#x0B36-#x0B39] | #x0B3D | [#x0B5C-#x0B5D] | [#x0B5F-#x0B61] | [#x0B85-#x0B8A] | [#x0B8E-#x0B90] | [#x0B92-#x0B95] | [#x0B99-#x0B9A] | #x0B9C | [#x0B9E-#x0B9F] | [#x0BA3-#x0BA4] | [#x0BA8-#x0BAA] | [#x0BAE-#x0BB5] | [#x0BB7-#x0BB9] | [#x0C05-#x0C0C] | [#x0C0E-#x0C10] | [#x0C12-#x0C28] | [#x0C2A-#x0C33] | [#x0C35-#x0C39] | [#x0C60-#x0C61] | [#x0C85-#x0C8C] | [#x0C8E-#x0C90] | [#x0C92-#x0CA8] | [#x0CAA-#x0CB3] | [#x0CB5-#x0CB9] | #x0CDE | [#x0CE0-#x0CE1] | [#x0D05-#x0D0C] | [#x0D0E-#x0D10] | [#x0D12-#x0D28] | [#x0D2A-#x0D39] | [#x0D60-#x0D61] | [#x0E01-#x0E2E] | #x0E30 | [#x0E32-#x0E33] | [#x0E40-#x0E45] | [#x0E81-#x0E82] | #x0E84 | [#x0E87-#x0E88] | #x0E8A | #x0E8D | [#x0E94-#x0E97] | [#x0E99-#x0E9F] | [#x0EA1-#x0EA3] | #x0EA5 | #x0EA7 | [#x0EAA-#x0EAB] | [#x0EAD-#x0EAE] | #x0EB0 | [#x0EB2-#x0EB3] | #x0EBD | [#x0EC0-#x0EC4] | [#x0F40-#x0F47] | [#x0F49-#x0F69] | [#x10A0-#x10C5] | [#x10D0-#x10F6] | #x1100 | [#x1102-#x1103] | [#x1105-#x1107] | #x1109 | [#x110B-#x110C] | [#x110E-#x1112] | #x113C | #x113E | #x1140 | #x114C | #x114E | #x1150 | [#x1154-#x1155] | #x1159 | [#x115F-#x1161] | #x1163 | #x1165 | #x1167 | #x1169 | [#x116D-#x116E] | [#x1172-#x1173] | #x1175 | #x119E | #x11A8 | #x11AB | [#x11AE-#x11AF] | [#x11B7-#x11B8] | #x11BA | [#x11BC-#x11C2] | #x11EB | #x11F0 | #x11F9 | [#x1E00-#x1E9B] | [#x1EA0-#x1EF9] | [#x1F00-#x1F15] | [#x1F18-#x1F1D] | [#x1F20-#x1F45] | [#x1F48-#x1F4D] | [#x1F50-#x1F57] | #x1F59 | #x1F5B | #x1F5D | [#x1F5F-#x1F7D] | [#x1F80-#x1FB4] | [#x1FB6-#x1FBC] | #x1FBE | [#x1FC2-#x1FC4] | [#x1FC6-#x1FCC] | [#x1FD0-#x1FD3] | [#x1FD6-#x1FDB] | [#x1FE0-#x1FEC] | [#x1FF2-#x1FF4] | [#x1FF6-#x1FFC] | #x2126 | [#x212A-#x212B] | #x212E | [#x2180-#x2182] | [#x3041-#x3094] | [#x30A1-#x30FA] | [#x3105-#x312C] | [#xAC00-#xD7A3]--- [86] Ideographic ::= [#x4E00-#x9FA5] | #x3007 | [#x3021-#x3029]--- [87] CombiningChar ::= [#x0300-#x0345] | [#x0360-#x0361] | [#x0483-#x0486] | [#x0591-#x05A1] | [#x05A3-#x05B9] | [#x05BB-#x05BD] | #x05BF | [#x05C1-#x05C2] | #x05C4 | [#x064B-#x0652] | #x0670 | [#x06D6-#x06DC] | [#x06DD-#x06DF] | [#x06E0-#x06E4] | [#x06E7-#x06E8] | [#x06EA-#x06ED] | [#x0901-#x0903] | #x093C | [#x093E-#x094C] | #x094D | [#x0951-#x0954] | [#x0962-#x0963] | [#x0981-#x0983] | #x09BC | #x09BE | #x09BF | [#x09C0-#x09C4] | [#x09C7-#x09C8] | [#x09CB-#x09CD] | #x09D7 | [#x09E2-#x09E3] | #x0A02 | #x0A3C | #x0A3E | #x0A3F | [#x0A40-#x0A42] | [#x0A47-#x0A48] | [#x0A4B-#x0A4D] | [#x0A70-#x0A71] | [#x0A81-#x0A83] | #x0ABC | [#x0ABE-#x0AC5] | [#x0AC7-#x0AC9] | [#x0ACB-#x0ACD] | [#x0B01-#x0B03] | #x0B3C | [#x0B3E-#x0B43] | [#x0B47-#x0B48] | [#x0B4B-#x0B4D] | [#x0B56-#x0B57] | [#x0B82-#x0B83] | [#x0BBE-#x0BC2] | [#x0BC6-#x0BC8] | [#x0BCA-#x0BCD] | #x0BD7 | [#x0C01-#x0C03] | [#x0C3E-#x0C44] | [#x0C46-#x0C48] | [#x0C4A-#x0C4D] | [#x0C55-#x0C56] | [#x0C82-#x0C83] | [#x0CBE-#x0CC4] | [#x0CC6-#x0CC8] | [#x0CCA-#x0CCD] | [#x0CD5-#x0CD6] | [#x0D02-#x0D03] | [#x0D3E-#x0D43] | [#x0D46-#x0D48] | [#x0D4A-#x0D4D] | #x0D57 | #x0E31 | [#x0E34-#x0E3A] | [#x0E47-#x0E4E] | #x0EB1 | [#x0EB4-#x0EB9] | [#x0EBB-#x0EBC] | [#x0EC8-#x0ECD] | [#x0F18-#x0F19] | #x0F35 | #x0F37 | #x0F39 | #x0F3E | #x0F3F | [#x0F71-#x0F84] | [#x0F86-#x0F8B] | [#x0F90-#x0F95] | #x0F97 | [#x0F99-#x0FAD] | [#x0FB1-#x0FB7] | #x0FB9 | [#x20D0-#x20DC] | #x20E1 | [#x302A-#x302F] | #x3099 | #x309A--- [88] Digit ::= [#x0030-#x0039] | [#x0660-#x0669] | [#x06F0-#x06F9] | [#x0966-#x096F] | [#x09E6-#x09EF] | [#x0A66-#x0A6F] | [#x0AE6-#x0AEF] | [#x0B66-#x0B6F] | [#x0BE7-#x0BEF] | [#x0C66-#x0C6F] | [#x0CE6-#x0CEF] | [#x0D66-#x0D6F] | [#x0E50-#x0E59] | [#x0ED0-#x0ED9] | [#x0F20-#x0F29]--- [89] Extender ::= #x00B7 | #x02D0 | #x02D1 | #x0387 | #x0640 | #x0E46 | #x0EC6 | #x3005 | [#x3031-#x3035] | [#x309D-#x309E] | [#x30FC-#x30FE]+document :: CommunicationParser ()+document = lexeme' $ communicationRule id (parens (some sentence)) >> return ()++sentence = lexeme' $ between (symbol' "(S") (symbol' ")") (some phrase)++phrase = lexeme' $ parens (tag >> some (tag <|> phrase)) >> return []++tag = lexicalItem++lexicalItem = lexeme' $ some notSpaceOrParen++notSpaceOrParen = satisfy (\c -> and [(not . isSpace) c, ('(' /= c), (')' /= c)])++lexeme' = lexeme space+symbol' = symbol space+parens = between (symbol' "(") (symbol' ")")
+ src/Data/Concrete/Prelude.hs view
@@ -0,0 +1,51 @@+module Data.Concrete.Prelude ( module Data.Concrete.Autogen.Communication_Types+ , module Data.Concrete.Autogen.Metadata_Types+ , module Data.Concrete.Autogen.Language_Types+ , module Data.Concrete.Autogen.Spans_Types+ , module Data.Concrete.Autogen.Structure_Types+ , module Data.Concrete.Autogen.Uuid_Types+ , module Data.Concrete.Autogen.Entities_Types+ , module Data.Concrete.Autogen.Cluster_Types+ , module Data.Concrete.Autogen.Email_Types+ , module Data.Concrete.Autogen.Audio_Types+ , module Data.Concrete.Autogen.Access_Types+ , module Data.Concrete.Autogen.Graph_Types+ , module Data.Concrete.Autogen.Learn_Types+ , module Data.Concrete.Autogen.Linking_Types+ , module Data.Concrete.Autogen.Nitf_Types+ , module Data.Concrete.Autogen.Results_Types+ , module Data.Concrete.Autogen.Twitter_Types++ , module Data.Concrete.Autogen.Lens.Section+ , module Data.Concrete.Autogen.Lens.Communication+ , module Data.Concrete.Autogen.Lens.AnnotationMetadata + ) where++import Data.Concrete.Autogen.Communication_Types+import Data.Concrete.Autogen.Structure_Types+import Data.Concrete.Autogen.Spans_Types+import Data.Concrete.Autogen.Metadata_Types+import Data.Concrete.Autogen.Uuid_Types+import Data.Concrete.Autogen.Language_Types+import Data.Concrete.Autogen.Entities_Types+import Data.Concrete.Autogen.Cluster_Types+import Data.Concrete.Autogen.Email_Types+import Data.Concrete.Autogen.Audio_Types+import Data.Concrete.Autogen.Access_Types+import Data.Concrete.Autogen.Graph_Types+import Data.Concrete.Autogen.Learn_Types+import Data.Concrete.Autogen.Linking_Types+import Data.Concrete.Autogen.Nitf_Types+import Data.Concrete.Autogen.Results_Types+import Data.Concrete.Autogen.Twitter_Types++import Data.Concrete.Autogen.Lens.Communication+import Data.Concrete.Autogen.Lens.Section+import Data.Concrete.Autogen.Lens.Sentence+import Data.Concrete.Autogen.Lens.Token+import Data.Concrete.Autogen.Lens.TaggedToken+import Data.Concrete.Autogen.Lens.LanguageIdentification+import Data.Concrete.Autogen.Lens.AnnotationMetadata+import Data.Concrete.Autogen.Lens.CommunicationMetadata+import Data.Concrete.Autogen.Lens.TextSpan+import Data.Concrete.Autogen.Lens.UUID
+ src/Data/Concrete/Services/Annotate.hs view
@@ -0,0 +1,2 @@+module Data.Concrete.Services.Annotate (+ ) where
src/Data/Concrete/Services/Fetch.hs view
@@ -2,6 +2,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE BangPatterns #-}+{-# LANGUAGE PackageImports #-} {-| Description: Implementations of FetchCommunicationService for various backends -}@@ -13,6 +14,7 @@ , TarFetch(..) , makeTarFetch , process+ , fetchDirect ) where import System.IO (Handle)@@ -24,10 +26,13 @@ import qualified Data.ByteString.Lazy as LBS import qualified Codec.Compression.GZip as GZip import qualified Codec.Compression.BZip as BZip-import qualified Codec.Archive.Zip as Zip+import qualified "zip-conduit" Codec.Archive.Zip as ZipC+import qualified "zip" Codec.Archive.Zip as Zip import qualified Codec.Archive.Tar as Tar import qualified Codec.Archive.Tar.Entry as Tar-import qualified Codec.Archive.Tar.Index as Tar+import qualified Codec.Archive.Tar.Index as TarIndex+import Codec.Archive.Tar.Index (TarIndex)+--import Data.Either (fromRight) import System.IO (openFile, IOMode(..), hTell) import System.FilePath (takeExtension) import Data.Either (rights)@@ -46,8 +51,22 @@ import Path.IO (resolveFile') import Control.Monad.IO.Class (liftIO) import qualified Data.Text.Lazy as T-import Control.Monad (liftM)+import Control.Monad (liftM, when)+import Conduit+import Data.Conduit+import Data.Conduit.Zlib (ungzip)+import Data.Conduit.BZlib (bunzip2)+import qualified Crypto.Hash.Conduit as CH+import qualified Data.Conduit.Tar as CT+import Control.DeepSeq +fetchDirect :: FetchCommunicationService_Iface a => a -> IO [Communication]+fetchDirect f = do+ n <- getCommunicationCount f+ is <- getCommunicationIDs f 0 n+ FetchResult cs <- fetch f (FetchRequest is Nothing)+ return $ V.toList cs+ -- | Handle-based Fetch backend newtype HandleFetch = HandleFetch Handle @@ -64,77 +83,144 @@ makeHandleFetch f = error "unimplemented" -- | Zip-based Fetch backend-newtype ZipFetch = ZipFetch ((Map String Zip.EntrySelector), String)+newtype ZipFetch = ZipFetch (String, Map Text String) instance Service_Iface ZipFetch where about _ = return $ ServiceInfo "Zip-backed FetchCommunicationService" "0.0.1" (Just "Haskell implementation") alive _ = return True instance FetchCommunicationService_Iface ZipFetch where- fetch (ZipFetch (ms, f)) (FetchRequest ii _) = do- f' <- resolveFile' f- let ids = map T.unpack (V.toList ii)- es = map (\i -> ms Map.! i) ids- ss <- liftIO $ Zip.withArchive f' (sequence $ map (liftM LBS.fromStrict . Zip.getEntry) es)- cs <- sequence $ map stringToComm ss- return $ default_FetchResult { fetchResult_communications=V.fromList cs }- getCommunicationIDs (ZipFetch (ms, f)) offset count = return $ V.fromList $ ((map (pack . fst)) . genericTake count . genericDrop offset . Map.toList) ms- getCommunicationCount (ZipFetch (ms, f)) = return ((genericLength . Map.toList) ms)+ fetch (ZipFetch (f, ms)) (FetchRequest ii _) = error "" --do+ -- f' <- resolveFile' f+ -- let ids = map T.unpack (V.toList ii)+ -- es = map (\i -> ms Map.! i) ids+ -- ss <- liftIO $ Zip.withArchive f' (sequence $ map (liftM LBS.fromStrict . Zip.getEntry) es)+ -- cs <- sequence $ map stringToComm ss+ -- return $ default_FetchResult { fetchResult_communications=V.fromList cs }+ getCommunicationIDs (ZipFetch (f, ms)) offset count = error "" -- return $ V.fromList $ ((map (pack . fst)) . genericTake count . genericDrop offset . Map.toList) ms+ getCommunicationCount (ZipFetch (f, ms)) = error "" --return ((genericLength . Map.toList) ms) -- | Create a Zip-based Fetch handler based on the given file makeZipFetch :: String -> IO ZipFetch makeZipFetch f = do f' <- resolveFile' f - es <- liftIO $ Zip.withArchive f' Zip.getEntries- ms <- Map.fromList <$> mapM (\e -> do- s <- liftIO $ Zip.withArchive f' (Zip.getEntry e)- c <- stringToComm (LBS.fromStrict s)- return (T.unpack $ communication_id c, e)) (Map.keys es)- return $ ZipFetch (ms, f)+ es <- liftIO $ ZipC.withArchive f ZipC.entryNames -- Zip.getEntries+ -- ms <- Map.fromList <$> mapM (\e -> do+ -- s <- liftIO $ Zip.withArchive f' (Zip.getEntry e)+ -- c <- stringToComm (LBS.fromStrict s)+ -- return (T.unpack $ communication_id c, e)) (Map.keys es)+ return $ ZipFetch (f, Map.fromList []) -- | Tar-based Fetch backend-newtype TarFetch = TarFetch (Handle, (LBS.ByteString -> LBS.ByteString), Tar.TarIndex, Map String FilePath)+newtype TarFetch = TarFetch (Handle, Map Text TarIndex.TarEntryOffset) -- [(String, Int, Int)])+ --(String, Int, Int)]+ deriving Show+--(Handle, (LBS.ByteString -> LBS.ByteString), Tar.TarIndex, Map String FilePath) instance Service_Iface TarFetch where about _ = return $ ServiceInfo "Tar-backed FetchCommunicationService" "0.0.1" (Just "Haskell implementation") alive _ = return True instance FetchCommunicationService_Iface TarFetch where- fetch (TarFetch (h, c, i, l)) ii = do- cc <- sequence $ map fetchOne ((map unpack . V.toList . fetchRequest_communicationIds) ii)- return $ default_FetchResult { fetchResult_communications=V.fromList cc }+ fetch (TarFetch (h, m)) ii = do+ let i = [m Map.! n | n <- ((V.toList . fetchRequest_communicationIds) ii)]+ i' <- mapM (\ o -> do+ e <- TarIndex.hReadEntry h o+ case Tar.entryContent e of+ Tar.NormalFile bs _ -> do+ stringToComm bs+ --return $!! (communication_id c, o)+ ) i+ return $ default_FetchResult { fetchResult_communications=V.fromList i' }++ --fetch (TarFetch (h, c, i, l)) ii = error "te"+ -- do+ -- cc <- sequence $ map fetchOne ((map unpack . V.toList . fetchRequest_communicationIds) ii)+ -- return $ default_FetchResult { fetchResult_communications=V.fromList cc }+ -- where+ -- fetchOne :: String -> IO Communication+ -- fetchOne p = do+ -- let (Just (Tar.TarFileEntry o)) = Tar.lookup i (l Map.! p)+ -- e <- Tar.hReadEntry h o+ -- stringToComm ((((\ (Tar.NormalFile bs _) -> bs) . Tar.entryContent) ) e)+ --getCommunicationIDs (TarFetch (_, _, _, l)) offset count = error "dd"+ getCommunicationIDs (TarFetch (_, m)) offset count = return $ (V.fromList . take count' . drop offset' . Map.keys) m where- fetchOne :: String -> IO Communication- fetchOne p = do- let (Just (Tar.TarFileEntry o)) = Tar.lookup i (l Map.! p)- e <- Tar.hReadEntry h o- stringToComm ((((\ (Tar.NormalFile bs _) -> bs) . Tar.entryContent) ) e)- getCommunicationIDs (TarFetch (_, _, _, l)) offset count = return $ V.fromList $ ((map (pack . fst)) . genericTake count . genericDrop offset . Map.toList) l- getCommunicationCount (TarFetch (_, _, _, l)) = return ((genericLength . Map.toList) l)+ count' = fromIntegral count+ offset' = fromIntegral offset+ -- return $ V.fromList $ ((map (pack . fst)) . genericTake count . genericDrop offset . Map.toList) l+ --getCommunicationCount (TarFetch (_, _, _, l)) = error "as"+ getCommunicationCount (TarFetch (_, m)) = return $ fromIntegral $ Map.size m+ -- return ((genericLength . Map.toList) l) -- | Create a Tar-based Fetch handler based on the given file makeTarFetch :: String -> IO TarFetch makeTarFetch f = do let c = case takeExtension f of- --".tgz" -> GZip.decompress- --".tbz2" -> BZip.decompress+ ".tgz" -> GZip.decompress+ ".tbz2" -> BZip.decompress ".tar" -> id h <- openFile f ReadMode- bs <- c <$> LBS.readFile f- let e = Tar.read bs- (l, i) <- build e+ t <- LBS.hGetContents h h' <- openFile f ReadMode- return $ TarFetch (h, c, i, Map.fromList l) -commFromEntry :: Tar.Entry -> IO Communication-commFromEntry e = stringToComm (((\ (Tar.NormalFile bs _) -> bs) . Tar.entryContent) e)+ let es = Tar.read t+ cs = []+ Right i = (liftM TarIndex.toList . TarIndex.build) es -build :: Tar.Entries e -> IO ([(String, FilePath)], Tar.TarIndex)-build = go ([], Tar.empty)- where- go :: ([(String, FilePath)], Tar.IndexBuilder) -> Tar.Entries e -> IO ([(String, FilePath)], Tar.TarIndex)- go (l, !builder) (Tar.Next e es) = do- c <- commFromEntry e- go (((unpack . communication_id) c, Tar.entryPath e):l, Tar.addNextEntry e builder) es- go (l, !builder) (Tar.Done) = do- return (l, Tar.finalise builder)+ --print i+ --print $ length i+ i' <- mapM (\(_, o) -> do+ e <- TarIndex.hReadEntry h' o+ case Tar.entryContent e of+ Tar.NormalFile bs _ -> do+ c <- stringToComm bs+ return $!! (communication_id c, o)+ ) i+ --let cs = Tar.foldEntries (\e l -> ((T.pack . Tar.entryPath) e):l) [] (\e -> []) (Tar.read t)+ -- cs <- Tar.foldEntries (\e !l -> do+ -- l' <- l+ -- let con = Tar.entryContent e+ -- case con of+ -- Tar.NormalFile bs _ -> do+ -- comm <- stringToComm bs+ -- return $!! (communication_id comm):l'+ -- _ -> return $!! l'+ -- ) (return []) (\e -> return []) (Tar.read t)+ + --is <- runConduitRes $ sourceFileBS f .| c .| CT.untar .| CT.withEntries commIdFromEntry .| sinkList+ --bs <- c <$> LBS.hGetContents h+ -- let e = Tar.read bs+ -- (l, i) <- build e+ -- h' <- openFile f ReadMode+ return $ TarFetch (h', Map.fromList i') --(fromRight (TarIndex.finalise Tar.empty). TarIndex.build . Tar.read) bs) --Tar.finalise Tar.empty)+ --is -- (h, c, i, is)++--entryName :: Monad m => CT.Header -> Conduit SBS.ByteString m (String, Int, Int)+--entryName c = yield $ (CT.headerFilePath c, CT.headerPayloadOffset c, CT.headerPayloadSize c)++--commFromEntry :: Tar.Entry -> IO Communication+commIdFromEntry e = when (CT.headerFileType e == CT.FTNormal) $ do+ --yield (communication_id default_Communication)+ c <- await+ case c of+ Just t -> do+ c' <- liftIO $ stringToComm (LBS.fromStrict t) -- default_Communication+ yield (communication_id c')+ _ -> yield ""+ --liftIO $ print c+ + --let c' = default_Communication+ --+ --yield "" -- (communication_id c')+ --stringToComm (((\ (Tar.NormalFile bs _) -> bs) . Tar.entryContent) e)++-- build :: Tar.Entries e -> IO ([(String, FilePath)], Tar.TarIndex)+-- build = go ([], Tar.empty)+-- where+-- go :: ([(String, FilePath)], Tar.IndexBuilder) -> Tar.Entries e -> IO ([(String, FilePath)], Tar.TarIndex)+-- go (l, !builder) (Tar.Next e es) = do+-- c <- commFromEntry e+-- go (((unpack . communication_id) c, Tar.entryPath e):l, Tar.addNextEntry e builder) es+-- go (l, !builder) (Tar.Done) = do+-- return (l, Tar.finalise builder)
src/Data/Concrete/Services/Store.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE DeriveGeneric, OverloadedStrings, FlexibleInstances #-}+{-# LANGUAGE DeriveGeneric, OverloadedStrings, FlexibleInstances, PackageImports #-} {-| Description: Implementations of StoreCommunicationService -}@@ -10,13 +10,15 @@ , makeTarStore , makeZipStore , makeHandleStore+ , storeDirect ) where import qualified Data.ByteString as SBS import qualified Data.ByteString.Lazy as LBS import qualified Codec.Compression.GZip as GZip import qualified Codec.Compression.BZip as BZip-import qualified Codec.Archive.Zip as Zip+import qualified "zip" Codec.Archive.Zip as Zip+import qualified "zip-conduit" Codec.Archive.Zip as ZipC import qualified Codec.Archive.Tar as Tar import qualified Codec.Archive.Tar.Entry as Tar import qualified Codec.Archive.Tar.Index as Tar@@ -37,6 +39,11 @@ import Control.Monad (liftM) import System.FilePath (takeExtension) import Control.Monad.IO.Class (liftIO)+import Conduit+import Data.Void (Void)++storeDirect :: StoreCommunicationService_Iface a => a -> [Communication] -> IO ()+storeDirect s cs = (sequence $ map (store s) cs) >> return () lift1st :: Monad m => (m a, b) -> m (a, b) lift1st (f, s) = do
src/Data/Concrete/Utils.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE DeriveGeneric, OverloadedStrings, FlexibleInstances #-}+{-# LANGUAGE DeriveGeneric, OverloadedStrings, FlexibleInstances, PackageImports #-} {-| Description: Common tools for working with Concrete data@@ -54,7 +54,7 @@ import qualified Codec.Compression.GZip as GZip import qualified Codec.Compression.BZip as BZip import qualified Codec.Archive.Tar as Tar-import qualified Codec.Archive.Zip as Zip+import qualified "zip" Codec.Archive.Zip as Zip import qualified Codec.Archive.Tar.Entry as Tar import qualified Codec.Archive.Tar.Index as Tar import Data.Time
tests/TestIngesters.hs view
@@ -20,6 +20,7 @@ import Data.Concrete.Utils (writeCommunication, showCommunication) import Data.Concrete.Parsers.Types (CommunicationParser) import Data.Concrete.Parsers (communicationParsers, ingest)+import Data.Concrete.Parsers.Utils (finalizeCommunication) import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..)) import Text.Printf (printf) import qualified Control.Concurrent as C@@ -36,63 +37,15 @@ import System.IO.Unsafe import qualified Codec.Compression.GZip as GZip import qualified Data.Vector as V---testFormat :: (String, (desc, CommunicationParser (), [String], String)) -> IO ()-testFormat (f, (d, p, c, i)) = do- let inputFile = printf "tests/data/example.%s.gz" f :: String- putStrLn (printf "\t%s" f :: String)- con <- connectToService "localhost" 9090- ih <- (liftM GZip.decompress . BS.readFile) inputFile- let (_, cp, _, _) = (Map.fromList communicationParsers) ! f- ingest (\c -> do- putStrLn $ printf "\t\t%s" (communication_id c)- StoreService.store con c) p (decodeUtf8 ih) c i "concrete-haskell unit test data"- return ()---testFetch :: IO ()-testFetch = do- putStrLn "Testing fetch service:"- con <- connectToService "localhost" 9091- c <- FetchService.getCommunicationCount con- putStrLn $ printf "\tFetch service reports %d Communications" c- ids <- FetchService.getCommunicationIDs con 0 c- putStrLn $ printf "\tReceived %d Communication IDs" (length ids)- FetchResult {..} <- FetchService.fetch con $ default_FetchRequest { fetchRequest_communicationIds=ids }- sequence $ map (putStrLn . printf "\t\tFetched Communication with ID == %s" . T.unpack . communication_id) (V.toList fetchResult_communications)- return ()- +import Conduit+import System.Exit (exitFailure)+ +testFormat (name, (desc, src, contentSects, idStr)) = do+ putStrLn $ "\nTesting parser for " ++ name+ let inputFile = printf "tests/data/example.%s.gz" name :: String+ txt <- (liftM GZip.decompress . BS.readFile) inputFile+ ingest src (\c -> print $ communication_id c) (\_ -> True) (decodeUtf8 txt) contentSects idStr "test comms" -main = do- putStrLn "\nTesting parsers + fetch and store services:"- let outputFile = "test.tar"- store <- C.forkIO $ do- case takeExtension outputFile of- ".zip" -> do- h <- makeZipStore outputFile- runConcreteService 9090 Store.process h- ".tar" -> do- h <- makeTarStore outputFile- runConcreteService 9090 Store.process h- _ -> do- h <- makeHandleStore outputFile- runConcreteService 9090 Store.process h- C.threadDelay 1000000- sequence $ map testFormat (communicationParsers)- C.killThread store- C.threadDelay 1000000- fetch <- C.forkIO $ do- case takeExtension outputFile of- ".zip" -> do- h <- makeZipFetch outputFile- runConcreteService 9091 Fetch.process h- ".tar" -> do- h <- makeTarFetch outputFile- runConcreteService 9091 Fetch.process h- _ -> do- h <- makeHandleFetch outputFile- runConcreteService 9091 Fetch.process h- C.threadDelay 1000000- testFetch- C.killThread fetch+main = do + mapM testFormat communicationParsers+ exitFailure
+ tests/TestSerialization.hs view
@@ -0,0 +1,22 @@+{-# LANGUAGE OverloadedStrings #-}+module Main (main) where++import Data.ByteString.Lazy (ByteString)+import qualified Data.ByteString.Lazy as BS+import Data.Map (toList, (!), keys)+import Data.Monoid ((<>))+import Data.List (intercalate)+import Control.Monad (void, join, liftM)+import Data.Text.Lazy (Text, unpack, take)+import qualified Data.Text.Lazy as T+import Data.Text.Lazy.Encoding (decodeUtf8)+import System.IO (stdin, stdout, stderr, openFile, Handle, IOMode(..), hPutStrLn)+import System.FilePath (takeExtension)+import qualified Codec.Compression.GZip as GZip+import Data.Concrete.Utils (writeCommunication, showCommunication)+import Data.Concrete.Parsers.Types (CommunicationParser)+import Data.Concrete.Parsers (communicationParsers, ingest)+import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..))+import Text.Printf (printf)++main = return ()
+ utils/ApplyAnnotators.hs view
@@ -0,0 +1,38 @@+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE FlexibleInstances #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE StandaloneDeriving #-}+{-# LANGUAGE TypeOperators #-}+++module Main (main) where+++import Control.Monad (liftM, foldM)+import Data.Concrete.Services (connectToService)+import Data.Concrete.Services.Store (makeTarStore, storeDirect)+import Data.Concrete.Services.Fetch (makeTarFetch, fetchDirect)+import Data.Concrete.Autogen.AnnotateCommunicationService_Client (annotate)+import Options.Generic+++data Parameters w = Parameters { inputFile :: w ::: String <?> "Input tar file"+ , outputFile :: w ::: String <?> "Output tar file"+ , host :: w ::: [String] <?> "Annotator host (may be specified multiple times)"+ , port :: w ::: [Int] <?> "Annotator port (may be specified multiple times)"+ } deriving (Generic)+++instance ParseRecord (Parameters Wrapped)+deriving instance Show (Parameters Unwrapped)+++main = do+ opts <- unwrapRecord "Run a tar file of Communications through a sequence of AnnotateCommunicationServices"+ source <- makeTarFetch $ inputFile opts+ comms <- fetchDirect source+ services <- mapM (uncurry connectToService) (zip (host opts) (port opts))+ comms' <- foldM (\cs s -> mapM (annotate s) cs) comms services+ oh <- makeTarStore $ outputFile opts+ storeDirect oh comms'
+ utils/BuildGraphs.hs view
@@ -0,0 +1,44 @@+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE FlexibleInstances #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE StandaloneDeriving #-}+{-# LANGUAGE TypeOperators #-}+++module Main (main) where+++import Control.Monad (void, join, liftM)+import Data.Concrete.Services.Store (makeTarStore, storeDirect)+import Data.Concrete.Services.Fetch (makeTarFetch, makeZipFetch, fetchDirect)+import Data.Concrete.Autogen.Communication_Types (Communication(..))+import Data.Concrete.Autogen.Graph_Types+import Data.Concrete.Autogen.AnnotateCommunicationService_Client (annotate)+import Options.Generic+import Conduit+import System.FilePath (takeExtension)++data Parameters w = Parameters { inputFile :: w ::: String <?> "Input tar file"+ , outputFile :: w ::: String <?> "Output tar file"+ , identifier :: w ::: String <?> "Section that is used to identify a node"+ , child :: w ::: Maybe String <?> "Section that identifies a node's child"+ , parent :: w ::: Maybe String <?> "Section that identifies a node's parent" + } deriving (Generic)+++instance ParseRecord (Parameters Wrapped)+deriving instance Show (Parameters Unwrapped)+++main = do+ ps <- unwrapRecord "Transform a file of Communications into a MultiGraph based on a particular link"+ --ih <- makeTarFetch $ inputFile ps+ cs <- join $ case takeExtension (inputFile ps) of+ ".zip" -> (liftM fetchDirect . makeZipFetch) (inputFile ps)+ ".tar" -> (liftM fetchDirect . makeTarFetch) (inputFile ps) +-- ".tar" -> fetchDirect <$> makeTarFetch $ inputFile ps --(liftM decompress . BS.hGetContents) stdin+ --foldOver (\x -> print x >> return ()) ih+ --cs <- fetchDirect ih+ --cs' <- runConduit $ yieldMany cs .| takeC 10 .| sinkList+ print $ length cs
utils/IngestCommunications.hs view
@@ -4,73 +4,70 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE StandaloneDeriving #-} {-# LANGUAGE TypeOperators #-}+{-# LANGUAGE ExtendedDefaultRules #-} module Main (main) where --import qualified Network as Net-import Data.ByteString.Lazy (ByteString) import qualified Data.ByteString.Lazy as BS import Data.Map.Strict (Map, toList, (!), keys) import qualified Data.Map.Strict as Map-import Data.Monoid ((<>)) import Data.Maybe (fromMaybe)-import Data.List (intercalate) import Control.Monad (void, join, liftM) import Data.Text.Lazy (Text, unpack, take)-import Data.Text.Lazy.Encoding (decodeUtf8)+import Data.Text.Lazy.Encoding (decodeUtf8, decodeLatin1)+import Data.Text.Encoding.Error (lenientDecode) import System.IO (stdin, stdout, stderr, openFile, Handle, IOMode(..), hPutStrLn, hClose) import System.FilePath (takeExtension)-import qualified Codec.Compression.GZip as GZip import Data.Concrete.Utils (writeCommunication, getCompressor, getDecompressor) import Data.Concrete.Services (connectToService)-import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..))-import qualified Data.Concrete.Utils as CU-import Data.Concrete.Parsers.Types (CommunicationParser)+import Data.Concrete.Services.Store (makeTarStore, makeHandleStore, makeZipStore, ZipStore(..), TarStore(..), HandleStore(..), storeDirect) import Data.Concrete.Parsers (communicationParsers, ingest)+import Data.Concrete.Parsers.Utils (unfoldParse, finalizeCommunication) import qualified Data.Concrete.Autogen.StoreCommunicationService_Client as StoreService-import qualified Data.Concrete.Autogen.Service_Client as Service-import Options.Generic+import Data.Concrete.Autogen.StoreCommunicationService_Iface (StoreCommunicationService_Iface(store))+import Options.Generic hiding (Text)+import Conduit -data Parameters w = Parameters { inputFile :: w ::: Maybe String <?> "Input file, possibly compressed (.bz2 or .gz)"- , commType :: w ::: String <?> "Value for the 'type' field of each Communication"- , commId :: w ::: String <?> "Template for the 'id' field of each Communication"- , contentSectionTypes :: w ::: [String] <?> "Section types that should count as 'content' rather than 'metadata'"- , format :: w ::: String <?> "Input format: (JSON, JSON-LINE, CSV)"- , outputFile :: w ::: Maybe String <?> "An output file (.txt, .gz, .bz2, .tgz, .tbz2, .zip)"+data Parameters w = Parameters { inputFile :: w ::: Maybe String <?> "Input text file, possibly compressed (.bz2 or .gz)"+ , commType :: w ::: Text <?> "Value for the 'type' field of each Communication"+ , commId :: w ::: Text <?> "Template for the 'id' field of each Communication"+ , contentSectionTypes :: w ::: [Text] <?> "Section types that should count as 'content' rather than 'metadata'"+ , format :: w ::: String <?> "Input format: (JSON-SEQUENCE, JSON-ARRAY, CSV, PTB, CONLL-U)"+ , outputFile :: w ::: Maybe String <?> "An output file (.tar)" , host :: w ::: Maybe String <?> "Host name for a StoreCommunicationService" , port :: w ::: Maybe Int <?> "Port for a StoreCommunicationService"+ , latin1 :: w ::: Maybe Bool <?> "Input is Latin-1 (8859) rather than UTF-8" } deriving (Generic) instance ParseRecord (Parameters Wrapped) deriving instance Show (Parameters Unwrapped) main = do- ps <- unwrapRecord "Ingest Concrete Communications from various formats"- let compress = getCompressor ((fromMaybe "" . inputFile) ps)- decompress = getDecompressor ((fromMaybe "" . outputFile) ps)- ih <- case inputFile ps of- Just f -> (liftM decompress . BS.readFile) f--- _ -> - -- _ -> BS.readFile f- Nothing -> (liftM decompress . BS.hGetContents) stdin- let (_, cp, _, _) = (Map.fromList communicationParsers) ! (format ps) + ps <- unwrapRecord "Ingest Concrete Communications from various formats"+ let decompress = getDecompressor ((fromMaybe "" . inputFile) ps)+ compress = getCompressor ((fromMaybe "" . outputFile) ps)+ content = contentSectionTypes ps+ cid = commId ps+ ctype = commType ps+ (_, cp, _, _) = (Map.fromList communicationParsers) ! (format ps)+ inFile = inputFile ps+ outFile = fromMaybe "" (outputFile ps)+ decode = if fromMaybe False (latin1 ps) == True then decodeLatin1 else decodeUtf8+ + t <- case inputFile ps of+ Just f -> (liftM decompress . BS.readFile) f+ Nothing -> (liftM decompress . BS.hGetContents) stdin+ cb <- case (outputFile ps, host ps, port ps) of (Just f, Nothing, Nothing) -> case takeExtension f of- ".gz" -> writeCommunication <$> openFile f WriteMode- ".bz2" -> writeCommunication <$> openFile f WriteMode- _ -> writeCommunication <$> openFile f WriteMode+ ".tar" -> store <$> makeTarStore f+ ".zip" -> store <$> makeZipStore f+ _ -> store <$> makeHandleStore f (Nothing, Just h, Just p) -> do con <- connectToService h p- return $ (\c -> do- --let p = CU.getSectionText c "parent_id"- -- i = CU.getSectionText c "id"- -- u = CU.getSectionText c "created_utc"- -- s = CU.getSectionText c "subreddit"- --print p- StoreService.store con c- )+ return $ StoreService.store con (Nothing, Nothing, Nothing) -> return $ writeCommunication stdout- _ -> error "Specify either an output file, a host and port, or nothing (for stdout)" - ingest cb cp (decodeUtf8 ih) (contentSectionTypes ps) (commId ps) (commType ps)+ _ -> error "Specify either an output file, a host and port, or nothing (for flat serialization to stdout)"++ ingest cp cb (\_ -> False) (decode t) content cid ctype
utils/InspectCommunications.hs view
@@ -14,8 +14,7 @@ import qualified Data.Concrete.Utils as CU import Data.Concrete.Autogen.Communication_Types (default_Communication, Communication(..)) import qualified Data.Concrete.Autogen.FetchCommunicationService_Client as FetchService-import Data.Concrete.Autogen.FetchCommunicationService_Iface (FetchCommunicationService_Iface(fetch, getCommunicationIDs, getCommunicationCount))-import Data.Concrete.Autogen.Access_Types (FetchRequest(..), default_FetchRequest, FetchResult(..))+import Data.Concrete.Autogen.Access_Types (FetchRequest(..)) import Data.Concrete.Services (connectToService) import qualified Data.Vector as V import Control.Monad (liftM, mapM, join)@@ -25,9 +24,7 @@ import System.IO (stdin, stdout, stderr, openFile, Handle, IOMode(..), hPutStrLn) import System.FilePath (takeExtension) import Options.Generic-import Data.Concrete.Services.Fetch (ZipFetch(..), TarFetch(..), process, makeTarFetch, makeZipFetch, makeHandleFetch) - data Parameters w = Parameters { inputFile :: w ::: Maybe String <?> "Input file, possibly compressed (.bz2 or .gz)" , start :: w ::: Maybe Int <?> "Index of first Communication to show" , end :: w ::: Maybe Int <?> "Index of last Communication to show"@@ -36,32 +33,32 @@ } deriving (Generic) - instance ParseRecord (Parameters Wrapped) deriving instance Show (Parameters Unwrapped) --fetchDirect :: FetchCommunicationService_Iface a => a -> IO FetchResult-fetchDirect h = do- c <- getCommunicationCount h- cs <- getCommunicationIDs h 0 c- fetch h $ default_FetchRequest { fetchRequest_communicationIds=cs }-- main = do ps <- unwrapRecord "Inspect Concrete Communications" let s = start ps e = end ps cb = (\x -> (putStrLn . T.unpack) (CU.showCommunication x))- FetchResult cs <- case inputFile ps of+ cs <- case inputFile ps of Nothing -> do con <- connectToService ((fromMaybe "0.0.0.0" . host) ps) ((fromMaybe 9090 . port) ps) c <- FetchService.getCommunicationCount con+ print c l <- FetchService.getCommunicationIDs con 0 c- FetchService.fetch con (FetchRequest l Nothing)+ --print l+ c' <- FetchService.fetch con (FetchRequest (V.drop 50 l) Nothing)+ print c'+ return () Just f -> case takeExtension f of- ".tar" -> join $ fetchDirect <$> makeTarFetch f- ".zip" -> join $ fetchDirect <$> makeZipFetch f- print $ length cs+ --".tar" -> CU.readCommunicationsFromTar <$> (BS.readFile f)+ --".tgz" -> CU.readCommunicationsFromTar <$> ((liftM GZip.decompress . BS.readFile) f)+ --".tbz2" -> CU.readCommunicationsFromTar <$> ((liftM BZip.decompress . BS.readFile) f)+ --".zip" -> return $ CU.readCommunicationsFromZip f+ --".bz2" -> CU.mapCommunicationsFromBytes cb s e <$> ((liftM BZip.decompress . BS.readFile) f)+ --".gz" -> CU.mapCommunicationsFromBytes cb s e <$> ((liftM GZip.decompress . BS.readFile) f)+ _ -> error "unimplemented" --CU.mapCommunicationsFromBytes cb s e <$> (BS.readFile f)+ --Nothing -> CU.readCommunicationsFromBytes <$> (BS.hGetContents stdin) return ()