packages feed

api-opentheory-unicode 1.0 → 1.1

raw patch · 216 files changed

+537/−119 lines, 216 filesbinary-addedPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

api-opentheory-unicode.cabal view
@@ -1,11 +1,12 @@ name: api-opentheory-unicode-version: 1.0+version: 1.1 category: Text synopsis: OpenTheory unicode character API license: MIT license-file: LICENSE cabal-version: >= 1.8.0.2 build-type: Simple+data-files: test/*.txt test/valid/*.txt test/invalid/*.txt author: Joe Leslie-Hurd <joe@gilith.com> maintainer: Joe Leslie-Hurd <joe@gilith.com> description:@@ -17,23 +18,18 @@     base >= 4.0 && < 5.0,     bytestring >= 0.9 && < 1.0,     opentheory-unicode >= 1.0 && < 2.0-   hs-source-dirs: src-   ghc-options: -Wall-   exposed-modules:     Unicode -executable api-opentheory-unicode-test+test-suite api-opentheory-unicode-test+  type: exitcode-stdio-1.0   build-depends:     base >= 4.0 && < 5.0,     bytestring >= 0.9 && < 1.0,     directory >= 1.0 && < 2.0,     opentheory-unicode >= 1.0 && < 2.0--  hs-source-dirs: src, testsrc-+  hs-source-dirs: src   ghc-options: -Wall--  main-is: Main.hs+  main-is: Test.hs
+ src/Test.hs view
@@ -0,0 +1,109 @@+{- |+module: Main+description: Testing the Unicode character library and API+license: MIT++maintainer: Joe Leslie-Hurd <joe@gilith.com>+stability: provisional+portability: portable+-}+module Main+  ( main )+where++import qualified Data.List as List+import qualified Data.Either as Either+import qualified Data.Word as Word+import qualified System.Directory as Directory++import Unicode+import qualified OpenTheory.Unicode as Unicode++demoLength :: Int+demoLength = 7621++testLength :: Int+testLength = 79++noInvalid :: [Either Word.Word8 Unicode.Unicode] -> Bool+noInvalid = all (Either.either (const False) (const True))++getTestFiles :: FilePath -> IO [FilePath]+getTestFiles d =+    do fs <- Directory.getDirectoryContents d+       let fs' = filter (List.isPrefixOf "test") fs+       return (map (\f -> d ++ "/" ++ f) fs')++readTestCharFile :: Maybe Int -> FilePath -> IO ()+readTestCharFile x f =+    do l <- decodeFile f+       let a = noInvalid l+       case x of+         Nothing ->+             if a+               then error $ "invalid file " ++ f ++ " was successfully decoded"+               else return ()+         Just n ->+             if not a+               then error $ "valid file " ++ f ++ " could not be decoded"+               else+                 if length l == n+                   then return ()+                   else+                     error $ "valid file " ++ f ++ " has bad line length: " +++                             show (length l)++readValidCharFile :: Int -> FilePath -> IO ()+readValidCharFile n = readTestCharFile (Just n)++readInvalidCharFile :: FilePath -> IO ()+readInvalidCharFile = readTestCharFile Nothing++partitionTestCharFile :: IO ()+partitionTestCharFile =+    do skip <- Directory.doesFileExist "test/valid/test061.txt"+       if skip+         then return ()+         else+           do putStrLn "\nsplitting UTF-8 test file into valid and invalid lines"+              cs <- decodeFile "test/test.txt"+              mapM_ outputLine (filter testLine (readLines 1 cs))+  where+    isNewline :: Either Word.Word8 Unicode.Unicode -> Bool+    isNewline (Left _) = False+    isNewline (Right c) = let n = Unicode.unUnicode c in n == 10 || n == 13++    chopNewline :: [Either Word.Word8 Unicode.Unicode] ->+                   [Either Word.Word8 Unicode.Unicode]+    chopNewline [] = []+    chopNewline (_ : cs) = cs++    readLines :: Int -> [Either Word.Word8 Unicode.Unicode] ->+                 [(Int,[Either Word.Word8 Unicode.Unicode])]+    readLines _ [] = []+    readLines lineno inp =+        let (line,inp') = List.span (not . isNewline) inp in+        let inp'' = chopNewline inp' in+        (lineno,line) : readLines (lineno + 1) inp''++    testLine :: (Int,[Either Word.Word8 Unicode.Unicode]) -> Bool+    testLine (lineno,line) = not (lineno < 61 || null line)++    outputLine :: (Int,[Either Word.Word8 Unicode.Unicode]) -> IO ()+    outputLine (lineno,line) =+        let valid = noInvalid line in+        let n = show lineno in+        let f = "test/" ++ (if valid then "valid" else "invalid") +++                "/test" ++ replicate (3 - length n) '0' ++ n ++ ".txt" in+        reencodeFile f line++main :: IO ()+main =+    do partitionTestCharFile+       readValidCharFile demoLength "test/demo.txt"+       validTestFiles <- getTestFiles "test/valid"+       invalidTestFiles <- getTestFiles "test/invalid"+       mapM_ (readValidCharFile testLength) validTestFiles+       mapM_ readInvalidCharFile invalidTestFiles+       putStrLn "  all tests pass"+       return ()
+ test/demo.txt view
@@ -0,0 +1,212 @@++UTF-8 encoded sample plain-text file+‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾++Markus Kuhn [ˈmaʳkʊs kuːn] <http://www.cl.cam.ac.uk/~mgk25/> — 2002-07-25+++The ASCII compatible UTF-8 encoding used in this plain-text file+is defined in Unicode, ISO 10646-1, and RFC 2279.+++Using Unicode/UTF-8, you can write in emails and source code things such as++Mathematics and sciences:++  ∮ E⋅da = Q,  n → ∞, ∑ f(i) = ∏ g(i),      ⎧⎡⎛┌─────┐⎞⎤⎫+                                            ⎪⎢⎜│a²+b³ ⎟⎥⎪+  ∀x∈ℝ: ⌈x⌉ = −⌊−x⌋, α ∧ ¬β = ¬(¬α ∨ β),    ⎪⎢⎜│───── ⎟⎥⎪+                                            ⎪⎢⎜⎷ c₈   ⎟⎥⎪+  ℕ ⊆ ℕ₀ ⊂ ℤ ⊂ ℚ ⊂ ℝ ⊂ ℂ,                   ⎨⎢⎜       ⎟⎥⎬+                                            ⎪⎢⎜ ∞     ⎟⎥⎪+  ⊥ < a ≠ b ≡ c ≤ d ≪ ⊤ ⇒ (⟦A⟧ ⇔ ⟪B⟫),      ⎪⎢⎜ ⎲     ⎟⎥⎪+                                            ⎪⎢⎜ ⎳aⁱ-bⁱ⎟⎥⎪+  2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm     ⎩⎣⎝i=1    ⎠⎦⎭++Linguistics and dictionaries:++  ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn+  Y [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ]++APL:++  ((V⍳V)=⍳⍴V)/V←,V    ⌷←⍳→⍴∆∇⊃‾⍎⍕⌈++Nicer typography in plain text files:++  ╔══════════════════════════════════════════╗+  ║                                          ║+  ║   • ‘single’ and “double” quotes         ║+  ║                                          ║+  ║   • Curly apostrophes: “We’ve been here” ║+  ║                                          ║+  ║   • Latin-1 apostrophe and accents: '´`  ║+  ║                                          ║+  ║   • ‚deutsche‘ „Anführungszeichen“       ║+  ║                                          ║+  ║   • †, ‡, ‰, •, 3–4, —, −5/+5, ™, …      ║+  ║                                          ║+  ║   • ASCII safety test: 1lI|, 0OD, 8B     ║+  ║                      ╭─────────╮         ║+  ║   • the euro symbol: │ 14.95 € │         ║+  ║                      ╰─────────╯         ║+  ╚══════════════════════════════════════════╝++Combining characters:++  STARGΛ̊TE SG-1, a = v̇ = r̈, a⃑ ⊥ b⃑++Greek (in Polytonic):++  The Greek anthem:++  Σὲ γνωρίζω ἀπὸ τὴν κόψη+  τοῦ σπαθιοῦ τὴν τρομερή,+  σὲ γνωρίζω ἀπὸ τὴν ὄψη+  ποὺ μὲ βία μετράει τὴ γῆ.++  ᾿Απ᾿ τὰ κόκκαλα βγαλμένη+  τῶν ῾Ελλήνων τὰ ἱερά+  καὶ σὰν πρῶτα ἀνδρειωμένη+  χαῖρε, ὦ χαῖρε, ᾿Ελευθεριά!++  From a speech of Demosthenes in the 4th century BC:++  Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι,+  ὅταν τ᾿ εἰς τὰ πράγματα ἀποβλέψω καὶ ὅταν πρὸς τοὺς+  λόγους οὓς ἀκούω· τοὺς μὲν γὰρ λόγους περὶ τοῦ+  τιμωρήσασθαι Φίλιππον ὁρῶ γιγνομένους, τὰ δὲ πράγματ᾿+  εἰς τοῦτο προήκοντα,  ὥσθ᾿ ὅπως μὴ πεισόμεθ᾿ αὐτοὶ+  πρότερον κακῶς σκέψασθαι δέον. οὐδέν οὖν ἄλλο μοι δοκοῦσιν+  οἱ τὰ τοιαῦτα λέγοντες ἢ τὴν ὑπόθεσιν, περὶ ἧς βουλεύεσθαι,+  οὐχὶ τὴν οὖσαν παριστάντες ὑμῖν ἁμαρτάνειν. ἐγὼ δέ, ὅτι μέν+  ποτ᾿ ἐξῆν τῇ πόλει καὶ τὰ αὑτῆς ἔχειν ἀσφαλῶς καὶ Φίλιππον+  τιμωρήσασθαι, καὶ μάλ᾿ ἀκριβῶς οἶδα· ἐπ᾿ ἐμοῦ γάρ, οὐ πάλαι+  γέγονεν ταῦτ᾿ ἀμφότερα· νῦν μέντοι πέπεισμαι τοῦθ᾿ ἱκανὸν+  προλαβεῖν ἡμῖν εἶναι τὴν πρώτην, ὅπως τοὺς συμμάχους+  σώσομεν. ἐὰν γὰρ τοῦτο βεβαίως ὑπάρξῃ, τότε καὶ περὶ τοῦ+  τίνα τιμωρήσεταί τις καὶ ὃν τρόπον ἐξέσται σκοπεῖν· πρὶν δὲ+  τὴν ἀρχὴν ὀρθῶς ὑποθέσθαι, μάταιον ἡγοῦμαι περὶ τῆς+  τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον.++  Δημοσθένους, Γ´ ᾿Ολυνθιακὸς++Georgian:++  From a Unicode conference invitation:++  გთხოვთ ახლავე გაიაროთ რეგისტრაცია Unicode-ის მეათე საერთაშორისო+  კონფერენციაზე დასასწრებად, რომელიც გაიმართება 10-12 მარტს,+  ქ. მაინცში, გერმანიაში. კონფერენცია შეჰკრებს ერთად მსოფლიოს+  ექსპერტებს ისეთ დარგებში როგორიცაა ინტერნეტი და Unicode-ი,+  ინტერნაციონალიზაცია და ლოკალიზაცია, Unicode-ის გამოყენება+  ოპერაციულ სისტემებსა, და გამოყენებით პროგრამებში, შრიფტებში,+  ტექსტების დამუშავებასა და მრავალენოვან კომპიუტერულ სისტემებში.++Russian:++  From a Unicode conference invitation:++  Зарегистрируйтесь сейчас на Десятую Международную Конференцию по+  Unicode, которая состоится 10-12 марта 1997 года в Майнце в Германии.+  Конференция соберет широкий круг экспертов по  вопросам глобального+  Интернета и Unicode, локализации и интернационализации, воплощению и+  применению Unicode в различных операционных системах и программных+  приложениях, шрифтах, верстке и многоязычных компьютерных системах.++Thai (UCS Level 2):++  Excerpt from a poetry on The Romance of The Three Kingdoms (a Chinese+  classic 'San Gua'):++  [----------------------------|------------------------]+    ๏ แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช  พระปกเกศกองบู๊กู้ขึ้นใหม่+  สิบสองกษัตริย์ก่อนหน้าแลถัดไป       สององค์ไซร้โง่เขลาเบาปัญญา+    ทรงนับถือขันทีเป็นที่พึ่ง           บ้านเมืองจึงวิปริตเป็นนักหนา+  โฮจิ๋นเรียกทัพทั่วหัวเมืองมา         หมายจะฆ่ามดชั่วตัวสำคัญ+    เหมือนขับไสไล่เสือจากเคหา      รับหมาป่าเข้ามาเลยอาสัญ+  ฝ่ายอ้องอุ้นยุแยกให้แตกกัน          ใช้สาวนั้นเป็นชนวนชื่นชวนใจ+    พลันลิฉุยกุยกีกลับก่อเหตุ          ช่างอาเพศจริงหนาฟ้าร้องไห้+  ต้องรบราฆ่าฟันจนบรรลัย           ฤๅหาใครค้ำชูกู้บรรลังก์ ฯ++  (The above is a two-column text. If combining characters are handled+  correctly, the lines of the second column should be aligned with the+  | character above.)++Ethiopian:++  Proverbs in the Amharic language:++  ሰማይ አይታረስ ንጉሥ አይከሰስ።+  ብላ ካለኝ እንደአባቴ በቆመጠኝ።+  ጌጥ ያለቤቱ ቁምጥና ነው።+  ደሀ በሕልሙ ቅቤ ባይጠጣ ንጣት በገደለው።+  የአፍ ወለምታ በቅቤ አይታሽም።+  አይጥ በበላ ዳዋ ተመታ።+  ሲተረጉሙ ይደረግሙ።+  ቀስ በቀስ፥ ዕንቁላል በእግሩ ይሄዳል።+  ድር ቢያብር አንበሳ ያስር።+  ሰው እንደቤቱ እንጅ እንደ ጉረቤቱ አይተዳደርም።+  እግዜር የከፈተውን ጉሮሮ ሳይዘጋው አይድርም።+  የጎረቤት ሌባ፥ ቢያዩት ይስቅ ባያዩት ያጠልቅ።+  ሥራ ከመፍታት ልጄን ላፋታት።+  ዓባይ ማደሪያ የለው፥ ግንድ ይዞ ይዞራል።+  የእስላም አገሩ መካ የአሞራ አገሩ ዋርካ።+  ተንጋሎ ቢተፉ ተመልሶ ባፉ።+  ወዳጅህ ማር ቢሆን ጨርስህ አትላሰው።+  እግርህን በፍራሽህ ልክ ዘርጋ።++Runes:++  ᚻᛖ ᚳᚹᚫᚦ ᚦᚫᛏ ᚻᛖ ᛒᚢᛞᛖ ᚩᚾ ᚦᚫᛗ ᛚᚪᚾᛞᛖ ᚾᚩᚱᚦᚹᛖᚪᚱᛞᚢᛗ ᚹᛁᚦ ᚦᚪ ᚹᛖᛥᚫ++  (Old English, which transcribed into Latin reads 'He cwaeth that he+  bude thaem lande northweardum with tha Westsae.' and means 'He said+  that he lived in the northern land near the Western Sea.')++Braille:++  ⡌⠁⠧⠑ ⠼⠁⠒  ⡍⠜⠇⠑⠹⠰⠎ ⡣⠕⠌++  ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠙⠑⠁⠙⠒ ⠞⠕ ⠃⠑⠛⠔ ⠺⠊⠹⠲ ⡹⠻⠑ ⠊⠎ ⠝⠕ ⠙⠳⠃⠞+  ⠱⠁⠞⠑⠧⠻ ⠁⠃⠳⠞ ⠹⠁⠞⠲ ⡹⠑ ⠗⠑⠛⠊⠌⠻ ⠕⠋ ⠙⠊⠎ ⠃⠥⠗⠊⠁⠇ ⠺⠁⠎+  ⠎⠊⠛⠝⠫ ⠃⠹ ⠹⠑ ⠊⠇⠻⠛⠹⠍⠁⠝⠂ ⠹⠑ ⠊⠇⠻⠅⠂ ⠹⠑ ⠥⠝⠙⠻⠞⠁⠅⠻⠂+  ⠁⠝⠙ ⠹⠑ ⠡⠊⠑⠋ ⠍⠳⠗⠝⠻⠲ ⡎⠊⠗⠕⠕⠛⠑ ⠎⠊⠛⠝⠫ ⠊⠞⠲ ⡁⠝⠙+  ⡎⠊⠗⠕⠕⠛⠑⠰⠎ ⠝⠁⠍⠑ ⠺⠁⠎ ⠛⠕⠕⠙ ⠥⠏⠕⠝ ⠰⡡⠁⠝⠛⠑⠂ ⠋⠕⠗ ⠁⠝⠹⠹⠔⠛ ⠙⠑+  ⠡⠕⠎⠑ ⠞⠕ ⠏⠥⠞ ⠙⠊⠎ ⠙⠁⠝⠙ ⠞⠕⠲++  ⡕⠇⠙ ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲++  ⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹+  ⠪⠝ ⠅⠝⠪⠇⠫⠛⠑⠂ ⠱⠁⠞ ⠹⠻⠑ ⠊⠎ ⠏⠜⠞⠊⠊⠥⠇⠜⠇⠹ ⠙⠑⠁⠙ ⠁⠃⠳⠞+  ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ ⡊ ⠍⠊⠣⠞ ⠙⠁⠧⠑ ⠃⠑⠲ ⠔⠊⠇⠔⠫⠂ ⠍⠹⠎⠑⠇⠋⠂ ⠞⠕+  ⠗⠑⠛⠜⠙ ⠁ ⠊⠕⠋⠋⠔⠤⠝⠁⠊⠇ ⠁⠎ ⠹⠑ ⠙⠑⠁⠙⠑⠌ ⠏⠊⠑⠊⠑ ⠕⠋ ⠊⠗⠕⠝⠍⠕⠝⠛⠻⠹+  ⠔ ⠹⠑ ⠞⠗⠁⠙⠑⠲ ⡃⠥⠞ ⠹⠑ ⠺⠊⠎⠙⠕⠍ ⠕⠋ ⠳⠗ ⠁⠝⠊⠑⠌⠕⠗⠎+  ⠊⠎ ⠔ ⠹⠑ ⠎⠊⠍⠊⠇⠑⠆ ⠁⠝⠙ ⠍⠹ ⠥⠝⠙⠁⠇⠇⠪⠫ ⠙⠁⠝⠙⠎+  ⠩⠁⠇⠇ ⠝⠕⠞ ⠙⠊⠌⠥⠗⠃ ⠊⠞⠂ ⠕⠗ ⠹⠑ ⡊⠳⠝⠞⠗⠹⠰⠎ ⠙⠕⠝⠑ ⠋⠕⠗⠲ ⡹⠳+  ⠺⠊⠇⠇ ⠹⠻⠑⠋⠕⠗⠑ ⠏⠻⠍⠊⠞ ⠍⠑ ⠞⠕ ⠗⠑⠏⠑⠁⠞⠂ ⠑⠍⠏⠙⠁⠞⠊⠊⠁⠇⠇⠹⠂ ⠹⠁⠞+  ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲++  (The first couple of paragraphs of "A Christmas Carol" by Dickens)++Compact font selection example text:++  ABCDEFGHIJKLMNOPQRSTUVWXYZ /0123456789+  abcdefghijklmnopqrstuvwxyz £©µÀÆÖÞßéöÿ+  –—‘“”„†•…‰™œŠŸž€ ΑΒΓΔΩαβγδω АБВГДабвгд+  ∀∂∈ℝ∧∪≡∞ ↑↗↨↻⇣ ┐┼╔╘░►☺♀ fi�⑀₂ἠḂӥẄɐː⍎אԱა++Greetings in various languages:++  Hello world, Καλημέρα κόσμε, コンニチハ++Box drawing alignment tests:                                          █+                                                                      ▉+  ╔══╦══╗  ┌──┬──┐  ╭──┬──╮  ╭──┬──╮  ┏━━┳━━┓  ┎┒┏┑   ╷  ╻ ┏┯┓ ┌┰┐    ▊ ╱╲╱╲╳╳╳+  ║┌─╨─┐║  │╔═╧═╗│  │╒═╪═╕│  │╓─╁─╖│  ┃┌─╂─┐┃  ┗╃╄┙  ╶┼╴╺╋╸┠┼┨ ┝╋┥    ▋ ╲╱╲╱╳╳╳+  ║│╲ ╱│║  │║   ║│  ││ │ ││  │║ ┃ ║│  ┃│ ╿ │┃  ┍╅╆┓   ╵  ╹ ┗┷┛ └┸┘    ▌ ╱╲╱╲╳╳╳+  ╠╡ ╳ ╞╣  ├╢   ╟┤  ├┼─┼─┼┤  ├╫─╂─╫┤  ┣┿╾┼╼┿┫  ┕┛┖┚     ┌┄┄┐ ╎ ┏┅┅┓ ┋ ▍ ╲╱╲╱╳╳╳+  ║│╱ ╲│║  │║   ║│  ││ │ ││  │║ ┃ ║│  ┃│ ╽ │┃  ░░▒▒▓▓██ ┊  ┆ ╎ ╏  ┇ ┋ ▎+  ║└─╥─┘║  │╚═╤═╝│  │╘═╪═╛│  │╙─╀─╜│  ┃└─╂─┘┃  ░░▒▒▓▓██ ┊  ┆ ╎ ╏  ┇ ┋ ▏+  ╚══╩══╝  └──┴──┘  ╰──┴──╯  ╰──┴──╯  ┗━━┻━━┛  ▗▄▖▛▀▜   └╌╌┘ ╎ ┗╍╍┛ ┋  ▁▂▃▄▅▆▇█+                                               ▝▀▘▙▄▟
+ test/invalid/test075.txt view
@@ -0,0 +1,1 @@+2.1.5  5 bytes (U-00200000):        "øˆ€€€"                                       |
+ test/invalid/test076.txt view
@@ -0,0 +1,1 @@+2.1.6  6 bytes (U-04000000):        "ü„€€€€"                                       |
+ test/invalid/test082.txt view
@@ -0,0 +1,1 @@+2.2.3  3 bytes (U-0000FFFF):        "￿"                                       |
+ test/invalid/test083.txt view
@@ -0,0 +1,1 @@+2.2.4  4 bytes (U-001FFFFF):        "÷¿¿¿"                                       |
+ test/invalid/test084.txt view
@@ -0,0 +1,1 @@+2.2.5  5 bytes (U-03FFFFFF):        "û¿¿¿¿"                                       |
+ test/invalid/test085.txt view
@@ -0,0 +1,1 @@+2.2.6  6 bytes (U-7FFFFFFF):        "ý¿¿¿¿¿"                                       |
+ test/invalid/test092.txt view
@@ -0,0 +1,1 @@+2.3.4  U-0010FFFF = f4 8f bf bf = "􏿿"                                         |
+ test/invalid/test093.txt view
@@ -0,0 +1,1 @@+2.3.5  U-00110000 = f4 90 80 80 = "ô€€"                                         |
+ test/invalid/test102.txt view
@@ -0,0 +1,1 @@+3.1.1  First continuation byte 0x80: "€"                                      |
+ test/invalid/test103.txt view
@@ -0,0 +1,1 @@+3.1.2  Last  continuation byte 0xbf: "¿"                                      |
+ test/invalid/test105.txt view
@@ -0,0 +1,1 @@+3.1.3  2 continuation bytes: "€¿"                                             |
+ test/invalid/test106.txt view
@@ -0,0 +1,1 @@+3.1.4  3 continuation bytes: "€¿€"                                            |
+ test/invalid/test107.txt view
@@ -0,0 +1,1 @@+3.1.5  4 continuation bytes: "€¿€¿"                                           |
+ test/invalid/test108.txt view
@@ -0,0 +1,1 @@+3.1.6  5 continuation bytes: "€¿€¿€"                                          |
+ test/invalid/test109.txt view
@@ -0,0 +1,1 @@+3.1.7  6 continuation bytes: "€¿€¿€¿"                                         |
+ test/invalid/test110.txt view
@@ -0,0 +1,1 @@+3.1.8  7 continuation bytes: "€¿€¿€¿€"                                        |
+ test/invalid/test114.txt view
@@ -0,0 +1,1 @@+   "€‚ƒ„…†‡ˆ‰Š‹ŒŽ                                                          |
+ test/invalid/test115.txt view
@@ -0,0 +1,1 @@+    ‘’“”•–—˜™š›œžŸ                                                          |
+ test/invalid/test116.txt view
@@ -0,0 +1,1 @@+     ¡¢£¤¥¦§¨©ª«¬­®¯                                                          |
+ test/invalid/test117.txt view
@@ -0,0 +1,1 @@+    °±²³´µ¶·¸¹º»¼½¾¿"                                                         |
+ test/invalid/test124.txt view
@@ -0,0 +1,1 @@+   "À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï                                           |
+ test/invalid/test125.txt view
@@ -0,0 +1,1 @@+    Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß "                                         |
+ test/invalid/test130.txt view
@@ -0,0 +1,1 @@+   "à á â ã ä å æ ç è é ê ë ì í î ï "                                         |
+ test/invalid/test135.txt view
@@ -0,0 +1,1 @@+   "ð ñ ò ó ô õ ö ÷ "                                                         |
+ test/invalid/test140.txt view
@@ -0,0 +1,1 @@+   "ø ù ú û "                                                                 |
+ test/invalid/test145.txt view
@@ -0,0 +1,1 @@+   "ü ý "                                                                     |
+ test/invalid/test153.txt view
@@ -0,0 +1,1 @@+3.3.1  2-byte sequence with last byte missing (U+0000):     "À"               |
+ test/invalid/test154.txt view
@@ -0,0 +1,1 @@+3.3.2  3-byte sequence with last byte missing (U+0000):     "à€"               |
+ test/invalid/test155.txt view
@@ -0,0 +1,1 @@+3.3.3  4-byte sequence with last byte missing (U+0000):     "ð€€"               |
+ test/invalid/test156.txt view
@@ -0,0 +1,1 @@+3.3.4  5-byte sequence with last byte missing (U+0000):     "ø€€€"               |
+ test/invalid/test157.txt view
@@ -0,0 +1,1 @@+3.3.5  6-byte sequence with last byte missing (U+0000):     "ü€€€€"               |
+ test/invalid/test158.txt view
@@ -0,0 +1,1 @@+3.3.6  2-byte sequence with last byte missing (U-000007FF): "ß"               |
+ test/invalid/test159.txt view
@@ -0,0 +1,1 @@+3.3.7  3-byte sequence with last byte missing (U-0000FFFF): "ï¿"               |
+ test/invalid/test160.txt view
@@ -0,0 +1,1 @@+3.3.8  4-byte sequence with last byte missing (U-001FFFFF): "÷¿¿"               |
+ test/invalid/test161.txt view
@@ -0,0 +1,1 @@+3.3.9  5-byte sequence with last byte missing (U-03FFFFFF): "û¿¿¿"               |
+ test/invalid/test162.txt view
@@ -0,0 +1,1 @@+3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF): "ý¿¿¿¿"               |
+ test/invalid/test169.txt view
@@ -0,0 +1,1 @@+   "Àà€ð€€ø€€€ü€€€€ßï¿÷¿¿û¿¿¿ý¿¿¿¿"                                                               |
+ test/invalid/test175.txt view
@@ -0,0 +1,1 @@+3.5.1  fe = "þ"                                                               |
+ test/invalid/test176.txt view
@@ -0,0 +1,1 @@+3.5.2  ff = "ÿ"                                                               |
+ test/invalid/test177.txt view
@@ -0,0 +1,1 @@+3.5.3  fe fe ff ff = "þþÿÿ"                                                   |
+ test/invalid/test207.txt view
@@ -0,0 +1,1 @@+4.1.1 U+002F = c0 af             = "À¯"                                        |
+ test/invalid/test208.txt view
@@ -0,0 +1,1 @@+4.1.2 U+002F = e0 80 af          = "à€¯"                                        |
+ test/invalid/test209.txt view
@@ -0,0 +1,1 @@+4.1.3 U+002F = f0 80 80 af       = "ð€€¯"                                        |
+ test/invalid/test210.txt view
@@ -0,0 +1,1 @@+4.1.4 U+002F = f8 80 80 80 af    = "ø€€€¯"                                        |
+ test/invalid/test211.txt view
@@ -0,0 +1,1 @@+4.1.5 U+002F = fc 80 80 80 80 af = "ü€€€€¯"                                        |
+ test/invalid/test220.txt view
@@ -0,0 +1,1 @@+4.2.1  U-0000007F = c1 bf             = "Á¿"                                   |
+ test/invalid/test221.txt view
@@ -0,0 +1,1 @@+4.2.2  U-000007FF = e0 9f bf          = "àŸ¿"                                   |
+ test/invalid/test222.txt view
@@ -0,0 +1,1 @@+4.2.3  U-0000FFFF = f0 8f bf bf       = "ð¿¿"                                   |
+ test/invalid/test223.txt view
@@ -0,0 +1,1 @@+4.2.4  U-001FFFFF = f8 87 bf bf bf    = "ø‡¿¿¿"                                   |
+ test/invalid/test224.txt view
@@ -0,0 +1,1 @@+4.2.5  U-03FFFFFF = fc 83 bf bf bf bf = "üƒ¿¿¿¿"                                   |
+ test/invalid/test232.txt view
@@ -0,0 +1,1 @@+4.3.1  U+0000 = c0 80             = ""                                       |
+ test/invalid/test233.txt view
@@ -0,0 +1,1 @@+4.3.2  U+0000 = e0 80 80          = "à€€"                                       |
+ test/invalid/test234.txt view
@@ -0,0 +1,1 @@+4.3.3  U+0000 = f0 80 80 80       = "ð€€€"                                       |
+ test/invalid/test235.txt view
@@ -0,0 +1,1 @@+4.3.4  U+0000 = f8 80 80 80 80    = "ø€€€€"                                       |
+ test/invalid/test236.txt view
@@ -0,0 +1,1 @@+4.3.5  U+0000 = fc 80 80 80 80 80 = "ü€€€€€"                                       |
+ test/invalid/test247.txt view
@@ -0,0 +1,1 @@+5.1.1  U+D800 = ed a0 80 = "í €"                                                |
+ test/invalid/test248.txt view
@@ -0,0 +1,1 @@+5.1.2  U+DB7F = ed ad bf = "í­¿"                                                |
+ test/invalid/test249.txt view
@@ -0,0 +1,1 @@+5.1.3  U+DB80 = ed ae 80 = "í®€"                                                |
+ test/invalid/test250.txt view
@@ -0,0 +1,1 @@+5.1.4  U+DBFF = ed af bf = "í¯¿"                                                |
+ test/invalid/test251.txt view
@@ -0,0 +1,1 @@+5.1.5  U+DC00 = ed b0 80 = "í°€"                                                |
+ test/invalid/test252.txt view
@@ -0,0 +1,1 @@+5.1.6  U+DF80 = ed be 80 = "í¾€"                                                |
+ test/invalid/test253.txt view
@@ -0,0 +1,1 @@+5.1.7  U+DFFF = ed bf bf = "í¿¿"                                                |
+ test/invalid/test257.txt view
@@ -0,0 +1,1 @@+5.2.1  U+D800 U+DC00 = ed a0 80 ed b0 80 = "𐀀"                               |
+ test/invalid/test258.txt view
@@ -0,0 +1,1 @@+5.2.2  U+D800 U+DFFF = ed a0 80 ed bf bf = "𐏿"                               |
+ test/invalid/test259.txt view
@@ -0,0 +1,1 @@+5.2.3  U+DB7F U+DC00 = ed ad bf ed b0 80 = "󯰀"                               |
+ test/invalid/test260.txt view
@@ -0,0 +1,1 @@+5.2.4  U+DB7F U+DFFF = ed ad bf ed bf bf = "í­¿í¿¿"                               |
+ test/invalid/test261.txt view
@@ -0,0 +1,1 @@+5.2.5  U+DB80 U+DC00 = ed ae 80 ed b0 80 = "󰀀"                               |
+ test/invalid/test262.txt view
@@ -0,0 +1,1 @@+5.2.6  U+DB80 U+DFFF = ed ae 80 ed bf bf = "󰏿"                               |
+ test/invalid/test263.txt view
@@ -0,0 +1,1 @@+5.2.7  U+DBFF U+DC00 = ed af bf ed b0 80 = "􏰀"                               |
+ test/invalid/test264.txt view
@@ -0,0 +1,1 @@+5.2.8  U+DBFF U+DFFF = ed af bf ed bf bf = "􏿿"                               |
+ test/invalid/test268.txt view
@@ -0,0 +1,1 @@+5.3.1  U+FFFE = ef bf be = "￾"                                                |
+ test/invalid/test269.txt view
@@ -0,0 +1,1 @@+5.3.2  U+FFFF = ef bf bf = "￿"                                                |
+ test/test.txt view

binary file changed (absent → 20334 bytes)

+ test/valid/test061.txt view
@@ -0,0 +1,1 @@+Here come the tests:                                                          |
+ test/valid/test062.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test063.txt view
@@ -0,0 +1,1 @@+1  Some correct UTF-8 text                                                    |
+ test/valid/test064.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test065.txt view
@@ -0,0 +1,1 @@+You should see the Greek word 'kosme':       "κόσμε"                          |
+ test/valid/test066.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test067.txt view
@@ -0,0 +1,1 @@+2  Boundary condition test cases                                              |
+ test/valid/test068.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test069.txt view
@@ -0,0 +1,1 @@+2.1  First possible sequence of a certain length                              |
+ test/valid/test070.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test071.txt view

binary file changed (absent → 79 bytes)

+ test/valid/test072.txt view
@@ -0,0 +1,1 @@+2.1.2  2 bytes (U-00000080):        "€"                                       |
+ test/valid/test073.txt view
@@ -0,0 +1,1 @@+2.1.3  3 bytes (U-00000800):        "ࠀ"                                       |
+ test/valid/test074.txt view
@@ -0,0 +1,1 @@+2.1.4  4 bytes (U-00010000):        "𐀀"                                       |
+ test/valid/test077.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test078.txt view
@@ -0,0 +1,1 @@+2.2  Last possible sequence of a certain length                               |
+ test/valid/test079.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test080.txt view
@@ -0,0 +1,1 @@+2.2.1  1 byte  (U-0000007F):        ""
+ test/valid/test081.txt view
@@ -0,0 +1,1 @@+2.2.2  2 bytes (U-000007FF):        "߿"                                       |
+ test/valid/test086.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test087.txt view
@@ -0,0 +1,1 @@+2.3  Other boundary conditions                                                |
+ test/valid/test088.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test089.txt view
@@ -0,0 +1,1 @@+2.3.1  U-0000D7FF = ed 9f bf = "퟿"                                            |
+ test/valid/test090.txt view
@@ -0,0 +1,1 @@+2.3.2  U-0000E000 = ee 80 80 = ""                                            |
+ test/valid/test091.txt view
@@ -0,0 +1,1 @@+2.3.3  U-0000FFFD = ef bf bd = "�"                                            |
+ test/valid/test094.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test095.txt view
@@ -0,0 +1,1 @@+3  Malformed sequences                                                        |
+ test/valid/test096.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test097.txt view
@@ -0,0 +1,1 @@+3.1  Unexpected continuation bytes                                            |
+ test/valid/test098.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test099.txt view
@@ -0,0 +1,1 @@+Each unexpected continuation byte should be separately signalled as a         |
+ test/valid/test100.txt view
@@ -0,0 +1,1 @@+malformed sequence of its own.                                                |
+ test/valid/test101.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test104.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test111.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test112.txt view
@@ -0,0 +1,1 @@+3.1.9  Sequence of all 64 possible continuation bytes (0x80-0xbf):            |
+ test/valid/test113.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test118.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test119.txt view
@@ -0,0 +1,1 @@+3.2  Lonely start characters                                                  |
+ test/valid/test120.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test121.txt view
@@ -0,0 +1,1 @@+3.2.1  All 32 first bytes of 2-byte sequences (0xc0-0xdf),                    |
+ test/valid/test122.txt view
@@ -0,0 +1,1 @@+       each followed by a space character:                                    |
+ test/valid/test123.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test126.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test127.txt view
@@ -0,0 +1,1 @@+3.2.2  All 16 first bytes of 3-byte sequences (0xe0-0xef),                    |
+ test/valid/test128.txt view
@@ -0,0 +1,1 @@+       each followed by a space character:                                    |
+ test/valid/test129.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test131.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test132.txt view
@@ -0,0 +1,1 @@+3.2.3  All 8 first bytes of 4-byte sequences (0xf0-0xf7),                     |
+ test/valid/test133.txt view
@@ -0,0 +1,1 @@+       each followed by a space character:                                    |
+ test/valid/test134.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test136.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test137.txt view
@@ -0,0 +1,1 @@+3.2.4  All 4 first bytes of 5-byte sequences (0xf8-0xfb),                     |
+ test/valid/test138.txt view
@@ -0,0 +1,1 @@+       each followed by a space character:                                    |
+ test/valid/test139.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test141.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test142.txt view
@@ -0,0 +1,1 @@+3.2.5  All 2 first bytes of 6-byte sequences (0xfc-0xfd),                     |
+ test/valid/test143.txt view
@@ -0,0 +1,1 @@+       each followed by a space character:                                    |
+ test/valid/test144.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test146.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test147.txt view
@@ -0,0 +1,1 @@+3.3  Sequences with last continuation byte missing                            |
+ test/valid/test148.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test149.txt view
@@ -0,0 +1,1 @@+All bytes of an incomplete sequence should be signalled as a single           |
+ test/valid/test150.txt view
@@ -0,0 +1,1 @@+malformed sequence, i.e., you should see only a single replacement            |
+ test/valid/test151.txt view
@@ -0,0 +1,1 @@+character in each of the next 10 tests. (Characters as in section 2)          |
+ test/valid/test152.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test163.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test164.txt view
@@ -0,0 +1,1 @@+3.4  Concatenation of incomplete sequences                                    |
+ test/valid/test165.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test166.txt view
@@ -0,0 +1,1 @@+All the 10 sequences of 3.3 concatenated, you should see 10 malformed         |
+ test/valid/test167.txt view
@@ -0,0 +1,1 @@+sequences being signalled:                                                    |
+ test/valid/test168.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test170.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test171.txt view
@@ -0,0 +1,1 @@+3.5  Impossible bytes                                                         |
+ test/valid/test172.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test173.txt view
@@ -0,0 +1,1 @@+The following two bytes cannot appear in a correct UTF-8 string               |
+ test/valid/test174.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test178.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test179.txt view
@@ -0,0 +1,1 @@+4  Overlong sequences                                                         |
+ test/valid/test180.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test181.txt view
@@ -0,0 +1,1 @@+The following sequences are not malformed according to the letter of          |
+ test/valid/test182.txt view
@@ -0,0 +1,1 @@+the Unicode 2.0 standard. However, they are longer then necessary and         |
+ test/valid/test183.txt view
@@ -0,0 +1,1 @@+a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8         |
+ test/valid/test184.txt view
@@ -0,0 +1,1 @@+decoder" should reject them just like malformed sequences for two             |
+ test/valid/test185.txt view
@@ -0,0 +1,1 @@+reasons: (1) It helps to debug applications if overlong sequences are         |
+ test/valid/test186.txt view
@@ -0,0 +1,1 @@+not treated as valid representations of characters, because this helps        |
+ test/valid/test187.txt view
@@ -0,0 +1,1 @@+to spot problems more quickly. (2) Overlong sequences provide                 |
+ test/valid/test188.txt view
@@ -0,0 +1,1 @@+alternative representations of characters, that could maliciously be          |
+ test/valid/test189.txt view
@@ -0,0 +1,1 @@+used to bypass filters that check only for ASCII characters. For              |
+ test/valid/test190.txt view
@@ -0,0 +1,1 @@+instance, a 2-byte encoded line feed (LF) would not be caught by a            |
+ test/valid/test191.txt view
@@ -0,0 +1,1 @@+line counter that counts only 0x0a bytes, but it would still be               |
+ test/valid/test192.txt view
@@ -0,0 +1,1 @@+processed as a line feed by an unsafe UTF-8 decoder later in the              |
+ test/valid/test193.txt view
@@ -0,0 +1,1 @@+pipeline. From a security point of view, ASCII compatibility of UTF-8         |
+ test/valid/test194.txt view
@@ -0,0 +1,1 @@+sequences means also, that ASCII characters are *only* allowed to be          |
+ test/valid/test195.txt view
@@ -0,0 +1,1 @@+represented by ASCII bytes in the range 0x00-0x7f. To ensure this             |
+ test/valid/test196.txt view
@@ -0,0 +1,1 @@+aspect of ASCII compatibility, use only "safe UTF-8 decoders" that            |
+ test/valid/test197.txt view
@@ -0,0 +1,1 @@+reject overlong UTF-8 sequences for which a shorter encoding exists.          |
+ test/valid/test198.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test199.txt view
@@ -0,0 +1,1 @@+4.1  Examples of an overlong ASCII character                                  |
+ test/valid/test200.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test201.txt view
@@ -0,0 +1,1 @@+With a safe UTF-8 decoder, all of the following five overlong                 |
+ test/valid/test202.txt view
@@ -0,0 +1,1 @@+representations of the ASCII character slash ("/") should be rejected         |
+ test/valid/test203.txt view
@@ -0,0 +1,1 @@+like a malformed UTF-8 sequence, for instance by substituting it with         |
+ test/valid/test204.txt view
@@ -0,0 +1,1 @@+a replacement character. If you see a slash below, you do not have a          |
+ test/valid/test205.txt view
@@ -0,0 +1,1 @@+safe UTF-8 decoder!                                                           |
+ test/valid/test206.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test212.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test213.txt view
@@ -0,0 +1,1 @@+4.2  Maximum overlong sequences                                               |
+ test/valid/test214.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test215.txt view
@@ -0,0 +1,1 @@+Below you see the highest Unicode value that is still resulting in an         |
+ test/valid/test216.txt view
@@ -0,0 +1,1 @@+overlong sequence if represented with the given number of bytes. This         |
+ test/valid/test217.txt view
@@ -0,0 +1,1 @@+is a boundary test for safe UTF-8 decoders. All five characters should        |
+ test/valid/test218.txt view
@@ -0,0 +1,1 @@+be rejected like malformed UTF-8 sequences.                                   |
+ test/valid/test219.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test225.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test226.txt view
@@ -0,0 +1,1 @@+4.3  Overlong representation of the NUL character                             |
+ test/valid/test227.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test228.txt view
@@ -0,0 +1,1 @@+The following five sequences should also be rejected like malformed           |
+ test/valid/test229.txt view
@@ -0,0 +1,1 @@+UTF-8 sequences and should not be treated like the ASCII NUL                  |
+ test/valid/test230.txt view
@@ -0,0 +1,1 @@+character.                                                                    |
+ test/valid/test231.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test237.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test238.txt view
@@ -0,0 +1,1 @@+5  Illegal code positions                                                     |
+ test/valid/test239.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test240.txt view
@@ -0,0 +1,1 @@+The following UTF-8 sequences should be rejected like malformed               |
+ test/valid/test241.txt view
@@ -0,0 +1,1 @@+sequences, because they never represent valid ISO 10646 characters and        |
+ test/valid/test242.txt view
@@ -0,0 +1,1 @@+a UTF-8 decoder that accepts them might introduce security problems           |
+ test/valid/test243.txt view
@@ -0,0 +1,1 @@+comparable to overlong UTF-8 sequences.                                       |
+ test/valid/test244.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test245.txt view
@@ -0,0 +1,1 @@+5.1 Single UTF-16 surrogates                                                  |
+ test/valid/test246.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test254.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test255.txt view
@@ -0,0 +1,1 @@+5.2 Paired UTF-16 surrogates                                                  |
+ test/valid/test256.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test265.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test266.txt view
@@ -0,0 +1,1 @@+5.3 Other illegal code positions                                              |
+ test/valid/test267.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test270.txt view
@@ -0,0 +1,1 @@+                                                                              |
+ test/valid/test271.txt view
@@ -0,0 +1,1 @@+THE END                                                                       |
− testsrc/Main.hs 
@@ -1,109 +0,0 @@-{- |-module: Main-description: Testing the Unicode character library and API-license: MIT--maintainer: Joe Leslie-Hurd <joe@gilith.com>-stability: provisional-portability: portable--}-module Main-  ( main )-where--import qualified Data.List as List-import qualified Data.Either as Either-import qualified Data.Word as Word-import qualified System.Directory as Directory--import Unicode-import qualified OpenTheory.Unicode as Unicode--demoLength :: Int-demoLength = 7621--testLength :: Int-testLength = 79--noInvalid :: [Either Word.Word8 Unicode.Unicode] -> Bool-noInvalid = all (Either.either (const False) (const True))--getTestFiles :: FilePath -> IO [FilePath]-getTestFiles d =-    do fs <- Directory.getDirectoryContents d-       let fs' = filter (List.isPrefixOf "test") fs-       return (map (\f -> d ++ "/" ++ f) fs')--readTestCharFile :: Maybe Int -> FilePath -> IO ()-readTestCharFile x f =-    do l <- decodeFile f-       let a = noInvalid l-       case x of-         Nothing ->-             if a-               then error $ "invalid file " ++ f ++ " was successfully decoded"-               else return ()-         Just n ->-             if not a-               then error $ "valid file " ++ f ++ " could not be decoded"-               else-                 if length l == n-                   then return ()-                   else-                     error $ "valid file " ++ f ++ " has bad line length: " ++-                             show (length l)--readValidCharFile :: Int -> FilePath -> IO ()-readValidCharFile n = readTestCharFile (Just n)--readInvalidCharFile :: FilePath -> IO ()-readInvalidCharFile = readTestCharFile Nothing--partitionTestCharFile :: IO ()-partitionTestCharFile =-    do skip <- Directory.doesFileExist "test/valid/test061.txt"-       if skip-         then return ()-         else-           do putStrLn "\nsplitting UTF-8 test file into valid and invalid lines"-              cs <- decodeFile "test/test.txt"-              mapM_ outputLine (filter testLine (readLines 1 cs))-  where-    isNewline :: Either Word.Word8 Unicode.Unicode -> Bool-    isNewline (Left _) = False-    isNewline (Right c) = let n = Unicode.unUnicode c in n == 10 || n == 13--    chopNewline :: [Either Word.Word8 Unicode.Unicode] ->-                   [Either Word.Word8 Unicode.Unicode]-    chopNewline [] = []-    chopNewline (_ : cs) = cs--    readLines :: Int -> [Either Word.Word8 Unicode.Unicode] ->-                 [(Int,[Either Word.Word8 Unicode.Unicode])]-    readLines _ [] = []-    readLines lineno inp =-        let (line,inp') = List.span (not . isNewline) inp in-        let inp'' = chopNewline inp' in-        (lineno,line) : readLines (lineno + 1) inp''--    testLine :: (Int,[Either Word.Word8 Unicode.Unicode]) -> Bool-    testLine (lineno,line) = not (lineno < 61 || null line)--    outputLine :: (Int,[Either Word.Word8 Unicode.Unicode]) -> IO ()-    outputLine (lineno,line) =-        let valid = noInvalid line in-        let n = show lineno in-        let f = "test/" ++ (if valid then "valid" else "invalid") ++-                "/test" ++ replicate (3 - length n) '0' ++ n ++ ".txt" in-        reencodeFile f line--main :: IO ()-main =-    do partitionTestCharFile-       readValidCharFile demoLength "test/demo.txt"-       validTestFiles <- getTestFiles "test/valid"-       invalidTestFiles <- getTestFiles "test/invalid"-       mapM_ (readValidCharFile testLength) validTestFiles-       mapM_ readInvalidCharFile invalidTestFiles-       putStrLn "  all tests pass"-       return ()