diff --git a/api-opentheory-unicode.cabal b/api-opentheory-unicode.cabal --- a/api-opentheory-unicode.cabal +++ b/api-opentheory-unicode.cabal @@ -1,11 +1,12 @@ name: api-opentheory-unicode -version: 1.0 +version: 1.1 category: Text synopsis: OpenTheory unicode character API license: MIT license-file: LICENSE cabal-version: >= 1.8.0.2 build-type: Simple +data-files: test/*.txt test/valid/*.txt test/invalid/*.txt author: Joe Leslie-Hurd maintainer: Joe Leslie-Hurd description: @@ -17,23 +18,18 @@ base >= 4.0 && < 5.0, bytestring >= 0.9 && < 1.0, opentheory-unicode >= 1.0 && < 2.0 - hs-source-dirs: src - ghc-options: -Wall - exposed-modules: Unicode -executable api-opentheory-unicode-test +test-suite api-opentheory-unicode-test + type: exitcode-stdio-1.0 build-depends: base >= 4.0 && < 5.0, bytestring >= 0.9 && < 1.0, directory >= 1.0 && < 2.0, opentheory-unicode >= 1.0 && < 2.0 - - hs-source-dirs: src, testsrc - + hs-source-dirs: src ghc-options: -Wall - - main-is: Main.hs + main-is: Test.hs diff --git a/src/Test.hs b/src/Test.hs new file mode 100644 --- /dev/null +++ b/src/Test.hs @@ -0,0 +1,109 @@ +{- | +module: Main +description: Testing the Unicode character library and API +license: MIT + +maintainer: Joe Leslie-Hurd +stability: provisional +portability: portable +-} +module Main + ( main ) +where + +import qualified Data.List as List +import qualified Data.Either as Either +import qualified Data.Word as Word +import qualified System.Directory as Directory + +import Unicode +import qualified OpenTheory.Unicode as Unicode + +demoLength :: Int +demoLength = 7621 + +testLength :: Int +testLength = 79 + +noInvalid :: [Either Word.Word8 Unicode.Unicode] -> Bool +noInvalid = all (Either.either (const False) (const True)) + +getTestFiles :: FilePath -> IO [FilePath] +getTestFiles d = + do fs <- Directory.getDirectoryContents d + let fs' = filter (List.isPrefixOf "test") fs + return (map (\f -> d ++ "/" ++ f) fs') + +readTestCharFile :: Maybe Int -> FilePath -> IO () +readTestCharFile x f = + do l <- decodeFile f + let a = noInvalid l + case x of + Nothing -> + if a + then error $ "invalid file " ++ f ++ " was successfully decoded" + else return () + Just n -> + if not a + then error $ "valid file " ++ f ++ " could not be decoded" + else + if length l == n + then return () + else + error $ "valid file " ++ f ++ " has bad line length: " ++ + show (length l) + +readValidCharFile :: Int -> FilePath -> IO () +readValidCharFile n = readTestCharFile (Just n) + +readInvalidCharFile :: FilePath -> IO () +readInvalidCharFile = readTestCharFile Nothing + +partitionTestCharFile :: IO () +partitionTestCharFile = + do skip <- Directory.doesFileExist "test/valid/test061.txt" + if skip + then return () + else + do putStrLn "\nsplitting UTF-8 test file into valid and invalid lines" + cs <- decodeFile "test/test.txt" + mapM_ outputLine (filter testLine (readLines 1 cs)) + where + isNewline :: Either Word.Word8 Unicode.Unicode -> Bool + isNewline (Left _) = False + isNewline (Right c) = let n = Unicode.unUnicode c in n == 10 || n == 13 + + chopNewline :: [Either Word.Word8 Unicode.Unicode] -> + [Either Word.Word8 Unicode.Unicode] + chopNewline [] = [] + chopNewline (_ : cs) = cs + + readLines :: Int -> [Either Word.Word8 Unicode.Unicode] -> + [(Int,[Either Word.Word8 Unicode.Unicode])] + readLines _ [] = [] + readLines lineno inp = + let (line,inp') = List.span (not . isNewline) inp in + let inp'' = chopNewline inp' in + (lineno,line) : readLines (lineno + 1) inp'' + + testLine :: (Int,[Either Word.Word8 Unicode.Unicode]) -> Bool + testLine (lineno,line) = not (lineno < 61 || null line) + + outputLine :: (Int,[Either Word.Word8 Unicode.Unicode]) -> IO () + outputLine (lineno,line) = + let valid = noInvalid line in + let n = show lineno in + let f = "test/" ++ (if valid then "valid" else "invalid") ++ + "/test" ++ replicate (3 - length n) '0' ++ n ++ ".txt" in + reencodeFile f line + +main :: IO () +main = + do partitionTestCharFile + readValidCharFile demoLength "test/demo.txt" + validTestFiles <- getTestFiles "test/valid" + invalidTestFiles <- getTestFiles "test/invalid" + mapM_ (readValidCharFile testLength) validTestFiles + mapM_ readInvalidCharFile invalidTestFiles + putStrLn " all tests pass" + return () diff --git a/test/demo.txt b/test/demo.txt new file mode 100644 --- /dev/null +++ b/test/demo.txt @@ -0,0 +1,212 @@ + +UTF-8 encoded sample plain-text file +‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ + +Markus Kuhn [ˈmaʳkʊs kuːn] — 2002-07-25 + + +The ASCII compatible UTF-8 encoding used in this plain-text file +is defined in Unicode, ISO 10646-1, and RFC 2279. + + +Using Unicode/UTF-8, you can write in emails and source code things such as + +Mathematics and sciences: + + ∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ⎧⎡⎛┌─────┐⎞⎤⎫ + ⎪⎢⎜│a²+b³ ⎟⎥⎪ + ∀x∈ℝ: ⌈x⌉ = −⌊−x⌋, α ∧ ¬β = ¬(¬α ∨ β), ⎪⎢⎜│───── ⎟⎥⎪ + ⎪⎢⎜⎷ c₈ ⎟⎥⎪ + ℕ ⊆ ℕ₀ ⊂ ℤ ⊂ ℚ ⊂ ℝ ⊂ ℂ, ⎨⎢⎜ ⎟⎥⎬ + ⎪⎢⎜ ∞ ⎟⎥⎪ + ⊥ < a ≠ b ≡ c ≤ d ≪ ⊤ ⇒ (⟦A⟧ ⇔ ⟪B⟫), ⎪⎢⎜ ⎲ ⎟⎥⎪ + ⎪⎢⎜ ⎳aⁱ-bⁱ⎟⎥⎪ + 2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm ⎩⎣⎝i=1 ⎠⎦⎭ + +Linguistics and dictionaries: + + ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn + Y [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ] + +APL: + + ((V⍳V)=⍳⍴V)/V←,V ⌷←⍳→⍴∆∇⊃‾⍎⍕⌈ + +Nicer typography in plain text files: + + ╔══════════════════════════════════════════╗ + ║ ║ + ║ • ‘single’ and “double” quotes ║ + ║ ║ + ║ • Curly apostrophes: “We’ve been here” ║ + ║ ║ + ║ • Latin-1 apostrophe and accents: '´` ║ + ║ ║ + ║ • ‚deutsche‘ „Anführungszeichen“ ║ + ║ ║ + ║ • †, ‡, ‰, •, 3–4, —, −5/+5, ™, … ║ + ║ ║ + ║ • ASCII safety test: 1lI|, 0OD, 8B ║ + ║ ╭─────────╮ ║ + ║ • the euro symbol: │ 14.95 € │ ║ + ║ ╰─────────╯ ║ + ╚══════════════════════════════════════════╝ + +Combining characters: + + STARGΛ̊TE SG-1, a = v̇ = r̈, a⃑ ⊥ b⃑ + +Greek (in Polytonic): + + The Greek anthem: + + Σὲ γνωρίζω ἀπὸ τὴν κόψη + τοῦ σπαθιοῦ τὴν τρομερή, + σὲ γνωρίζω ἀπὸ τὴν ὄψη + ποὺ μὲ βία μετράει τὴ γῆ. + + ᾿Απ᾿ τὰ κόκκαλα βγαλμένη + τῶν ῾Ελλήνων τὰ ἱερά + καὶ σὰν πρῶτα ἀνδρειωμένη + χαῖρε, ὦ χαῖρε, ᾿Ελευθεριά! + + From a speech of Demosthenes in the 4th century BC: + + Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι, + ὅταν τ᾿ εἰς τὰ πράγματα ἀποβλέψω καὶ ὅταν πρὸς τοὺς + λόγους οὓς ἀκούω· τοὺς μὲν γὰρ λόγους περὶ τοῦ + τιμωρήσασθαι Φίλιππον ὁρῶ γιγνομένους, τὰ δὲ πράγματ᾿ + εἰς τοῦτο προήκοντα, ὥσθ᾿ ὅπως μὴ πεισόμεθ᾿ αὐτοὶ + πρότερον κακῶς σκέψασθαι δέον. οὐδέν οὖν ἄλλο μοι δοκοῦσιν + οἱ τὰ τοιαῦτα λέγοντες ἢ τὴν ὑπόθεσιν, περὶ ἧς βουλεύεσθαι, + οὐχὶ τὴν οὖσαν παριστάντες ὑμῖν ἁμαρτάνειν. ἐγὼ δέ, ὅτι μέν + ποτ᾿ ἐξῆν τῇ πόλει καὶ τὰ αὑτῆς ἔχειν ἀσφαλῶς καὶ Φίλιππον + τιμωρήσασθαι, καὶ μάλ᾿ ἀκριβῶς οἶδα· ἐπ᾿ ἐμοῦ γάρ, οὐ πάλαι + γέγονεν ταῦτ᾿ ἀμφότερα· νῦν μέντοι πέπεισμαι τοῦθ᾿ ἱκανὸν + προλαβεῖν ἡμῖν εἶναι τὴν πρώτην, ὅπως τοὺς συμμάχους + σώσομεν. ἐὰν γὰρ τοῦτο βεβαίως ὑπάρξῃ, τότε καὶ περὶ τοῦ + τίνα τιμωρήσεταί τις καὶ ὃν τρόπον ἐξέσται σκοπεῖν· πρὶν δὲ + τὴν ἀρχὴν ὀρθῶς ὑποθέσθαι, μάταιον ἡγοῦμαι περὶ τῆς + τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον. + + Δημοσθένους, Γ´ ᾿Ολυνθιακὸς + +Georgian: + + From a Unicode conference invitation: + + გთხოვთ ახლავე გაიაროთ რეგისტრაცია Unicode-ის მეათე საერთაშორისო + კონფერენციაზე დასასწრებად, რომელიც გაიმართება 10-12 მარტს, + ქ. მაინცში, გერმანიაში. კონფერენცია შეჰკრებს ერთად მსოფლიოს + ექსპერტებს ისეთ დარგებში როგორიცაა ინტერნეტი და Unicode-ი, + ინტერნაციონალიზაცია და ლოკალიზაცია, Unicode-ის გამოყენება + ოპერაციულ სისტემებსა, და გამოყენებით პროგრამებში, შრიფტებში, + ტექსტების დამუშავებასა და მრავალენოვან კომპიუტერულ სისტემებში. + +Russian: + + From a Unicode conference invitation: + + Зарегистрируйтесь сейчас на Десятую Международную Конференцию по + Unicode, которая состоится 10-12 марта 1997 года в Майнце в Германии. + Конференция соберет широкий круг экспертов по вопросам глобального + Интернета и Unicode, локализации и интернационализации, воплощению и + применению Unicode в различных операционных системах и программных + приложениях, шрифтах, верстке и многоязычных компьютерных системах. + +Thai (UCS Level 2): + + Excerpt from a poetry on The Romance of The Three Kingdoms (a Chinese + classic 'San Gua'): + + [----------------------------|------------------------] + ๏ แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช พระปกเกศกองบู๊กู้ขึ้นใหม่ + สิบสองกษัตริย์ก่อนหน้าแลถัดไป สององค์ไซร้โง่เขลาเบาปัญญา + ทรงนับถือขันทีเป็นที่พึ่ง บ้านเมืองจึงวิปริตเป็นนักหนา + โฮจิ๋นเรียกทัพทั่วหัวเมืองมา หมายจะฆ่ามดชั่วตัวสำคัญ + เหมือนขับไสไล่เสือจากเคหา รับหมาป่าเข้ามาเลยอาสัญ + ฝ่ายอ้องอุ้นยุแยกให้แตกกัน ใช้สาวนั้นเป็นชนวนชื่นชวนใจ + พลันลิฉุยกุยกีกลับก่อเหตุ ช่างอาเพศจริงหนาฟ้าร้องไห้ + ต้องรบราฆ่าฟันจนบรรลัย ฤๅหาใครค้ำชูกู้บรรลังก์ ฯ + + (The above is a two-column text. If combining characters are handled + correctly, the lines of the second column should be aligned with the + | character above.) + +Ethiopian: + + Proverbs in the Amharic language: + + ሰማይ አይታረስ ንጉሥ አይከሰስ። + ብላ ካለኝ እንደአባቴ በቆመጠኝ። + ጌጥ ያለቤቱ ቁምጥና ነው። + ደሀ በሕልሙ ቅቤ ባይጠጣ ንጣት በገደለው። + የአፍ ወለምታ በቅቤ አይታሽም። + አይጥ በበላ ዳዋ ተመታ። + ሲተረጉሙ ይደረግሙ። + ቀስ በቀስ፥ ዕንቁላል በእግሩ ይሄዳል። + ድር ቢያብር አንበሳ ያስር። + ሰው እንደቤቱ እንጅ እንደ ጉረቤቱ አይተዳደርም። + እግዜር የከፈተውን ጉሮሮ ሳይዘጋው አይድርም። + የጎረቤት ሌባ፥ ቢያዩት ይስቅ ባያዩት ያጠልቅ። + ሥራ ከመፍታት ልጄን ላፋታት። + ዓባይ ማደሪያ የለው፥ ግንድ ይዞ ይዞራል። + የእስላም አገሩ መካ የአሞራ አገሩ ዋርካ። + ተንጋሎ ቢተፉ ተመልሶ ባፉ። + ወዳጅህ ማር ቢሆን ጨርስህ አትላሰው። + እግርህን በፍራሽህ ልክ ዘርጋ። + +Runes: + + ᚻᛖ ᚳᚹᚫᚦ ᚦᚫᛏ ᚻᛖ ᛒᚢᛞᛖ ᚩᚾ ᚦᚫᛗ ᛚᚪᚾᛞᛖ ᚾᚩᚱᚦᚹᛖᚪᚱᛞᚢᛗ ᚹᛁᚦ ᚦᚪ ᚹᛖᛥᚫ + + (Old English, which transcribed into Latin reads 'He cwaeth that he + bude thaem lande northweardum with tha Westsae.' and means 'He said + that he lived in the northern land near the Western Sea.') + +Braille: + + ⡌⠁⠧⠑ ⠼⠁⠒ ⡍⠜⠇⠑⠹⠰⠎ ⡣⠕⠌ + + ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠙⠑⠁⠙⠒ ⠞⠕ ⠃⠑⠛⠔ ⠺⠊⠹⠲ ⡹⠻⠑ ⠊⠎ ⠝⠕ ⠙⠳⠃⠞ + ⠱⠁⠞⠑⠧⠻ ⠁⠃⠳⠞ ⠹⠁⠞⠲ ⡹⠑ ⠗⠑⠛⠊⠌⠻ ⠕⠋ ⠙⠊⠎ ⠃⠥⠗⠊⠁⠇ ⠺⠁⠎ + ⠎⠊⠛⠝⠫ ⠃⠹ ⠹⠑ ⠊⠇⠻⠛⠹⠍⠁⠝⠂ ⠹⠑ ⠊⠇⠻⠅⠂ ⠹⠑ ⠥⠝⠙⠻⠞⠁⠅⠻⠂ + ⠁⠝⠙ ⠹⠑ ⠡⠊⠑⠋ ⠍⠳⠗⠝⠻⠲ ⡎⠊⠗⠕⠕⠛⠑ ⠎⠊⠛⠝⠫ ⠊⠞⠲ ⡁⠝⠙ + ⡎⠊⠗⠕⠕⠛⠑⠰⠎ ⠝⠁⠍⠑ ⠺⠁⠎ ⠛⠕⠕⠙ ⠥⠏⠕⠝ ⠰⡡⠁⠝⠛⠑⠂ ⠋⠕⠗ ⠁⠝⠹⠹⠔⠛ ⠙⠑ + ⠡⠕⠎⠑ ⠞⠕ ⠏⠥⠞ ⠙⠊⠎ ⠙⠁⠝⠙ ⠞⠕⠲ + + ⡕⠇⠙ ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ + + ⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹ + ⠪⠝ ⠅⠝⠪⠇⠫⠛⠑⠂ ⠱⠁⠞ ⠹⠻⠑ ⠊⠎ ⠏⠜⠞⠊⠊⠥⠇⠜⠇⠹ ⠙⠑⠁⠙ ⠁⠃⠳⠞ + ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ ⡊ ⠍⠊⠣⠞ ⠙⠁⠧⠑ ⠃⠑⠲ ⠔⠊⠇⠔⠫⠂ ⠍⠹⠎⠑⠇⠋⠂ ⠞⠕ + ⠗⠑⠛⠜⠙ ⠁ ⠊⠕⠋⠋⠔⠤⠝⠁⠊⠇ ⠁⠎ ⠹⠑ ⠙⠑⠁⠙⠑⠌ ⠏⠊⠑⠊⠑ ⠕⠋ ⠊⠗⠕⠝⠍⠕⠝⠛⠻⠹ + ⠔ ⠹⠑ ⠞⠗⠁⠙⠑⠲ ⡃⠥⠞ ⠹⠑ ⠺⠊⠎⠙⠕⠍ ⠕⠋ ⠳⠗ ⠁⠝⠊⠑⠌⠕⠗⠎ + ⠊⠎ ⠔ ⠹⠑ ⠎⠊⠍⠊⠇⠑⠆ ⠁⠝⠙ ⠍⠹ ⠥⠝⠙⠁⠇⠇⠪⠫ ⠙⠁⠝⠙⠎ + ⠩⠁⠇⠇ ⠝⠕⠞ ⠙⠊⠌⠥⠗⠃ ⠊⠞⠂ ⠕⠗ ⠹⠑ ⡊⠳⠝⠞⠗⠹⠰⠎ ⠙⠕⠝⠑ ⠋⠕⠗⠲ ⡹⠳ + ⠺⠊⠇⠇ ⠹⠻⠑⠋⠕⠗⠑ ⠏⠻⠍⠊⠞ ⠍⠑ ⠞⠕ ⠗⠑⠏⠑⠁⠞⠂ ⠑⠍⠏⠙⠁⠞⠊⠊⠁⠇⠇⠹⠂ ⠹⠁⠞ + ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ + + (The first couple of paragraphs of "A Christmas Carol" by Dickens) + +Compact font selection example text: + + ABCDEFGHIJKLMNOPQRSTUVWXYZ /0123456789 + abcdefghijklmnopqrstuvwxyz £©µÀÆÖÞßéöÿ + –—‘“”„†•…‰™œŠŸž€ ΑΒΓΔΩαβγδω АБВГДабвгд + ∀∂∈ℝ∧∪≡∞ ↑↗↨↻⇣ ┐┼╔╘░►☺♀ fi�⑀₂ἠḂӥẄɐː⍎אԱა + +Greetings in various languages: + + Hello world, Καλημέρα κόσμε, コンニチハ + +Box drawing alignment tests: █ + ▉ + ╔══╦══╗ ┌──┬──┐ ╭──┬──╮ ╭──┬──╮ ┏━━┳━━┓ ┎┒┏┑ ╷ ╻ ┏┯┓ ┌┰┐ ▊ ╱╲╱╲╳╳╳ + ║┌─╨─┐║ │╔═╧═╗│ │╒═╪═╕│ │╓─╁─╖│ ┃┌─╂─┐┃ ┗╃╄┙ ╶┼╴╺╋╸┠┼┨ ┝╋┥ ▋ ╲╱╲╱╳╳╳ + ║│╲ ╱│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╿ │┃ ┍╅╆┓ ╵ ╹ ┗┷┛ └┸┘ ▌ ╱╲╱╲╳╳╳ + ╠╡ ╳ ╞╣ ├╢ ╟┤ ├┼─┼─┼┤ ├╫─╂─╫┤ ┣┿╾┼╼┿┫ ┕┛┖┚ ┌┄┄┐ ╎ ┏┅┅┓ ┋ ▍ ╲╱╲╱╳╳╳ + ║│╱ ╲│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╽ │┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▎ + ║└─╥─┘║ │╚═╤═╝│ │╘═╪═╛│ │╙─╀─╜│ ┃└─╂─┘┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▏ + ╚══╩══╝ └──┴──┘ ╰──┴──╯ ╰──┴──╯ ┗━━┻━━┛ ▗▄▖▛▀▜ └╌╌┘ ╎ ┗╍╍┛ ┋ ▁▂▃▄▅▆▇█ + ▝▀▘▙▄▟ diff --git a/test/invalid/test075.txt b/test/invalid/test075.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test075.txt @@ -0,0 +1,1 @@ +2.1.5 5 bytes (U-00200000): "øˆ€€€" | diff --git a/test/invalid/test076.txt b/test/invalid/test076.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test076.txt @@ -0,0 +1,1 @@ +2.1.6 6 bytes (U-04000000): "ü„€€€€" | diff --git a/test/invalid/test082.txt b/test/invalid/test082.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test082.txt @@ -0,0 +1,1 @@ +2.2.3 3 bytes (U-0000FFFF): "￿" | diff --git a/test/invalid/test083.txt b/test/invalid/test083.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test083.txt @@ -0,0 +1,1 @@ +2.2.4 4 bytes (U-001FFFFF): "÷¿¿¿" | diff --git a/test/invalid/test084.txt b/test/invalid/test084.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test084.txt @@ -0,0 +1,1 @@ +2.2.5 5 bytes (U-03FFFFFF): "û¿¿¿¿" | diff --git a/test/invalid/test085.txt b/test/invalid/test085.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test085.txt @@ -0,0 +1,1 @@ +2.2.6 6 bytes (U-7FFFFFFF): "ý¿¿¿¿¿" | diff --git a/test/invalid/test092.txt b/test/invalid/test092.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test092.txt @@ -0,0 +1,1 @@ +2.3.4 U-0010FFFF = f4 8f bf bf = "􏿿" | diff --git a/test/invalid/test093.txt b/test/invalid/test093.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test093.txt @@ -0,0 +1,1 @@ +2.3.5 U-00110000 = f4 90 80 80 = "ô€€" | diff --git a/test/invalid/test102.txt b/test/invalid/test102.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test102.txt @@ -0,0 +1,1 @@ +3.1.1 First continuation byte 0x80: "€" | diff --git a/test/invalid/test103.txt b/test/invalid/test103.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test103.txt @@ -0,0 +1,1 @@ +3.1.2 Last continuation byte 0xbf: "¿" | diff --git a/test/invalid/test105.txt b/test/invalid/test105.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test105.txt @@ -0,0 +1,1 @@ +3.1.3 2 continuation bytes: "€¿" | diff --git a/test/invalid/test106.txt b/test/invalid/test106.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test106.txt @@ -0,0 +1,1 @@ +3.1.4 3 continuation bytes: "€¿€" | diff --git a/test/invalid/test107.txt b/test/invalid/test107.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test107.txt @@ -0,0 +1,1 @@ +3.1.5 4 continuation bytes: "€¿€¿" | diff --git a/test/invalid/test108.txt b/test/invalid/test108.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test108.txt @@ -0,0 +1,1 @@ +3.1.6 5 continuation bytes: "€¿€¿€" | diff --git a/test/invalid/test109.txt b/test/invalid/test109.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test109.txt @@ -0,0 +1,1 @@ +3.1.7 6 continuation bytes: "€¿€¿€¿" | diff --git a/test/invalid/test110.txt b/test/invalid/test110.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test110.txt @@ -0,0 +1,1 @@ +3.1.8 7 continuation bytes: "€¿€¿€¿€" | diff --git a/test/invalid/test114.txt b/test/invalid/test114.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test114.txt @@ -0,0 +1,1 @@ + "€‚ƒ„…†‡ˆ‰Š‹ŒŽ | diff --git a/test/invalid/test115.txt b/test/invalid/test115.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test115.txt @@ -0,0 +1,1 @@ + ‘’“”•–—˜™š›œžŸ | diff --git a/test/invalid/test116.txt b/test/invalid/test116.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test116.txt @@ -0,0 +1,1 @@ +  ¡¢£¤¥¦§¨©ª«¬­®¯ | diff --git a/test/invalid/test117.txt b/test/invalid/test117.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test117.txt @@ -0,0 +1,1 @@ + °±²³´µ¶·¸¹º»¼½¾¿" | diff --git a/test/invalid/test124.txt b/test/invalid/test124.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test124.txt @@ -0,0 +1,1 @@ + "À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï | diff --git a/test/invalid/test125.txt b/test/invalid/test125.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test125.txt @@ -0,0 +1,1 @@ + Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß " | diff --git a/test/invalid/test130.txt b/test/invalid/test130.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test130.txt @@ -0,0 +1,1 @@ + "à á â ã ä å æ ç è é ê ë ì í î ï " | diff --git a/test/invalid/test135.txt b/test/invalid/test135.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test135.txt @@ -0,0 +1,1 @@ + "ð ñ ò ó ô õ ö ÷ " | diff --git a/test/invalid/test140.txt b/test/invalid/test140.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test140.txt @@ -0,0 +1,1 @@ + "ø ù ú û " | diff --git a/test/invalid/test145.txt b/test/invalid/test145.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test145.txt @@ -0,0 +1,1 @@ + "ü ý " | diff --git a/test/invalid/test153.txt b/test/invalid/test153.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test153.txt @@ -0,0 +1,1 @@ +3.3.1 2-byte sequence with last byte missing (U+0000): "À" | diff --git a/test/invalid/test154.txt b/test/invalid/test154.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test154.txt @@ -0,0 +1,1 @@ +3.3.2 3-byte sequence with last byte missing (U+0000): "à€" | diff --git a/test/invalid/test155.txt b/test/invalid/test155.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test155.txt @@ -0,0 +1,1 @@ +3.3.3 4-byte sequence with last byte missing (U+0000): "ð€€" | diff --git a/test/invalid/test156.txt b/test/invalid/test156.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test156.txt @@ -0,0 +1,1 @@ +3.3.4 5-byte sequence with last byte missing (U+0000): "ø€€€" | diff --git a/test/invalid/test157.txt b/test/invalid/test157.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test157.txt @@ -0,0 +1,1 @@ +3.3.5 6-byte sequence with last byte missing (U+0000): "ü€€€€" | diff --git a/test/invalid/test158.txt b/test/invalid/test158.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test158.txt @@ -0,0 +1,1 @@ +3.3.6 2-byte sequence with last byte missing (U-000007FF): "ß" | diff --git a/test/invalid/test159.txt b/test/invalid/test159.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test159.txt @@ -0,0 +1,1 @@ +3.3.7 3-byte sequence with last byte missing (U-0000FFFF): "ï¿" | diff --git a/test/invalid/test160.txt b/test/invalid/test160.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test160.txt @@ -0,0 +1,1 @@ +3.3.8 4-byte sequence with last byte missing (U-001FFFFF): "÷¿¿" | diff --git a/test/invalid/test161.txt b/test/invalid/test161.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test161.txt @@ -0,0 +1,1 @@ +3.3.9 5-byte sequence with last byte missing (U-03FFFFFF): "û¿¿¿" | diff --git a/test/invalid/test162.txt b/test/invalid/test162.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test162.txt @@ -0,0 +1,1 @@ +3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF): "ý¿¿¿¿" | diff --git a/test/invalid/test169.txt b/test/invalid/test169.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test169.txt @@ -0,0 +1,1 @@ + "Àà€ð€€ø€€€ü€€€€ßï¿÷¿¿û¿¿¿ý¿¿¿¿" | diff --git a/test/invalid/test175.txt b/test/invalid/test175.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test175.txt @@ -0,0 +1,1 @@ +3.5.1 fe = "þ" | diff --git a/test/invalid/test176.txt b/test/invalid/test176.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test176.txt @@ -0,0 +1,1 @@ +3.5.2 ff = "ÿ" | diff --git a/test/invalid/test177.txt b/test/invalid/test177.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test177.txt @@ -0,0 +1,1 @@ +3.5.3 fe fe ff ff = "þþÿÿ" | diff --git a/test/invalid/test207.txt b/test/invalid/test207.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test207.txt @@ -0,0 +1,1 @@ +4.1.1 U+002F = c0 af = "À¯" | diff --git a/test/invalid/test208.txt b/test/invalid/test208.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test208.txt @@ -0,0 +1,1 @@ +4.1.2 U+002F = e0 80 af = "à€¯" | diff --git a/test/invalid/test209.txt b/test/invalid/test209.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test209.txt @@ -0,0 +1,1 @@ +4.1.3 U+002F = f0 80 80 af = "ð€€¯" | diff --git a/test/invalid/test210.txt b/test/invalid/test210.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test210.txt @@ -0,0 +1,1 @@ +4.1.4 U+002F = f8 80 80 80 af = "ø€€€¯" | diff --git a/test/invalid/test211.txt b/test/invalid/test211.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test211.txt @@ -0,0 +1,1 @@ +4.1.5 U+002F = fc 80 80 80 80 af = "ü€€€€¯" | diff --git a/test/invalid/test220.txt b/test/invalid/test220.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test220.txt @@ -0,0 +1,1 @@ +4.2.1 U-0000007F = c1 bf = "Á¿" | diff --git a/test/invalid/test221.txt b/test/invalid/test221.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test221.txt @@ -0,0 +1,1 @@ +4.2.2 U-000007FF = e0 9f bf = "àŸ¿" | diff --git a/test/invalid/test222.txt b/test/invalid/test222.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test222.txt @@ -0,0 +1,1 @@ +4.2.3 U-0000FFFF = f0 8f bf bf = "ð¿¿" | diff --git a/test/invalid/test223.txt b/test/invalid/test223.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test223.txt @@ -0,0 +1,1 @@ +4.2.4 U-001FFFFF = f8 87 bf bf bf = "ø‡¿¿¿" | diff --git a/test/invalid/test224.txt b/test/invalid/test224.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test224.txt @@ -0,0 +1,1 @@ +4.2.5 U-03FFFFFF = fc 83 bf bf bf bf = "üƒ¿¿¿¿" | diff --git a/test/invalid/test232.txt b/test/invalid/test232.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test232.txt @@ -0,0 +1,1 @@ +4.3.1 U+0000 = c0 80 = "À€" | diff --git a/test/invalid/test233.txt b/test/invalid/test233.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test233.txt @@ -0,0 +1,1 @@ +4.3.2 U+0000 = e0 80 80 = "à€€" | diff --git a/test/invalid/test234.txt b/test/invalid/test234.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test234.txt @@ -0,0 +1,1 @@ +4.3.3 U+0000 = f0 80 80 80 = "ð€€€" | diff --git a/test/invalid/test235.txt b/test/invalid/test235.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test235.txt @@ -0,0 +1,1 @@ +4.3.4 U+0000 = f8 80 80 80 80 = "ø€€€€" | diff --git a/test/invalid/test236.txt b/test/invalid/test236.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test236.txt @@ -0,0 +1,1 @@ +4.3.5 U+0000 = fc 80 80 80 80 80 = "ü€€€€€" | diff --git a/test/invalid/test247.txt b/test/invalid/test247.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test247.txt @@ -0,0 +1,1 @@ +5.1.1 U+D800 = ed a0 80 = "í €" | diff --git a/test/invalid/test248.txt b/test/invalid/test248.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test248.txt @@ -0,0 +1,1 @@ +5.1.2 U+DB7F = ed ad bf = "í­¿" | diff --git a/test/invalid/test249.txt b/test/invalid/test249.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test249.txt @@ -0,0 +1,1 @@ +5.1.3 U+DB80 = ed ae 80 = "í®€" | diff --git a/test/invalid/test250.txt b/test/invalid/test250.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test250.txt @@ -0,0 +1,1 @@ +5.1.4 U+DBFF = ed af bf = "í¯¿" | diff --git a/test/invalid/test251.txt b/test/invalid/test251.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test251.txt @@ -0,0 +1,1 @@ +5.1.5 U+DC00 = ed b0 80 = "í°€" | diff --git a/test/invalid/test252.txt b/test/invalid/test252.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test252.txt @@ -0,0 +1,1 @@ +5.1.6 U+DF80 = ed be 80 = "í¾€" | diff --git a/test/invalid/test253.txt b/test/invalid/test253.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test253.txt @@ -0,0 +1,1 @@ +5.1.7 U+DFFF = ed bf bf = "í¿¿" | diff --git a/test/invalid/test257.txt b/test/invalid/test257.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test257.txt @@ -0,0 +1,1 @@ +5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80 = "𐀀" | diff --git a/test/invalid/test258.txt b/test/invalid/test258.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test258.txt @@ -0,0 +1,1 @@ +5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf = "𐏿" | diff --git a/test/invalid/test259.txt b/test/invalid/test259.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test259.txt @@ -0,0 +1,1 @@ +5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80 = "󯰀" | diff --git a/test/invalid/test260.txt b/test/invalid/test260.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test260.txt @@ -0,0 +1,1 @@ +5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf = "í­¿í¿¿" | diff --git a/test/invalid/test261.txt b/test/invalid/test261.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test261.txt @@ -0,0 +1,1 @@ +5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80 = "󰀀" | diff --git a/test/invalid/test262.txt b/test/invalid/test262.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test262.txt @@ -0,0 +1,1 @@ +5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf = "󰏿" | diff --git a/test/invalid/test263.txt b/test/invalid/test263.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test263.txt @@ -0,0 +1,1 @@ +5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80 = "􏰀" | diff --git a/test/invalid/test264.txt b/test/invalid/test264.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test264.txt @@ -0,0 +1,1 @@ +5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf = "􏿿" | diff --git a/test/invalid/test268.txt b/test/invalid/test268.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test268.txt @@ -0,0 +1,1 @@ +5.3.1 U+FFFE = ef bf be = "￾" | diff --git a/test/invalid/test269.txt b/test/invalid/test269.txt new file mode 100644 --- /dev/null +++ b/test/invalid/test269.txt @@ -0,0 +1,1 @@ +5.3.2 U+FFFF = ef bf bf = "￿" | diff --git a/test/test.txt b/test/test.txt new file mode 100644 Binary files /dev/null and b/test/test.txt differ diff --git a/test/valid/test061.txt b/test/valid/test061.txt new file mode 100644 --- /dev/null +++ b/test/valid/test061.txt @@ -0,0 +1,1 @@ +Here come the tests: | diff --git a/test/valid/test062.txt b/test/valid/test062.txt new file mode 100644 --- /dev/null +++ b/test/valid/test062.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test063.txt b/test/valid/test063.txt new file mode 100644 --- /dev/null +++ b/test/valid/test063.txt @@ -0,0 +1,1 @@ +1 Some correct UTF-8 text | diff --git a/test/valid/test064.txt b/test/valid/test064.txt new file mode 100644 --- /dev/null +++ b/test/valid/test064.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test065.txt b/test/valid/test065.txt new file mode 100644 --- /dev/null +++ b/test/valid/test065.txt @@ -0,0 +1,1 @@ +You should see the Greek word 'kosme': "κόσμε" | diff --git a/test/valid/test066.txt b/test/valid/test066.txt new file mode 100644 --- /dev/null +++ b/test/valid/test066.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test067.txt b/test/valid/test067.txt new file mode 100644 --- /dev/null +++ b/test/valid/test067.txt @@ -0,0 +1,1 @@ +2 Boundary condition test cases | diff --git a/test/valid/test068.txt b/test/valid/test068.txt new file mode 100644 --- /dev/null +++ b/test/valid/test068.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test069.txt b/test/valid/test069.txt new file mode 100644 --- /dev/null +++ b/test/valid/test069.txt @@ -0,0 +1,1 @@ +2.1 First possible sequence of a certain length | diff --git a/test/valid/test070.txt b/test/valid/test070.txt new file mode 100644 --- /dev/null +++ b/test/valid/test070.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test071.txt b/test/valid/test071.txt new file mode 100644 Binary files /dev/null and b/test/valid/test071.txt differ diff --git a/test/valid/test072.txt b/test/valid/test072.txt new file mode 100644 --- /dev/null +++ b/test/valid/test072.txt @@ -0,0 +1,1 @@ +2.1.2 2 bytes (U-00000080): "€" | diff --git a/test/valid/test073.txt b/test/valid/test073.txt new file mode 100644 --- /dev/null +++ b/test/valid/test073.txt @@ -0,0 +1,1 @@ +2.1.3 3 bytes (U-00000800): "ࠀ" | diff --git a/test/valid/test074.txt b/test/valid/test074.txt new file mode 100644 --- /dev/null +++ b/test/valid/test074.txt @@ -0,0 +1,1 @@ +2.1.4 4 bytes (U-00010000): "𐀀" | diff --git a/test/valid/test077.txt b/test/valid/test077.txt new file mode 100644 --- /dev/null +++ b/test/valid/test077.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test078.txt b/test/valid/test078.txt new file mode 100644 --- /dev/null +++ b/test/valid/test078.txt @@ -0,0 +1,1 @@ +2.2 Last possible sequence of a certain length | diff --git a/test/valid/test079.txt b/test/valid/test079.txt new file mode 100644 --- /dev/null +++ b/test/valid/test079.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test080.txt b/test/valid/test080.txt new file mode 100644 --- /dev/null +++ b/test/valid/test080.txt @@ -0,0 +1,1 @@ +2.2.1 1 byte (U-0000007F): "" diff --git a/test/valid/test081.txt b/test/valid/test081.txt new file mode 100644 --- /dev/null +++ b/test/valid/test081.txt @@ -0,0 +1,1 @@ +2.2.2 2 bytes (U-000007FF): "߿" | diff --git a/test/valid/test086.txt b/test/valid/test086.txt new file mode 100644 --- /dev/null +++ b/test/valid/test086.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test087.txt b/test/valid/test087.txt new file mode 100644 --- /dev/null +++ b/test/valid/test087.txt @@ -0,0 +1,1 @@ +2.3 Other boundary conditions | diff --git a/test/valid/test088.txt b/test/valid/test088.txt new file mode 100644 --- /dev/null +++ b/test/valid/test088.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test089.txt b/test/valid/test089.txt new file mode 100644 --- /dev/null +++ b/test/valid/test089.txt @@ -0,0 +1,1 @@ +2.3.1 U-0000D7FF = ed 9f bf = "퟿" | diff --git a/test/valid/test090.txt b/test/valid/test090.txt new file mode 100644 --- /dev/null +++ b/test/valid/test090.txt @@ -0,0 +1,1 @@ +2.3.2 U-0000E000 = ee 80 80 = "" | diff --git a/test/valid/test091.txt b/test/valid/test091.txt new file mode 100644 --- /dev/null +++ b/test/valid/test091.txt @@ -0,0 +1,1 @@ +2.3.3 U-0000FFFD = ef bf bd = "�" | diff --git a/test/valid/test094.txt b/test/valid/test094.txt new file mode 100644 --- /dev/null +++ b/test/valid/test094.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test095.txt b/test/valid/test095.txt new file mode 100644 --- /dev/null +++ b/test/valid/test095.txt @@ -0,0 +1,1 @@ +3 Malformed sequences | diff --git a/test/valid/test096.txt b/test/valid/test096.txt new file mode 100644 --- /dev/null +++ b/test/valid/test096.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test097.txt b/test/valid/test097.txt new file mode 100644 --- /dev/null +++ b/test/valid/test097.txt @@ -0,0 +1,1 @@ +3.1 Unexpected continuation bytes | diff --git a/test/valid/test098.txt b/test/valid/test098.txt new file mode 100644 --- /dev/null +++ b/test/valid/test098.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test099.txt b/test/valid/test099.txt new file mode 100644 --- /dev/null +++ b/test/valid/test099.txt @@ -0,0 +1,1 @@ +Each unexpected continuation byte should be separately signalled as a | diff --git a/test/valid/test100.txt b/test/valid/test100.txt new file mode 100644 --- /dev/null +++ b/test/valid/test100.txt @@ -0,0 +1,1 @@ +malformed sequence of its own. | diff --git a/test/valid/test101.txt b/test/valid/test101.txt new file mode 100644 --- /dev/null +++ b/test/valid/test101.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test104.txt b/test/valid/test104.txt new file mode 100644 --- /dev/null +++ b/test/valid/test104.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test111.txt b/test/valid/test111.txt new file mode 100644 --- /dev/null +++ b/test/valid/test111.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test112.txt b/test/valid/test112.txt new file mode 100644 --- /dev/null +++ b/test/valid/test112.txt @@ -0,0 +1,1 @@ +3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf): | diff --git a/test/valid/test113.txt b/test/valid/test113.txt new file mode 100644 --- /dev/null +++ b/test/valid/test113.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test118.txt b/test/valid/test118.txt new file mode 100644 --- /dev/null +++ b/test/valid/test118.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test119.txt b/test/valid/test119.txt new file mode 100644 --- /dev/null +++ b/test/valid/test119.txt @@ -0,0 +1,1 @@ +3.2 Lonely start characters | diff --git a/test/valid/test120.txt b/test/valid/test120.txt new file mode 100644 --- /dev/null +++ b/test/valid/test120.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test121.txt b/test/valid/test121.txt new file mode 100644 --- /dev/null +++ b/test/valid/test121.txt @@ -0,0 +1,1 @@ +3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf), | diff --git a/test/valid/test122.txt b/test/valid/test122.txt new file mode 100644 --- /dev/null +++ b/test/valid/test122.txt @@ -0,0 +1,1 @@ + each followed by a space character: | diff --git a/test/valid/test123.txt b/test/valid/test123.txt new file mode 100644 --- /dev/null +++ b/test/valid/test123.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test126.txt b/test/valid/test126.txt new file mode 100644 --- /dev/null +++ b/test/valid/test126.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test127.txt b/test/valid/test127.txt new file mode 100644 --- /dev/null +++ b/test/valid/test127.txt @@ -0,0 +1,1 @@ +3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef), | diff --git a/test/valid/test128.txt b/test/valid/test128.txt new file mode 100644 --- /dev/null +++ b/test/valid/test128.txt @@ -0,0 +1,1 @@ + each followed by a space character: | diff --git a/test/valid/test129.txt b/test/valid/test129.txt new file mode 100644 --- /dev/null +++ b/test/valid/test129.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test131.txt b/test/valid/test131.txt new file mode 100644 --- /dev/null +++ b/test/valid/test131.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test132.txt b/test/valid/test132.txt new file mode 100644 --- /dev/null +++ b/test/valid/test132.txt @@ -0,0 +1,1 @@ +3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7), | diff --git a/test/valid/test133.txt b/test/valid/test133.txt new file mode 100644 --- /dev/null +++ b/test/valid/test133.txt @@ -0,0 +1,1 @@ + each followed by a space character: | diff --git a/test/valid/test134.txt b/test/valid/test134.txt new file mode 100644 --- /dev/null +++ b/test/valid/test134.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test136.txt b/test/valid/test136.txt new file mode 100644 --- /dev/null +++ b/test/valid/test136.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test137.txt b/test/valid/test137.txt new file mode 100644 --- /dev/null +++ b/test/valid/test137.txt @@ -0,0 +1,1 @@ +3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb), | diff --git a/test/valid/test138.txt b/test/valid/test138.txt new file mode 100644 --- /dev/null +++ b/test/valid/test138.txt @@ -0,0 +1,1 @@ + each followed by a space character: | diff --git a/test/valid/test139.txt b/test/valid/test139.txt new file mode 100644 --- /dev/null +++ b/test/valid/test139.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test141.txt b/test/valid/test141.txt new file mode 100644 --- /dev/null +++ b/test/valid/test141.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test142.txt b/test/valid/test142.txt new file mode 100644 --- /dev/null +++ b/test/valid/test142.txt @@ -0,0 +1,1 @@ +3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd), | diff --git a/test/valid/test143.txt b/test/valid/test143.txt new file mode 100644 --- /dev/null +++ b/test/valid/test143.txt @@ -0,0 +1,1 @@ + each followed by a space character: | diff --git a/test/valid/test144.txt b/test/valid/test144.txt new file mode 100644 --- /dev/null +++ b/test/valid/test144.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test146.txt b/test/valid/test146.txt new file mode 100644 --- /dev/null +++ b/test/valid/test146.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test147.txt b/test/valid/test147.txt new file mode 100644 --- /dev/null +++ b/test/valid/test147.txt @@ -0,0 +1,1 @@ +3.3 Sequences with last continuation byte missing | diff --git a/test/valid/test148.txt b/test/valid/test148.txt new file mode 100644 --- /dev/null +++ b/test/valid/test148.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test149.txt b/test/valid/test149.txt new file mode 100644 --- /dev/null +++ b/test/valid/test149.txt @@ -0,0 +1,1 @@ +All bytes of an incomplete sequence should be signalled as a single | diff --git a/test/valid/test150.txt b/test/valid/test150.txt new file mode 100644 --- /dev/null +++ b/test/valid/test150.txt @@ -0,0 +1,1 @@ +malformed sequence, i.e., you should see only a single replacement | diff --git a/test/valid/test151.txt b/test/valid/test151.txt new file mode 100644 --- /dev/null +++ b/test/valid/test151.txt @@ -0,0 +1,1 @@ +character in each of the next 10 tests. (Characters as in section 2) | diff --git a/test/valid/test152.txt b/test/valid/test152.txt new file mode 100644 --- /dev/null +++ b/test/valid/test152.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test163.txt b/test/valid/test163.txt new file mode 100644 --- /dev/null +++ b/test/valid/test163.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test164.txt b/test/valid/test164.txt new file mode 100644 --- /dev/null +++ b/test/valid/test164.txt @@ -0,0 +1,1 @@ +3.4 Concatenation of incomplete sequences | diff --git a/test/valid/test165.txt b/test/valid/test165.txt new file mode 100644 --- /dev/null +++ b/test/valid/test165.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test166.txt b/test/valid/test166.txt new file mode 100644 --- /dev/null +++ b/test/valid/test166.txt @@ -0,0 +1,1 @@ +All the 10 sequences of 3.3 concatenated, you should see 10 malformed | diff --git a/test/valid/test167.txt b/test/valid/test167.txt new file mode 100644 --- /dev/null +++ b/test/valid/test167.txt @@ -0,0 +1,1 @@ +sequences being signalled: | diff --git a/test/valid/test168.txt b/test/valid/test168.txt new file mode 100644 --- /dev/null +++ b/test/valid/test168.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test170.txt b/test/valid/test170.txt new file mode 100644 --- /dev/null +++ b/test/valid/test170.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test171.txt b/test/valid/test171.txt new file mode 100644 --- /dev/null +++ b/test/valid/test171.txt @@ -0,0 +1,1 @@ +3.5 Impossible bytes | diff --git a/test/valid/test172.txt b/test/valid/test172.txt new file mode 100644 --- /dev/null +++ b/test/valid/test172.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test173.txt b/test/valid/test173.txt new file mode 100644 --- /dev/null +++ b/test/valid/test173.txt @@ -0,0 +1,1 @@ +The following two bytes cannot appear in a correct UTF-8 string | diff --git a/test/valid/test174.txt b/test/valid/test174.txt new file mode 100644 --- /dev/null +++ b/test/valid/test174.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test178.txt b/test/valid/test178.txt new file mode 100644 --- /dev/null +++ b/test/valid/test178.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test179.txt b/test/valid/test179.txt new file mode 100644 --- /dev/null +++ b/test/valid/test179.txt @@ -0,0 +1,1 @@ +4 Overlong sequences | diff --git a/test/valid/test180.txt b/test/valid/test180.txt new file mode 100644 --- /dev/null +++ b/test/valid/test180.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test181.txt b/test/valid/test181.txt new file mode 100644 --- /dev/null +++ b/test/valid/test181.txt @@ -0,0 +1,1 @@ +The following sequences are not malformed according to the letter of | diff --git a/test/valid/test182.txt b/test/valid/test182.txt new file mode 100644 --- /dev/null +++ b/test/valid/test182.txt @@ -0,0 +1,1 @@ +the Unicode 2.0 standard. However, they are longer then necessary and | diff --git a/test/valid/test183.txt b/test/valid/test183.txt new file mode 100644 --- /dev/null +++ b/test/valid/test183.txt @@ -0,0 +1,1 @@ +a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8 | diff --git a/test/valid/test184.txt b/test/valid/test184.txt new file mode 100644 --- /dev/null +++ b/test/valid/test184.txt @@ -0,0 +1,1 @@ +decoder" should reject them just like malformed sequences for two | diff --git a/test/valid/test185.txt b/test/valid/test185.txt new file mode 100644 --- /dev/null +++ b/test/valid/test185.txt @@ -0,0 +1,1 @@ +reasons: (1) It helps to debug applications if overlong sequences are | diff --git a/test/valid/test186.txt b/test/valid/test186.txt new file mode 100644 --- /dev/null +++ b/test/valid/test186.txt @@ -0,0 +1,1 @@ +not treated as valid representations of characters, because this helps | diff --git a/test/valid/test187.txt b/test/valid/test187.txt new file mode 100644 --- /dev/null +++ b/test/valid/test187.txt @@ -0,0 +1,1 @@ +to spot problems more quickly. (2) Overlong sequences provide | diff --git a/test/valid/test188.txt b/test/valid/test188.txt new file mode 100644 --- /dev/null +++ b/test/valid/test188.txt @@ -0,0 +1,1 @@ +alternative representations of characters, that could maliciously be | diff --git a/test/valid/test189.txt b/test/valid/test189.txt new file mode 100644 --- /dev/null +++ b/test/valid/test189.txt @@ -0,0 +1,1 @@ +used to bypass filters that check only for ASCII characters. For | diff --git a/test/valid/test190.txt b/test/valid/test190.txt new file mode 100644 --- /dev/null +++ b/test/valid/test190.txt @@ -0,0 +1,1 @@ +instance, a 2-byte encoded line feed (LF) would not be caught by a | diff --git a/test/valid/test191.txt b/test/valid/test191.txt new file mode 100644 --- /dev/null +++ b/test/valid/test191.txt @@ -0,0 +1,1 @@ +line counter that counts only 0x0a bytes, but it would still be | diff --git a/test/valid/test192.txt b/test/valid/test192.txt new file mode 100644 --- /dev/null +++ b/test/valid/test192.txt @@ -0,0 +1,1 @@ +processed as a line feed by an unsafe UTF-8 decoder later in the | diff --git a/test/valid/test193.txt b/test/valid/test193.txt new file mode 100644 --- /dev/null +++ b/test/valid/test193.txt @@ -0,0 +1,1 @@ +pipeline. From a security point of view, ASCII compatibility of UTF-8 | diff --git a/test/valid/test194.txt b/test/valid/test194.txt new file mode 100644 --- /dev/null +++ b/test/valid/test194.txt @@ -0,0 +1,1 @@ +sequences means also, that ASCII characters are *only* allowed to be | diff --git a/test/valid/test195.txt b/test/valid/test195.txt new file mode 100644 --- /dev/null +++ b/test/valid/test195.txt @@ -0,0 +1,1 @@ +represented by ASCII bytes in the range 0x00-0x7f. To ensure this | diff --git a/test/valid/test196.txt b/test/valid/test196.txt new file mode 100644 --- /dev/null +++ b/test/valid/test196.txt @@ -0,0 +1,1 @@ +aspect of ASCII compatibility, use only "safe UTF-8 decoders" that | diff --git a/test/valid/test197.txt b/test/valid/test197.txt new file mode 100644 --- /dev/null +++ b/test/valid/test197.txt @@ -0,0 +1,1 @@ +reject overlong UTF-8 sequences for which a shorter encoding exists. | diff --git a/test/valid/test198.txt b/test/valid/test198.txt new file mode 100644 --- /dev/null +++ b/test/valid/test198.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test199.txt b/test/valid/test199.txt new file mode 100644 --- /dev/null +++ b/test/valid/test199.txt @@ -0,0 +1,1 @@ +4.1 Examples of an overlong ASCII character | diff --git a/test/valid/test200.txt b/test/valid/test200.txt new file mode 100644 --- /dev/null +++ b/test/valid/test200.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test201.txt b/test/valid/test201.txt new file mode 100644 --- /dev/null +++ b/test/valid/test201.txt @@ -0,0 +1,1 @@ +With a safe UTF-8 decoder, all of the following five overlong | diff --git a/test/valid/test202.txt b/test/valid/test202.txt new file mode 100644 --- /dev/null +++ b/test/valid/test202.txt @@ -0,0 +1,1 @@ +representations of the ASCII character slash ("/") should be rejected | diff --git a/test/valid/test203.txt b/test/valid/test203.txt new file mode 100644 --- /dev/null +++ b/test/valid/test203.txt @@ -0,0 +1,1 @@ +like a malformed UTF-8 sequence, for instance by substituting it with | diff --git a/test/valid/test204.txt b/test/valid/test204.txt new file mode 100644 --- /dev/null +++ b/test/valid/test204.txt @@ -0,0 +1,1 @@ +a replacement character. If you see a slash below, you do not have a | diff --git a/test/valid/test205.txt b/test/valid/test205.txt new file mode 100644 --- /dev/null +++ b/test/valid/test205.txt @@ -0,0 +1,1 @@ +safe UTF-8 decoder! | diff --git a/test/valid/test206.txt b/test/valid/test206.txt new file mode 100644 --- /dev/null +++ b/test/valid/test206.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test212.txt b/test/valid/test212.txt new file mode 100644 --- /dev/null +++ b/test/valid/test212.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test213.txt b/test/valid/test213.txt new file mode 100644 --- /dev/null +++ b/test/valid/test213.txt @@ -0,0 +1,1 @@ +4.2 Maximum overlong sequences | diff --git a/test/valid/test214.txt b/test/valid/test214.txt new file mode 100644 --- /dev/null +++ b/test/valid/test214.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test215.txt b/test/valid/test215.txt new file mode 100644 --- /dev/null +++ b/test/valid/test215.txt @@ -0,0 +1,1 @@ +Below you see the highest Unicode value that is still resulting in an | diff --git a/test/valid/test216.txt b/test/valid/test216.txt new file mode 100644 --- /dev/null +++ b/test/valid/test216.txt @@ -0,0 +1,1 @@ +overlong sequence if represented with the given number of bytes. This | diff --git a/test/valid/test217.txt b/test/valid/test217.txt new file mode 100644 --- /dev/null +++ b/test/valid/test217.txt @@ -0,0 +1,1 @@ +is a boundary test for safe UTF-8 decoders. All five characters should | diff --git a/test/valid/test218.txt b/test/valid/test218.txt new file mode 100644 --- /dev/null +++ b/test/valid/test218.txt @@ -0,0 +1,1 @@ +be rejected like malformed UTF-8 sequences. | diff --git a/test/valid/test219.txt b/test/valid/test219.txt new file mode 100644 --- /dev/null +++ b/test/valid/test219.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test225.txt b/test/valid/test225.txt new file mode 100644 --- /dev/null +++ b/test/valid/test225.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test226.txt b/test/valid/test226.txt new file mode 100644 --- /dev/null +++ b/test/valid/test226.txt @@ -0,0 +1,1 @@ +4.3 Overlong representation of the NUL character | diff --git a/test/valid/test227.txt b/test/valid/test227.txt new file mode 100644 --- /dev/null +++ b/test/valid/test227.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test228.txt b/test/valid/test228.txt new file mode 100644 --- /dev/null +++ b/test/valid/test228.txt @@ -0,0 +1,1 @@ +The following five sequences should also be rejected like malformed | diff --git a/test/valid/test229.txt b/test/valid/test229.txt new file mode 100644 --- /dev/null +++ b/test/valid/test229.txt @@ -0,0 +1,1 @@ +UTF-8 sequences and should not be treated like the ASCII NUL | diff --git a/test/valid/test230.txt b/test/valid/test230.txt new file mode 100644 --- /dev/null +++ b/test/valid/test230.txt @@ -0,0 +1,1 @@ +character. | diff --git a/test/valid/test231.txt b/test/valid/test231.txt new file mode 100644 --- /dev/null +++ b/test/valid/test231.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test237.txt b/test/valid/test237.txt new file mode 100644 --- /dev/null +++ b/test/valid/test237.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test238.txt b/test/valid/test238.txt new file mode 100644 --- /dev/null +++ b/test/valid/test238.txt @@ -0,0 +1,1 @@ +5 Illegal code positions | diff --git a/test/valid/test239.txt b/test/valid/test239.txt new file mode 100644 --- /dev/null +++ b/test/valid/test239.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test240.txt b/test/valid/test240.txt new file mode 100644 --- /dev/null +++ b/test/valid/test240.txt @@ -0,0 +1,1 @@ +The following UTF-8 sequences should be rejected like malformed | diff --git a/test/valid/test241.txt b/test/valid/test241.txt new file mode 100644 --- /dev/null +++ b/test/valid/test241.txt @@ -0,0 +1,1 @@ +sequences, because they never represent valid ISO 10646 characters and | diff --git a/test/valid/test242.txt b/test/valid/test242.txt new file mode 100644 --- /dev/null +++ b/test/valid/test242.txt @@ -0,0 +1,1 @@ +a UTF-8 decoder that accepts them might introduce security problems | diff --git a/test/valid/test243.txt b/test/valid/test243.txt new file mode 100644 --- /dev/null +++ b/test/valid/test243.txt @@ -0,0 +1,1 @@ +comparable to overlong UTF-8 sequences. | diff --git a/test/valid/test244.txt b/test/valid/test244.txt new file mode 100644 --- /dev/null +++ b/test/valid/test244.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test245.txt b/test/valid/test245.txt new file mode 100644 --- /dev/null +++ b/test/valid/test245.txt @@ -0,0 +1,1 @@ +5.1 Single UTF-16 surrogates | diff --git a/test/valid/test246.txt b/test/valid/test246.txt new file mode 100644 --- /dev/null +++ b/test/valid/test246.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test254.txt b/test/valid/test254.txt new file mode 100644 --- /dev/null +++ b/test/valid/test254.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test255.txt b/test/valid/test255.txt new file mode 100644 --- /dev/null +++ b/test/valid/test255.txt @@ -0,0 +1,1 @@ +5.2 Paired UTF-16 surrogates | diff --git a/test/valid/test256.txt b/test/valid/test256.txt new file mode 100644 --- /dev/null +++ b/test/valid/test256.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test265.txt b/test/valid/test265.txt new file mode 100644 --- /dev/null +++ b/test/valid/test265.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test266.txt b/test/valid/test266.txt new file mode 100644 --- /dev/null +++ b/test/valid/test266.txt @@ -0,0 +1,1 @@ +5.3 Other illegal code positions | diff --git a/test/valid/test267.txt b/test/valid/test267.txt new file mode 100644 --- /dev/null +++ b/test/valid/test267.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test270.txt b/test/valid/test270.txt new file mode 100644 --- /dev/null +++ b/test/valid/test270.txt @@ -0,0 +1,1 @@ + | diff --git a/test/valid/test271.txt b/test/valid/test271.txt new file mode 100644 --- /dev/null +++ b/test/valid/test271.txt @@ -0,0 +1,1 @@ +THE END | diff --git a/testsrc/Main.hs b/testsrc/Main.hs deleted file mode 100644 --- a/testsrc/Main.hs +++ /dev/null @@ -1,109 +0,0 @@ -{- | -module: Main -description: Testing the Unicode character library and API -license: MIT - -maintainer: Joe Leslie-Hurd -stability: provisional -portability: portable --} -module Main - ( main ) -where - -import qualified Data.List as List -import qualified Data.Either as Either -import qualified Data.Word as Word -import qualified System.Directory as Directory - -import Unicode -import qualified OpenTheory.Unicode as Unicode - -demoLength :: Int -demoLength = 7621 - -testLength :: Int -testLength = 79 - -noInvalid :: [Either Word.Word8 Unicode.Unicode] -> Bool -noInvalid = all (Either.either (const False) (const True)) - -getTestFiles :: FilePath -> IO [FilePath] -getTestFiles d = - do fs <- Directory.getDirectoryContents d - let fs' = filter (List.isPrefixOf "test") fs - return (map (\f -> d ++ "/" ++ f) fs') - -readTestCharFile :: Maybe Int -> FilePath -> IO () -readTestCharFile x f = - do l <- decodeFile f - let a = noInvalid l - case x of - Nothing -> - if a - then error $ "invalid file " ++ f ++ " was successfully decoded" - else return () - Just n -> - if not a - then error $ "valid file " ++ f ++ " could not be decoded" - else - if length l == n - then return () - else - error $ "valid file " ++ f ++ " has bad line length: " ++ - show (length l) - -readValidCharFile :: Int -> FilePath -> IO () -readValidCharFile n = readTestCharFile (Just n) - -readInvalidCharFile :: FilePath -> IO () -readInvalidCharFile = readTestCharFile Nothing - -partitionTestCharFile :: IO () -partitionTestCharFile = - do skip <- Directory.doesFileExist "test/valid/test061.txt" - if skip - then return () - else - do putStrLn "\nsplitting UTF-8 test file into valid and invalid lines" - cs <- decodeFile "test/test.txt" - mapM_ outputLine (filter testLine (readLines 1 cs)) - where - isNewline :: Either Word.Word8 Unicode.Unicode -> Bool - isNewline (Left _) = False - isNewline (Right c) = let n = Unicode.unUnicode c in n == 10 || n == 13 - - chopNewline :: [Either Word.Word8 Unicode.Unicode] -> - [Either Word.Word8 Unicode.Unicode] - chopNewline [] = [] - chopNewline (_ : cs) = cs - - readLines :: Int -> [Either Word.Word8 Unicode.Unicode] -> - [(Int,[Either Word.Word8 Unicode.Unicode])] - readLines _ [] = [] - readLines lineno inp = - let (line,inp') = List.span (not . isNewline) inp in - let inp'' = chopNewline inp' in - (lineno,line) : readLines (lineno + 1) inp'' - - testLine :: (Int,[Either Word.Word8 Unicode.Unicode]) -> Bool - testLine (lineno,line) = not (lineno < 61 || null line) - - outputLine :: (Int,[Either Word.Word8 Unicode.Unicode]) -> IO () - outputLine (lineno,line) = - let valid = noInvalid line in - let n = show lineno in - let f = "test/" ++ (if valid then "valid" else "invalid") ++ - "/test" ++ replicate (3 - length n) '0' ++ n ++ ".txt" in - reencodeFile f line - -main :: IO () -main = - do partitionTestCharFile - readValidCharFile demoLength "test/demo.txt" - validTestFiles <- getTestFiles "test/valid" - invalidTestFiles <- getTestFiles "test/invalid" - mapM_ (readValidCharFile testLength) validTestFiles - mapM_ readInvalidCharFile invalidTestFiles - putStrLn " all tests pass" - return ()