diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,67 @@ # Revision history for djot +## 0.1.4.3 -- 2026-09-23 + + * Fix dropped attribute when attaching to last word of merged Strs. + E.g. in `x y.z{.c}`, the class was silently discarded. + + * Fix implicit reference labels spanning multiple lines. + Two bugs conspired to make `[link\ntext][]` resolve to an empty href + even when a "link text" reference was defined. + + * Fix resumable attribute parser ignoring new input on resume. + Previously a multiline block attribute followed by an indented + block never finished parsing and the attributes were silently + dropped, e.g.: + + {#id .class + style="color:red"} + A paragraph + + * Avoid quadratic inline parsing by caching chunk count. Previously + parsing a paragraph was quadratic in the number of lines. + + * Speed up HTML escaping by copying unescaped runs wholesale. + + * Wrap letter list markers around after 26. + + * Fix off-by-one in tab column computation. + + * Propagate inline parse errors in `parseTextLines` instead of crashing. + + * Make NoPos an identity for `<>`, so Monoid Pos is lawful. + Previously NoPos was absorbing `(NoPos <> p = NoPos)`, + which would silently discard positions in any law-relying generic code + (mconcat, fold). The only internal uses of `<>` on Pos (merging + adjacent Strs) always combine two Pos or two NoPos values, so this + does not change any parsing or rendering behavior. + + * Fix UTF-8 truncation in `inlinesToByteString`. + + * Improve classification of autolinks. Previously `` + was rendered as a plain URL link, not an email link. It's an email + link if it contains an '@' preceded by a character other than ':'. + + * Preserve soft breaks for CR-only line endings. + + * Add tests for whitespace collapsing in quoted attribute values. + + * Fix off-by-one in `pAtMost`. + + * Skip empty chunks when advancing to the next chunk. + + * Speed up djot rendering of Str inlines: + + + Rewrite `escapeDjot` to work directly on ByteStrings, copying + runs of unescapable characters wholesale instead of processing + character by character through String. (Equivalence with the + old implementation checked with 100,000 QuickCheck cases.) + + Skip the smart-punctuation replacement passes when the + ByteString contains no 0xE2 byte (the first byte of all the + UTF-8 sequences involved), avoiding five `T.replace` traversals + per Str in the common case. + + Avoid emitting an empty literal after each single-space chunk. + ## 0.1.4.2 -- 2026-08-27 * Djot renderer: emit raw blocks/inlines for formats other than djot (#18). diff --git a/djot.cabal b/djot.cabal --- a/djot.cabal +++ b/djot.cabal @@ -1,6 +1,6 @@ cabal-version: 3.0 name: djot -version: 0.1.4.2 +version: 0.1.4.3 synopsis: Parser and renderer for djot light markup syntax. description: Djot () is a light markup language. This package provides a data structure to represent diff --git a/src/Djot/AST.hs b/src/Djot/AST.hs --- a/src/Djot/AST.hs +++ b/src/Djot/AST.hs @@ -116,11 +116,13 @@ data Pos = NoPos | Pos Int Int Int Int -- start line, start col, end line, end col deriving (Show, Eq, Ord, Typeable, Data, Generic, Lift) +-- | @p1 <> p2@ spans from the start of @p1@ to the end of @p2@. +-- 'NoPos' is an identity. instance Semigroup Pos where Pos sl1 sc1 _ _ <> Pos _ _ el2 ec2 = Pos sl1 sc1 el2 ec2 - NoPos <> _ = NoPos - _ <> NoPos = NoPos + NoPos <> x = x + x <> NoPos = x instance Monoid Pos where mappend = (<>) @@ -194,13 +196,13 @@ else let sblen = B8.length (B8.filter (\c -> c < '\128' || c >= '\192') sb) (pos1', pos2') = - case pos1 <> pos2 of - NoPos -> (NoPos, NoPos) - Pos sl sc el ec -> - (Pos sl sc el (ec - sblen), - Pos sl (sc + sblen + 1) el ec) + case (pos1, pos2) of + (Pos sl1 sc1 el1 ec1, Pos _ _ el2 ec2) -> + (Pos sl1 sc1 el1 (ec1 - sblen), + Pos el1 (ec1 - sblen + 1) el2 ec2) + _ -> (NoPos, NoPos) in Many ((as' Seq.|> Node pos1' mempty (Str sa) - Seq.|> Node pos2' attr (Str (sb <> t))) <> bs') + Seq.|> Node pos2' attr' (Str (sb <> t))) <> bs') | attr == attr' -> Many (as' <> (Node (pos1 <> pos2) attr (Str (s <> t)) Seq.<| bs')) (as' Seq.:> Node pos attr (Str s), Node _ _ HardBreak Seq.:< _) @@ -430,9 +432,9 @@ Superscript ils -> inlinesToByteString ils Subscript ils -> inlinesToByteString ils Quoted SingleQuotes ils -> - "\x2018" <> inlinesToByteString ils <> "\x2019" + "\226\128\152" <> inlinesToByteString ils <> "\226\128\153" Quoted DoubleQuotes ils -> - "\x201C" <> inlinesToByteString ils <> "\x201D" + "\226\128\156" <> inlinesToByteString ils <> "\226\128\157" Verbatim bs -> bs Math DisplayMath bs -> "$$" <> bs <> "$$" Math InlineMath bs -> "$" <> bs <> "$" @@ -446,4 +448,4 @@ FootnoteReference bs -> "[" <> bs <> "]" SoftBreak -> "\n" HardBreak -> "\n" - NonBreakingSpace -> "\160" + NonBreakingSpace -> "\194\160" diff --git a/src/Djot/Attributes.hs b/src/Djot/Attributes.hs --- a/src/Djot/Attributes.hs +++ b/src/Djot/Attributes.hs @@ -15,7 +15,6 @@ import qualified Data.ByteString.Char8 as B8 import Data.ByteString.Char8 ( (!?) ) import Data.Typeable (Typeable) -import Data.Maybe (fromMaybe) -- import Debug.Trace @@ -86,10 +85,12 @@ -- | Resumable parser, returning parts in reverse order. parseAttributes :: Maybe AttrParserState -> ByteString -> AttrParseResult parseAttributes mbState bs = - case go (fromMaybe AttrParserState{ aState = START - , subject = bs - , offset = 0 - , parts = [] } mbState) of + case go (case mbState of + Nothing -> AttrParserState{ aState = START + , subject = bs + , offset = 0 + , parts = [] } + Just st -> st{ subject = bs, offset = 0 }) of AttrParserState{ aState = DONE, parts = attparts, offset = off } -> Done (attrPartsToAttr attparts, off) AttrParserState{ aState = FAIL, offset = off } -> Failed off @@ -123,6 +124,8 @@ case nextc of '"' -> go st{ aState = SCANNING, offset = off + 1 } '\\' -> go st{ aState = SCANNING_ESCAPE, offset = off + 1 } + -- runs of whitespace (including newlines) collapse to a + -- single space, as in djot.js c | isWs c -> let st' = skipWhile isWs st in go st'{ parts = AttrValue " " : parts st' } diff --git a/src/Djot/Blocks.hs b/src/Djot/Blocks.hs --- a/src/Djot/Blocks.hs +++ b/src/Djot/Blocks.hs @@ -36,23 +36,27 @@ parseDoc :: ParseOptions -> ByteString -> Either String Doc parseDoc opts bs = do - case parse pDoc PState{ psParseOptions = opts - , psContainerStack = - NonEmpty.fromList - [emptyContainer{ containerSpec = docSpec }] - , psReferenceMap = mempty - , psAutoReferenceMap = mempty - , psNoteMap = mempty - , psLastAttributeLine = 0 - , psAttributes = mempty - , psAttrParserState = Nothing - , psIds = mempty - , psAutoIds = mempty - , psLastColumnPrevLine = 0 - , psLastLine = 1 - } [Chunk{ chunkLine = 1, chunkColumn = 1, chunkBytes = bs }] of - Just doc -> Right doc - Nothing -> Left "Parse failure." + case parse ((,) <$> pDoc <*> (psParseError <$> getState)) + PState{ psParseOptions = opts + , psContainerStack = + NonEmpty.fromList + [emptyContainer{ containerSpec = docSpec }] + , psReferenceMap = mempty + , psAutoReferenceMap = mempty + , psNoteMap = mempty + , psLastAttributeLine = 0 + , psAttributes = mempty + , psAttrParserState = Nothing + , psIds = mempty + , psAutoIds = mempty + , psLastColumnPrevLine = 0 + , psLastLine = 1 + , psParseError = Nothing + } [Chunk{ chunkLine = 1, chunkColumn = 1, chunkBytes = bs }] of + Just (_, Just err) -> Left err + Just (doc, Nothing) -> Right doc + Nothing -> Left "Parse failure. The djot block parser should accept any\ + \ input, so this is a bug: please report it." data BlockType = Normal | ListItem | CaptionBlock | Document @@ -745,6 +749,7 @@ , blockContainsBlock = Nothing , blockContainsLines = True , blockClose = \container -> do + updateState $ \st -> st{ psAttrParserState = Nothing } let bs = foldMap chunkBytes $ containerText container case parseAttributes Nothing bs of Done (attr, off) @@ -857,7 +862,12 @@ parseTextLines :: Container -> P Inlines parseTextLines cont = do opts <- psParseOptions <$> getState - either error pure . parseInlines opts $ containerText cont + case parseInlines opts (containerText cont) of + Right ils -> pure ils + Left msg -> do -- record error; parseDoc will return Left + updateState $ \st -> + st{ psParseError = psParseError st <|> Just msg } + pure mempty emptyContainer :: Container emptyContainer = @@ -924,6 +934,7 @@ , psAutoIds :: Set ByteString , psLastColumnPrevLine :: Int , psLastLine :: Int + , psParseError :: Maybe String } type P = Parser PState diff --git a/src/Djot/Djot.hs b/src/Djot/Djot.hs --- a/src/Djot/Djot.hs +++ b/src/Djot/Djot.hs @@ -12,7 +12,6 @@ import Djot.AST import Djot.Options (RenderOptions(..)) import Data.Char (ord, chr, isSpace) -import Djot.Parse (utf8ToStr) import Data.ByteString (ByteString) import qualified Data.ByteString.Char8 as B8 import qualified Data.Set as Set @@ -100,7 +99,7 @@ {-# INLINE escapeDjot #-} escapeDjot :: EscapeContext -> ByteString -> Text escapeDjot Normal bs - | B8.any escapable bs = T.pack. go . utf8ToStr $ bs + | B8.any escapable bs = fromUtf8 $ B8.concat $ go bs | otherwise = fromUtf8 bs where escapable c = c == '[' || c == ']' || c == '<' || c == '>' || @@ -108,28 +107,39 @@ c == '-' || c == '^' || c == '~' || c == '*' || c == '_' || c == '\''|| c == '"' || c == '.' || c == '|' || c == '`' || c == '\\' - go [] = [] - go ('$':c:cs) - | c == '`' = '\\' : '$' : c : go cs - | otherwise = '$' : go (c : cs) - go ('-':cs) = - case cs of - '-':_ -> '\\' : '-' : go cs - _ -> '-' : go cs - go ('.':cs) = - case cs of - '.':'.':_ -> '\\' : '.' : go cs - _ -> '.' : go cs - go (c:':':cs) - | c /= ']' - , case cs of - [] -> True - (' ':_) -> True - _ -> False - = (if escapable c then ('\\' :) else id) $ c : ':' : go cs - go (c:cs) - | escapable c = '\\' : c : go cs - | otherwise = c : go cs + -- Copy runs of unescapable bytes wholesale; handle each escapable + -- byte with the lookahead (and, for ':', lookbehind) rules below. + go s = + case B8.findIndex escapable s of + Nothing -> [s] + Just i -> + let c = B8.index s i + rest = B8.drop (i + 1) s + in (if i == 0 then id else (B8.take i s :)) $ + handle c (i > 0) rest + -- endOrSpace r: the escapable char is at the end or followed by space + endOrSpace r = B8.null r || B8.head r == ' ' + handle c hasPrev rest = + case c of + '$' | B8.take 1 rest == "`" -> "\\$`" : go (B8.drop 1 rest) + | B8.null rest -> ["\\$"] + | otherwise -> "$" : go rest + '-' | B8.take 1 rest == "-" -> "\\-" : go rest + | otherwise -> "-" : go rest + '.' | B8.take 2 rest == ".." -> "\\." : go rest + | otherwise -> "." : go rest + ':' -- unescaped when preceded by an unescapable byte and + -- followed by space or end; "::" before space or end gets + -- only the first colon escaped + | hasPrev, endOrSpace rest -> ":" : go rest + | B8.take 1 rest == ":", endOrSpace (B8.drop 1 rest) + -> "\\::" : go (B8.drop 1 rest) + | otherwise -> "\\:" : go rest + _ | c /= ']' + , B8.take 1 rest == ":" + , endOrSpace (B8.drop 1 rest) + -> B8.pack ['\\', c, ':'] : go (B8.drop 1 rest) + | otherwise -> B8.pack ['\\', c] : go rest newtype BlockAttr = BlockAttr Attr @@ -338,8 +348,8 @@ formatNumber :: OrderedListStyle -> Int -> Layout.Doc Text formatNumber Decimal n = literal (T.pack (show n)) -formatNumber LetterUpper n = literal (T.singleton (chr (ord 'A' + n - 1))) -formatNumber LetterLower n = literal (T.singleton (chr (ord 'a' + n - 1))) +formatNumber LetterUpper n = literal (T.singleton (chr (ord 'A' + (n - 1) `mod` 26))) +formatNumber LetterLower n = literal (T.singleton (chr (ord 'a' + (n - 1) `mod` 26))) formatNumber RomanUpper n = literal $ toRomanNumeral n formatNumber RomanLower n = literal $ T.toLower (toRomanNumeral n) @@ -368,11 +378,16 @@ toLayout (Node _pos attr il) = (<>) <$> case il of Str bs -> do - let fixSmart = T.replace "\x2014" "---" . + let fixSmart + -- all the smart characters are UTF-8 sequences + -- starting with 0xE2: + | B8.elem '\xE2' bs = + T.replace "\x2014" "---" . T.replace "\x2013" "--" . T.replace "\x2026" "..." . T.replace "\x2019" "'" . T.replace "\x201C" "\"" + | otherwise = id let chunks = T.groupBy (\c d -> (c /= ' ' && d /= ' ') || (c == ' ' && d == ' ')) @@ -380,7 +395,8 @@ let toChunk ch = case T.uncons ch of Just (' ', rest) - -> afterBreak "{}" <> space <> literal rest + | T.null rest -> afterBreak "{}" <> space + | otherwise -> afterBreak "{}" <> space <> literal rest _ -> literal ch pure $ hcat $ map toChunk chunks SoftBreak -> do diff --git a/src/Djot/Html.hs b/src/Djot/Html.hs --- a/src/Djot/Html.hs +++ b/src/Djot/Html.hs @@ -68,32 +68,30 @@ (Link (str (strToUtf8 "\8617\65038")) (Direct ("#fnref" <> num))) -{-# INLINE escapeHtml #-} escapeHtml :: ByteString -> Builder escapeHtml bs = - if hasEscapable bs - then B.foldl' go mempty bs - else byteString bs + case B.uncons rest of + Nothing -> byteString before + Just (w, rest') -> byteString before <> escaped w <> escapeHtml rest' where - hasEscapable = B.any (\w -> w == 38 || w == 60 || w == 62) - go b 38 = b <> byteString "&" - go b 60 = b <> byteString "<" - go b 62 = b <> byteString ">" - go b c = b <> word8 c + (before, rest) = B.break (\w -> w == 38 || w == 60 || w == 62) bs + escaped 38 = byteString "&" + escaped 60 = byteString "<" + escaped 62 = byteString ">" + escaped w = word8 w -- unreachable -{-# INLINE escapeHtmlAttribute #-} escapeHtmlAttribute :: ByteString -> Builder escapeHtmlAttribute bs = - if hasEscapable bs - then B.foldl' go mempty bs - else byteString bs + case B.uncons rest of + Nothing -> byteString before + Just (w, rest') -> byteString before <> escaped w <> escapeHtmlAttribute rest' where - hasEscapable = B.any (\w -> w == 38 || w == 60 || w == 62 || w == 34) - go b 38 = b <> byteString "&" - go b 60 = b <> byteString "<" - go b 62 = b <> byteString ">" - go b 34 = b <> byteString """ - go b c = b <> word8 c + (before, rest) = B.break (\w -> w == 38 || w == 60 || w == 62 || w == 34) bs + escaped 38 = byteString "&" + escaped 60 = byteString "<" + escaped 62 = byteString ">" + escaped 34 = byteString """ + escaped w = word8 w -- unreachable data BState = BState { noteMap :: NoteMap diff --git a/src/Djot/Inlines.hs b/src/Djot/Inlines.hs --- a/src/Djot/Inlines.hs +++ b/src/Djot/Inlines.hs @@ -164,6 +164,7 @@ '-' -> pHyphens '.' -> pEllipses '\n' -> pSoftBreak + '\r' -> pSoftBreak _ -> mzero) <|> pSpecial ) <|> pWords @@ -372,10 +373,14 @@ res <- byteStringOf $ skipSome $ skipSatisfyByte (\c -> c /= '>' && c /= '<') asciiChar '>' let url = B8.filter (\c -> c /= '\n' && c /= '\r') res - case B8.find (\c -> c == '@' || c == ':' || c == '.') url of - Just '@' -> pure $ emailLink url - Just _ -> pure $ urlLink url - Nothing -> mzero + -- an email link contains an '@' preceded by a character other than ':' + -- (cf. djot.js, which tests /[^:]@/) + let isEmail = or $ B8.zipWith (\a b -> b == '@' && a /= ':') url (B.drop 1 url) + if isEmail + then pure $ emailLink url + else case B8.find (\c -> c == ':' || c == '.') url of + Just _ -> pure $ urlLink url + Nothing -> mzero pLinkOrSpan :: P Inlines pLinkOrSpan = do @@ -430,7 +435,7 @@ pure $ Reference label pAtMost :: Int -> P () -> P () -pAtMost n pa = optional_ (pa *> when (n > 0) (pAtMost ( n - 1 ) pa)) +pAtMost n pa = when (n > 0) $ optional_ (pa *> pAtMost (n - 1) pa) pOpenDoubleQuote :: P () pOpenDoubleQuote = do diff --git a/src/Djot/Parse.hs b/src/Djot/Parse.hs --- a/src/Djot/Parse.hs +++ b/src/Djot/Parse.hs @@ -100,6 +100,7 @@ data ParserState a = ParserState { chunks :: [Chunk] + , chunkCount :: !Int -- ^ length of chunks (cached to avoid O(n) length) , subject :: !ByteString , offset :: !Int , line :: !Int @@ -114,6 +115,7 @@ parse parser ustate chunks'' = snd <$> runParser parser ParserState { chunks = chunks' + , chunkCount = length chunks' , subject = bs , offset = 0 , line = startline @@ -122,7 +124,7 @@ where (chunks', bs, startline, startcol) = - case chunks'' of + case dropWhile (B.null . chunkBytes) chunks'' of [] -> ([], mempty, 1, 0) (c:cs) -> (cs, chunkBytes c, chunkLine c, chunkColumn c) @@ -135,8 +137,9 @@ unsafeAdvanceByte :: ParserState s -> ParserState s unsafeAdvanceByte st | offset st + 1 >= B.length (subject st) - , c:cs <- chunks st + , (emptyChunks, c:cs) <- span (B.null . chunkBytes) (chunks st) = st{ chunks = cs + , chunkCount = chunkCount st - (length emptyChunks + 1) , subject = chunkBytes c , offset = 0 , line = chunkLine c @@ -147,7 +150,7 @@ , line = line st + 1 , column = 1 } 9 -> st{ offset = offset st + 1 - , column = column st + (4 - (column st `mod` 4)) } + , column = column st + (4 - ((column st - 1) `mod` 4)) } !w | w < 0x80 -> st{ offset = offset st + 1 , column = column st + 1 } -- utf8 multibyte: only count byte 1: @@ -288,22 +291,25 @@ withByteString :: Parser s a -> Parser s (a, ByteString) withByteString pa = Parser $ \st -> case runParser pa st of - Just (st', x) -> Just (st', (x, B8.take (offset st' - offset st) - (B8.drop (offset st) (subject st)))) + Just (st', x) -> Just (st', (x, consumedByteString st st')) Nothing -> Nothing -- | Returns bytestring consumed by parse. byteStringOf :: Parser s a -> Parser s ByteString byteStringOf pa = Parser $ \st -> case runParser pa st of - Just (st', _) -> Just (st', - case length (chunks st) - length (chunks st') of - 0 -> B8.take (offset st' - offset st) (B8.drop (offset st) (subject st)) - n -> - B8.drop (offset st) (subject st) <> - foldMap chunkBytes (take (n - 1) (chunks st)) <> - B8.take (offset st') (subject st')) + Just (st', _) -> Just (st', consumedByteString st st') Nothing -> Nothing + +-- | Bytestring consumed between two parser states. +consumedByteString :: ParserState s -> ParserState s -> ByteString +consumedByteString st st' = + case chunkCount st - chunkCount st' of + 0 -> B8.take (offset st' - offset st) (B8.drop (offset st) (subject st)) + n -> + B8.drop (offset st) (subject st) <> + foldMap chunkBytes (take (n - 1) (chunks st)) <> + B8.take (offset st') (subject st') -- | Succeeds if first parser succeeds and second fails, returning -- first parser's value. diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -35,6 +35,8 @@ | (fp, ts) <- tests , takeFileName fp /= "raw.test"] , testGroup "Djot.Parse" parserTests + , testGroup "Djot.AST" astTests + , testGroup "djot writer" writerTests , testGroup "sourcepos" sourcePosTests , testGroup "Fuzz" [testProperty "parses all inputs" @@ -52,8 +54,80 @@ (toChunks $ strToUtf8 "ǎ老bc") @?= Just '老') , testProperty "UTF8 conversion round-trips" (\s -> utf8ToStr (strToUtf8 s) == s) + , testCase "empty chunk mid-stream does not cause premature EOF" + (parse (satisfy (=='a') *> satisfy (=='b')) () + [ Chunk{ chunkBytes = "a", chunkLine = 1, chunkColumn = 0 } + , Chunk{ chunkBytes = "", chunkLine = 2, chunkColumn = 0 } + , Chunk{ chunkBytes = "b", chunkLine = 3, chunkColumn = 0 } + ] @?= Just 'b') + , testCase "leading empty chunk does not cause premature EOF" + (parse (satisfy (=='a')) () + [ Chunk{ chunkBytes = "", chunkLine = 1, chunkColumn = 0 } + , Chunk{ chunkBytes = "a", chunkLine = 2, chunkColumn = 0 } + ] @?= Just 'a') ] +astTests :: [TestTree] +astTests = + [ testCase "NoPos is an identity for <>" $ do + Pos 1 1 2 5 <> NoPos @?= Pos 1 1 2 5 + NoPos <> Pos 1 1 2 5 @?= Pos 1 1 2 5 + , testCase "<> on Pos spans both arguments" $ + Pos 1 1 1 4 <> Pos 2 1 2 7 @?= Pos 1 1 2 7 + , testCase "inlinesToByteString emits valid UTF-8" $ do + inlinesToByteString (singleQuoted (str "a")) @?= + strToUtf8 "\x2018\&a\x2019" + inlinesToByteString (doubleQuoted (str "a")) @?= + strToUtf8 "\x201C\&a\x201D" + inlinesToByteString nonBreakingSpace @?= strToUtf8 "\xA0" + , testCase "image alt text with smart quotes is valid UTF-8" $ + convertNoPos "![a \"b\" c](url)\n" @?= + "

\"a

\n" + , testCase "auto identifier with smart quotes is valid UTF-8" $ + convertNoPos "# Say \"hi\"\n" @?= + "
\n

Say \x201Chi\x201D

\n
\n" + , testCase "autolink with dot before @ is an email link" $ + convertNoPos "\n" @?= + "

user.name@example.com

\n" + , testCase "autolink with @ only after : is a url link" $ + convertNoPos "\n" @?= + "

x:@example.com

\n" + , testCase "CR-only line ending produces a soft break" $ + convertNoPos "a\rb\n" @?= "

a\nb

\n" + , testCase "CRLF line ending produces a soft break" $ + convertNoPos "a\r\nb\n" @?= "

a\nb

\n" + , testCase "whitespace runs in quoted attribute values collapse (as in djot.js)" $ + convertNoPos "{k=\"a b\"}\npara\n" @?= + "

para

\n" + , testCase "newline in quoted attribute value becomes a space" $ + convertNoPos "{k=\"a\n b\"}\npara\n" @?= + "

para

\n" + , testCase "reference labels of no more than 400 bytes" $ do + let label400 = BL.pack (replicate 400 'x') + convertNoPos ("[a][" <> label400 <> "]\n") @?= "

a

\n" + convertNoPos ("[a][" <> label400 <> "y]\n") @?= + "

[a][" <> fromUtf8 (label400 <> "y") <> "]

\n" + ] + +convertNoPos :: BL.ByteString -> TL.Text +convertNoPos = either mempty (fromUtf8 . toLazyByteString . + renderHtml RenderOptions{ preserveSoftBreaks = True }) + . parseDoc ParseOptions{ sourcePositions = NoSourcePos } + . BL.toStrict + +writerTests :: [TestTree] +writerTests = + [ testCase "letter list style wraps around after 26" $ + render Nothing (renderDjot RenderOptions{ preserveSoftBreaks = True } + mempty{ docBlocks = Djot.AST.orderedList + OrderedListAttributes{ orderedListStyle = LetterUpper + , orderedListDelim = RightPeriod + , orderedListStart = 27 } + Tight + [para (str "one"), para (str "two")] }) + @?= "A. one\nB. two\n" + ] + sourcePosTests :: [TestTree] sourcePosTests = let convert = either mempty (fromUtf8 . toLazyByteString . @@ -65,6 +139,12 @@ , testCase "attr after *" $ convert "*{.foo}\n" @?= "

*

\n" + , testCase "attr on last word of merged strs" $ + convert "x y.z{.c}\n" @?= + "

x y.z

\n" + , testCase "tab advances to next tab stop (1-based)" $ + convert "a\tb *c*\n" @?= + "

a\tb c

\n" , testCase "no newline at end" $ convert "foo" @?= "

foo

\n" diff --git a/test/attributes.test b/test/attributes.test --- a/test/attributes.test +++ b/test/attributes.test @@ -13,7 +13,16 @@

(some text)

``` +The last word may come from several parsed inline elements +(here, "y", ".", and "z"): + ``` +x y.z{.c} +. +

x y.z

+``` + +``` [some text]{.attr} .

some text

@@ -117,6 +126,31 @@ A paragraph .

A paragraph

+``` + +The block after multiline attributes may itself be indented: + +``` +{#id .class + style="color:red"} + A paragraph +. +

A paragraph

+``` + +A multiline attribute block may be followed by another one: + +``` +{.a + .b} +one + +{.c + .d} +two +. +

one

+

two

``` If the attribute block can't be parsed as attributes, it will be diff --git a/test/links_and_images.test b/test/links_and_images.test --- a/test/links_and_images.test +++ b/test/links_and_images.test @@ -145,6 +145,18 @@

link and link

``` +An implicit reference label may span multiple lines: + +``` +[link +text][] + +[link text]: url +. +

link +text

+``` + ``` ![basic _image_](url) .