evilgooinajar
/
htmlq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: ignore empty words in tokenisation

main
evilgooinajar 2025-01-19 14:00:13 +01:00
parent e5af12a9af
commit 601d3779c5
1 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
html.lua
View File

@ -238,7 +238,9 @@ function M.tokenise( content )
end end
end end
if not word:match("^%s*$") then
table.insert( TOKENS, {type="WORD", value=word}) table.insert( TOKENS, {type="WORD", value=word})
end
else else
table.insert( TOKENS, {type="TEXT", value=text_memory} ) table.insert( TOKENS, {type="TEXT", value=text_memory} )
end end
@ -271,8 +273,10 @@ function M.tokenise( content )
end end
end end
if not word:match("^%s*$") then
table.insert( TOKENS, {type="WORD", value=word}) table.insert( TOKENS, {type="WORD", value=word})
text_memory = "" text_memory = ""
end
goto continue goto continue
end end
@ -418,7 +422,7 @@ function M.parse_tokens_into_document( TOKENS )
name = token.value:match("([%w-]+)") name = token.value:match("([%w-]+)")
if name == nil then if name == nil then
error("Unrecognised word: " .. name) error("Unrecognised word: " .. tostring(name) .. " (Token ".. tostring(i) .." , type=" .. tostring(token.type) .. ", value=" .. tostring(token.value) .. ")")
end end
current_doc_element.attributes[name] = true current_doc_element.attributes[name] = true