new syntax aware algorithm with debuggingsyntax_aware_debug

This is just for archiving the debugging code

1 files changed, 141 insertions, 56 deletions

diff --git a/spellcheck.lua b/spellcheck.lua
index 52a2e54..00eee5a 100644
--- a/spellcheck.lua
+++ b/spellcheck.lua
@@ -93,6 +93,35 @@ local function typo_iter(text, typos, ignored)
 	end
 end
 
+function serializeTable(val, name, skipnewlines, depth)
+    skipnewlines = skipnewlines or false
+    depth = depth or 0
+
+    local tmp = string.rep(" ", depth)
+
+    if name then tmp = tmp .. name .. " = " end
+
+    if type(val) == "table" then
+        tmp = tmp .. "{" .. (not skipnewlines and "\n" or "")
+
+        for k, v in pairs(val) do
+            tmp =  tmp .. serializeTable(v, k, skipnewlines, depth + 1) .. "," .. (not skipnewlines and "\n" or "")
+        end
+
+        tmp = tmp .. string.rep(" ", depth) .. "}"
+    elseif type(val) == "number" then
+        tmp = tmp .. tostring(val)
+    elseif type(val) == "string" then
+        tmp = tmp .. string.format("%q", val)
+    elseif type(val) == "boolean" then
+        tmp = tmp .. (val and "true" or "false")
+    else
+        tmp = tmp .. "\"[inserializeable datatype:" .. type(val) .. "]\""
+    end
+
+    return tmp
+end
+
 local last_viewport, last_typos = nil, ""
 
 vis.events.subscribe(vis.events.WIN_HIGHLIGHT, function(win)
@@ -122,21 +151,42 @@ end)
 local wrapped_lex_funcs = {}
 
 local wrap_lex_func = function(old_lex_func)
+	local old_viewport = vis.win.viewport
+	local old_viewport_text = ""
+	local old_typos = {}
+
+	local logged_yet = false
+	local log_file = io.open("spellcheck.log", "w")
+
+	local function log(msg)
+		log_file:write(msg)
+		log_file:flush()
+	end
+
 	return function(lexer, data, index, redrawtime_max)
-		-- vis:info("hooked lexer.lex")
+		log(string.format("spellchecking called with %s %s %d %d\n",
+			lexer, data, index, redrawtime_max))
+
 		local tokens, timedout = old_lex_func(lexer, data, index, redrawtime_max)
 
-		-- quit early if the lexer already took to long
+		-- quit early if the lexer already took to long or we should highlight an empty data
 		-- TODO: investigate further if timedout is actually set by the lexer.
 		--       As I understand lpeg.match used by lexer.lex timedout will always be nil
-		if timeout then
+		if data == "" or timeout then
+			log(string.format("lexer timed out returning: %s %s\n", tokens, timedout))
 			return tokens, timedout
 		end
 
-		local log = nil
+		-- this may not be the window we are currently highlighting
 		local win = vis.win
-		local cmd = spellcheck.list_cmd:format(spellcheck.lang)
-		local new_tokens = {}
+
+		-- TODO: find a way to prevent typo highlight to disappear while in command prompt
+		if not win.syntax or not win.file then
+			log(string.format("win has no syntax returning: %s %s\n", tokens, timedout))
+			return tokens, timedout
+		end
+
+		log(string.format("spellcheckng file: %s with syntax:%s\n", win.file.name, win.syntax))
 
 		-- get file position we lex
 		-- duplicated code with vis-std.lua
@@ -145,61 +195,96 @@ local wrap_lex_func = function(old_lex_func)
 		local horizon = viewport.start < horizon_max and viewport.start or horizon_max
 		local view_start = viewport.start
 		local lex_start = viewport.start - horizon
-		local token_end = lex_start + (tokens[#tokens] or 1) - 1
-
-		for i = 1, #tokens - 1, 2 do
-			local token_start = lex_start + (tokens[i-1] or 1) - 1
-			local token_end = tokens[i+1]
-			local token_range = {start = token_start, finish = token_end - 1}
-
-			-- check if token is visable
-			if token_start >= view_start or token_end > view_start then
-				local token_name = tokens[i]
-				-- token is not listed for spellchecking just add it to the token stream
-				if not spellcheck.check_tokens[token_name] then
-					table.insert(new_tokens, tokens[i])
+		local lex_end = lex_start + (tokens[#tokens] or 1) - 1
+
+		log(string.format("spellcheck viewport: {s=%d, f=%d}\n", view_start, viewport.finish))
+
+		local new_tokens = {}
+
+		local viewport_text = data:sub(view_start)
+
+		local typos = ""
+		if old_viewport.start ~= view_start
+			or old_viewport.finish ~= viewport.finish
+			or old_viewport_text ~= viewport_text
+		then
+			typos = get_typos(viewport_text)
+			old_typos = typos
+			old_viewport = viewport
+			old_viewport_text = viewport_text
+			log(string.format("use new typos: %s\n", typos))
+		else
+			typos = old_typos
+			log(string.format("use old typos: %s\n", typos))
+		end
+
+		local typo_stream = {}
+		local i = 1
+		for typo, start, finish in typo_iter(viewport_text, typos, ignored) do
+			table.insert(typo_stream, {typo, start, finish})
+			local typo_start = view_start + start
+			local typo_end = view_start + finish
+			log(string.format("cur typo %s %d %d\n", typo, typo_start, typo_end))
+			repeat
+				local token_type = tokens[i]
+				local token_start = lex_start + (tokens[i-1] or 1) - 1
+				local token_end = tokens[i+1]
+
+				if not token_type or not token_end or not token_start then
+					break
+				end
+
+				-- the current token ends before our typo -> append to stream
+				-- or is not spellchecked
+				if token_end < typo_start or not spellcheck.check_tokens[token_type] then
+					table.insert(new_tokens, token_type)
 					table.insert(new_tokens, token_end)
-				-- spellcheck the token
+					log(string.format("untouched token %s %d\n", token_type, token_end))
+
+					log("advance token stream\n")
+					i = i + 2
+				-- typo and checked token overlap
 				else
-					local ret, stdout, stderr = vis:pipe(win.file, token_range, cmd)
-					if ret ~= 0 then
-						vis:info("calling cmd: `" .. cmd .. "` failed ("..ret..")")
-					-- we got misspellings
-					elseif stdout then
-						local typo_iter = stdout:gmatch("(.-)\n")
-						local token_content = win.file:content(token_range)
-
-						-- current position in token_content
-						local index = 1
-						for typo in typo_iter do
-							if not ignored[typo] then
-								local start, finish = token_content:find(typo, index, true)
-								-- split token
-								local pre_typo_end = start - 1
-								-- correct part before typo
-								if pre_typo_end > index then
-									table.insert(new_tokens, token_name)
-									table.insert(new_tokens, token_start + pre_typo_end)
-								end
-								-- typo
-								-- TODO make style configurable
-								table.insert(new_tokens, vis.lexers.ERROR)
-								table.insert(new_tokens, token_start + finish + 1)
-								index = finish
-							end
-						end
-						-- rest which is not already inserted into the token stream
-						table.insert(new_tokens, token_name)
-						table.insert(new_tokens, token_end)
-					-- found no misspellings just add it to the token stream
-					else
-						table.insert(new_tokens, token_name)
-						table.insert(new_tokens, token_end)
+					log(string.format("touched token %s %d\n", token_type, token_end))
+					local pre_typo_end = typo_start - 1
+					-- unchanged token part before typo
+					if pre_typo_end > token_start then
+						table.insert(new_tokens, token_type)
+						table.insert(new_tokens, pre_typo_end + 1)
 					end
-					-- comment with mispellings anf oter stuf
+
+					-- highlight typo
+					table.insert(new_tokens, vis.lexers.ERROR)
+					table.insert(new_tokens, typo_end + 1)
 				end
-			end
+			until(not token_end or token_end > typo_end)
+			log("advance typo stream\n")
 		end
+
+		-- add all tokens left after we handled all typos
+		for i = i, #tokens, 1 do
+			table.insert(new_tokens, tokens[i])
+		end
+
+		if not logged_yet then
+			local typo_log = io.open("typo.log", "w")
+			typo_log:write(serializeTable(typo_stream))
+			typo_log:close()
+
+			local token_log = io.open("tokens.log", "w")
+			token_log:write(serializeTable(tokens))
+			token_log:close()
+
+			local new_token_log = io.open("new_tokens.log", "w")
+			new_token_log:write(serializeTable(new_tokens))
+			new_token_log:close()
+
+			-- log:close()
+			logged_yet = true
+		end
+
+		log("--- finished spellchecking ---\n\n")
+
 		return new_tokens, timedout
 	end
 end