Compact map files in distibution to only include what we actually need

appveyor.yml

@@ -138,6 +138,8 @@ after_build:
   - copy /Y enc\build\lua\macroapi_manual.pl\macroapi_manual.pl.chm ci\%configuration%.%platform_name%\Encyclopedia
   #copy misc docs and addons
   - xcopy /e /q /y /k extra\* ci\%configuration%.%platform_name%\
+  #compact maps
+  - far\tools\stacktrace.py ci\%configuration%.%platform_name%
   #test & archive 
   - cd ci\%configuration%.%platform_name%
   #Run macrotest.lua, on failure %ERRORLEVEL% is non zero

far/changelog

@@ -1,3 +1,8 @@
+--------------------------------------------------------------------------------
+drkns 2025-01-19 20:58:24+00:00 - build 6418
+
+1. Compact map files in distibution to only include what we actually need.
+
 --------------------------------------------------------------------------------
 drkns 2025-01-17 14:41:31+00:00 - build 6417
 

far/map_file.cpp

@@ -150,13 +150,17 @@ map_file::info map_file::get(uintptr_t const Address)
 enum class map_format
 {
 	unknown,
+	mini,
 	msvc,
 	clang,
 	gcc,
 };
 
 static auto determine_format(string_view const Str)
 {
+	if (Str == L"MINIMAP"sv)
+		return map_format::mini;
+
 	if (Str.starts_with(L' '))
 		return map_format::msvc;
 
@@ -181,6 +185,49 @@ static auto determine_format(std::istream& Stream)
 	return determine_format(Lines.begin()->Str);
 }
 
+static void read_mini(std::istream& Stream, unordered_string_set& Files, std::map<uintptr_t, map_file::line>& Symbols)
+{
+	RegExp ReObject, ReSymbol;
+	ReObject.Compile(L"^(\\d+) (.+)$"sv, OP_OPTIMIZE);
+	ReSymbol.Compile(L"^([0-9A-Fa-f]+) (\\d+) (.+)$"sv, OP_OPTIMIZE);
+
+	regex_match Match;
+	auto& m = Match.Matches;
+	m.reserve(4);
+
+	std::unordered_map<size_t, string> ObjNames;
+	bool FilesSeen = false, SymbolsSeen = false;
+
+	for (const auto& i: enum_lines(Stream, CP_UTF8))
+	{
+		if (i.Str.empty())
+			continue;
+
+		if (!FilesSeen)
+			FilesSeen = i.Str == L"FILES"sv;
+
+		if (!SymbolsSeen)
+			SymbolsSeen = i.Str == L"SYMBOLS"sv;
+
+		if (FilesSeen && !SymbolsSeen && ReObject.Search(i.Str, Match))
+		{
+			ObjNames.try_emplace(from_string<size_t>(get_match(i.Str, m[1])), get_match(i.Str, m[2]));
+			continue;
+		}
+
+		if (SymbolsSeen && ReSymbol.Search(i.Str, Match))
+		{
+			map_file::line Line;
+			Line.Name = get_match(i.Str, m[3]);
+			const auto FileIndex = from_string<size_t>(get_match(i.Str, m[2]));
+			Line.File = std::to_address(Files.emplace(ObjNames.find(FileIndex)->second).first);
+			const auto Address = from_string<uintptr_t>(get_match(i.Str, m[1]), {}, 16);
+			Symbols.try_emplace(Address, std::move(Line));
+			continue;
+		}
+	}
+}
+
 static void read_vc(std::istream& Stream, unordered_string_set& Files, std::map<uintptr_t, map_file::line>& Symbols)
 {
 	RegExp ReBase, ReSymbol;
@@ -189,7 +236,7 @@ static void read_vc(std::istream& Stream, unordered_string_set& Files, std::map<
 
 	regex_match Match;
 	auto& m = Match.Matches;
-	m.reserve(3);
+	m.reserve(4);
 
 	uintptr_t BaseAddress{};
 
@@ -232,7 +279,7 @@ static void read_clang(std::istream& Stream, unordered_string_set& Files, std::m
 
 	regex_match Match;
 	auto& m = Match.Matches;
-	m.reserve(2);
+	m.reserve(3);
 
 	string ObjName;
 
@@ -268,7 +315,7 @@ static void read_gcc(std::istream& Stream, unordered_string_set& Files, std::map
 
 	regex_match Match;
 	auto& m = Match.Matches;
-	m.reserve(2);
+	m.reserve(3);
 
 	const auto BaseAddress = 0x1000;
 
@@ -304,6 +351,9 @@ void map_file::read(std::istream& Stream)
 {
 	switch (determine_format(Stream))
 	{
+	case map_format::mini:
+		return read_mini(Stream, m_Files, m_Symbols);
+
 	case map_format::msvc:
 		return read_vc(Stream, m_Files, m_Symbols);
 

far/tools/stacktrace.py

@@ -1,8 +1,10 @@
 import bisect
 import ctypes
+import os
 import re
 import sys
 
+
 def undecorate(name):
 	UNDNAME_NO_FUNCTION_RETURNS = 0x0004
 	UNDNAME_NO_ALLOCATION_MODEL = 0x0008
@@ -25,6 +27,38 @@ def undecorate(name):
 	return str(out.value, "utf-8") if ctypes.windll.dbghelp.UnDecorateSymbolName(ctypes.create_string_buffer(bytes(name, "utf-8")), out, buffer_size, flags) != 0 else name
 
 
+def parse_mini(map_file, map_data):
+	re_object = re.compile(r"^(\d+) (.+)\s+$")
+	re_symbol = re.compile(r"^([0-9A-Fa-f]+) (\d+) (.+)\s+$")
+
+	obj_names = {}
+	files_seen = False
+	symbols_seen = False
+
+	for line in map_file:
+		if len(line) == 0:
+			continue
+
+		if not files_seen:
+			files_seen = line.startswith("FILES")
+
+		if not symbols_seen:
+			symbols_seen = line.startswith("SYMBOLS")
+
+		if files_seen and not symbols_seen:
+			m = re_object.search(line)
+			if m is not None:
+				obj_names.setdefault(int(m.group(1)), m.group(2))
+				continue;
+
+		if symbols_seen:
+			m = re_symbol.search(line)
+			if m is not None:
+				Address = int(m.group(1), 16);
+				file_name = obj_names[int(m.group(2))]
+				map_data.setdefault(Address, (m.group(3), file_name))
+
+
 def parse_vc(map_file, map_data):
 	re_base = re.compile(r"^ +Preferred load address is ([0-9A-Fa-f]+)\s+$")
 	re_symbol = re.compile(r"^ +[0-9A-Fa-f]+:[0-9A-Fa-f]+ +([^ ]+) +([0-9A-Fa-f]+) .+ ([^ ]+)\s+$")
@@ -42,18 +76,18 @@ def parse_vc(map_file, map_data):
 			Address = int(m.group(2), 16);
 			if Address >= BaseAddress:
 				Address -= BaseAddress
-			map_data[Address] = (m.group(1), m.group(3))
+			map_data.setdefault(Address, (m.group(1), m.group(3)))
 
 
 def parse_clang(map_file, map_data):
-	re_object = re.compile(r"^[0-9A-Fa-f]+ [0-9A-Fa-f]+ +[0-9]+         (.+)\s+$")
+	re_object = re.compile(r"^[0-9A-Fa-f]+ [0-9A-Fa-f]+ +[0-9]+         (.+):(.+)\s+$")
 	re_symbol = re.compile(r"^([0-9A-Fa-f]+) [0-9A-Fa-f]+     0                 (.+)\s+$")
 	objname = None
 
 	for line in map_file:
 		m = re_symbol.search(line)
 		if m is not None:
-			map_data[int(m.group(1), 16)] = (m.group(2), objname)
+			map_data.setdefault(int(m.group(1), 16), (m.group(2), objname))
 			continue
 
 		m = re_object.search(line)
@@ -80,28 +114,34 @@ def parse_gcc(map_file, map_data):
 
 		m = re_symbol.search(line)
 		if m is not None:
-			map_data[int(m.group(1), 16) + BaseAddress] = (m.group(2), file_name)
+			map_data.setdefault(int(m.group(1), 16) + BaseAddress, (m.group(2), file_name))
 			continue
 
 		last_line = line
 
 
-def show_stack():
+def read_map(map_file_name):
 	map_data = dict()
 
-	with open(sys.argv[1], encoding="utf-8") as map_file:
+	with open(map_file_name, 'r', encoding="utf-8") as map_file:
 		header = map_file.readline()
 		map_file.seek(0)
-		if header.startswith("Address  Size     Align Out     In      Symbol"):
+		if header.startswith("MINIMAP"):
+			parse_mini(map_file, map_data)
+		elif header.startswith("Address  Size     Align Out     In      Symbol"):
 			parse_clang(map_file, map_data)
 		elif len(header.strip()) == 0:
 			parse_gcc(map_file, map_data)
 		else:
 			parse_vc(map_file, map_data)
 
-	keys = sorted(map_data.keys())
+	return sorted(map_data.keys()), map_data
 
-	for i in sys.argv[2:]:
+
+def show_stack(map_file_name, frames):
+	keys, map_data = read_map(map_file_name)
+
+	for i in frames:
 		address = int(i, 16)
 		key_index = bisect.bisect(keys, address)
 		if keys[key_index] > address:
@@ -110,5 +150,45 @@ def show_stack():
 		print("{0}: {1}+{2:X} ({3})".format(i, undecorate(data[0]), address - keys[key_index], data[1]))
 
 
+def compact_map(map_file_name):
+		keys, map_data = read_map(map_file_name)
+
+		tmp_name = map_file_name + ".tmp"
+		with open(tmp_name, 'w', encoding="utf-8") as out_file:
+			out_file.write("MINIMAP\n")
+
+			files = set()
+			for key in keys:
+				data = map_data[key]
+				file = data[1]
+				files.add(file)
+
+			out_file.write("\nFILES\n")
+			files_map = {}
+			for index, file in enumerate(sorted(files)):
+				files_map[file] = index
+				out_file.write(f"{index} {file}\n")
+
+			out_file.write("\nSYMBOLS\n")
+			for key in keys:
+				data = map_data[key]
+				out_file.write(f"{key:X} {files_map[data[1]]} {data[0]}\n")
+
+		os.replace(tmp_name, map_file_name)
+
+
+def compact_maps(directory_name):
+	for root, dirnames, filenames in os.walk(directory_name):
+		for filename in filenames:
+			if filename.lower().endswith(".map"):
+				compact_map(os.path.join(root, filename))
+
+
+def main():
+	if len(sys.argv) == 2:
+		compact_maps(sys.argv[1])
+	else:
+		show_stack(sys.argv[1], sys.argv[2:])
+
 if __name__ == "__main__":
-	show_stack()
+	main()

far/vbuild.m4

@@ -1 +1 @@
-6417
+6418

misc/nightly/installer.sh

@@ -2,10 +2,13 @@
 
 cd misc/msi-installer || exit 1
 
+python far/tools/stacktrace.py outfinalnew32
 WINEDLLOVERRIDES="msi=n" wine cmd /c ../../installer.32.bat
 
+python far/tools/stacktrace.py outfinalnew64
 WINEDLLOVERRIDES="msi=n" wine cmd /c ../../installer.64.bat
 
+python far/tools/stacktrace.py outfinalnewARM64
 WINEDLLOVERRIDES="msi=n" wine cmd /c ../../installer.ARM64.bat
 
 cd ../..