Add: better long-paths support on Windows (nzbgetcom#441)

- fixed multiple bugs related to bad support of long-paths on Windows

cmake/par2-turbo.cmake

@@ -28,7 +28,7 @@ ExternalProject_add(
 	par2-turbo
 	PREFIX			par2-turbo
 	GIT_REPOSITORY	https://github.com/nzbgetcom/par2cmdline-turbo.git
-	GIT_TAG			v1.1.1-nzbget
+	GIT_TAG			v1.1.1-nzbget-20241128
 	TLS_VERIFY		TRUE
 	GIT_SHALLOW		TRUE
 	GIT_PROGRESS	TRUE

daemon/extension/ManifestFile.cpp

@@ -19,28 +19,34 @@
 
 #include "nzbget.h"
 
-#include <fstream>
 #include "ManifestFile.h"
 #include "Json.h"
 #include "FileSystem.h"
+#include "Log.h"
 
 namespace ManifestFile
 {
-	const char* MANIFEST_FILE = "manifest.json";
-	const char* DEFAULT_SECTION_NAME = "options";
-
 	bool Load(Manifest& manifest, const char* directory)
 	{
-		BString<1024> path("%s%c%s", directory, PATH_SEPARATOR, MANIFEST_FILE);
-		std::ifstream fs(path);
-		if (!fs.is_open())
-			return false;
+		std::string path = std::string(directory) + PATH_SEPARATOR + MANIFEST_FILE;
+		DiskFile file;
+		if (!file.Open(path.c_str(), DiskFile::omRead)) return false;
 
-		Json::ErrorCode ec;
-		Json::JsonValue jsonValue = Json::Deserialize(fs, ec);
-		if (ec)
+		std::string jsonStr;
+		char buffer[BUFFER_SIZE];
+		while (file.ReadLine(buffer, BUFFER_SIZE))
+		{
+			jsonStr += buffer;
+		}
+
+		auto desRes = Json::Deserialize(jsonStr);
+		if (!desRes.has_value())
+		{
+			error("Failed to parse %s. Syntax error.", path.c_str());
 			return false;
+		}
 
+		Json::JsonValue jsonValue = std::move(desRes.value());
 		Json::JsonObject json = jsonValue.as_object();
 
 		if (!ValidateAndSet(json, manifest))

daemon/extension/ManifestFile.h

@@ -30,8 +30,9 @@ namespace ManifestFile
 {
 	using SelectOption = std::variant<double, std::string>;
 
-	extern const char* MANIFEST_FILE;
-	extern const char* DEFAULT_SECTION_NAME;
+	inline constexpr size_t BUFFER_SIZE = 4096;
+	inline const char* MANIFEST_FILE = "manifest.json";
+	inline const char* DEFAULT_SECTION_NAME = "options";
 
 	struct Section
 	{

daemon/main/nzbget.h

@@ -56,7 +56,7 @@
 #pragma warning(disable:4800) // 'type' : forcing value to bool 'true' or 'false' (performance warning)
 #pragma warning(disable:4267) // 'var' : conversion from 'size_t' to 'type', possible loss of data
 
-#define popen _popen
+#define popen _wpopen
 #define pclose _pclose
 
 #endif

daemon/postprocess/ParChecker.cpp

@@ -71,7 +71,7 @@ class Repairer final : public Par2::Par2Repairer, public ParChecker::AbstractRep
 			? g_Options->GetParBuffer()
 			: 0;
 	}
-	Par2::Result PreProcess(const char *parFilename);
+	Par2::Result PreProcess(const std::string& parFilename);
 	Par2::Result Process(bool dorepair);
 	virtual Repairer* GetRepairer() { return this; }
 
@@ -108,30 +108,30 @@ class RepairCreatorPacket : public Par2::CreatorPacket
 	friend class ParChecker;
 };
 
-Par2::Result Repairer::PreProcess(const char *parFilename)
+Par2::Result Repairer::PreProcess(const std::string& parFilename)
 {
 	std::string memParam = "-m" + std::to_string(m_memToUse);
 	std::string threadsParam = "-t" + std::to_string(m_threadsToUse);
 
 	if (g_Options->GetParScan() == Options::psFull)
 	{
-		BString<1024> wildcardParam(parFilename, 1024);
+		BString<1024> wildcardParam(parFilename.c_str(), 1024);
 		char* basename = FileSystem::BaseFileName(wildcardParam);
 		if (basename != wildcardParam && strlen(basename) > 0)
 		{
 			basename[0] = '*';
 			basename[1] = '\0';
 		}
 
-		const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename, wildcardParam };
+		const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename.c_str(), wildcardParam };
 		if (!m_commandLine.Parse(7, (char**)argv))
 		{
 			return Par2::eInvalidCommandLineArguments;
 		}
 	}
 	else
 	{
-		const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename };
+		const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename.c_str() };
 		if (!m_commandLine.Parse(6, (char**)argv))
 		{
 			return Par2::eInvalidCommandLineArguments;
@@ -294,7 +294,9 @@ ParChecker::EStatus ParChecker::RunParCheckAll()
 
 		if (!IsStopped())
 		{
-			BString<1024> fullParFilename( "%s%c%s", m_destDir.c_str(), PATH_SEPARATOR, parFilename.c_str());
+			std::string path = m_destDir + PATH_SEPARATOR + parFilename;
+			std::string fullParFilename = FileSystem::GetRealPath(path)
+				.value_or(std::move(path));
 
 			int baseLen = 0;
 			ParParser::ParseParFilename(parFilename.c_str(), true, &baseLen, nullptr);
@@ -304,7 +306,7 @@ ParChecker::EStatus ParChecker::RunParCheckAll()
 			BString<1024> parInfoName("%s%c%s", m_nzbName.c_str(), PATH_SEPARATOR, *infoName);
 			SetInfoName(parInfoName);
 
-			EStatus status = RunParCheck(fullParFilename);
+			EStatus status = RunParCheck(std::move(fullParFilename));
 
 			// accumulate total status, the worst status has priority
 			if (allStatus > status)
@@ -317,10 +319,10 @@ ParChecker::EStatus ParChecker::RunParCheckAll()
 	return allStatus;
 }
 
-ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename)
+ParChecker::EStatus ParChecker::RunParCheck(std::string parFilename)
 {
 	Cleanup();
-	m_parFilename = parFilename ? parFilename : "";
+	m_parFilename = std::move(parFilename);
 	m_stage = ptLoadingPars;
 	m_processedCount = 0;
 	m_extraFiles = 0;
@@ -484,7 +486,7 @@ int ParChecker::PreProcessPar()
 			m_repairer = std::make_unique<Repairer>(this);
 		}
 
-		res = GetRepairer()->PreProcess(m_parFilename.c_str());
+		res = GetRepairer()->PreProcess(m_parFilename);
 		debug("ParChecker: PreProcess-result=%i", res);
 
 		if (IsStopped())
@@ -1081,7 +1083,12 @@ void ParChecker::CheckEmptyFiles()
 		if (sourcefile && sourcefile->GetDescriptionPacket())
 		{
 			// GetDescriptionPacket()->FileName() returns a temp string object, which we need to hold for a while
-			std::string filenameObj = sourcefile->GetDescriptionPacket()->FileName();
+			std::string filenameObj = Par2::DescriptionPacket::TranslateFilenameFromPar2ToLocal(
+				m_parCout, 
+				m_parCerr, 
+				Par2::nlNormal,
+				sourcefile->GetDescriptionPacket()->FileName()
+			);
 			if (!filenameObj.empty() && !IsProcessedFile(filenameObj.c_str()))
 			{
 				bool ignore = Util::MatchFileExt(filenameObj.c_str(), g_Options->GetParIgnoreExt(), ",;");

daemon/postprocess/ParChecker.h

@@ -212,7 +212,7 @@ class ParChecker
 
 	void Cleanup();
 	EStatus RunParCheckAll();
-	EStatus RunParCheck(const char* parFilename);
+	EStatus RunParCheck(std::string parFilename);
 	int PreProcessPar();
 	bool LoadMainParBak();
 	int ProcessMorePars();

daemon/postprocess/ParRenamer.cpp

@@ -178,7 +178,12 @@ void ParRenamer::LoadParFile(const char* parFilename)
 			m_hasDamagedParFiles = true;
 			continue;
 		}
-		std::string filename = sourceFile->GetDescriptionPacket()->FileName();
+		std::string filename = Par2::DescriptionPacket::TranslateFilenameFromPar2ToLocal(
+			m_nout, 
+			m_nout,
+			Par2::nlNormal,
+			sourceFile->GetDescriptionPacket()->FileName()
+		);
 		std::string hash = sourceFile->GetDescriptionPacket()->Hash16k().print();
 
 		bool exists = std::find_if(m_fileHashList.begin(), m_fileHashList.end(),

daemon/postprocess/ParRenamer.h

@@ -48,6 +48,10 @@ class ParRenamer
 	int GetStageProgress() { return m_stageProgress; }
 
 private:
+	class NullStreamBuf final : public std::streambuf {};
+	NullStreamBuf m_nullbuf;
+	std::ostream m_nout{&m_nullbuf};
+
 	class FileHash
 	{
 	public:

daemon/system/SystemInfo.cpp

@@ -195,7 +195,7 @@ namespace System
 
 		std::string path = FileSystem::ExtractFilePathFromCmd(cmd);
 
-		auto result = FileSystem::GetFileRealPath(path);
+		auto result = FileSystem::GetRealPath(path);
 
 		if (result.has_value() && FileSystem::FileExists(result.value().c_str()))
 		{

daemon/util/FileSystem.cpp

@@ -24,6 +24,10 @@
 #include "Util.h"
 #include "Log.h"
 
+#ifdef WIN32
+#include "utf8.h"
+#endif
+
 const char* RESERVED_DEVICE_NAMES[] = { "CON", "PRN", "AUX", "NUL",
 	"COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9",
 	"LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", NULL };
@@ -56,28 +60,49 @@ void FileSystem::NormalizePathSeparators(char* path)
 	}
 }
 
-std::optional<std::string> FileSystem::GetFileRealPath(const std::string& path)
+std::optional<std::string> FileSystem::GetRealPath(const std::string& path)
 {
+	if (path.empty()) return std::nullopt;
+
 #ifdef WIN32
-	char buffer[MAX_PATH];
-	DWORD len = GetFullPathName(path.c_str(), MAX_PATH, buffer, nullptr);
-	if (len != 0)
+	auto res = Utf8::Utf8ToWide(path);
+	if (!res.has_value()) return std::nullopt;
+
+	std::wstring wpath = std::move(res.value());
+
+	if (wpath.size() > MAX_DIR_PATH && std::wcsncmp(wpath.c_str(), L"\\\\", 2) == 0)
+	{
+		wpath = L"\\\\?\\UNC" + wpath;
+	}
+	else if (wpath.size() > MAX_DIR_PATH)
 	{
-		return std::optional{ buffer };
+		wpath = L"\\\\?\\" + wpath;
 	}
+
+	DWORD len = GetFullPathNameW(wpath.c_str(), 0, nullptr, nullptr);
+	if (len == 0) return std::nullopt;
+
+	std::wstring buffer(len, '\0');
+	len = GetFullPathNameW(wpath.c_str(), len, &buffer[0], nullptr);
+	if (len == 0) return std::nullopt;
+
+	return Utf8::WideToUtf8(buffer);
 #else
 	if (char* realPath = realpath(path.c_str(), nullptr))
 	{
 		std::string res = realPath;
 		free(realPath);
-		return std::optional(std::move(res));
+		return res;
 	}
 #endif
 
 	return std::nullopt;
 }
 
 #ifdef WIN32
+
+const size_t FileSystem::MAX_DIR_PATH = 248;
+
 bool FileSystem::ForceDirectories(const char* path, CString& errmsg)
 {
 	errmsg.Clear();
@@ -621,7 +646,7 @@ std::string FileSystem::ExtractFilePathFromCmd(const std::string& path)
 {
 	if (path.empty())
 	{
-		return std::string(path);
+		return path;
 	}
 
 	size_t lastSeparatorPos = path.find_last_of(PATH_SEPARATOR);

daemon/util/FileSystem.h

@@ -40,7 +40,7 @@ class FileSystem
 	static char* BaseFileName(const char* filename);
 	static bool SameFilename(const char* filename1, const char* filename2);
 	static void NormalizePathSeparators(char* path);
-	static std::optional<std::string> GetFileRealPath(const std::string& path);
+	static std::optional<std::string> GetRealPath(const std::string& path);
 	static bool LoadFileIntoBuffer(const char* filename, CharBuffer& buffer, bool addTrailingNull);
 	static bool SaveBufferIntoFile(const char* filename, const char* buffer, int bufLen);
 	static bool AllocateFile(const char* filename, int64 size, bool sparse, CString& errmsg);
@@ -96,6 +96,8 @@ class FileSystem
 	static CString WidePathToUtfPath(const wchar_t* wpath);
 	static CString MakeCanonicalPath(const char* filename);
 	static bool NeedLongPath(const char* path);
+
+	static const size_t MAX_DIR_PATH;
 #endif
 };
 

daemon/util/Json.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <https://nzbget.com>.
  *
- *  Copyright (C) 2023 Denis <[email protected]>
+ *  Copyright (C) 2023-2024 Denis <[email protected]>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,32 +21,37 @@
 
 #include "Json.h"
 
-namespace Json {
-	JsonValue Deserialize(std::istream& is, ErrorCode& ec)
+namespace Json
+{
+	std::optional<JsonValue> Deserialize(std::basic_istream<char>& is) noexcept
 	{
+		ErrorCode ec;
 		StreamParser parser;
 		std::string line;
-
 		while (std::getline(is, line))
 		{
 			parser.write(line, ec);
-			if (ec)
-			{
-				return nullptr;
-			}
+			if (ec) return std::nullopt;
 		}
 
 		parser.finish(ec);
 
-		if (ec)
-		{
-			return nullptr;
-		}
+		if (ec) return std::nullopt;
 
 		return parser.release();
 	}
 
-	std::string Serialize(const JsonObject& json)
+	std::optional<JsonValue> Deserialize(const std::string& jsonStr) noexcept
+	{
+		ErrorCode ec;
+		JsonValue value = parse(jsonStr, ec);
+
+		if (ec) return std::nullopt;
+
+		return value;
+	}
+
+	std::string Serialize(const JsonObject& json) noexcept
 	{
 		return serialize(json);
 	}