fix: fix filename mismatch for empty file inode reuse (#1101)

* fix filename mismatch for empty file inode reuse Signed-off-by: Tao Yu <[email protected]> * remove uninitialized file size print * add logtail initilaization completed log Signed-off-by: Tao Yu <[email protected]> --------- Signed-off-by: Tao Yu <[email protected]>
alibaba · Sep 3, 2023 · ea4bbae · ea4bbae
1 parent c3c2f1a
commit ea4bbae
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -45,5 +45,5 @@ your changes, such as:
 - [public] [both] [updated] Flusher Kafka V2: support send the message with headers to kafka
 - [public] [both] [fixed] fix multiline is splitted if not flushed to disk together
 - [public] [both] [fixed] fix line is truncated if \0 is in the middle of line
+- [public] [both] [fixed] fix filename being mismatched to the deleted file if the deleted file size is 0 and their inode is same
 - [public] [both] [fixed] fix config server panic caused by concurrent read and write shared object
-
diff --git a/core/logtail.cpp b/core/logtail.cpp
@@ -288,11 +288,12 @@ void do_worker_process() {
     appInfoJson["update_time"] = GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S");
     std::string appInfo = appInfoJson.toStyledString();
     OverwriteFile(GetProcessExecutionDir() + STRING_FLAG(app_info_file), appInfo);
-    APSARA_LOG_INFO(sLogger, ("Logtail started, appInfo", appInfo));
+    APSARA_LOG_INFO(sLogger, ("appInfo", appInfo));
 
     ConfigManager::GetInstance()->InitUpdateConfig(configExistFlag);
     ConfigManager::GetInstance()->RegisterHandlers();
     EventDispatcher::GetInstance()->AddExistedCheckPointFileEvents();
+    APSARA_LOG_INFO(sLogger, ("Logtail started", "initialization completed"));
 
     // [Main thread] Run the Dispatch routine.
     EventDispatcher::GetInstance()->Dispatch();

diff --git a/core/logtail_windows.cpp b/core/logtail_windows.cpp
@@ -200,11 +200,12 @@ void do_worker_process() {
     appInfoJson["update_time"] = GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S");
     std::string appInfo = appInfoJson.toStyledString();
     OverwriteFile(GetProcessExecutionDir() + STRING_FLAG(app_info_file), appInfo);
-    APSARA_LOG_INFO(sLogger, ("Logtail started, appInfo", appInfo));
+    APSARA_LOG_INFO(sLogger, ("appInfo", appInfo));
 
     ConfigManager::GetInstance()->InitUpdateConfig(configExistFlag);
     ConfigManager::GetInstance()->RegisterHandlers();
     EventDispatcher::GetInstance()->AddExistedCheckPointFileEvents();
+    APSARA_LOG_INFO(sLogger, ("Logtail started", "initialization completed"));
 
     EventDispatcher::GetInstance()->Dispatch();
 }

diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp
@@ -424,6 +424,7 @@ LogFileReader::LogFileReader(const string& projectName,
     mHostLogPathDir = hostLogPathDir;
     mHostLogPathFile = hostLogPathFile;
     mHostLogPath = PathJoin(hostLogPathDir, hostLogPathFile);
+    // mRealLogPath = mHostLogPath; fix it in 1.8
     mTailLimit = tailLimit;
     mLastFilePos = 0;
     mLastFileSize = 0;
@@ -469,6 +470,7 @@ LogFileReader::LogFileReader(const std::string& projectName,
     mHostLogPathDir = hostLogPathDir;
     mHostLogPathFile = hostLogPathFile;
     mHostLogPath = PathJoin(hostLogPathDir, hostLogPathFile);
+    // mRealLogPath = mHostLogPath; fix it in 1.8
     mTailLimit = tailLimit;
     mLastFilePos = 0;
     mLastFileSize = 0;
@@ -1119,8 +1121,7 @@ bool LogFileReader::UpdateFilePtr() {
                          ("open file succeeded, project", mProjectName)("logstore", mCategory)("config", mConfigName)(
                              "log reader queue name", mHostLogPath)("file device", ToString(mDevInode.dev))(
                              "file inode", ToString(mDevInode.inode))("file signature", mLastFileSignatureHash)(
-                             "real file path", mRealLogPath)("file size", mLastFileSize)("last file position",
-                                                                                         mLastFilePos));
+                             "real file path", mRealLogPath)("last file position", mLastFilePos));
                 return true;
             } else {
                 mLogFileOp.Close();
@@ -1151,9 +1152,9 @@ bool LogFileReader::UpdateFilePtr() {
             GloablFileDescriptorManager::GetInstance()->OnFileOpen(this);
             LOG_INFO(sLogger,
                      ("open file succeeded, project", mProjectName)("logstore", mCategory)("config", mConfigName)(
-                         "log reader queue name", mHostLogPath)("file device", ToString(mDevInode.dev))(
-                         "file inode", ToString(mDevInode.inode))("file signature", mLastFileSignatureHash)(
-                         "file size", mLastFileSize)("last file position", mLastFilePos));
+                         "log reader queue name",
+                         mHostLogPath)("file device", ToString(mDevInode.dev))("file inode", ToString(mDevInode.inode))(
+                         "file signature", mLastFileSignatureHash)("last file position", mLastFilePos));
             return true;
         } else {
             mLogFileOp.Close();
@@ -1282,6 +1283,11 @@ bool LogFileReader::CheckFileSignatureAndOffset(int64_t& fileSize) {
         }
     }
 
+    // If file size is 0 and filename is changed, we cannot judge if the inode is reused by signature,
+    // so we just recreate the reader to avoid filename mismatch
+    if (mLastFileSize == 0 && mRealLogPath != mHostLogPath) {
+        return false;
+    }
     fileSize = endSize;
     mLastFileSize = endSize;
     bool sigCheckRst = CheckAndUpdateSignature(string(firstLine), mLastFileSignatureHash, mLastFileSignatureSize);
@@ -2007,6 +2013,11 @@ LogFileReader::FileCompareResult LogFileReader::CompareToFile(const string& file
         sigStr[readSize] = '\0';
         uint64_t sigHash = mLastFileSignatureHash;
         uint32_t sigSize = mLastFileSignatureSize;
+        // If file size is 0 and filename is changed, we cannot judge if the inode is reused by signature,
+        // so we just recreate the reader to avoid filename mismatch
+        if (mLastFileSize == 0 && filePath != mHostLogPath) {
+            return FileCompareResult_SigChange;
+        }
         bool sigSameRst = CheckAndUpdateSignature(string(sigStr), sigHash, sigSize);
         if (!sigSameRst) {
             return FileCompareResult_SigChange;