use a fallback date if Date header is missing

If a message doesn't have a usable Date header, we can fall back to
using the modtime of the file for Maildir, or the delivery time which
should have been written to the From line in mbox.

If all else fails, use a fixed date in the past, rather than Time.now.
This will prevent the mails from popping back to the top of the thread
list every time Sup reloads them.

Fixes #596.

lib/sup/maildir.rb

@@ -114,6 +114,10 @@ def raw_message id
     with_file_for(id) { |f| f.read }
   end
 
+  def fallback_date_for_message id
+    File.mtime File.join(@dir, id)
+  end
+
   ## XXX use less memory
   def poll
     added = []

lib/sup/mbox.rb

@@ -1,5 +1,6 @@
 require 'uri'
 require 'set'
+require 'time'
 
 module Redwood
 
@@ -137,6 +138,26 @@ def each_raw_message_line offset
     end
   end
 
+  def fallback_date_for_message offset
+    ## This is a bit awkward... We treat the From line as a delimiter,
+    ## not part of the message. So the offset is pointing *after* the
+    ## From line for the desired message. With a bit of effort we can
+    ## scan backwards to find its From line and extract a date from it.
+    buf = @mutex.synchronize do
+      ensure_open
+      start = offset
+      loop do
+        start = (start - 200).clamp 0, 2**64
+        @f.seek start
+        buf = @f.read (offset - start)
+        break buf if buf.include? ?\n or start == 0
+      end
+    end
+    BREAK_RE.match buf.lines.last do |m|
+      Time.strptime m[1], "%a %b %d %H:%M:%S %Y"
+    end
+  end
+
   def default_labels
     [:inbox, :unread]
   end

lib/sup/message.rb

@@ -103,16 +103,10 @@ def parse_header encoded_header
     when Time
       date
     when String
-      begin
-        Time.parse date
-      rescue ArgumentError
-        #debug "faking mangled date header for #{@id} (orig #{header['date'].inspect} gave error: #{e.message})"
-        Time.now
-      end
-    else
-      #debug "faking non-existent date header for #{@id}"
-      Time.now
+      Time.parse date rescue nil
     end
+    @date = location.fallback_date if @date.nil?
+    @date = Time.utc 1970, 1, 1 if @date.nil?
 
     subj = header["subject"]
     subj = subj ? subj.fix_encoding! : nil
@@ -808,6 +802,10 @@ def parsed_message
     source.load_message info
   end
 
+  def fallback_date
+    source.fallback_date_for_message info
+  end
+
   def valid?
     source.valid? info
   end

test/dummy_source.rb

@@ -59,6 +59,10 @@ def each_raw_message_line id
       end
     end
   end
+
+  def fallback_date_for_message id
+    Time.utc 2001, 2, 3, 4, 56, 57
+  end
 end
 
 end

test/integration/test_maildir.rb

@@ -34,7 +34,9 @@ def create_a_maildir(extra='')
   end
 
   def create_a_maildir_email(folder, content)
-    File.write(File.join(folder, "#{Time.now.to_f}.hostname:2,S"), content)
+    filename = File.join folder, "#{Time.now.to_f}.hostname:2,S"
+    File.write filename, content
+    filename
   end
 
   def start_sup_and_add_source(source)
@@ -74,5 +76,17 @@ def test_can_index_a_maildir_directory_with_special_characters
 
   end
 
+  def test_missing_date_header
+    ## The message is missing a Date header so we should use its modtime
+    ## as a fallback.
+    fallback_date = Time.new 2004, 4, 19, 11, 12, 13
+    maildir = create_a_maildir
+    filename = create_a_maildir_email(File.join(maildir, 'cur'), @test_message_1)
+    File.utime fallback_date, fallback_date, filename
+    start_sup_and_add_source Maildir.new "maildir:#{maildir}"
+
+    messages_in_index = Index.instance.enum_for(:each_message).to_a
+    assert_equal fallback_date, messages_in_index.first.date
+  end
 end
 

test/integration/test_mbox.rb

@@ -69,4 +69,14 @@ def test_can_index_a_mbox_directory_with_special_characters
 
   end
 
+  def test_missing_date_header
+    ## The message is missing a Date header so we should use envelope date
+    ## stored in the From line as a fallback.
+    fallback_date = Time.new 2009, 4, 27, 12, 56, 18
+    mbox = create_a_mbox
+    start_sup_and_add_source MBox.new "mbox:#{mbox}"
+
+    messages_in_index = Index.instance.enum_for(:each_message).to_a
+    assert_equal fallback_date, messages_in_index.first.date
+  end
 end