Merge pull request #145 from sisimai/v4.25.0p1

V4.25.0p1

ChangeLog.md

@@ -3,6 +3,18 @@ RELEASE NOTES for Ruby version of Sisimai
 - releases: "https://github.com/sisimai/rb-Sisimai/releases"
 - download: "https://rubygems.org/gems/sisimai"
 
+v4.25.0p1
+--------------------------------------------------------------------------------
+- release: ""
+- version: ""
+- changes:
+  - Check the format of the value of `Message-Id` header for detecting a bounce
+    mail from Exim or not.
+  - Call `Sisimai::Rhost::FrancePTT` module when the value of `rhost` includes
+    `.wanadoo.fr`.
+  - Fix code at `Sisimai::Message::Email.takeapart` method to decode a Subject
+    header of the original message.
+
 v4.25.0
 --------------------------------------------------------------------------------
 - release: "Tue,  9 Apr 2019 11:22:22 +0900 (JST)"

lib/sisimai/bite/email/exim.rb

@@ -139,10 +139,13 @@ def headerlist;  return %w[x-failed-recipients]; end
       #                                   part or nil if it failed to parse or
       #                                   the arguments are missing
       def scan(mhead, mbody)
-        # :'message-id' => %r/\A[<]\w+[-]\w+[-]\w+[@].+\z/,
-        return nil if     mhead['from'] =~ /[@].+[.]mail[.]ru[>]?/
-        return nil unless mhead['from'].start_with?('Mail Delivery System')
-        return nil unless mhead['subject'] =~ %r{(?:
+        return nil if mhead['from'] =~ /[@].+[.]mail[.]ru[>]?/
+
+        # Message-Id: <[email protected]>
+        match  = 0
+        match += 1 if mhead['from'].start_with?('Mail Delivery System')
+        match += 1 if mhead['message-id'].to_s =~ %r/\A[<]\w{7}[-]\w{6}[-]\w{2}[@]/
+        match += 1 if mhead['subject'] =~ %r{(?:
            Mail[ ]delivery[ ]failed(:[ ]returning[ ]message[ ]to[ ]sender)?
           |Warning:[ ]message[ ].+[ ]delayed[ ]+
           |Delivery[ ]Status[ ]Notification
@@ -151,6 +154,7 @@ def scan(mhead, mbody)
           |error[(]s[)][ ]in[ ]forwarding[ ]or[ ]filtering
           )
         }x
+        return nil if match < 2
 
         require 'sisimai/rfc1894'
         fieldtable = Sisimai::RFC1894.FIELDTABLE

lib/sisimai/message/email.rb

@@ -263,7 +263,7 @@ def self.takeapart(heads)
 
         # 1. Scrub to avoid "invalid byte sequence in UTF-8" exception (#82)
         # 2. Convert from string to hash reference
-        heads = heads.scrub('?').gsub(/^[>]+[ ]/m, '')
+        heads = heads.scrub('?').gsub(/^[>]+[ ]/m, '').gsub(/=[ ]+=/, "=\n =")
 
         previousfn = '' # Previous field name
         asciiarmor = {} # Header names which has MIME encoded value

lib/sisimai/reason/suspend.rb

@@ -20,6 +20,7 @@ class << self
           'recipient rejected: temporarily inactive',
           'recipient suspend the service',
           'this account has been disabled or discontinued',
+          'this mailbox is disabled',
           'user suspended',   # http://mail.163.com/help/help_spam_16.htm
           'vdelivermail: account is locked email bounced',
         ]

lib/sisimai/rhost.rb

@@ -14,6 +14,7 @@ class << self
         'mailstore1.secureserver.net' => 'GoDaddy',
         'laposte.net'                 => 'FrancePTT',
         'orange.fr'                   => 'FrancePTT',
+        'wanadoo.fr'                  => 'FrancePTT',
         'lsean.ezweb.ne.jp'           => 'KDDI',
         'msmx.au.com'                 => 'KDDI',
         '.qq.com'                     => 'TencentQQ',

lib/sisimai/version.rb

@@ -1,4 +1,4 @@
 # Define the version number of Sisimai
 module Sisimai
-  VERSION = '4.25.0'.freeze
+  VERSION = '4.25.0p1'.freeze
 end

set-of-emails/maildir/bsd/email-postfix-57.eml

@@ -0,0 +1,127 @@
+Received: from BY2NAM03HT158.eop-NAM03.prod.protection.outlook.com
+    (2603:1096:100:19::20) by SLXP216MB0381.KORP216.PROD.OUTLOOK.COM with HTTPS
+    via SL2P216CA0058.KORP216.PROD.OUTLOOK.COM; Wed, 17 Apr 2019 05:39:02 +0000
+Received: from BY2NAM03FT015.eop-NAM03.prod.protection.outlook.com
+    (10.152.84.59) by BY2NAM03HT158.eop-NAM03.prod.protection.outlook.com
+    (10.152.85.21) with Microsoft SMTP Server (version=TLS1_2,
+    cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id 15.20.1771.16; Wed, 17 Apr
+    2019 05:39:02 +0000
+Authentication-Results: spf=none (sender IP is 192.0.2.202)
+    smtp.helo=rokujo.cr.nyaan.jp; outlook.com; dkim=none (message not signed)
+    header.d=none;outlook.com; dmarc=none action=none
+    header.from=rokujo.cr.nyaan.jp;
+Received-SPF: None (protection.outlook.com: rokujo.cr.nyaan.jp does not
+    designate permitted sender hosts)
+Received: from rokujo.cr.nyaan.jp (192.0.2.202) by
+    BY2NAM03FT015.mail.protection.outlook.com (10.152.84.212) with Microsoft SMTP
+    Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id
+    15.20.1771.16 via Frontend Transport; Wed, 17 Apr 2019 05:39:01 +0000
+X-IncomingTopHeaderMarker: OriginalChecksum:A1CD8B3776EE3FF5CEA4C998409E1F69E1E831B693CE099E69B68E391941E6F9;UpperCasedChecksum:D30D6C927BA1C67867F6BB364EC03CB75A3E42993CEA79E53991E20FA99D064A;SizeAsReceived:517;Count:9
+Received: by rokujo.cr.nyaan.jp (Postfix)
+    id 44kWJ43kJDz1yxHF; Wed, 17 Apr 2019 14:39:00 +0900 (JST)
+Date: Wed, 17 Apr 2019 14:39:00 +0900
+From: Mail Delivery System <[email protected]>
+Subject: Undelivered Mail Returned to Sender
+To: <[email protected]>
+Auto-Submitted: auto-replied
+Content-Type: multipart/report; report-type=delivery-status;
+    boundary="44kWHZ2S3Qz1yxHC.1555479540/rokujo.cr.nyaan.jp"
+Message-ID: <[email protected]>
+X-IncomingHeaderCount: 9
+Return-Path: <>
+X-MS-Exchange-Organization-ExpirationStartTime: 17 Apr 2019 05:39:01.5236
+    (UTC)
+X-MS-Exchange-Organization-ExpirationStartTimeReason: OriginalSubmit
+X-MS-Exchange-Organization-ExpirationInterval: 2:00:00:00.0000000
+X-MS-Exchange-Organization-ExpirationIntervalReason: OriginalSubmit
+X-MS-Exchange-Organization-Network-Message-Id: 761e4ae2-81c3-4302-eec4-08d6c2f6fefc
+X-EOPAttributedMessage: 0
+X-EOPTenantAttributedMessage: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa:0
+X-MS-Exchange-Organization-MessageDirectionality: Incoming
+X-Forefront-Antispam-Report: EFV:NLI;
+X-MS-Exchange-Organization-AuthSource:
+    BY2NAM03FT015.eop-NAM03.prod.protection.outlook.com
+X-MS-Exchange-Organization-AuthAs: Anonymous
+X-MS-PublicTrafficType: Email
+X-MS-UserLastLogonTime: 4/17/2019 5:34:26 AM
+X-MS-Office365-Filtering-Correlation-Id: 761e4ae2-81c3-4302-eec4-08d6c2f6fefc
+X-Microsoft-Antispam:
+BCL:0;PCL:0;RULEID:(2390118)(5000112)(711020)(4605104)(610169)(8291501072);SRVR:BY2NAM03HT158;
+X-MS-TrafficTypeDiagnostic: BY2NAM03HT158:
+X-MS-Exchange-EOPDirect: true
+X-Sender-IP: 192.0.2.202
+X-SID-PRA: [email protected]
+X-SID-Result: NONE
+X-MS-Exchange-Organization-PCL: 2
+X-OriginatorOrg: outlook.com
+X-MS-Exchange-CrossTenant-OriginalArrivalTime: 17 Apr 2019 05:39:01.3474
+    (UTC)
+X-MS-Exchange-CrossTenant-Network-Message-Id: 761e4ae2-81c3-4302-eec4-08d6c2f6fefc
+X-MS-Exchange-CrossTenant-Id: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa
+X-MS-Exchange-CrossTenant-FromEntityHeader: Internet
+X-MS-Exchange-Transport-CrossTenantHeadersStamped: BY2NAM03HT158
+X-MS-Exchange-Transport-EndToEndLatency: 00:00:01.4290992
+X-MS-Exchange-Processed-By-BccFoldering: 15.20.1813.000
+X-Microsoft-Antispam-Mailbox-Delivery:
+abwl:0;wl:0;pcwl:0;kl:0;iwl:0;dwl:0;dkl:0;rwl:0;ucf:0;jmr:0;ex:0;psp:0;auth:0;dest:J;OFR:SpamFilterAuthJ;ENG:(5062000261)(5061607266)(5061608174)(1002269)(4900115)(8390100)(8394020)(8376100)(8383139)(8397020)(4920090)(6360004)(4950130)(4990090)(9140004);RF:JunkEmail;
+X-Message-Info:
+    qoGN4b5S4yqdjbFs0cudHJzI1xw5DQG6TA6ATxQvABdTK52ZC1ChFIE+udWOEBijGHe1fWjxGh/LgC2fOJcsrl7CNnjXDN6RSlZ9QKKG7tzZUUdGr+lV9YiPhCcR8uU0/LRDE5tASvq8b/ad/di5z4EWLCcPlruYgXBhB6XTQAzWbC5r2Km1XDIqAdJIKsNPtdJWuEP5IcOpCBd7YuqwdQ==
+X-Message-Delivery: Vj0xLjE7dXM9MDtsPTA7YT0wO0Q9MjtHRD0yO1NDTD02
+X-Microsoft-Antispam-Message-Info:
+    lmynbZQP96t2HiZRaqmspUnxgWmUEB5NkBE9FEaHIarSbVnfAjH4w5J8DR8B6DZM6kqqUgxcrqFhoEHpecGKm+C63MB/yqHLlPu0buaLGUXRA5rg9erBbaMOvlkepeH7m0aTxsaxOk86FKSruhBl2v/wGkbn4VvZnG8JM8XtsmgI3OZipeh+bCKou5/b7eJKCHWFpjoD/RdKxgDey+rwo514bPdtd6vR/Se7VYeDaeTS9jwUMw9aenEnqYN+PUTgyeK4Uf8JW++OlHDsgzhJ8oMo7duSNKxWjEoaK7tLr0fPOT2caZJpadHavz5R+G3AHX6viQKssj2XVZxAuIEK/svqQxncyAFJ1AP8mGH9YXxffFfPgjRYG4cs55qIypfq9JTd21jEIvjw0namnZPApyox3aL/bjP8kB7FQ/kY3CX1LXwTtMNS+mqSj9QEXFqaN9MRMWMAe3ZlSU/0ecJbJO/HafJaoefcXT1MujlTlzJ/0b+Yk4EcnLCgIH/ZfJVJPjViJHZD5WbDRpPujHyRRluOVe2YYLauyEid/tQHGeRm//CLf6Y/H5Fxb7DrxeyBOw8w62eZ+/giP/zC7QIuk0E9p5UaoGVBI5KDTTOqryQtGrJd8x3p62ZJWDzQmgCF+dVTGn0QeI6NF936QhUwtq62zy8n4hpI5RSx9gHukC+hd12vTH8F1WftZdEYvyT2jL1ESYblt5HMISA+uvjAqG8W/7API/K0JUMMiNRlnPZTNj2O/46UbQWZdwyXPvD6+Pl6QzGKrpP5xpUWPsvQ3A==
+MIME-Version: 1.0
+
+--44kWHZ2S3Qz1yxHC.1555479540/rokujo.cr.nyaan.jp
+Content-Description: Notification
+Content-Type: text/plain; charset="us-ascii"
+
+This is the mail system at host rokujo.cr.nyaan.jp.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+The mail system
+
+<[email protected]>: host mx.cr.nyaan.jp[203.0.113.22] said: 550 5.2.1
+<[email protected]>... User Unknown (in reply to RCPT TO command)
+
+--44kWHZ2S3Qz1yxHC.1555479540/rokujo.cr.nyaan.jp
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; rokujo.cr.nyaan.jp
+X-Postfix-Queue-ID: 44kWHZ2S3Qz1yxHC
+X-Postfix-Sender: rfc822; [email protected]
+Arrival-Date: Wed, 17 Apr 2019 14:38:21 +0900 (JST)
+
+Final-Recipient: rfc822; [email protected]
+Original-Recipient: rfc822;[email protected]
+Action: failed
+Status: 5.2.1
+Remote-MTA: dns; mx.cr.nyaan.jp
+Diagnostic-Code: smtp; 550 5.2.1 <[email protected]>... User Unknown
+
+--44kWHZ2S3Qz1yxHC.1555479540/rokujo.cr.nyaan.jp
+Content-Description: Undelivered Message
+Content-Type: message/rfc822
+
+Return-Path: <[email protected]>
+Received: from [127.0.0.1] (localhost [127.0.0.1])
+by rokujo.cr.nyaan.jp (Postfix) with ESMTP id 44kWHZ2S3Qz1yxHC
+for <[email protected]>; Wed, 17 Apr 2019 14:38:21 +0900 (JST)
+Subject: Nyaan
+To: <[email protected]>
+From: <[email protected]>
+Message-ID: <[email protected]>
+Date: Wed, 17 Apr 2019 14:38:21 +0900
+Content-Type: text/plain
+MIME-Version: 1.0
+
+Nyaan
+
+--44kWHZ2S3Qz1yxHC.1555479540/rokujo.cr.nyaan.jp--

spec/sisimai/bite/email/private-exim_spec.rb

@@ -181,6 +181,9 @@
   { 'n' => '01181', 'r' => /mailererror/ },
   { 'n' => '01182', 'r' => /userunknown/ },
   { 'n' => '01183', 'r' => /mailboxfull/ },
+  { 'n' => '01184', 'r' => /userunknown/ },
+  { 'n' => '01185', 'r' => /suspend/ },
+  { 'n' => '01184', 'r' => /userunknown/ },
 ]
 Sisimai::Bite::Email::Code.maketest(enginename, isexpected, true)
 

spec/sisimai/bite/email/private-office365_spec.rb

@@ -17,6 +17,8 @@
   { 'n' => '01013', 'r' => /networkerror/ },
   { 'n' => '01014', 'r' => /userunknown/ },
   { 'n' => '01015', 'r' => /userunknown/ },
+  { 'n' => '01016', 'r' => /userunknown/ },
+  { 'n' => '01017', 'r' => /mailboxfull/ },
 ]
 Sisimai::Bite::Email::Code.maketest(enginename, isexpected, true)
 

spec/sisimai/bite/email/private-postfix_spec.rb

@@ -213,6 +213,7 @@
   { 'n' => '01209', 'r' => /networkerror/ },
   { 'n' => '01210', 'r' => /blocked/ },
   { 'n' => '01211', 'r' => /userunknown/ },
+  { 'n' => '01212', 'r' => /userunknown/ },
 ]
 Sisimai::Bite::Email::Code.maketest(enginename, isexpected, true)
 

spec/sisimai/bite/email/public-office365_spec.rb

@@ -12,6 +12,8 @@
   { 'n' => '08', 's' => /\A5[.]4[.]316\z/,'r' => /expired/,      'b' => /\A1\z/ },
   { 'n' => '09', 's' => /\A5[.]1[.]351\z/,'r' => /userunknown/,  'b' => /\A0\z/ },
   { 'n' => '10', 's' => /\A5[.]1[.]351\z/,'r' => /userunknown/,  'b' => /\A0\z/ },
+  { 'n' => '11', 's' => /\A5[.]1[.]1\z/,  'r' => /userunknown/,  'b' => /\A0\z/ },
+  { 'n' => '12', 's' => /\A5[.]2[.]2\z/,  'r' => /mailboxfull/,  'b' => /\A1\z/ },
 ]
 Sisimai::Bite::Email::Code.maketest(enginename, isexpected)
 

spec/sisimai/bite/email/public-postfix_spec.rb

@@ -47,6 +47,7 @@
   { 'n' => '54', 's' => /\A5[.]7[.]1\z/,   'r' => /rejected/,      'b' => /\A1\z/ },
   { 'n' => '55', 's' => /\A5[.]0[.]0\z/,   'r' => /toomanyconn/,   'b' => /\A1\z/ },
   { 'n' => '56', 's' => /\A4[.]4[.]2\z/,   'r' => /networkerror/,  'b' => /\A1\z/ },
+  { 'n' => '57', 's' => /\A5[.]2[.]1\z/,   'r' => /userunknown/,   'b' => /\A0\z/ },
 ]
 Sisimai::Bite::Email::Code.maketest(enginename, isexpected)
 

spec/sisimai/bite/email/public-rfc3464_spec.rb

@@ -14,7 +14,6 @@
   { 'n' => '26', 's' => /\A5[.]1[.]1\z/,     'r' => /userunknown/, 'a' => /RFC3464/, 'b' => /\A0\z/ },
   { 'n' => '28', 's' => /\A2[.]1[.]5\z/,     'r' => /delivered/,   'a' => /RFC3464/, 'b' => /\A-1\z/ },
   { 'n' => '29', 's' => /\A5[.]5[.]0\z/,     'r' => /syntaxerror/, 'a' => /RFC3464/, 'b' => /\A1\z/ },
-  { 'n' => '33', 's' => /\A5[.]2[.]0\z/,     'r' => /spamdetected/,'a' => /RFC3464/, 'b' => /\A1\z/ },
   { 'n' => '34', 's' => /\A5[.]0[.]\d+\z/,   'r' => /networkerror/,'a' => /RFC3464/, 'b' => /\A1\z/ },
   { 'n' => '35', 's' => /\A5[.]0[.]0\z/,     'r' => /filtered/,    'a' => /RFC3464/, 'b' => /\A1\z/ },
   { 'n' => '36', 's' => /\A4[.]0[.]0\z/,     'r' => /expired/,     'a' => /RFC3464/, 'b' => /\A1\z/ },

spec/sisimai/mail/maildir_spec.rb

@@ -3,7 +3,7 @@
 
 describe Sisimai::Mail::Maildir do
   samplemaildir = './set-of-emails/maildir/bsd'
-  allofthefiles = 436
+  allofthefiles = 439
   let(:mailobj) { Sisimai::Mail::Maildir.new(samples) }
   let(:mockobj) { Sisimai::Mail::Maildir.new(invalid) }
 

spec/sisimai/rhost/franceptt_spec.rb

@@ -8,14 +8,15 @@
   rs = {
     '01' => { 'status' => %r/\A5[.]1[.]1\z/, 'reason' => %r/userunknown/ },
     '02' => { 'status' => %r/\A5[.]5[.]0\z/, 'reason' => %r/userunknown/ },
+    '03' => { 'status' => %r/\A5[.]2[.]0\z/, 'reason' => %r/spamdetected/ },
   }
   describe 'bounce mail from FrancePTT' do
     rs.each_key.each do |n|
       emailfn = sprintf('./set-of-emails/maildir/bsd/rhost-franceptt-%02d.eml', n)
       next unless File.exist?(emailfn)
 
       mailbox = Sisimai::Mail.new(emailfn)
-      mtahost = %r/(?:smtp-in[.]orange[.]fr|smtpz4[.]laposte[.]net)/
+      mtahost = %r/(?:smtp-in[.]orange[.]fr|smtpz4[.]laposte[.]net|smtp[.]wanadoo[.]fr)/
       next unless mailbox
 
       while r = mailbox.read do
@@ -27,15 +28,15 @@
         it('has From line in "from" accessor' ) { expect(mesg.from.size).to be > 0 }
 
         mesg.ds.each do |e|
-          example('spec is "SMTP"') { expect(e['spec']).to be == 'SMTP' }
+          example('spec is a String') { expect(e['spec']).to be_a ::String }
           example 'recipient is email address' do
             expect(e['recipient']).to match(/\A.+[@].+[.].+\z/)
           end
           example('status is DSN') { expect(e['status']).to match(/\A\d[.]\d[.]\d\z/) }
           example('command is SMTP command') { expect(e['command']).to match(/\A[A-Z]{4}\z/) }
-          example('date is not empty') { expect(e['date']).not_to be_empty }
+          example('date is a String') { expect(e['date']).to be_a ::String }
           example('diagnosis is not empty') { expect(e['diagnosis']).not_to be_empty }
-          example('action is not empty') { expect(e['action']).not_to be_empty }
+          example('action is a String') { expect(e['action']).to be_a ::String }
           example('rhost is ' + e['rhost']) { expect(e['rhost']).to match(mtahost) }
           example('alias exists') { expect(e.key?('alias')).to be true }
           example('agent matches Email::*') { expect(e['agent']).to match(/\AEmail::.+/) }