[whimsy.git] [3/37] Commit 01d5a6b: Better recovery

[Sam Ruby <ru...@apache.org>]

Commit 01d5a6b8724927d383963bb704b3bfd45ad827fb:
    Better recovery


Branch: refs/heads/secmail
Author: Sam Ruby <ru...@intertwingly.net>
Committer: Sam Ruby <ru...@intertwingly.net>
Pusher: rubys <ru...@apache.org>

------------------------------------------------------------
config.rb                                                    | ++++++++++ 
parsemail.rb                                                 | ++++++++ ----
------------------------------------------------------------
44 changes: 36 additions, 8 deletions.
------------------------------------------------------------


diff --git a/config.rb b/config.rb
new file mode 100644
index 0000000..8c4b802
--- /dev/null
+++ b/config.rb
@@ -0,0 +1,20 @@
+#
+# Where to find the archive
+#
+
+SOURCE = 'minotaur.apache.org:/home/apmail/private-arch/officers-secretary'
+
+#
+# Where to save the archive locally
+#
+
+ARCHIVE = File.basename(SOURCE)
+
+#
+# What to use as a hash for mail
+#
+require 'digest'
+def hashmail(message)
+  Digest::SHA1.hexdigest(mail[/^Message-ID:.*/i] || mail)[0..9]
+end
+
diff --git a/parsemail.rb b/parsemail.rb
index 3af7b64..22581cb 100644
--- a/parsemail.rb
+++ b/parsemail.rb
@@ -4,8 +4,11 @@
 # Parse (and optionally fetch) officer-secretary emails for later
 # processing.
 #
-
-SOURCE = 'minotaur.apache.org:/home/apmail/private-arch/officers-secretary'
+# Care is taken to recover from improperly formed emails, including:
+#   * Malformed message ids
+#   * Improper encoding
+#   * Invalid from addresses
+#
 
 require 'mail'
 require 'zlib'
@@ -14,18 +17,25 @@
 require 'stringio'
 require 'time'
 
+require_relative 'config'
+
 database = File.basename(SOURCE)
 
 Dir.chdir File.dirname(File.expand_path(__FILE__))
 
 if ARGV.include? '--fetch' or not Dir.exist? database
-  system "rsync -av --no-motd --delete --exclude='*.yml' #{SOURCE} ."
+  system "rsync -av --no-motd --delete --exclude='*.yml' #{SOURCE}/ #{ARCHIVE}/"
 end
 
 # common header logic for messages and attachments
 def headers(part)
   # extract all fields from the mail (recovering from bad encoding issues)
   fields = part.header_fields.map do |field|
+    begin
+      next [field.name, field.to_s] if field.to_s.valid_encoding?
+    rescue
+    end
+
     if field.value and field.value.valid_encoding?
       [field.name, field.value]
     else
@@ -82,12 +92,10 @@ def headers(part)
 
     # process each
     mails.each do |mail|
-      # extract id, skip if already processed
-      id = mail[/^Message-ID: <(.*?)>\s*$/i, 1]
-      next if id and mbox[id]
-      mail = Mail.read_from_string(mail)
-      id ||= mail.message_id
+      # compute id, skip if already processed
+      id = hashmail(mail)
       next if mbox[id]
+      mail = Mail.read_from_string(mail)
 
       # parse from address
       begin