Bump version

dmarcreceiver/commands.py

@@ -4,9 +4,12 @@ import zipfile
 from datetime import datetime
 from email import message_from_file
 from tempfile import TemporaryDirectory
-from lxml.etree import parse as parse_xml
+from lxml.etree import parse as parse_xml, tounicode as xml_tree_to_unicode
-from dmarcreceiver.model import DBSession, Report, ReportError, ReportRecord, OverrideReason, DKIMResult, SPFResult, metadata, init_model
+from dmarcreceiver.model import DBSession, Report, ReportXML, \
+    ReportError, ReportRecord, OverrideReason, DKIMResult, SPFResult, metadata, init_model
 import transaction
+import gzip
+from io import BytesIO
 
 from .util import sendmail, install_exception_handler
 from .config import config
@@ -24,13 +27,15 @@ def parse_metadata(tree):
     date_begin = datetime.fromtimestamp(int(date_begin))
     date_end, = tree.xpath('date_range/end/text()')
     date_end = datetime.fromtimestamp(int(date_end))
+    errors = [ ReportError(txt) for txt in tree.xpath('errors/text()') ]
     return Report(
         org_name=org_name,
         email=email,
         extra_contact_info=extra_contact_info,
         report_id=report_id,
         date_begin=date_begin,
-        date_end=date_end
+        date_end=date_end,
+        errors=errors
     )
 
 def scoop_elements(obj, tree, *elements):
@@ -64,16 +69,15 @@ def parse_report(f):
         scoop_elements(record, record_node.xpath('./identifiers')[0], 'envelope_to', 'header_from')
         for dkim_node in record_node.xpath('./auth_results/dkim'):
             dkim = DKIMResult()
-            scoop_elements(dkim, dkim_node, 'domain', 'result', 'human_result')
+            scoop_elements(dkim, dkim_node, 'domain', 'selector', 'result', 'human_result')
-            t = dkim_node.xpath('./selector/text()')
-            if len(t) > 0:
-                dkim.human_result = t[0]
             record.dkim_results.append(dkim)
         for spf_node in record_node.xpath('./auth_results/spf'):
             spf = SPFResult()
             scoop_elements(spf, spf_node, 'domain', 'result')
             record.spf_results.append(spf)
         report.records.append(record)
+    xml = xml_tree_to_unicode(tree)
+    report.original = ReportXML(xml=xml)
     DBSession.add(report)
     transaction.commit()
 
@@ -82,32 +86,7 @@ def read_config_if_present(args):
         if os.access(args.config_file, os.R_OK):
             config.read_file(open(args.config_file, 'rt'))
 
-def receive_report(args):
+def extract_zip(msg):
-    read_config_if_present(args)
-    init_model()
-
-    # read email message from stdin
-    msg = message_from_file(sys.stdin)
-
-    # if not running on a tty, install email-based exception handler
-    if not sys.stderr.isatty():
-        install_exception_handler(msg)
-
-    # check for zip file
-    content_type = msg['content-type']
-    if content_type.find(';') != -1:
-        content_type = content_type.split(';',1)[0]
-
-    if content_type != 'application/zip':
-        # not a zip file - bounce to postmaster
-        bounce_address = config.get('bounce_address')
-        if args.bounce_address:
-            bounce_address = args.bounce_address
-        if bounce_address:
-            msg['To'] = bounce_address
-            sendmail(msg)
-        return
-
     with TemporaryDirectory(
         prefix=os.path.splitext(os.path.basename(sys.argv[0]))[0] + '-'
     ) as tempdir:
@@ -122,7 +101,52 @@ def receive_report(args):
             report_fn = os.path.join(tempdir, os.path.basename(namelist[0]))
             z.extract(namelist[0], path=tempdir)
         with open(report_fn, 'rb') as f:
-            parse_report(f)
+            data = f.read()
+    return data
+
+def extract_gzip(msg):
+    return gzip.decompress(msg.get_payload(decode=True))
+
+def receive_report(args):
+    read_config_if_present(args)
+    init_model()
+
+    # read email message from stdin
+    msg = message_from_file(sys.stdin)
+
+    # if not running on a tty, install email-based exception handler
+    if not sys.stderr.isatty():
+        install_exception_handler(msg)
+
+    xml_content = None
+    for part in msg.walk():
+        # check for zip file
+        content_type = part['content-type']
+
+        if content_type is None:
+            continue
+
+        if content_type.find(';') != -1:
+            content_type = content_type.split(';',1)[0]
+
+        if content_type == 'application/zip':
+            xml_content = extract_zip(part)
+            break
+        elif content_type == 'application/gzip':
+            xml_content = extract_gzip(part)
+            break
+
+    if xml_content is None:
+        # not a zip file - bounce to postmaster
+        bounce_address = config.get('bounce_address')
+        if args.bounce_address:
+            bounce_address = args.bounce_address
+        if bounce_address:
+            msg['To'] = bounce_address
+            sendmail(msg)
+        return
+
+    parse_report(BytesIO(xml_content))
 
 def init(args):
     read_config_if_present(args)

dmarcreceiver/model.py

@@ -1,6 +1,7 @@
 import zope.sqlalchemy as zsqla
-from sqlalchemy import Column, Integer, String, Unicode, Enum, CheckConstraint, ForeignKey, DateTime, create_engine
+from sqlalchemy import Column, Integer, String, Unicode, \
-from sqlalchemy.orm import scoped_session, sessionmaker, relationship
+    UnicodeText, Enum, CheckConstraint, ForeignKey, DateTime, create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker, relationship, backref
 import sqlalchemy.types as satypes
 import sqlalchemy.dialects.postgresql as dpg
 from sqlalchemy.ext.declarative import declarative_base
@@ -24,9 +25,19 @@ def init_model():
 Alignment = Enum('r', 's', name='alignment')
 Disposition = Enum('none', 'quarantine', 'reject', name='disposition')
 DMARCResult = Enum('pass', 'fail', name='dmarc_result')
-PolicyOverride = Enum('forwarded', 'sampled_out', 'trusted_forwarder', 'other', name='policy_override')
+PolicyOverride = Enum(
-SPFResultType = Enum('none', 'neutral', 'pass', 'fail', 'softfail', 'temperror', 'permerror', name='spf_result')
+    'forwarded', 'sampled_out', 'trusted_forwarder',
-DKIMResultType = Enum('none', 'pass', 'fail', 'policy', 'neutral', 'temperror', 'permerror', name='dkim_result')
+    'mailing_list', 'local_policy', 'other',
+    name='policy_override'
+)
+SPFResultType = Enum(
+    'none', 'neutral', 'pass', 'fail', 'softfail', 'temperror', 'permerror',
+    name='spf_result'
+)
+DKIMResultType = Enum(
+    'none', 'pass', 'fail', 'policy', 'neutral', 'temperror', 'permerror',
+    name='dkim_result'
+)
 
 class INET(satypes.TypeDecorator):
     impl = satypes.CHAR
@@ -50,15 +61,23 @@ class Report(DeclarativeBase):
     adkim = Column(Alignment, nullable=False)
     aspf = Column(Alignment, nullable=False)
     p = Column(Disposition, nullable=False)
-    sp = Column(Disposition, nullable=False)
+    sp = Column(Disposition)
     pct = Column(Integer, CheckConstraint('pct >= 0 AND pct <= 100'), nullable=False)
 
+class ReportXML(DeclarativeBase):
+    __tablename__ = 'report_xml'
+    report_id = Column(Integer, ForeignKey(Report.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False, primary_key=True)
+    xml = Column(UnicodeText, nullable=False)
+    report = relationship(Report, backref=backref('original', uselist=False), uselist=False)
+
 class ReportError(DeclarativeBase):
     __tablename__ = 'report_errors'
     id = Column(Integer, primary_key=True)
     report_id = Column(Integer, ForeignKey(Report.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False)
     error = Column(String, nullable=False)
 
+    report = relationship(Report, backref='errors')
+
 class ReportRecord(DeclarativeBase):
     __tablename__ = 'report_records'
     id = Column(Integer, primary_key=True)
@@ -87,6 +106,7 @@ class DKIMResult(DeclarativeBase):
     id = Column(Integer, primary_key=True)
     record_id = Column(Integer, ForeignKey(ReportRecord.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False)
     domain = Column(String, nullable=False)
+    selector = Column(String, nullable=True)
     result = Column(DKIMResultType, nullable=False)
     human_result = Column(String, nullable=True)
 

setup.py

@@ -11,12 +11,15 @@ except ImportError:
 
 install_requires=[
     'lxml',
-    'sqlalchemy'
+    'sqlalchemy',
+    'transaction',
+    'zope.interface',
+    'zope.sqlalchemy',
 ]
 
 setup(
     name='DMARCReceiver',
-    version='1.0',
+    version='1.4',
     description='Receive DMARC reports',
     author='David Baer',
     author_email='david@amyanddavid.net',