Initial import

.gitignore

@@ -0,0 +1,109 @@
+#### joe made this: http://goel.io/joe
+
+#####=== Python ===#####
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+

dmarcreceiver/commands.py

@@ -0,0 +1,126 @@
+
+import sys, os
+import zipfile
+from datetime import datetime
+from email import message_from_file
+from tempfile import TemporaryDirectory
+from lxml.etree import parse as parse_xml
+from dmarcreceiver.model import DBSession, Report, ReportError, ReportRecord, OverrideReason, DKIMResult, SPFResult, metadata, init_model
+import transaction
+
+from .util import sendmail
+from .config import config
+
+def parse_metadata(tree):
+    org_name, = tree.xpath('org_name/text()')
+    email, = tree.xpath('email/text()')
+    extra_contact_info = tree.xpath('extra_contact_info/text()')
+    if len(extra_contact_info) == 0:
+        extra_contact_info = None
+    else:
+        extra_contact_info = extra_contact_info[0]
+    report_id, = tree.xpath('report_id/text()')
+    date_begin, = tree.xpath('date_range/begin/text()')
+    date_begin = datetime.fromtimestamp(int(date_begin))
+    date_end, = tree.xpath('date_range/end/text()')
+    date_end = datetime.fromtimestamp(int(date_end))
+    return Report(
+        org_name=org_name,
+        email=email,
+        extra_contact_info=extra_contact_info,
+        report_id=report_id,
+        date_begin=date_begin,
+        date_end=date_end
+    )
+
+def scoop_elements(obj, tree, *elements):
+    for elt in elements:
+        try:
+            value, = tree.xpath('./' + elt + '/text()')
+        except ValueError:
+            value = None
+        else:
+            if all([ c.isdigit() for c in value ]):
+                value = int(value)
+        setattr(obj, elt, value)
+
+def parse_report(f):
+    tree = parse_xml(f)
+
+    metadata_tree, = tree.xpath('/feedback/report_metadata')
+    report = parse_metadata(metadata_tree)
+    scoop_elements(report, tree.xpath('/feedback/policy_published')[0], 'domain', 'adkim', 'aspf', 'p', 'sp', 'pct')
+    for record_node in tree.xpath('/feedback/record'):
+        record = ReportRecord()
+        scoop_elements(record, record_node.xpath('./row')[0], 'source_ip', 'count')
+        scoop_elements(record, record_node.xpath('./row/policy_evaluated')[0], 'disposition', 'dkim', 'spf')
+        for reason_node in record_node.xpath('./row/policy_evaluated/reason'):
+            reason = OverrideReason()
+            reason.policy_override_type, = reason_node.xpath('./type/text()')
+            comment = reason_node.xpath('./comment/text()')
+            if len(comment) > 0:
+                reason.comment = comment[0]
+            record.override_reasons.append(reason)
+        scoop_elements(record, record_node.xpath('./identifiers')[0], 'envelope_to', 'header_from')
+        for dkim_node in record_node.xpath('./auth_results/dkim'):
+            dkim = DKIMResult()
+            scoop_elements(dkim, dkim_node, 'domain', 'result', 'human_result')
+            t = dkim_node.xpath('./selector/text()')
+            if len(t) > 0:
+                dkim.human_result = t[0]
+            record.dkim_results.append(dkim)
+        for spf_node in record_node.xpath('./auth_results/spf'):
+            spf = SPFResult()
+            scoop_elements(spf, spf_node, 'domain', 'result')
+            record.spf_results.append(spf)
+        report.records.append(record)
+    DBSession.add(report)
+    transaction.commit()
+
+def read_config_if_present(args):
+    if args.config_file:
+        if os.access(args.config_file, os.R_OK):
+            config.read_file(open(args.config_file, 'rt'))
+
+def receive_report(args):
+    read_config_if_present(args)
+    init_model()
+
+    # read email message from stdin
+    msg = message_from_file(sys.stdin)
+
+    # check for zip file
+    content_type = msg['content-type']
+    if content_type.find(';') != -1:
+        content_type = content_type.split(';',1)[0]
+
+    if content_type != 'application/zip':
+        # not a zip file - bounce to postmaster
+        bounce_address = config.get('bounce_address')
+        if args.bounce_address:
+            bounce_address = args.bounce_address
+        if bounce_address:
+            msg['To'] = bounce_address
+            sendmail(msg)
+        return
+
+    with TemporaryDirectory(
+        prefix=os.path.splitext(os.path.basename(sys.argv[0]))[0] + '-'
+    ) as tempdir:
+        filename = msg.get_filename()
+        if filename is None:
+            filename = 'report.zip'
+        fn = os.path.join(tempdir, filename)
+        with open(fn, 'wb') as f:
+            f.write(msg.get_payload(decode=True))
+        with zipfile.ZipFile(fn, 'r') as z:
+            namelist = z.namelist()
+            report_fn = os.path.join(tempdir, os.path.basename(namelist[0]))
+            z.extract(namelist[0], path=tempdir)
+        with open(report_fn, 'rb') as f:
+            parse_report(f)
+
+def init(args):
+    read_config_if_present(args)
+    init_model()
+    metadata.create_all()

dmarcreceiver/config.py

@@ -0,0 +1,15 @@
+class _Config(dict):
+    def read_file(self, f):
+        for line in f:
+            line = line.strip()
+            if line.find('#') != -1:
+                line = line[:line.find('#')]
+            if line.find('=') != -1:
+                name, value = line.split('=', 1)
+                name = name.strip()
+                value = value.strip()
+                self[name] = value
+
+config = _Config()
+
+__all__ = [ 'config' ]

dmarcreceiver/model.py

@@ -0,0 +1,107 @@
+from zope.sqlalchemy import ZopeTransactionExtension
+from sqlalchemy import Column, Integer, String, Unicode, Enum, CheckConstraint, ForeignKey, DateTime, create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker, relationship
+import sqlalchemy.types as satypes
+import sqlalchemy.dialects.postgresql as dpg
+from sqlalchemy.ext.declarative import declarative_base
+
+maker = sessionmaker(autoflush=True, autocommit=False,
+                     extension=ZopeTransactionExtension())
+DBSession = scoped_session(maker)
+
+DeclarativeBase = declarative_base()
+metadata = DeclarativeBase.metadata
+
+def init_model():
+    from sqlalchemy import create_engine
+    from dmarcreceiver.config import config
+
+    engine = create_engine(config['db_uri'])
+    DBSession.configure(bind=engine)
+    metadata.bind = engine
+    return DBSession
+
+Alignment = Enum('r', 's')
+Disposition = Enum('none', 'quarantine', 'reject')
+DMARCResult = Enum('pass', 'fail')
+PolicyOverride = Enum('forwarded', 'sampled_out', 'trusted_forwarder', 'other')
+SPFResultType = Enum('none', 'neutral', 'pass', 'fail', 'softfail', 'temperror', 'permerror')
+DKIMResultType = Enum('none', 'pass', 'fail', 'policy', 'neutral', 'temperror', 'permerror')
+
+class INET(satypes.TypeDecorator):
+    impl = satypes.CHAR
+
+    def load_dialect_impl(self, dialect):
+        if dialect.name == 'postgresql':
+            return dialect.type_descriptor(dpg.INET())
+        else:
+            return dialect.type_descriptor(satypes.CHAR(40))
+
+class Report(DeclarativeBase):
+    __tablename__ = 'reports'
+    id = Column(Integer, primary_key=True)
+    org_name = Column(String, nullable=False)
+    email = Column(String, nullable=False)
+    extra_contact_info = Column(String, nullable=True)
+    report_id = Column(String, nullable=False)
+    date_begin = Column(DateTime, nullable=False)
+    date_end = Column(DateTime, nullable=False)
+    domain = Column(String, nullable=False)
+    adkim = Column(Alignment, nullable=False)
+    aspf = Column(Alignment, nullable=False)
+    p = Column(Disposition, nullable=False)
+    sp = Column(Disposition, nullable=False)
+    pct = Column(Integer, CheckConstraint('pct >= 0 AND pct <= 100'), nullable=False)
+
+class ReportError(DeclarativeBase):
+    __tablename__ = 'report_errors'
+    id = Column(Integer, primary_key=True)
+    report_id = Column(Integer, ForeignKey(Report.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False)
+    error = Column(String, nullable=False)
+
+class ReportRecord(DeclarativeBase):
+    __tablename__ = 'report_records'
+    id = Column(Integer, primary_key=True)
+    report_id = Column(Integer, ForeignKey(Report.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False)
+    source_ip = Column(INET, nullable=False)
+    count = Column(Integer, nullable=False, default=0)
+    disposition = Column(Disposition, nullable=True)
+    dkim = Column(DMARCResult, nullable=True)
+    spf = Column(DMARCResult, nullable=True)
+    envelope_to = Column(String, nullable=True)
+    header_from = Column(String, nullable=False)
+
+    report = relationship(Report, backref='records')
+
+class OverrideReason(DeclarativeBase):
+    __tablename__ = 'report_record_override_reasons'
+    id = Column(Integer, primary_key=True)
+    record_id = Column(Integer, ForeignKey(ReportRecord.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False)
+    policy_override_type = Column(PolicyOverride, nullable=False)
+    comment = Column(String, nullable=True)
+
+    record = relationship(ReportRecord, backref='override_reasons')
+
+class DKIMResult(DeclarativeBase):
+    __tablename__ = 'report_record_dkim_results'
+    id = Column(Integer, primary_key=True)
+    record_id = Column(Integer, ForeignKey(ReportRecord.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False)
+    domain = Column(String, nullable=False)
+    result = Column(DKIMResultType, nullable=False)
+    human_result = Column(String, nullable=True)
+
+    record = relationship(ReportRecord, backref='dkim_results')
+
+class SPFResult(DeclarativeBase):
+    __tablename__ = 'report_record_spf_results'
+    id = Column(Integer, primary_key=True)
+    record_id = Column(Integer, ForeignKey(ReportRecord.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False)
+    domain = Column(String, nullable=False)
+    result = Column(SPFResultType, nullable=False)
+
+    record = relationship(ReportRecord, backref='spf_results')
+
+if __name__ == '__main__':
+    engine = create_engine('sqlite:///data.db')
+    metadata.bind = engine
+    DBSession.configure(bind=engine)

dmarcreceiver/util.py

@@ -0,0 +1,16 @@
+import os
+import subprocess
+
+def sendmail(msg):
+    sendmail_executable = None
+    for pth in ( '/usr/sbin', '/usr/bin', '/usr/local/sbin', '/usr/local/bin', '/sbin', '/bin'):
+        if os.access(os.path.join(pth, 'sendmail'), os.X_OK):
+            sendmail_executable = os.path.join(pth, 'sendmail')
+            break
+    if sendmail_executable is None:
+        raise FileNotFoundError('Could not find sendmail executable')
+
+    pipe = subprocess.Popen([ sendmail_executable, '-t' ], stdin=subprocess.PIPE)
+    stdout, stderr = pipe.communicate(msg.as_bytes)
+    return pipe.wait()
+

scripts/dmarc-receive

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+import argparse
+from dmarcreceiver.commands import receive_report, init
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-b', dest='bounce_address', help='Forwarding address for messages that are not DMARC reports.')
+    parser.add_argument('-f', dest='config_file', default='/etc/dmarc-receive.conf', help='Configuration file')
+    parser.add_argument('-i', action='store_true', dest='init', default=False, help='Initialize database')
+    args = parser.parse_args()
+    if args.init:
+        init(args)
+    else:
+        receive_report(args)
+

setup.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+import sys
+
+try:
+    from setuptools import setup, find_packages
+except ImportError:
+    from ez_setup import use_setuptools
+    use_setuptools()
+    from setuptools import setup, find_packages
+
+install_requires=[
+    'lxml',
+    'sqlalchemy'
+]
+
+setup(
+    name='DMARCReceiver',
+    version='0.9',
+    description='Receive DMARC reports',
+    author='David Baer',
+    author_email='david@amyanddavid.net',
+    packages=find_packages(),
+    scripts=['scripts/dmarc-receive'],
+    install_requires=install_requires,
+    include_package_data=True,
+    zip_safe=True
+)