commit 93ed219b2980d9846845bde76cd0cd10b71b54c4 Author: David Baer Date: Wed Jan 22 18:02:35 2020 -0500 Initial import diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7e842be --- /dev/null +++ b/.gitignore @@ -0,0 +1,109 @@ +#### joe made this: http://goel.io/joe + +#####=== Python ===##### + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + diff --git a/dmarcreceiver/__init__.py b/dmarcreceiver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dmarcreceiver/commands.py b/dmarcreceiver/commands.py new file mode 100644 index 0000000..cd8114a --- /dev/null +++ b/dmarcreceiver/commands.py @@ -0,0 +1,126 @@ + +import sys, os +import zipfile +from datetime import datetime +from email import message_from_file +from tempfile import TemporaryDirectory +from lxml.etree import parse as parse_xml +from dmarcreceiver.model import DBSession, Report, ReportError, ReportRecord, OverrideReason, DKIMResult, SPFResult, metadata, init_model +import transaction + +from .util import sendmail +from .config import config + +def parse_metadata(tree): + org_name, = tree.xpath('org_name/text()') + email, = tree.xpath('email/text()') + extra_contact_info = tree.xpath('extra_contact_info/text()') + if len(extra_contact_info) == 0: + extra_contact_info = None + else: + extra_contact_info = extra_contact_info[0] + report_id, = tree.xpath('report_id/text()') + date_begin, = tree.xpath('date_range/begin/text()') + date_begin = datetime.fromtimestamp(int(date_begin)) + date_end, = tree.xpath('date_range/end/text()') + date_end = datetime.fromtimestamp(int(date_end)) + return Report( + org_name=org_name, + email=email, + extra_contact_info=extra_contact_info, + report_id=report_id, + date_begin=date_begin, + date_end=date_end + ) + +def scoop_elements(obj, tree, *elements): + for elt in elements: + try: + value, = tree.xpath('./' + elt + '/text()') + except ValueError: + value = None + else: + if all([ c.isdigit() for c in value ]): + value = int(value) + setattr(obj, elt, value) + +def parse_report(f): + tree = parse_xml(f) + + metadata_tree, = tree.xpath('/feedback/report_metadata') + report = parse_metadata(metadata_tree) + scoop_elements(report, tree.xpath('/feedback/policy_published')[0], 'domain', 'adkim', 'aspf', 'p', 'sp', 'pct') + for record_node in tree.xpath('/feedback/record'): + record = ReportRecord() + scoop_elements(record, record_node.xpath('./row')[0], 'source_ip', 'count') + scoop_elements(record, record_node.xpath('./row/policy_evaluated')[0], 'disposition', 'dkim', 'spf') + for reason_node in record_node.xpath('./row/policy_evaluated/reason'): + reason = OverrideReason() + reason.policy_override_type, = reason_node.xpath('./type/text()') + comment = reason_node.xpath('./comment/text()') + if len(comment) > 0: + reason.comment = comment[0] + record.override_reasons.append(reason) + scoop_elements(record, record_node.xpath('./identifiers')[0], 'envelope_to', 'header_from') + for dkim_node in record_node.xpath('./auth_results/dkim'): + dkim = DKIMResult() + scoop_elements(dkim, dkim_node, 'domain', 'result', 'human_result') + t = dkim_node.xpath('./selector/text()') + if len(t) > 0: + dkim.human_result = t[0] + record.dkim_results.append(dkim) + for spf_node in record_node.xpath('./auth_results/spf'): + spf = SPFResult() + scoop_elements(spf, spf_node, 'domain', 'result') + record.spf_results.append(spf) + report.records.append(record) + DBSession.add(report) + transaction.commit() + +def read_config_if_present(args): + if args.config_file: + if os.access(args.config_file, os.R_OK): + config.read_file(open(args.config_file, 'rt')) + +def receive_report(args): + read_config_if_present(args) + init_model() + + # read email message from stdin + msg = message_from_file(sys.stdin) + + # check for zip file + content_type = msg['content-type'] + if content_type.find(';') != -1: + content_type = content_type.split(';',1)[0] + + if content_type != 'application/zip': + # not a zip file - bounce to postmaster + bounce_address = config.get('bounce_address') + if args.bounce_address: + bounce_address = args.bounce_address + if bounce_address: + msg['To'] = bounce_address + sendmail(msg) + return + + with TemporaryDirectory( + prefix=os.path.splitext(os.path.basename(sys.argv[0]))[0] + '-' + ) as tempdir: + filename = msg.get_filename() + if filename is None: + filename = 'report.zip' + fn = os.path.join(tempdir, filename) + with open(fn, 'wb') as f: + f.write(msg.get_payload(decode=True)) + with zipfile.ZipFile(fn, 'r') as z: + namelist = z.namelist() + report_fn = os.path.join(tempdir, os.path.basename(namelist[0])) + z.extract(namelist[0], path=tempdir) + with open(report_fn, 'rb') as f: + parse_report(f) + +def init(args): + read_config_if_present(args) + init_model() + metadata.create_all() diff --git a/dmarcreceiver/config.py b/dmarcreceiver/config.py new file mode 100644 index 0000000..426a011 --- /dev/null +++ b/dmarcreceiver/config.py @@ -0,0 +1,15 @@ +class _Config(dict): + def read_file(self, f): + for line in f: + line = line.strip() + if line.find('#') != -1: + line = line[:line.find('#')] + if line.find('=') != -1: + name, value = line.split('=', 1) + name = name.strip() + value = value.strip() + self[name] = value + +config = _Config() + +__all__ = [ 'config' ] diff --git a/dmarcreceiver/model.py b/dmarcreceiver/model.py new file mode 100644 index 0000000..f74e801 --- /dev/null +++ b/dmarcreceiver/model.py @@ -0,0 +1,107 @@ +from zope.sqlalchemy import ZopeTransactionExtension +from sqlalchemy import Column, Integer, String, Unicode, Enum, CheckConstraint, ForeignKey, DateTime, create_engine +from sqlalchemy.orm import scoped_session, sessionmaker, relationship +import sqlalchemy.types as satypes +import sqlalchemy.dialects.postgresql as dpg +from sqlalchemy.ext.declarative import declarative_base + +maker = sessionmaker(autoflush=True, autocommit=False, + extension=ZopeTransactionExtension()) +DBSession = scoped_session(maker) + +DeclarativeBase = declarative_base() +metadata = DeclarativeBase.metadata + +def init_model(): + from sqlalchemy import create_engine + from dmarcreceiver.config import config + + engine = create_engine(config['db_uri']) + DBSession.configure(bind=engine) + metadata.bind = engine + return DBSession + +Alignment = Enum('r', 's') +Disposition = Enum('none', 'quarantine', 'reject') +DMARCResult = Enum('pass', 'fail') +PolicyOverride = Enum('forwarded', 'sampled_out', 'trusted_forwarder', 'other') +SPFResultType = Enum('none', 'neutral', 'pass', 'fail', 'softfail', 'temperror', 'permerror') +DKIMResultType = Enum('none', 'pass', 'fail', 'policy', 'neutral', 'temperror', 'permerror') + +class INET(satypes.TypeDecorator): + impl = satypes.CHAR + + def load_dialect_impl(self, dialect): + if dialect.name == 'postgresql': + return dialect.type_descriptor(dpg.INET()) + else: + return dialect.type_descriptor(satypes.CHAR(40)) + +class Report(DeclarativeBase): + __tablename__ = 'reports' + id = Column(Integer, primary_key=True) + org_name = Column(String, nullable=False) + email = Column(String, nullable=False) + extra_contact_info = Column(String, nullable=True) + report_id = Column(String, nullable=False) + date_begin = Column(DateTime, nullable=False) + date_end = Column(DateTime, nullable=False) + domain = Column(String, nullable=False) + adkim = Column(Alignment, nullable=False) + aspf = Column(Alignment, nullable=False) + p = Column(Disposition, nullable=False) + sp = Column(Disposition, nullable=False) + pct = Column(Integer, CheckConstraint('pct >= 0 AND pct <= 100'), nullable=False) + +class ReportError(DeclarativeBase): + __tablename__ = 'report_errors' + id = Column(Integer, primary_key=True) + report_id = Column(Integer, ForeignKey(Report.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False) + error = Column(String, nullable=False) + +class ReportRecord(DeclarativeBase): + __tablename__ = 'report_records' + id = Column(Integer, primary_key=True) + report_id = Column(Integer, ForeignKey(Report.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False) + source_ip = Column(INET, nullable=False) + count = Column(Integer, nullable=False, default=0) + disposition = Column(Disposition, nullable=True) + dkim = Column(DMARCResult, nullable=True) + spf = Column(DMARCResult, nullable=True) + envelope_to = Column(String, nullable=True) + header_from = Column(String, nullable=False) + + report = relationship(Report, backref='records') + +class OverrideReason(DeclarativeBase): + __tablename__ = 'report_record_override_reasons' + id = Column(Integer, primary_key=True) + record_id = Column(Integer, ForeignKey(ReportRecord.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False) + policy_override_type = Column(PolicyOverride, nullable=False) + comment = Column(String, nullable=True) + + record = relationship(ReportRecord, backref='override_reasons') + +class DKIMResult(DeclarativeBase): + __tablename__ = 'report_record_dkim_results' + id = Column(Integer, primary_key=True) + record_id = Column(Integer, ForeignKey(ReportRecord.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False) + domain = Column(String, nullable=False) + result = Column(DKIMResultType, nullable=False) + human_result = Column(String, nullable=True) + + record = relationship(ReportRecord, backref='dkim_results') + +class SPFResult(DeclarativeBase): + __tablename__ = 'report_record_spf_results' + id = Column(Integer, primary_key=True) + record_id = Column(Integer, ForeignKey(ReportRecord.id, onupdate='CASCADE', ondelete='CASCADE'), nullable=False) + domain = Column(String, nullable=False) + result = Column(SPFResultType, nullable=False) + + record = relationship(ReportRecord, backref='spf_results') + +if __name__ == '__main__': + engine = create_engine('sqlite:///data.db') + metadata.bind = engine + DBSession.configure(bind=engine) diff --git a/dmarcreceiver/util.py b/dmarcreceiver/util.py new file mode 100644 index 0000000..335847d --- /dev/null +++ b/dmarcreceiver/util.py @@ -0,0 +1,16 @@ +import os +import subprocess + +def sendmail(msg): + sendmail_executable = None + for pth in ( '/usr/sbin', '/usr/bin', '/usr/local/sbin', '/usr/local/bin', '/sbin', '/bin'): + if os.access(os.path.join(pth, 'sendmail'), os.X_OK): + sendmail_executable = os.path.join(pth, 'sendmail') + break + if sendmail_executable is None: + raise FileNotFoundError('Could not find sendmail executable') + + pipe = subprocess.Popen([ sendmail_executable, '-t' ], stdin=subprocess.PIPE) + stdout, stderr = pipe.communicate(msg.as_bytes) + return pipe.wait() + diff --git a/scripts/dmarc-receive b/scripts/dmarc-receive new file mode 100755 index 0000000..69d5fcb --- /dev/null +++ b/scripts/dmarc-receive @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +import argparse +from dmarcreceiver.commands import receive_report, init + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-b', dest='bounce_address', help='Forwarding address for messages that are not DMARC reports.') + parser.add_argument('-f', dest='config_file', default='/etc/dmarc-receive.conf', help='Configuration file') + parser.add_argument('-i', action='store_true', dest='init', default=False, help='Initialize database') + args = parser.parse_args() + if args.init: + init(args) + else: + receive_report(args) + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d10d14a --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +import sys + +try: + from setuptools import setup, find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + from setuptools import setup, find_packages + +install_requires=[ + 'lxml', + 'sqlalchemy' +] + +setup( + name='DMARCReceiver', + version='0.9', + description='Receive DMARC reports', + author='David Baer', + author_email='david@amyanddavid.net', + packages=find_packages(), + scripts=['scripts/dmarc-receive'], + install_requires=install_requires, + include_package_data=True, + zip_safe=True +)