Skip to content
Snippets Groups Projects
bms.py 65.2 KiB
Newer Older
  • Learn to ignore specific revisions
  • #!/usr/bin/env python
    
    # A simple milter that has grown quite a bit.
    
    # $Log$
    
    # Revision 1.109  2007/06/23 20:53:05  customdesigned
    # Ban IPs based on too many invalid recipients in a connection.  Requires
    # configuring check_user.  Tighten HELO best_guess policy.
    #
    
    # Revision 1.108  2007/04/19 16:02:43  customdesigned
    # Do not process valid SRS recipients as delayed_failure.
    #
    
    # Revision 1.107  2007/04/15 01:01:13  customdesigned
    # Ban ips with too many bad rcpts on a connection.
    #
    
    # Revision 1.105  2007/04/13 17:20:09  customdesigned
    # Check access_file at startup.  Compress rcpt to log.
    #
    
    # Revision 1.104  2007/04/05 17:59:07  customdesigned
    # Stop querying gossip server twice.
    #
    
    # Revision 1.103  2007/04/02 18:37:25  customdesigned
    # Don't disable gossip for temporary error.
    #
    
    # Revision 1.102  2007/03/30 18:13:41  customdesigned
    # Report bestguess and helo-spf as key-value pairs in Received-SPF
    # instead of in their own headers.
    #
    
    # Revision 1.101  2007/03/29 03:06:10  customdesigned
    # Don't count DSN and unqualified MAIL FROM as internal_domain.
    #
    
    # Revision 1.100  2007/03/24 00:30:24  customdesigned
    # Do not CBV for internal domains.
    #
    
    # Revision 1.99  2007/03/23 22:39:10  customdesigned
    # Get SMTP-Auth policy from access_file.
    #
    
    # Revision 1.98  2007/03/21 04:02:13  customdesigned
    # Properly log From: and Sender:
    #
    
    # Revision 1.97  2007/03/18 02:32:21  customdesigned
    # Gossip configuration options: client or standalone with optional peers.
    #
    
    # Revision 1.96  2007/03/17 21:22:48  customdesigned
    # New delayed DSN pattern.  Retab (expandtab).
    #
    
    # Revision 1.95  2007/03/03 19:18:57  customdesigned
    # Fix continuing findsrs when srs.reverse fails.
    #
    
    # Revision 1.94  2007/03/03 18:46:26  customdesigned
    # Improve delayed failure detection.
    #
    
    # Revision 1.93  2007/02/07 23:21:26  customdesigned
    # Use re for auto-reply recognition.
    #
    
    # Revision 1.92  2007/01/26 03:47:23  customdesigned
    # Handle null in header value.
    #
    
    # Revision 1.91  2007/01/25 22:47:25  customdesigned
    # Persist blacklisting from delayed DSNs.
    #
    
    # Revision 1.90  2007/01/23 19:46:20  customdesigned
    # Add private relay.
    #
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    # Revision 1.89  2007/01/22 02:46:01  customdesigned
    # Convert tabs to spaces.
    #
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    # Revision 1.88  2007/01/19 23:31:38  customdesigned
    # Move parse_header to Milter.utils.
    # Test case for delayed DSN parsing.
    # Fix plock when source missing or cannot set owner/group.
    #
    
    # Revision 1.87  2007/01/18 16:48:44  customdesigned
    # Doc update.
    # Parse From header for delayed failure detection.
    # Don't check reputation of trusted host.
    # Track IP reputation only when missing PTR.
    #
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    # Revision 1.86  2007/01/16 05:17:29  customdesigned
    # REJECT after data for blacklisted emails - so in case of mistakes, a
    # legitimate sender will know what happened.
    #
    
    # Revision 1.85  2007/01/11 04:31:26  customdesigned
    # Negative feedback for bad headers.  Purge cache logs on startup.
    #
    
    # Revision 1.84  2007/01/10 04:44:25  customdesigned
    # Documentation updates.
    #
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    # Revision 1.83  2007/01/08 23:20:54  customdesigned
    # Get user feedback.
    #
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    # Revision 1.82  2007/01/06 04:21:30  customdesigned
    # Add config file to spfmilter
    #
    
    # Revision 1.81  2007/01/05 23:33:55  customdesigned
    # Make blacklist an AddrCache
    #
    
    # Revision 1.80  2007/01/05 23:12:12  customdesigned
    # Move parse_addr, iniplist, ip4re to Milter.utils
    #
    
    # Revision 1.79  2007/01/05 21:25:40  customdesigned
    # Move AddrCache to Milter package.
    #
    
    # Revision 1.78  2007/01/04 18:01:10  customdesigned
    # Do plain CBV when template missing.
    #
    
    # Revision 1.77  2006/12/31 03:07:20  customdesigned
    # Use HELO identity if good when MAILFROM is bad.
    #
    
    # Revision 1.76  2006/12/30 18:58:53  customdesigned
    # Skip reputation/whitelist/blacklist when rejecting on SPF.  Add X-Hello-SPF.
    #
    
    # Revision 1.75  2006/12/28 01:54:32  customdesigned
    # Reject on bad_reputation or blacklist and nodspam.  Match valid helo like
    # PTR for guessed SPF pass.
    #
    
    # Revision 1.74  2006/12/19 00:59:30  customdesigned
    # Add archive option to wiretap.
    #
    
    # Revision 1.73  2006/12/04 18:47:03  customdesigned
    # Reject multiple recipients to DSN.
    # Auto-disable gossip on DB error.
    #
    
    # Revision 1.72  2006/11/22 16:31:22  customdesigned
    # SRS domains were missing srs_reject check when SES was active.
    #
    
    # Revision 1.71  2006/11/22 01:03:28  customdesigned
    # Replace last use of deprecated rfc822 module.
    #
    
    # Revision 1.70  2006/11/21 18:45:49  customdesigned
    # Update a use of deprecated rfc822.  Recognize report-type=delivery-status
    
    # Author: Stuart D. Gathman <stuart@bmsi.com>
    
    # Copyright 2001,2002,2003,2004,2005 Business Management Systems, Inc.
    # This code is under the GNU General Public License.  See COPYING for details.
    
    
    import sys
    import os
    import StringIO
    import mime
    import email.Errors
    import Milter
    import tempfile
    import time
    
    import socket
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    import re
    
    import shutil
    
    import Milter.dsn as dsn
    from Milter.dynip import is_dynip as dynip
    
    from Milter.utils import iniplist,parse_addr,parse_header,ip4re,addr2bin
    
    from Milter.config import MilterConfigParser
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    
    
    from fnmatch import fnmatchcase
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    from email.Utils import getaddresses,parseaddr
    
    # Import gossip if available
    try:
      import gossip
    
      import gossip.client
      import gossip.server
    
    except: gossip = None
    
    
    # Import pysrs if available
    try:
      import SRS
      srsre = re.compile(r'^SRS[01][+-=]',re.IGNORECASE)
    except: SRS = None
    
    try:
      import SES
    except: SES = None
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    
    # Import spf if available
    
    try: import spf
    except: spf = None
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    
    
    # Sometimes, MTAs reply to our DSN.  We recognize this type of reply/DSN
    # and check for the original recipient SRS encoded in Message-ID.
    # If found, we blacklist that recipient.
    
    _subjpats = (
    
     r'^failure notice',
    
     r'^returned mail',
    
     r'\bdelivery\b.*\bfail',
     r'\bdelivery problem',
     r'\bnot\s+be\s+delivered',
    
     r'^failed', r'^mail failed',
    
     r'^fallo en la entrega',
    
    refaildsn = re.compile('|'.join(_subjpats),re.IGNORECASE)
    
    # We don't want to whitelist recipients of Autoreplys and other robots.
    # There doesn't seem to be a foolproof way to recognize these, so
    # we use this heuristic.  The worst that can happen is someone won't get
    # whitelisted when they should, or we'll whitelist some spammer for a while.
    _autopats = (
     r'^read:',
     r'\bautoreply:\b',
     r'^return receipt',
     r'^Your message\b.*\bawaits moderator approval'
    )
    reautoreply = re.compile('|'.join(_autopats),re.IGNORECASE)
    
    
    # Thanks to Chris Liechti for config parsing suggestions
    
    # Global configuration defaults suitable for test framework.
    socketname = "/tmp/pythonsock"
    reject_virus_from = ()
    wiretap_users = {}
    discard_users = {}
    wiretap_dest = None
    
    mail_archive = None
    _archive_lock = None
    
    blind_wiretap = True
    check_user = {}
    block_forward = {}
    hide_path = ()
    log_headers = False
    block_chinese = False
    
    case_sensitive_localpart = False
    
    spam_words = ()
    porn_words = ()
    
    banned_exts = mime.extlist.split(',')
    scan_zip = False
    
    scan_html = True
    scan_rfc822 = True
    internal_connect = ()
    trusted_relay = ()
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    private_relay = ()
    
    internal_domains = ()
    
    hello_blacklist = ()
    smart_alias = {}
    dspam_dict = None
    dspam_users = {}
    dspam_userdir = None
    dspam_exempt = {}
    dspam_whitelist = {}
    
    whitelist_senders = {}
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    dspam_screener = ()
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    dspam_internal = True   # True if internal mail should be dspammed
    
    dspam_reject = ()
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    dspam_sizelimit = 180000
    
    srs = None
    
    srs_reject_spoofed = False
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    srs_domain = None
    
    spf_reject_neutral = ()
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    spf_accept_softfail = ()
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    spf_best_guess = False
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    spf_reject_noptr = False
    
    supply_sender = False
    access_file = None
    
    Stuart Gathman's avatar
    Stuart Gathman committed
    timeout = 600
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            stream=sys.stdout,
            level=logging.INFO,
            format='%(asctime)s %(message)s',
            datefmt='%Y%b%d %H:%M:%S'
    
    def read_config(list):
      cp = MilterConfigParser({
        'tempdir': "/var/log/milter/save",
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        'socket': "/var/run/milter/pythonsock",
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        'timeout': '600',
    
        'scan_html': 'no',
        'scan_rfc822': 'yes',
    
        'block_chinese': 'no',
        'log_headers': 'no',
        'blind_wiretap': 'yes',
        'maxage': '8',
        'hashlength': '8',
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        'reject_spoofed': 'no',
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        'reject_noptr': 'no',
    
        'supply_sender': 'no',
    
        'best_guess': 'no',
    
        'dspam_internal': 'yes',
        'case_sensitive_localpart': 'no'
    
      })
      cp.read(list)
    
      tempfile.tempdir = cp.get('milter','tempdir')
    
      global socketname, timeout, check_user, log_headers
      global internal_connect, internal_domains, trusted_relay, hello_blacklist
    
    Stuart Gathman's avatar
    Stuart Gathman committed
      global case_sensitive_localpart, private_relay
    
      socketname = cp.get('milter','socket')
    
    Stuart Gathman's avatar
    Stuart Gathman committed
      timeout = cp.getint('milter','timeout')
    
      check_user = cp.getaddrset('milter','check_user')
    
      log_headers = cp.getboolean('milter','log_headers')
      internal_connect = cp.getlist('milter','internal_connect')
      internal_domains = cp.getlist('milter','internal_domains')
      trusted_relay = cp.getlist('milter','trusted_relay')
    
    Stuart Gathman's avatar
    Stuart Gathman committed
      private_relay = cp.getlist('milter','private_relay')
    
      hello_blacklist = cp.getlist('milter','hello_blacklist')
    
      case_sensitive_localpart = cp.getboolean('milter','case_sensitive_localpart')
    
    
      # defang section
      global scan_rfc822, scan_html, block_chinese, scan_zip, block_forward
      global banned_exts, porn_words, spam_words
      if cp.has_section('defang'):
        section = 'defang'
    
        # for backward compatibility,
        # banned extensions defaults to empty only when defang section exists
        banned_exts = cp.getlist(section,'banned_exts')
    
    Stuart Gathman's avatar
    Stuart Gathman committed
      else: # use milter section if no defang section for compatibility
    
        section = 'milter'
      scan_rfc822 = cp.getboolean(section,'scan_rfc822')
      scan_zip = cp.getboolean(section,'scan_zip')
      scan_html = cp.getboolean(section,'scan_html')
      block_chinese = cp.getboolean(section,'block_chinese')
      block_forward = cp.getaddrset(section,'block_forward')
      porn_words = cp.getlist(section,'porn_words')
      spam_words = cp.getlist(section,'spam_words')
    
      # scrub section
      global hide_path, reject_virus_from
    
      hide_path = cp.getlist('scrub','hide_path')
    
      reject_virus_from = cp.getlist('scrub','reject_virus_from')
    
      global blind_wiretap,wiretap_users,wiretap_dest,discard_users,mail_archive
    
      blind_wiretap = cp.getboolean('wiretap','blind')
      wiretap_users = cp.getaddrset('wiretap','users')
      discard_users = cp.getaddrset('wiretap','discard')
      wiretap_dest = cp.getdefault('wiretap','dest')
      if wiretap_dest: wiretap_dest = '<%s>' % wiretap_dest
    
      mail_archive = cp.getdefault('wiretap','archive')
    
      for sa,v in [
          (k,cp.get('wiretap',k)) for k in cp.getlist('wiretap','smart_alias')
        ] + (cp.has_section('smart_alias') and cp.items('smart_alias',True) or []):
        print sa,v
        sm = [q.strip() for q in v.split(',')]
    
        if len(sm) < 2:
    
          milter_log.warning('malformed smart alias: %s',sa)
    
          continue
        if len(sm) == 2: sm.append(sa)
    
        if case_sensitive_localpart:
          key = (sm[0],sm[1])
        else:
          key = (sm[0].lower(),sm[1].lower())
    
        smart_alias[key] = sm[2:]
    
    
      global dspam_dict, dspam_users, dspam_userdir, dspam_exempt, dspam_internal
    
      global dspam_screener,dspam_whitelist,dspam_reject,dspam_sizelimit
    
      global whitelist_senders
      whitelist_senders = cp.getaddrset('dspam','whitelist_senders')
    
      dspam_dict = cp.getdefault('dspam','dspam_dict')
      dspam_exempt = cp.getaddrset('dspam','dspam_exempt')
      dspam_whitelist = cp.getaddrset('dspam','dspam_whitelist')
      dspam_users = cp.getaddrdict('dspam','dspam_users')
      dspam_userdir = cp.getdefault('dspam','dspam_userdir')
    
    Stuart Gathman's avatar
    Stuart Gathman committed
      dspam_screener = cp.getlist('dspam','dspam_screener')
    
      dspam_reject = cp.getlist('dspam','dspam_reject')
    
      dspam_internal = cp.getboolean('dspam','dspam_internal')
    
      if cp.has_option('dspam','dspam_sizelimit'):
        dspam_sizelimit = cp.getint('dspam','dspam_sizelimit')
    
    
      # spf section
      global spf_reject_neutral,spf_best_guess,SRS,spf_reject_noptr
    
      global spf_accept_softfail,spf_accept_fail,supply_sender,access_file
    
      if spf:
        spf.DELEGATE = cp.getdefault('spf','delegate')
        spf_reject_neutral = cp.getlist('spf','reject_neutral')
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        spf_accept_softfail = cp.getlist('spf','accept_softfail')
    
        spf_accept_fail = cp.getlist('spf','accept_fail')
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        spf_best_guess = cp.getboolean('spf','best_guess')
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        spf_reject_noptr = cp.getboolean('spf','reject_noptr')
    
        supply_sender = cp.getboolean('spf','supply_sender')
        access_file = cp.getdefault('spf','access_file')
    
        trusted_forwarder = cp.getlist('spf','trusted_forwarder')
    
      srs_config = cp.getdefault('srs','config')
      if srs_config: cp.read([srs_config])
      srs_secret = cp.getdefault('srs','secret')
      if SRS and srs_secret:
    
        global ses,srs,srs_reject_spoofed,srs_domain,banned_users
    
        database = cp.getdefault('srs','database')
        srs_reject_spoofed = cp.getboolean('srs','reject_spoofed')
        maxage = cp.getint('srs','maxage')
        hashlength = cp.getint('srs','hashlength')
        separator = cp.getdefault('srs','separator','=')
        if database:
          import SRS.DB
          srs = SRS.DB.DB(database=database,secret=srs_secret,
            maxage=maxage,hashlength=hashlength,separator=separator)
        else:
          srs = SRS.Guarded.Guarded(secret=srs_secret,
            maxage=maxage,hashlength=hashlength,separator=separator)
    
        if SES:
          ses = SES.new(secret=srs_secret,expiration=maxage)
    
          srs_domain = set(cp.getlist('srs','ses'))
    
          srs_domain.update(cp.getlist('srs','srs'))
    
          srs_domain = set(cp.getlist('srs','srs'))
        srs_domain.update(cp.getlist('srs','sign'))
        srs_domain.add(cp.getdefault('srs','fwdomain'))
    
        banned_users = cp.getlist('srs','banned_users')
    
      if gossip:
        global gossip_node
        if cp.has_option('gossip','server'):
          server = cp.get('gossip','server')
          host,port = gossip.splitaddr(server)
          gossip_node = gossip.client.Gossip(host,port)
        else:
          gossip_node = gossip.server.Gossip('gossip4.db',1000)
          for p in cp.getlist('gossip','peers'):
            host,port = gossip.splitaddr(p)
            gossip_node.peers.append(gossip.server.Peer(host,port))
    
    
    def findsrs(fp):
      lastln = None
      for ln in fp:
        if lastln:
          if ln[0].isspace() and ln[0] != '\n':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            lastln += ln
            continue
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            name,val = lastln.rstrip().split(None,1)
            pos = val.find('<SRS')
            if pos >= 0:
    
              end = val.find('>',pos+4)
    
              return srs.reverse(val[pos+1:end])
          except: pass
    
        lnl = ln.lower()
        if lnl.startswith('action:'):
          if lnl.split()[-1] != 'failed': break
    
        for k in ('message-id:','x-mailer:','sender:','references:'):
    
          if lnl.startswith(k):
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            lastln = ln
            break
    
    class SPFPolicy(object):
    
      "Get SPF policy by result from sendmail style access file."
    
      def __init__(self,sender):
        self.sender = sender
        self.domain = sender.split('@')[-1].lower()
    
        if access_file:
          try: acf = anydbm.open(access_file,'r')
          except: acf = None
        else: acf = None
        self.acf = acf
    
    
      def close(self):
        if self.acf:
          self.acf.close()
    
    
      def getPolicy(self,pfx):
        acf = self.acf
        if not acf: return None
        try:
    
          return acf[pfx + self.sender]
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            return acf[pfx + self.domain]
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            try:
              return acf[pfx]
            except KeyError:
              return None
    
    
      def getFailPolicy(self):
    
        policy = self.getPolicy('spf-fail:')
    
        if not policy:
          if self.domain in spf_accept_fail:
            policy = 'CBV'
          else:
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            policy = 'REJECT'
    
        return policy
    
      def getNonePolicy(self):
    
        policy = self.getPolicy('spf-none:')
    
        if not policy:
          if spf_reject_noptr:
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            policy = 'REJECT'
    
          else:
            policy = 'CBV'
        return policy
    
      def getSoftfailPolicy(self):
    
        policy = self.getPolicy('spf-softfail:')
    
        if not policy:
          if self.domain in spf_accept_softfail:
            policy = 'OK'
          elif self.domain in spf_reject_neutral:
            policy = 'REJECT'
          else:
            policy = 'CBV'
        return policy
    
      def getNeutralPolicy(self):
    
        policy = self.getPolicy('spf-neutral:')
    
        if not policy:
          if self.domain in spf_reject_neutral:
            policy = 'REJECT'
          policy = 'OK'
        return policy
    
      def getPermErrorPolicy(self):
    
        policy = self.getPolicy('spf-permerror:')
    
        if not policy:
          policy = 'REJECT'
        return policy
    
      def getPassPolicy(self):
    
        policy = self.getPolicy('spf-pass:')
    
        if not policy:
          policy = 'OK'
        return policy
    
    
    from Milter.cache import AddrCache
    
    cbv_cache = AddrCache(renew=7)
    
    cbv_cache.load('send_dsn.log',age=30)
    
    auto_whitelist = AddrCache(renew=30)
    
    auto_whitelist.load('auto_whitelist.log',age=120)
    
    blacklist = AddrCache(renew=30)
    blacklist.load('blacklist.log',age=60)
    
    class bmsMilter(Milter.Milter):
    
    Stuart Gathman's avatar
    Stuart Gathman committed
      """Milter to replace attachments poisonous to Windows with a WARNING message,
         check SPF, and other anti-forgery features, and implement wiretapping
         and smart alias redirection."""
    
    
      def log(self,*msg):
    
        milter_log.info('[%d] %s',self.id,' '.join([str(m) for m in msg]))
    
    
      def __init__(self):
        self.tempname = None
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        self.mailfrom = None        # sender in SMTP form
        self.canon_from = None      # sender in end user form
    
        self.fp = None
        self.bodysize = 0
        self.id = Milter.uniqueID()
    
      # delrcpt can only be called from eom().  This accumulates recipient
      # changes which can then be applied by alter_recipients()
      def del_recipient(self,rcpt):
        rcpt = rcpt.lower()
        if not rcpt in self.discard_list:
          self.discard_list.append(rcpt)
    
      # addrcpt can only be called from eom().  This accumulates recipient
      # changes which can then be applied by alter_recipients()
      def add_recipient(self,rcpt):
        rcpt = rcpt.lower()
        if not rcpt in self.redirect_list:
          self.redirect_list.append(rcpt)
    
      # addheader can only be called from eom().  This accumulates added headers
      # which can then be applied by alter_headers()
    
      def add_header(self,name,val,idx=-1):
        self.new_headers.append((name,val,idx))
    
        self.log('%s: %s' % (name,val))
    
      def connect(self,hostname,unused,hostaddr):
        self.internal_connection = False
        self.trusted_relay = False
    
        # sometimes people put extra space in sendmail config, so we strip
        self.receiver = self.getsymval('j').strip()
    
        if hostaddr and len(hostaddr) > 0:
          ipaddr = hostaddr[0]
    
          if iniplist(ipaddr,internal_connect):
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.internal_connection = True
    
          if iniplist(ipaddr,trusted_relay):
            self.trusted_relay = True
    
        else: ipaddr = ''
        self.connectip = ipaddr
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        self.missing_ptr = dynip(hostname,self.connectip)
    
        if self.internal_connection:
          connecttype = 'INTERNAL'
        else:
          connecttype = 'EXTERNAL'
        if self.trusted_relay:
          connecttype += ' TRUSTED'
    
        if self.missing_ptr:
          connecttype += ' DYN'
    
        self.log("connect from %s at %s %s" % (hostname,hostaddr,connecttype))
    
        if addr2bin(ipaddr) in banned_ips:
          self.log("REJECT: BANNED IP")
          self.setreply('550','5.7.1', 'Banned for dictionary attacks')
          return Milter.REJECT
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        self.hello_name = None
        self.connecthost = hostname
    
        if hostname == 'localhost' and not ipaddr.startswith('127.') \
        or hostname == '.':
          self.log("REJECT: PTR is",hostname)
          self.setreply('550','5.7.1', '"%s" is not a reasonable PTR name'%hostname)
          return Milter.REJECT
    
        return Milter.CONTINUE
    
      def hello(self,hostname):
        self.hello_name = hostname
        self.log("hello from %s" % hostname)
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        if ip4re.match(hostname):
          self.log("REJECT: numeric hello name:",hostname)
          self.setreply('550','5.7.1','hello name cannot be numeric ip')
          return Milter.REJECT
    
        if not self.internal_connection and hostname in hello_blacklist:
          self.log("REJECT: spam from self:",hostname)
    
          self.setreply('550','5.7.1',
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            'Your mail server lies.  Its name is *not* %s.' % hostname)
    
          return Milter.REJECT
    
        if hostname == 'GC':
          n = gc.collect()
          self.log("gc:",n,' unreachable objects')
    
          self.log("auto-whitelist:",len(auto_whitelist),' entries')
          self.log("cbv_cache:",len(cbv_cache),' entries')
    
          self.setreply('550','5.7.1','%d unreachable objects'%n)
          return Milter.REJECT
    
        return Milter.CONTINUE
    
    
      def smart_alias(self,to):
        if smart_alias:
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            t = parse_addr(to)
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            t = parse_addr(to.lower())
    
          if len(t) == 2:
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            ct = '@'.join(t)
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            ct = t[0]
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            cf = self.canon_from
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            cf = self.canon_from.lower()
    
          cf0 = cf.split('@',1)
          if len(cf0) == 2:
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            cf0 = '@' + cf0[1]
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            cf0 = cf
    
          for key in ((cf,ct),(cf0,ct)):
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            if smart_alias.has_key(key):
              self.del_recipient(to)
              for t in smart_alias[key]:
                self.add_recipient('<%s>'%t)
    
      def offense(self,inc=1):
        self.offenses += inc
        if self.offenses > 3:
          try:
            ip = addr2bin(self.connectip)
            if ip not in banned_ips:
              banned_ips.add(ip)
              print >>open('banned_ips','a'),self.connectip
          except: pass
        return Milter.REJECT
    
    
      # multiple messages can be received on a single connection
      # envfrom (MAIL FROM in the SMTP protocol) seems to mark the start
      # of each message.
      def envfrom(self,f,*str):
        self.log("mail from",f,str)
        self.fp = StringIO.StringIO()
        self.tempname = None
        self.mailfrom = f
        self.forward = True
        self.bodysize = 0
        self.hidepath = False
        self.discard = False
        self.dspam = True
    
        self.whitelist = False
    
        self.blacklist = False
    
        self.reject_spam = True
        self.data_allowed = True
    
        self.delayed_failure = None
    
        self.trust_received = self.trusted_relay
    
        self.trust_spf = self.trusted_relay
    
        self.redirect_list = []
        self.discard_list = []
        self.new_headers = []
        self.recipients = []
    
        self.cbv_needed = None
    
        self.whitelist_sender = False
    
        self.postmaster_reply = False
    
        t = parse_addr(f)
        if len(t) == 2: t[1] = t[1].lower()
    
        self.canon_from = '@'.join(t)
    
        # Some braindead MTAs can't be relied upon to properly flag DSNs.
        # This heuristic tries to recognize such.
        self.is_bounce = (f == '<>' or t[0].lower() in banned_users
            #and t[1] == self.hello_name
        )
    
    
        # Check SMTP AUTH, also available:
    
        #   auth_authen  authenticated user
    
        #   auth_author  (ESMTP AUTH= param)
        #   auth_ssf     (connection security, 0 = unencrypted)
        #   auth_type    (authentication method, CRAM-MD5, DIGEST-MD5, PLAIN, etc)
    
        # cipher_bits  SSL encryption strength
        # cert_subject SSL cert subject
        # verify       SSL cert verified
    
    
        self.user = self.getsymval('{auth_authen}')
        if self.user:
    
          # Very simple SMTP AUTH policy by defaul:
          #   any successful authentication is considered INTERNAL
          # FIXME: configure allowed MAIL FROM by user
    
          self.internal_connection = True
    
          self.log(
            "SMTP AUTH:",self.user, self.getsymval('{auth_type}'),
            "sslbits =",self.getsymval('{cipher_bits}'),
            "ssf =",self.getsymval('{auth_ssf}'), "INTERNAL"
          )
          if self.getsymval('{verify}'):
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.log("SSL AUTH:",
              self.getsymval('{cert_subject}'),
              "verify =",self.getsymval('{verify}')
            )
    
        self.fp.write('From %s %s\n' % (self.canon_from,time.ctime()))
    
        self.internal_domain = False
    
        if len(t) == 2:
          user,domain = t
    
          for pat in internal_domains:
    
            if fnmatchcase(domain,pat):
              self.internal_domain = True
              break
    
          if self.internal_connection:
    
            if self.user:
              p = SPFPolicy('%s@%s'%(self.user,domain))
    
              policy = p.getPolicy('smtp-auth:')
    
            else:
              policy = None
            if policy:
              if policy != 'OK':
                self.log("REJECT: unauthorized user",self.user,
                    "at",self.connectip,"sending MAIL FROM",self.canon_from)
                self.setreply('550','5.7.1',
                  'SMTP user %s is not authorized to use MAIL FROM %s.' %
                  (self.user,self.canon_from)
                )
    
            elif internal_domains and not self.internal_domain:
              self.log("REJECT: zombie PC at ",self.connectip,
                  " sending MAIL FROM ",self.canon_from)
              self.setreply('550','5.7.1',
              'Your PC is using an unauthorized MAIL FROM.',
              'It is either badly misconfigured or controlled by organized crime.'
              )
              return Milter.REJECT
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            wl_users = whitelist_senders.get(domain,())
            if user in wl_users or '' in wl_users:
              self.whitelist_sender = True
              
    
          self.rejectvirus = domain in reject_virus_from
          if user in wiretap_users.get(domain,()):
            self.add_recipient(wiretap_dest)
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.smart_alias(wiretap_dest)
    
          if user in discard_users.get(domain,()):
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.discard = True
    
          exempt_users = dspam_whitelist.get(domain,())
          if user in exempt_users or '' in exempt_users:
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.dspam = False
    
        else:
          self.rejectvirus = False
    
          domain = None
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        if not self.hello_name:
          self.log("REJECT: missing HELO")
          self.setreply('550','5.7.1',"It's polite to say HELO first.")
          return Milter.REJECT
    
        self.umis = None
    
    Stuart Gathman's avatar
    Stuart Gathman committed
        if not (self.internal_connection or self.trusted_relay)     \
            and self.connectip and spf:
    
          if rc != Milter.CONTINUE:
            if rc != Milter.TEMPFAIL: self.offense()
            return rc
    
        # FIXME: parse Received-SPF from trusted_relay for SPF result
    
        res = self.spf and self.spf_guess
        hres = self.spf and self.spf_helo
    
        if auto_whitelist.has_key(self.canon_from):
    
          if res == 'pass' or self.trusted_relay:
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.whitelist = True
            self.log("WHITELIST",self.canon_from)
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.log("PROBATION",self.canon_from)
    
        elif cbv_cache.has_key(self.canon_from) and cbv_cache[self.canon_from] \
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            or domain in blacklist:
    
            if not dspam_userdir:
              if domain in blacklist:
                self.log('REJECT: BLACKLIST',self.canon_from)
                self.setreply('550','5.7.1', 'Sender email local blacklist')
              else:
                res = cbv_cache[self.canon_from]
                desc = "CBV: %d %s" % res[:2]
                self.log('REJECT:',desc)
                self.setreply('550','5.7.1',*desc.splitlines())
              return Milter.REJECT
            self.blacklist = True
            self.log("BLACKLIST",self.canon_from)
    
        else:
          global gossip
          if gossip and domain and rc == Milter.CONTINUE \
    
    Stuart Gathman's avatar
    Stuart Gathman committed
              and not (self.internal_connection or self.trusted_relay):
            if self.spf and self.spf.result == 'pass':
              qual = 'SPF'
            elif res == 'pass':
              qual = 'GUESS'
            elif hres == 'pass':
              qual = 'HELO'
              domain = self.spf.h
    
            else:   
              # No good identity: blame purported domain.  Qualify by SPF
              # result so NEUTRAL will get separate reputation from SOFTFAIL.
    
              qual = res
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            try:
              umis = gossip.umis(domain+qual,self.id+time.time())
    
              res = gossip_node.query(umis,domain,qual,1)
              if res:
    
                res,hdr,val = res
    
                self.add_header(hdr,val)
                a = val.split(',')
                self.reputation = int(a[-2])
                self.confidence = int(a[-1])
                self.umis = umis
    
                # We would like to reject on bad reputation here, but we
                # need to give special consideration to postmaster.  So
                # we have to wait until envrcpt().  Perhaps an especially
                # bad reputation could be rejected here.
                if self.reputation < -70 and self.confidence > 5:
    
                  self.log('REJECT: REPUTATION')
    
                  self.setreply('550','5.7.1',
                    'Your domain has been sending nothing but spam')
                  return Milter.REJECT
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            except:
              gossip = None
              raise
    
    
      def check_spf(self):
    
        receiver = self.receiver
    
        for tf in trusted_forwarder:
          q = spf.query(self.connectip,'',tf,receiver=receiver,strict=False)
          res,code,txt = q.check()
    
          if res == 'none':
            res,code,txt = q.best_guess('v=spf1 a mx')
    
          if res == 'pass':
            self.log("TRUSTED_FORWARDER:",tf)
            break
        else:
          q = spf.query(self.connectip,self.canon_from,self.hello_name,
    
    Stuart Gathman's avatar
    Stuart Gathman committed
              receiver=receiver,strict=False)
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            'SPF fail: see http://openspf.org/why.html?sender=%s&ip=%s' % (q.s,q.i))
    
        if res in ('unknown','permerror') and q.perm_error and q.perm_error.ext:
    
    Stuart Gathman's avatar
    Stuart Gathman committed
          self.cbv_needed = (q,res) # report SPF syntax error to sender
          res,code,txt = q.perm_error.ext   # extended (lax processing) result
    
        p = SPFPolicy(q.s)
    
        # FIXME: try:finally to close policy db, or reuse with lock
    
        if res not in ('pass','error','temperror'):
    
    Stuart Gathman's avatar
    Stuart Gathman committed
          if self.mailfrom != '<>':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            # check hello name via spf unless spf pass
            h = spf.query(self.connectip,'',self.hello_name,receiver=receiver)
            hres,hcode,htxt = h.check()
    
            # FIXME: in a few cases, rejecting on HELO neutral causes problems
            # for senders forced to use their braindead ISPs email service.
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            if hres in ('deny','fail','neutral','softfail'):
              self.log('REJECT: hello SPF: %s 550 %s' % (hres,htxt))
              self.setreply('550','5.7.1',htxt,
                "The hostname given in your MTA's HELO response is not listed",
                "as a legitimate MTA in the SPF records for your domain.  If you",
                "get this bounce, the message was not in fact a forgery, and you",
                "should IMMEDIATELY notify your email administrator of the problem."
              )
              return Milter.REJECT
            if hres == 'none' and spf_best_guess \
              and not dynip(self.hello_name,self.connectip):
    
              # HELO must match more exactly.  Don't match PTR or zombies
              # will be able to get a best_guess pass on their ISPs domain.
              hres,hcode,htxt = h.best_guess('v=spf1 a mx')
    
          else:
            hres,hcode,htxt = res,code,txt
    
          if self.internal_domain and res == 'none':
            # we don't accept our own domains externally without an SPF record
            self.log('REJECT: spam from self',q.o)
            self.setreply('550','5.7.1',"I hate talking to myself!")
            return Milter.REJECT
    
    Stuart Gathman's avatar
    Stuart Gathman committed
          if spf_best_guess and res == 'none':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            #self.log('SPF: no record published, guessing')
            q.set_default_explanation(
                    'SPF guess: see http://openspf.org/why.html')
            # best_guess should not result in fail
            if self.missing_ptr:
              # ignore dynamic PTR for best guess
              res,code,txt = q.best_guess('v=spf1 a/24 mx/24')
            else:
              res,code,txt = q.best_guess()
            if res != 'pass' and hres == 'pass' and spf.domainmatch([q.h],q.o):
              res = 'pass'  # get a guessed pass for valid matching HELO 
    
          if self.missing_ptr and ores == 'none' and res != 'pass' \
    
    Stuart Gathman's avatar
    Stuart Gathman committed
                    and hres != 'pass':
            # this bad boy has no credentials whatsoever
            policy = p.getNonePolicy()
            if policy == 'CBV':
              if self.mailfrom != '<>':
                self.cbv_needed = (q,ores)  # accept, but inform sender via DSN
    
    	  self.offenses = 3    # ban ip if any bad recipient
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            elif policy != 'OK':
              self.log('REJECT: no PTR, HELO or SPF')
              self.setreply('550','5.7.1',
    
        "You must have a valid HELO or publish SPF: http://www.openspf.org ",
        "Contact your mail administrator IMMEDIATELY!  Your mail server is ",
        "severely misconfigured.  It has no PTR record (dynamic PTR records ",
        "that contain your IP don't count), an invalid or dynamic HELO, ",
        "and no SPF record."
    
    Stuart Gathman's avatar
    Stuart Gathman committed
              )
              return Milter.REJECT
    
        if res in ('deny', 'fail'):
    
          policy = p.getFailPolicy()
    
          if policy == 'CBV':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            if self.mailfrom != '<>':
              self.cbv_needed = (q,res)
    
          elif policy != 'OK':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.log('REJECT: SPF %s %i %s' % (res,code,txt))
            self.setreply(str(code),'5.7.1',txt)
            # A proper SPF fail error message would read:
            # forger.biz [1.2.3.4] is not allowed to send mail with the domain
            # "forged.org" in the sender address.  Contact <postmaster@forged.org>.
            return Milter.REJECT
    
        if res == 'softfail':
          policy = p.getSoftfailPolicy()
    
          if policy == 'CBV':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            if self.mailfrom != '<>':
              self.cbv_needed = (q,res)
    
          elif policy != 'OK':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.log('REJECT: SPF %s %i %s' % (res,code,txt))
            self.setreply('550','5.7.1',
              'SPF softfail: If you get this Delivery Status Notice, your email',
              'was probably legitimate.  Your administrator has published SPF',
              'records in a testing mode.  The SPF record reported your email as',
              'a forgery, which is a mistake if you are reading this.  Please',
              'notify your administrator of the problem immediately.'
            )
            return Milter.REJECT
    
        if res == 'neutral':
    
          policy = p.getNeutralPolicy()
    
          if policy == 'CBV':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            if self.mailfrom != '<>':
              self.cbv_needed = (q,res)
              # FIXME: this makes Received-SPF show wrong result
    
          elif policy != 'OK':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.log('REJECT: SPF neutral for',q.s)
            self.setreply('550','5.7.1',
              'mail from %s must pass SPF: http://openspf.org/why.html' % q.o,
              'The %s domain is one that spammers love to forge.  Due to' % q.o,
              'the volume of forged mail, we can only accept mail that',
              'the SPF record for %s explicitly designates as legitimate.' % q.o,
              'Sending your email through the recommended outgoing SMTP',
              'servers for %s should accomplish this.' % q.o
            )
            return Milter.REJECT
    
        if res in ('unknown','permerror'):
          policy = p.getPermErrorPolicy()
    
          if policy == 'CBV':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            if self.mailfrom != '<>':
              self.cbv_needed = (q,res)
    
          elif policy != 'OK':
    
    Stuart Gathman's avatar
    Stuart Gathman committed
            self.log('REJECT: SPF %s %i %s' % (res,code,txt))
            # latest SPF draft recommends 5.5.2 instead of 5.7.1
            self.setreply(str(code),'5.5.2',txt,
              'There is a fatal syntax error in the SPF record for %s' % q.o,
              'We cannot accept mail from %s until this is corrected.' % q.o
            )
            return Milter.REJECT
    
        if res in ('error','temperror'):
    
    Stuart Gathman's avatar
    Stuart Gathman committed
          self.log('TEMPFAIL: SPF %s %i %s' % (res,code,txt))
    
          self.setreply(str(code),'4.3.0',txt)