From 9fb3ad70d4782e6bd5e33b1c8662f85cc918dc8a Mon Sep 17 00:00:00 2001
From: Stuart Gathman <stuart@gathman.org>
Date: Tue, 31 May 2005 18:23:49 +0000
Subject: [PATCH] Development changes since 0.7.2

---
 Milter/__init__.py | 208 ++++++++++++++++++++++++++++
 Milter/dsn.py      | 168 ++++++++++++++++++++++
 Milter/dynip.py    |  87 ++++++++++++
 NEWS               |   1 -
 README             |  11 --
 TODO               |  12 ++
 bms.py             | 338 ++++++++++++++++++++-------------------------
 milter.cfg         |  44 +++---
 milter.html        |  57 +++++---
 milter.spec        |  14 +-
 mime.py            | 338 +++++++++++++--------------------------------
 sample.py          |   2 +-
 setup.py           |   2 +-
 spf.py             | 192 ++++++++++++++-----------
 testbms.py         |  33 +++--
 testmime.py        |  54 ++++++--
 testsample.py      |  12 +-
 17 files changed, 973 insertions(+), 600 deletions(-)
 create mode 100755 Milter/__init__.py
 create mode 100644 Milter/dsn.py
 create mode 100644 Milter/dynip.py

diff --git a/Milter/__init__.py b/Milter/__init__.py
new file mode 100755
index 0000000..88a2214
--- /dev/null
+++ b/Milter/__init__.py
@@ -0,0 +1,208 @@
+
+# Author: Stuart D. Gathman <stuart@bmsi.com>
+# Copyright 2001 Business Management Systems, Inc.
+# This code is under GPL.  See COPYING for details.
+
+import os
+import milter
+import thread
+
+from milter import ACCEPT,CONTINUE,REJECT,DISCARD,TEMPFAIL,	\
+  set_flags, setdbg, setbacklog, settimeout, \
+  ADDHDRS, CHGBODY, ADDRCPT, DELRCPT, CHGHDRS,	\
+  V1_ACTS, V2_ACTS, CURR_ACTS
+
+try: from milter import QUARANTINE
+except: pass
+
+_seq_lock = thread.allocate_lock()
+_seq = 0
+
+def uniqueID():
+  """Return a sequence number unique to this process.
+  """
+  global _seq
+  _seq_lock.acquire()
+  seqno = _seq = _seq + 1
+  _seq_lock.release()
+  return seqno
+  
+class Milter:
+  """A simple class interface to the milter module.
+  """
+  def _setctx(self,ctx):
+    self.__ctx = ctx
+    if ctx:
+      ctx.setpriv(self)
+
+  # user replaceable callbacks
+  def log(self,*msg):
+    print 'Milter:',
+    for i in msg: print i,
+    print
+
+  def connect(self,hostname,unused,hostaddr):
+    "Called for each connection to sendmail."
+    self.log("connect from %s at %s" % (hostname,hostaddr))
+    return CONTINUE
+
+  def hello(self,hostname):
+    "Called after the HELO command."
+    self.log("hello from %s" % hostname)
+    return CONTINUE
+
+  def envfrom(self,f,*str):
+    """Called to begin each message.
+    f -> string		message sender
+    str -> tuple	additional ESMTP parameters
+    """
+    self.log("mail from",f,str)
+    return CONTINUE
+
+  def envrcpt(self,to,*str):
+    "Called for each message recipient."
+    self.log("rcpt to",to,str)
+    return CONTINUE
+
+  def header(self,field,value):
+    "Called for each message header."
+    self.log("%s: %s" % (field,value))
+    return CONTINUE
+
+  def eoh(self):
+    "Called after all headers are processed."
+    self.log("eoh")
+    return CONTINUE
+
+  def body(self,unused):
+    "Called to transfer the message body."
+    return CONTINUE
+
+  def eom(self):
+    "Called at the end of message."
+    self.log("eom")
+    return CONTINUE
+
+  def abort(self):
+    "Called if the connection is terminated abnormally."
+    self.log("abort")
+    return CONTINUE
+
+  def close(self):
+    "Called at the end of connection, even if aborted."
+    self.log("close")
+    return CONTINUE
+
+  # Milter methods which can be invoked from callbacks
+  def getsymval(self,sym):
+    return self.__ctx.getsymval(sym)
+
+  # If sendmail does not support setmlreply, then only the
+  # first msg line is used.
+  def setreply(self,rcode,xcode=None,msg=None,*ml):
+    return self.__ctx.setreply(rcode,xcode,msg,*ml)
+
+  # Milter methods which can only be called from eom callback.
+  def addheader(self,field,value):
+    return self.__ctx.addheader(field,value)
+
+  def chgheader(self,field,idx,value):
+    return self.__ctx.chgheader(field,idx,value)
+
+  def addrcpt(self,rcpt):
+    return self.__ctx.addrcpt(rcpt)
+
+  def delrcpt(self,rcpt):
+    return self.__ctx.delrcpt(rcpt)
+
+  def replacebody(self,body):
+    return self.__ctx.replacebody(body)
+
+  # When quarantined, a message goes into the mailq as if to be delivered,
+  # but delivery is deferred until the message is unquarantined.
+  def quarantine(self,reason):
+    return self.__ctx.quarantine(reason)
+
+  def progress(self):
+    return self.__ctx.progress()
+
+factory = Milter
+
+def connectcallback(ctx,hostname,family,hostaddr):
+  m = factory()
+  m._setctx(ctx)
+  return m.connect(hostname,family,hostaddr)
+
+def closecallback(ctx):
+  m = ctx.getpriv()
+  if not m: return CONTINUE
+  rc = m.close()
+  m._setctx(None)	# release milterContext
+  return rc
+
+def envcallback(c,args):
+  """Convert ESMTP parms to keyword parameters.
+  Can be used in the envfrom and/or envrcpt callbacks to process
+  ESMTP parameters as python keyword parameters."""
+  kw = {}
+  for s in args[1:]:
+    pos = s.find('=')
+    if pos > 0:
+      kw[s[:pos]] = s[pos+1:]
+  return apply(c,args,kw)
+
+def runmilter(name,socketname,timeout = 0):
+  # This bit is here on the assumption that you will be starting this filter
+  # before sendmail.  If sendmail is not running and the socket already exists,
+  # libmilter will throw a warning.  If sendmail is running, this is still
+  # safe if there are no messages currently being processed.  It's safer to
+  # shutdown sendmail, kill the filter process, restart the filter, and then
+  # restart sendmail.
+  pos = socketname.find(':')
+  if pos > 1:
+    s = socketname[:pos]
+    fname = socketname[pos+1:]
+  else:
+    s = "unix"
+    fname = socketname
+  if s == "unix" or s == "local":
+    print "Removing %s" % fname
+    try:
+      os.unlink(fname)
+    except:
+      pass
+
+  # The default flags set include everything
+  # milter.set_flags(milter.ADDHDRS)
+  milter.set_connect_callback(connectcallback)
+  milter.set_helo_callback(lambda ctx, host: ctx.getpriv().hello(host))
+  milter.set_envfrom_callback(lambda ctx,*str:
+  	ctx.getpriv().envfrom(*str))
+#  	envcallback(ctx.getpriv().envfrom,str))
+  milter.set_envrcpt_callback(lambda ctx,*str:
+  	ctx.getpriv().envrcpt(*str))
+#  	envcallback(ctx.getpriv().envrcpt,str))
+  milter.set_header_callback(lambda ctx,fld,val:
+  	ctx.getpriv().header(fld,val))
+  milter.set_eoh_callback(lambda ctx: ctx.getpriv().eoh())
+  milter.set_body_callback(lambda ctx,chunk: ctx.getpriv().body(chunk))
+  milter.set_eom_callback(lambda ctx: ctx.getpriv().eom())
+  milter.set_abort_callback(lambda ctx: ctx.getpriv().abort())
+  milter.set_close_callback(closecallback)
+
+  milter.setconn(socketname)
+  if timeout > 0: milter.settimeout(timeout)
+  # The name *must* match the X line in sendmail.cf (supposedly)
+  milter.register(name)
+  start_seq = _seq
+  try:
+    milter.main()
+  except milter.error:
+    if start_seq == _seq: raise	# couldn't start
+    # milter has been running for a while, but now it can't start new threads
+    raise milter.error("out of thread resources")
+
+__all__ = globals().copy()
+for priv in ('os','milter','thread','factory','_seq','_seq_lock'):
+  del __all__[priv]
+__all__ = __all__.keys()
diff --git a/Milter/dsn.py b/Milter/dsn.py
new file mode 100644
index 0000000..1865689
--- /dev/null
+++ b/Milter/dsn.py
@@ -0,0 +1,168 @@
+import smtplib
+import spf
+import socket
+from email.Message import Message
+
+nospf_msg = """This is an automatically generated Delivery Status Notification.
+
+THIS IS A WARNING MESSAGE ONLY.
+
+YOU DO *NOT* NEED TO RESEND YOUR MESSAGE.
+
+Delivery to the following recipients has been delayed.
+
+	%(rcpt)s
+
+Subject: %(subject)s 
+
+Someone at IP address %(connectip)s sent an email claiming
+to be from %(sender)s.  
+
+If that wasn't you, then your domain, %(sender_domain)s,
+was forged - i.e. used without your knowlege or authorization by
+someone attempting to steal your mail identity.  This is a very
+serious problem, and you need to provide authentication for your
+SMTP (email) servers to prevent criminals from forging your
+domain.  The simplest step is usually to publish an SPF record
+with your Sender Policy.  
+
+For more information, see: http://spfhelp.net
+
+I hate to annoy you with a DSN (Delivery Status
+Notification) from a possibly forged email, but since you
+have not published a sender policy, there is no other way
+of bringing this to your attention.
+
+If it *was* you that sent the email, then your email domain
+or configuration is in error.  If you don't know anything
+about mail servers, then pass this on to your SMTP (mail)
+server administrator.  We have accepted the email anyway, in
+case it is important, but we couldn't find anything about
+the mail submitter at %(connectip)s to distinguish it from a
+zombie (compromised/infected computer - usually a Windows
+PC).  There was no PTR record for its IP address (PTR names
+that contain the IP address don't count).  RFC2821 requires
+that your hello name be a FQN (Fully Qualified domain Name,
+i.e. at least one dot) that resolves to the IP address of
+the mail sender.  In addition, just like for PTR, we don't
+accept a helo name that contains the IP, since this doesn't
+help to identify you.  The hello name you used,
+%(heloname)s, was invalid.
+
+Furthermore, there was no SPF record for the sending domain
+%(sender_domain)s.  We even tried to find its IP in any A or
+MX records for your domain, but that failed also.  We really
+should reject mail from anonymous mail clients, but in case
+it is important, we are accepting it anyway.
+
+We are sending you this message to alert you to the fact that
+
+Either - Someone is forging your domain.
+Or - You have problems with your email configuration.
+Or - Possibly both.
+
+If you need further assistance, please do not hesitate to
+contact me again.
+
+Kind regards,
+Stuart D. Gathman
+postmaster@%(receiver)s
+"""
+
+softfail_msg = """
+This is an automatically generated Delivery Status Notification.
+
+THIS IS A WARNING MESSAGE ONLY.
+
+YOU DO *NOT* NEED TO RESEND YOUR MESSAGE.
+
+Delivery to the following recipients has been delayed.
+
+       %(rcpt)s
+
+Subject: %(subject)s
+Received-SPF: %(spf_result)s
+"""
+
+def send_dsn(mailfrom,receiver,msg=None):
+  "Send DSN.  If msg is None, do callback verification."
+  user,domain = mailfrom.split('@')
+  q = spf.query(None,None,None)
+  mxlist = q.dns(domain,'MX')
+  if not mxlist:
+    mxlist = (0,domain),
+  else:
+    mxlist.sort()
+  smtp = smtplib.SMTP()
+  for prior,host in mxlist:
+    try:
+      smtp.connect(host)
+      code,resp = smtp.helo(receiver)
+      # some wiley spammers have MX records that resolve to 127.0.0.1
+      if resp.split()[0] == receiver:
+        return (553,'Fraudulent MX for %s' % domain)
+      if not (200 <= code <= 299):
+	raise SMTPHeloError(code, resp)
+      if msg:
+        try:
+	  smtp.sendmail('<>',mailfrom,msg)
+	except smtplib.SMTPSenderRefused:
+	  # does not accept DSN, try postmaster (at the risk of mail loops)
+	  smtp.sendmail('<postmaster@%s>'%receiver,mailfrom,msg)
+      else:	# CBV
+	code,resp = smtp.docmd('MAIL FROM: <>')
+	if code != 250:
+	  raise SMTPSenderRefused(code, resp, '<>')
+	code,resp = smtp.rcpt(mailfrom)
+	if code not in (250,251):
+	  return (code,resp)		# permanent error
+	smtp.quit()
+      return None			# success
+    except smtplib.SMTPRecipientsRefused,x:
+      return x.recipients[mailfrom]	# permanent error
+    except smtplib.SMTPSenderRefused,x:
+      return x		# does not accept DSN
+    except smtplib.SMTPDataError,x:
+      return x				# permanent error
+    except smtplib.SMTPException:
+      pass		# any other error, try next MX
+    except socket.error:
+      pass		# MX didn't accept connections, try next one
+    smtp.close()
+  return (450,'No MX servers available')	# temp error
+
+def create_msg(q,rcptlist,origmsg):
+  heloname = q.h
+  sender = q.s
+  connectip = q.i
+  receiver = q.r
+  sender_domain = q.o
+  rcpt = '\n\t'.join(rcptlist)
+  try: subject = origmsg['Subject']
+  except: subject = '(none)'
+  try:
+    spf_result = origmsg['Received-SPF']
+    if not spf_result.startswith('softfail'):
+      spf_result = None
+  except: spf_result = None
+  msg = Message()
+  msg.add_header('To',sender)
+  msg.add_header('From','postmaster@%s'%receiver)
+  msg.add_header('Auto-Submitted','auto-generated (configuration error)')
+  msg.set_type('text/plain')
+  if spf_result:
+    msg.add_header('Subject','SPF softfail (POSSIBLE FORGERY)')
+    msg.set_payload(softfail_msg % locals())
+  else:
+    msg.add_header('Subject','Critical mail server configuration error')
+    msg.set_payload(nospf_msg % locals())
+  return msg
+
+if __name__ == '__main__':
+  q = spf.query('192.168.9.50',
+  'SRS0=pmeHL=RH=bmsi.com=stuart@bmsi.com',
+  'bmsred.bmsi.com',receiver='mail.bmsi.com')
+  msg = create_msg(q,'charlie@jsconnor.com')
+  #print msg.as_string()
+  # print send_dsn(f,msg.as_string())
+  print send_dsn(q.s,'mail.bmsi.com',msg.as_string())
diff --git a/Milter/dynip.py b/Milter/dynip.py
new file mode 100644
index 0000000..1337c03
--- /dev/null
+++ b/Milter/dynip.py
@@ -0,0 +1,87 @@
+# examples we don't yet recognize:
+#
+# wiley-268-8196.roadrunner.nf.net at ('205.251.174.46', 4810)
+# cbl-sd-02-79.aster.com.do at ('200.88.62.79', 4153)
+
+import re
+
+ip3 = re.compile('[0-9]{1,3}')
+hpats = (
+ 'h[0-9a-f]{12}[.]',
+ 'h\d*n\d*c\d*o\d*\.',
+ 'pcp\d{6,10}pcs[.]',
+ 'no-reverse',
+ 'S[0-9a-f]{16}[.][a-z]{2}[.]',
+ 'user<3>\.',
+ '[Cc]ust<3>\.',
+ '^<3>\.',
+ 'ppp[^.]*<3>\.',
+ '-ppp\d*\.',
+ '\d*-<3>\.',
+ '[0-9a-f]{1,3}-<3>\.',
+ 'p<3>\.pool',
+ 'h<3>\.',
+ 'xdsl-\d*\.',
+ '-\d*-\d*\.',
+ '\.adsl\.',
+ '\.cable\.'
+)
+rehmac = re.compile('|'.join(hpats))
+
+def is_dynip(host,addr):
+  """Return True if hostname is for a dynamic ip.
+  Examples:
+
+  >>> is_dynip('post3.fabulousdealz.com','69.60.99.112')
+  False
+  >>> is_dynip('adsl-69-208-201-177.dsl.emhril.ameritech.net','69.208.201.177')
+  True
+  >>> is_dynip('[1.2.3.4]','1.2.3.4')
+  True
+  """
+  if host.startswith('[') and host.endswith(']'):
+    return True
+  if addr:
+    if host.find(addr) >= 0: return True
+    a = addr.split('.')
+    ia = map(int,a)
+    h = host
+    m = ip3.findall(host)
+    if m:
+      g = map(int,m)
+      ia3 = (ia[1:],ia[:3])
+      if g[-3:] in ia3: return True
+      if g[0] == ia[3] and g[1:3] == ia[:2]: return True
+      if g[-2:] == ia[2:]: return True
+      g.reverse()
+      if g[:3] in ia3: return True
+      if g[:2] == ia[2:]: return True
+      if ia[2:] in (g[:2],g[-2:]): return True
+      for m in ip3.finditer(host):
+        if int(m.group()) == ia[3]:
+	  h = host[:m.start()] + '<3>' + host[m.end():]
+	  break
+    if rehmac.search(h): return True
+    if host.find(''.join(a[:3])) >= 0: return True
+    if host.find(''.join(a[1:])) >= 0: return True
+    x = "%02x%02x%02x%02x" % tuple(ia)
+    if host.lower().find(x) >= 0: return True
+  return False
+
+if __name__ == '__main__':
+  import fileinput
+  import sets
+  seen = sets.Set()
+  for ln in fileinput.input():
+    a = ln.split()
+    if a[3:5] == ['connect','from']:
+      host = a[5]
+      if host.startswith('[') and host.endswith(']'):
+	continue	# no PTR
+      ip = a[7][2:-2]
+      if ip in seen: continue
+      seen.add(ip)
+      if is_dynip(host,ip):
+        print '%s\t%s DYN' % (ip,host)
+      else:
+        print '%s\t%s' % (ip,host)
diff --git a/NEWS b/NEWS
index 3b44bf7..5ec2771 100644
--- a/NEWS
+++ b/NEWS
@@ -5,7 +5,6 @@ Here is a history of user visible changes to Python milter.
 	Three strikes and yer out rule.
 	Block softfail by default when no PTR or HELO
 	Return unknown for null mechanism
-	Return unknown for invalid ip address in mechanism
 	Try best guess on HELO also
 	Expand setreply for common errors
 	make rhsbl.m4 hack available for sendmail.mc
diff --git a/README b/README
index 8f564e8..5df6c93 100644
--- a/README
+++ b/README
@@ -92,7 +92,6 @@ milter.  This milter's socket is a unix-domain socket in the filesystem.
 See libmilter/README for the definitive list of options.
 NB: The name is specified in two places: here, in sendmail's cf file, and
 in the milter itself.  Make sure the two match.
-NB: OpenBSD must use an inet socket.  See the web page for details.
 NB: The above lines can be added in your .mc file with this line:
 
 INPUT_MAIL_FILTER(`pythonfilter', `S=local:/home/username/pythonsock')
@@ -124,16 +123,6 @@ and headers at
 
 http://www.bmsi.com/linux/sendmail-rh72.spec
 
-OpenBSD Notes
--------------
-
-Sendmail is broken on OpenBSD for unix domain sockets.  You must use an
-inet socket for milter.  The sendmail.cf 'X' config line would look like:
-
-Xpythonfilter,        S=inet:1234@localhost
-
-and the sample milter needs to be modified accordingly.
-
 IPv6 Notes
 ----------
 
diff --git a/TODO b/TODO
index 2fae047..0389f45 100644
--- a/TODO
+++ b/TODO
@@ -1,3 +1,15 @@
+Defer TEMPERROR in SPF evaluation - give precedence to security
+(only defer for PASS mechanisms).
+
+Allow multiple recipients for MAIL FROM: <> by default.
+
+Option to add Received-SPF header, but never reject on SPF.
+
+Option to configure banned extension list for mime.py.  Default to empty.
+
+Create null config that does nothing - except maybe add Received-SPF
+headers.  Many admins would like to turn features on one at a time.
+
 Checking in mime.py;
 /bms/cvs/milter/mime.py,v  <--  mime.py
 new revision: 1.56; previous revision: 1.55
diff --git a/bms.py b/bms.py
index 0c04474..946056b 100644
--- a/bms.py
+++ b/bms.py
@@ -1,6 +1,33 @@
 #!/usr/bin/env python
 # A simple milter.
 # $Log$
+# Revision 1.134  2005/05/25 15:36:43  stuart
+# Use dynip module.
+# Support smart aliasing of wiretap destination.
+# Always send DSN for SOFTFAIL.
+# Close forged bounce loophole when there are no headers.
+#
+# Revision 1.133  2005/03/16 21:58:04  stuart
+# Auto DSN feature.
+#
+# Revision 1.132  2005/02/12 02:11:10  stuart
+# Pass unit tests with python2.4.
+#
+# Revision 1.131  2005/02/11 18:34:13  stuart
+# Handle garbage after quote in boundary.
+#
+# Revision 1.130  2005/02/10 01:10:58  stuart
+# Fixed MimeMessage.ismodified()
+#
+# Revision 1.129  2005/02/10 00:56:48  stuart
+# Runs with python2.4.  Defang not working correctly - more work needed.
+#
+# Revision 1.128  2005/02/09 17:53:34  stuart
+# Optionally run dspam on internal mail.
+#
+# Revision 1.127  2004/12/03 14:26:21  stuart
+# Mark DYN PTR, REJECT softfail, log Received-SPF from trusted MTA.
+#
 # Revision 1.126  2004/11/24 14:39:38  stuart
 # Also accept softfail if valid PTR or HELO.
 #
@@ -151,102 +178,6 @@
 # Revision 1.79  2003/12/04 23:46:06  stuart
 # Release 0.6.4
 #
-# Revision 1.78  2003/12/04 23:20:24  stuart
-# Make headerChange handle deleting absent header
-#
-# Revision 1.77  2003/12/04 22:01:40  stuart
-# Limit size of messages which will be dspammed.  This works around a bug
-# in dspam-2.6.5.2 where it scans large binary attachments.  I've never
-# seen really big spam anyway.
-#
-# Revision 1.76  2003/12/04 21:44:33  stuart
-# Pass header changes from Dspam to sendmail
-#
-# Revision 1.75  2003/11/25 17:43:07  stuart
-# Update FAQ.
-#
-# Revision 1.74  2003/11/25 17:36:58  stuart
-# dspam_reject
-#
-# Revision 1.73  2003/11/24 15:46:00  stuart
-# Missing global for dspam_whitelist
-#
-# Revision 1.72  2003/11/22 02:52:07  stuart
-# Handle multiple x-dspam-recipients properly on false positive
-#
-# Revision 1.71  2003/11/22 02:49:57  stuart
-# dspam whitelist
-#
-# Revision 1.70  2003/11/09 03:53:34  stuart
-# Don't block delivery of defanged false positives.
-#
-# Revision 1.69  2003/11/08 22:47:04  stuart
-# Exempt entire domains with '@domain.com'
-#
-# Revision 1.68  2003/11/02 03:06:16  stuart
-# Adjust error codes again.
-#
-# Revision 1.67  2003/11/02 03:01:46  stuart
-# Adjust SMTP error codes after careful reading of standard.
-#
-# Revision 1.66  2003/11/02 01:56:43  stuart
-# Use busy SMTP code.
-#
-# Revision 1.65  2003/11/02 01:44:11  stuart
-# Suppress traceback for Dspam lock timeouts
-#
-# Revision 1.64  2003/10/28 01:00:19  stuart
-# Dspam internal mail for dspam users
-#
-# Revision 1.63  2003/10/25 02:10:34  stuart
-# Match hostname for internal connection test, even if no ipaddr.
-#
-# Revision 1.62  2003/10/24 04:34:52  stuart
-# Fix for not saving defang of false positive triggered rejecting it
-# as a virus from self.
-#
-# Revision 1.61  2003/10/22 22:03:14  stuart
-# Apply dspam_exempt to screening
-#
-# Revision 1.60  2003/10/22 21:58:42  stuart
-# Don't save false positives as defang file.
-#
-# Revision 1.59  2003/10/22 05:02:27  stuart
-# Add support for dspam screeners
-#
-# Revision 1.58  2003/10/16 22:19:24  stuart
-# Redirect Dspam logging to bms milter
-#
-# Revision 1.57  2003/10/10 00:15:04  stuart
-# DISCARD message if quarrantined for any recipient.
-#
-# Revision 1.56  2003/10/06 19:30:27  stuart
-# REJECT messages with boundard errors
-#
-# Revision 1.55  2003/10/03 18:20:31  stuart
-# Opt-out feature to exempt certain recipients from header filtering.
-#
-# Revision 1.54  2003/09/22 13:36:04  stuart
-# Release 0.6.1
-#
-# Revision 1.53  2003/09/06 07:08:36  stuart
-# dspam support improvements.
-#
-# Revision 1.51  2003/09/02 00:27:27  stuart
-# Should have full milter based dspam support working
-#
-# Revision 1.50  2003/08/26 06:08:17  stuart
-# Use new python boolean since we now require 2.2.2
-#
-# Revision 1.49  2003/08/26 05:45:51  stuart
-# Fix conditional import of dspam.  Update web page.
-#
-# Revision 1.48  2003/08/26 05:10:43  stuart
-# Readability tweaks
-#
-# Revision 1.47  2003/08/26 05:01:38  stuart
-# Release 0.6.0
-#
 # Author: Stuart D. Gathman <stuart@bmsi.com>
 # Copyright 2001 Business Management Systems, Inc.
 # This code is under GPL.  See COPYING for details.
@@ -262,6 +193,8 @@ import tempfile
 import ConfigParser
 import time
 import re
+import Milter.dsn as dsn
+from Milter.dynip import is_dynip as dynip
 
 from fnmatch import fnmatchcase
 from email.Header import decode_header
@@ -320,6 +253,11 @@ spf_accept_softfail = ()
 spf_best_guess = False
 spf_reject_noptr = False
 timeout = 600
+cbv_cache = {}
+try:
+  for rcpt in open('send_dsn.log'):
+    cbv_cache[rcpt.strip()] = None
+except IOError: pass
 
 class MilterConfigParser(ConfigParser.ConfigParser):
 
@@ -380,7 +318,8 @@ def read_config(list):
     'hashlength': '8',
     'reject_spoofed': 'no',
     'reject_noptr': 'no',
-    'best_guess': 'no'
+    'best_guess': 'no',
+    'dspam_internal': 'yes'
   })
   cp.read(list)
   tempfile.tempdir = cp.get('milter','tempdir')
@@ -423,7 +362,7 @@ def read_config(list):
     key = (sm[0],sm[1])
     smart_alias[key] = sm[2:]
 
-  global dspam_dict, dspam_users, dspam_userdir, dspam_exempt
+  global dspam_dict, dspam_users, dspam_userdir, dspam_exempt, dspam_internal
   global dspam_screener,dspam_whitelist,dspam_reject,dspam_sizelimit
   global spf_reject_neutral,spf_best_guess,SRS,spf_reject_noptr
   global spf_accept_softfail
@@ -434,6 +373,7 @@ def read_config(list):
   dspam_userdir = cp.getdefault('dspam','dspam_userdir')
   dspam_screener = cp.getlist('dspam','dspam_screener')
   dspam_reject = cp.getlist('dspam','dspam_reject')
+  dspam_internal = cp.getboolean('dspam','dspam_internal')
   if cp.has_option('dspam','dspam_sizelimit'):
     dspam_sizelimit = cp.getint('dspam','dspam_sizelimit')
 
@@ -488,43 +428,6 @@ def parse_header(val):
   except LookupError: pass
   return val
 
-ip3 = re.compile('([0-9]{1,3})[.-]([0-9]{1,3})[.-]([0-9]{1,3})')
-rehmac = re.compile('h[0-9a-f]{12}[.]|pcp[0-9]{6,10}pcs[.]|no-reverse')
-
-def dynip(host,addr):
-  """Return True if hostname is for a dynamic ip.
-  Examples:
-
-  >>> is_dynip('post3.fabulousdealz.com','69.60.99.112')
-  False
-  >>> is_dynip('adsl-69-208-201-177.dsl.emhril.ameritech.net','69.208.201.177')
-  True
-  """
-  if host.startswith('[') and host.endswith(']'):
-    return True
-  if addr:
-    if host.find(addr) >= 0: return True
-    a = addr.split('.')
-    m = ip3.search(host)
-    if m:
-      g = list(m.groups())
-      if g == a[1:] or g == a[:3]: return True
-      g.reverse()
-      if g == a[1:] or g == a[:3]: return True
-    if rehmac.search(host): return True
-    if host.find("-%s." % '-'.join(a[2:])) >= 0: return True
-    if host.find("w%s." % '-'.join(a[:2])) >= 0: return True
-    if host.find(''.join(a[:3])) >= 0: return True
-    if host.find(''.join(a[1:])) >= 0: return True
-    x = "%02x%02x%02x%02x" % tuple(map(int,a))
-    if host.lower().find(x) >= 0: return True
-    z = [n.zfill(3) for n in a]
-    if host.find('-'.join(z)) >= 0: return True
-    if host.find("-%s." % '-'.join(z[2:])) >= 0: return True
-    if host.find("%s." % ''.join(z[2:])) >= 0: return True
-    if host.find(''.join(z)) >= 0: return True
-  return False
-
 class bmsMilter(Milter.Milter):
   """Milter to replace attachments poisonous to Windows with a WARNING message,
      check SPF, and other anti-forgery features, and implement wiretapping
@@ -577,9 +480,8 @@ class bmsMilter(Milter.Milter):
 	if fnmatchcase(ipaddr,pat):
 	  self.trusted_relay = True
 	  break
-      self.connectip = ipaddr
-    else:
-      self.connectip = None
+    else: ipaddr = ''
+    self.connectip = ipaddr
     self.missing_ptr = dynip(hostname,self.connectip)
     for pat in internal_connect:
       if fnmatchcase(hostname,pat):
@@ -591,9 +493,16 @@ class bmsMilter(Milter.Milter):
       connecttype = 'EXTERNAL'
     if self.trusted_relay:
       connecttype += ' TRUSTED'
+    if self.missing_ptr:
+      connecttype += ' DYN'
     self.log("connect from %s at %s %s" % (hostname,hostaddr,connecttype))
     self.hello_name = None
     self.connecthost = hostname
+    if hostname == 'localhost' and not ipaddr.startswith('127.') \
+    or hostname == '.':
+      self.log("REJECT: PTR is",hostname)
+      self.setreply('550','5.7.1', '"%s" is not a reasonable PTR name'%hostname)
+      return Milter.REJECT
     return Milter.CONTINUE
 
   def hello(self,hostname):
@@ -609,6 +518,25 @@ class bmsMilter(Milter.Milter):
       return Milter.REJECT
     return Milter.CONTINUE
 
+  def smart_alias(self,to):
+    if smart_alias:
+      t = parse_addr(to.lower())
+      if len(t) == 2:
+	ct = '@'.join(t)
+      else:
+	ct = t[0]
+      cf = self.canon_from
+      cf0 = cf.split('@',1)
+      if len(cf0) == 2:
+	cf0 = '@' + cf0[1]
+      else:
+	cf0 = cf
+      for key in ((cf,ct),(cf0,ct)):
+	if smart_alias.has_key(key):
+	  self.del_recipient(to)
+	  for t in smart_alias[key]:
+	    self.add_recipient('<%s>'%t)
+
   # multiple messages can be received on a single connection
   # envfrom (MAIL FROM in the SMTP protocol) seems to mark the start
   # of each message.
@@ -625,10 +553,12 @@ class bmsMilter(Milter.Milter):
     self.reject_spam = True
     self.data_allowed = True
     self.trust_received = self.trusted_relay
+    self.trust_spf = self.trusted_relay
     self.redirect_list = []
     self.discard_list = []
     self.new_headers = []
     self.recipients = []
+    self.cbv_needed = None
     t = parse_addr(f.lower())
     self.canon_from = '@'.join(t)
     self.fp.write('From %s %s\n' % (self.canon_from,time.ctime()))
@@ -643,6 +573,7 @@ class bmsMilter(Milter.Milter):
       self.rejectvirus = domain in reject_virus_from
       if user in wiretap_users.get(domain,()):
         self.add_recipient(wiretap_dest)
+	self.smart_alias(wiretap_dest)
       if user in discard_users.get(domain,()):
 	self.discard = True
       exempt_users = dspam_whitelist.get(domain,())
@@ -662,14 +593,14 @@ class bmsMilter(Milter.Milter):
   def check_spf(self):
     t = parse_addr(self.mailfrom)
     if len(t) == 2: t[1] = t[1].lower()
-    q = spf.query(self.connectip,'@'.join(t),self.hello_name)
+    receiver = self.receiver
+    q = spf.query(self.connectip,'@'.join(t),self.hello_name,receiver=receiver)
     q.set_default_explanation('SPF fail: see http://spf.pobox.com/why.html')
     res,code,txt = q.check()
-    receiver = self.receiver
     if res in ('none', 'softfail'):
       if self.mailfrom != '<>':
 	# check hello name via spf
-	h = spf.query(self.connectip,'',self.hello_name)
+	h = spf.query(self.connectip,'',self.hello_name,receiver=receiver)
 	hres,hcode,htxt = h.check()
 	if hres in ('deny','fail','neutral','softfail'):
 	  self.log('REJECT: hello SPF: %s 550 %s' % (hres,htxt))
@@ -695,32 +626,39 @@ class bmsMilter(Milter.Milter):
 	else:
 	  res,code,txt = q.best_guess()
 	receiver += ': guessing'
-      if self.missing_ptr and res in ('neutral', 'none')	\
-      	and spf_reject_noptr and hres != 'pass':
-        self.log('REJECT: no PTR, HELO or SPF')
-	self.setreply('550','5.7.1',
-  'You must have a reverse lookup or publish SPF: http://spf.pobox.com',
-  'Contact your mail administrator IMMEDIATELY!  Your mail server is',
-  'severely misconfigured.  It has no PTR record (dynamic PTR records',
-  "that contain your IP don't count), an invalid HELO, and no SPF record."
-	)
-	return Milter.REJECT
+      if self.missing_ptr and res in ('neutral', 'none') and hres != 'pass':
+	if spf_reject_noptr:
+	  self.log('REJECT: no PTR, HELO or SPF')
+	  self.setreply('550','5.7.1',
+    'You must have a reverse lookup or publish SPF: http://spf.pobox.com',
+    'Contact your mail administrator IMMEDIATELY!  Your mail server is',
+    'severely misconfigured.  It has no PTR record (dynamic PTR records',
+    "that contain your IP don't count), an invalid HELO, and no SPF record."
+	  )
+	  return Milter.REJECT
+        if self.mailfrom != '<>':
+	  self.cbv_needed = q
     if res in ('deny', 'fail'):
       self.log('REJECT: SPF %s %i %s' % (res,code,txt))
       self.setreply(str(code),'5.7.1',txt)
+      # A proper SPF fail error message would read:
+      # forger.biz [1.2.3.4] is not allowed to send mail with the domain
+      # "forged.org" in the sender address.  Contact <postmaster@forged.org>.
       return Milter.REJECT
     if res == 'softfail' and not q.o in spf_accept_softfail:
-      if self.missing_ptr and spf_reject_noptr and hres != 'pass':
-	self.log('TEMPFAIL: SPF %s 450 %s' % (res,txt))
-	self.setreply('450','4.3.0',
-	  'SPF softfail: will keep trying until your SPF record is fixed.',
-	  'If you get this Delivery Status Notice, your email was probably',
-	  'legitimate.  Your administrator has published SPF records in a',
-	  'testing mode.  The SPF record reported your email as a forgery,',
-	  'which is a mistake if you are reading this.  Please notify your',
-	  'administrator of the problem immediately.'
-	)
-	return Milter.TEMPFAIL
+      if self.missing_ptr and hres != 'pass':
+        if spf_reject_noptr or q.o in spf_reject_neutral:
+	  self.log('REJECT: SPF %s %i %s' % (res,code,txt))
+	  self.setreply('550','5.7.1',
+	    'SPF softfail: If you get this Delivery Status Notice, your email',
+	    'was probably legitimate.  Your administrator has published SPF',
+	    'records in a testing mode.  The SPF record reported your email as',
+	    'a forgery, which is a mistake if you are reading this.  Please',
+	    'notify your administrator of the problem immediately.'
+	  )
+	  return Milter.REJECT
+      if self.mailfrom != '<>':
+	self.cbv_needed = q
     if res == 'neutral' and q.o in spf_reject_neutral:
       self.log('REJECT: SPF neutral for',q.s)
       self.setreply('550','5.7.1',
@@ -789,19 +727,7 @@ class bmsMilter(Milter.Milter):
         self.hidepath = True
       if not domain in dspam_reject:
         self.reject_spam = False
-    if smart_alias:
-      cf = self.canon_from
-      cf0 = cf.split('@',1)
-      if len(cf0) == 2:
-	cf0 = '@' + cf0[1]
-      else:
-	cf0 = cf
-      ct = '@'.join(t)
-      for key in ((cf,ct),(cf0,ct)):
-	if smart_alias.has_key(key):
-	  self.del_recipient(to)
-	  for t in smart_alias[key]:
-	    self.add_recipient('<%s>'%t)
+    self.smart_alias(to)
     #rcpt = self.getsymval("{rcpt_addr}")
     #self.log("rcpt-addr",rcpt);
     return Milter.CONTINUE
@@ -811,7 +737,7 @@ class bmsMilter(Milter.Milter):
     lname = name.lower()
     # val is decoded header value
     if lname == 'subject':
-
+      
       # check for common spam keywords
       for wrd in spam_words:
         if val.find(wrd) >= 0:
@@ -861,17 +787,28 @@ class bmsMilter(Milter.Milter):
     elif self.trust_received and lname == 'received':
       self.trust_received = False
       self.log('%s: %s' % (name,val.splitlines()[0]))
+    elif self.trust_spf and lname == 'received-spf':
+      self.trust_spf = False
+      self.log('%s: %s' % (name,val.splitlines()[0]))
     return Milter.CONTINUE
 
+  def forged_bounce(self):
+    if len(self.recipients) > 1:
+      self.log('REJECT: Multiple bounce recipients')
+      self.setreply('550','5.7.1','Multiple bounce recipients')
+    else:
+      self.log('REJECT: bounce with no SRS encoding')
+      self.setreply('550','5.7.1',
+      "I did not send you that message. Please consider implementing SPF",
+      "(http://spf.pobox.com) to avoid bouncing mail to spoofed senders.",
+      "Thank you."
+      )
+    return Milter.REJECT
+    
   def header(self,name,hval):
     if not self.data_allowed:
-      if len(self.recipients) > 1:
-	self.log('REJECT: Multiple bounce recipients')
-	self.setreply('550','5.7.1','Multiple bounce recipients')
-      else:
-	self.log('REJECT: bounce with no SRS encoding')
-	self.setreply('550','5.7.1',"I did not send you that message.")
-      return Milter.REJECT
+      return self.forged_bounce()
+	  
     lname = name.lower()
     # decode near ascii text to unobfuscate
     val = parse_header(hval)
@@ -897,6 +834,8 @@ class bmsMilter(Milter.Milter):
 
   def eoh(self):
     if not self.fp: return Milter.TEMPFAIL	# not seen by envfrom
+    if not self.data_allowed:
+      return self.forged_bounce()
     for name,val in self.new_headers:
       self.fp.write("%s: %s\n" % (name,val))	# add new headers to buffer
     self.fp.write("\n")				# terminate headers
@@ -945,7 +884,8 @@ class bmsMilter(Milter.Milter):
     # don't let a tricky virus slip one past us
     if scan_rfc822:
       msg = msg.get_submsg()
-      if msg: return mime.check_attachments(msg,self._chk_attach)
+      if isinstance(msg,email.Message.Message):
+	return mime.check_attachments(msg,self._chk_attach)
     return Milter.CONTINUE
 
   def alter_recipients(self,discard_list,redirect_list):
@@ -1047,11 +987,12 @@ class bmsMilter(Milter.Milter):
 
       # analyze all mail for dangerous attachments and scripts
       self.fp.seek(0)
-      msg = mime.MimeMessage(self.fp)
+      msg = mime.message_from_file(self.fp)
       # pass header changes in top level message to sendmail
       msg.headerchange = self._headerChange
 
       # filter leaf attachments through _chk_attach
+      assert not msg.ismodified()
       rc = mime.check_attachments(msg,self._chk_attach)
     except:	# milter crashed trying to analyze mail
       exc_type,exc_value = sys.exc_info()[0:2]
@@ -1106,6 +1047,33 @@ class bmsMilter(Milter.Milter):
     for name,val in self.new_headers:
       self.addheader(name,val)
 
+    if self.cbv_needed:
+      sender = self.cbv_needed.s
+      cached = cbv_cache.has_key(sender)
+      if cached:
+	self.log('CBV:',sender,'(cached)')
+        res = cbv_cache[sender]
+      else:
+	self.log('CBV:',sender)
+	m = dsn.create_msg(self.cbv_needed,self.recipients,msg)
+        m = m.as_string()
+        print >>open('last_dsn','w'),m
+	res = dsn.send_dsn(sender,self.receiver,m)
+      if res:
+        desc = "CBV: %d %s" % res[:2]
+        if 400 <= res[0] < 500:
+	  self.log('TEMPFAIL:',desc)
+          self.setreply('450','4.2.0',*desc.splitlines())
+	  return Milter.TEMPFAIL
+	cbv_cache[sender] = res
+	self.log('REJECT:',desc)
+        self.setreply('550','5.7.1',*desc.splitlines())
+	return Milter.REJECT
+      cbv_cache[sender] = res
+      if not cached:
+	print >>open('send_dsn.log','a'),sender # log who we sent DSNs to
+      self.cbv_needed = None
+
     if not defanged and not spam_checked:
       os.remove(self.tempname)
       self.tempname = None	# prevent re-removal
diff --git a/milter.cfg b/milter.cfg
index adbd0ed..43f0e53 100644
--- a/milter.cfg
+++ b/milter.cfg
@@ -1,4 +1,3 @@
-# features intended to filter or block incoming mail
 [milter]
 # the socket used to communicate with sendmail.  Must match sendmail.cf
 ;socket=/var/run/milter/pythonsock
@@ -6,7 +5,27 @@
 tempdir = /var/log/milter/save
 # how long to wait for a response from sendmail before giving up 
 ;timeout=600
+log_headers = 0
+# connection ips and hostnames are matched against this glob style list
+# to recognize internal senders
+;internal_connect = 192.168.*.*
+
+# mail that is not an internal_connect and claims to be from an
+# internal domain is rejected.  You should enable SPF instead if you can.
+# SPF is much more comprehensive and flexible.
+;internal_domains = mycorp.com
 
+# connections from a trusted relay can trust the first Received header
+# SPF checks are bypassed for internal connections and trusted relays.
+;trusted_relay = 1.2.3.4, 66.12.34.56
+
+# reject external senders with hello names no legit external sender would use
+# SPF will do this also, but listing your own domain and mailserver here
+# will save some DNS lookups when rejecting certain viruses.
+;hello_blacklist = mycorp.com, 66.12.34.56
+
+# features intended to filter or block incoming mail
+;[defang]
 # do virus scanning on attached messages also
 scan_rfc822 = 1
 # Comment out scripts in HTML attachments.  Can be CPU intensive.
@@ -15,7 +34,6 @@ scan_html = 0
 block_chinese = 1
 # list users who hate forwarded mail
 ;block_forward = egghead@mycorp.com, busybee@mycorp.com
-log_headers = 0
 # Reject mail for domains mentioned unless user is mentioned here also
 ;check_user = joe@mycorp.com, mary@mycorp.com, file:bigcorp.com
 # reject mail with these case insensitive strings in the subject
@@ -24,28 +42,10 @@ porn_words = penis, breast, pussy, horse cock, porn, xenical, diet pill, d1ck,
 	p-e-n-i-s, hydrocodone, vicodin, xanax, vicod1n, x@nax, diazepam,
 	v1@gra, xan@x, cialis, ci@lis, fr�e, x�nax, val�um, v�lium, via-gra,
 	x@n3x, vicod3n, pen�s, c0d1n, phentermine, en1arge, dip1oma, v1codin,
-	valium, rolex
+	valium, rolex, sexual
 # reject mail with these case sensitive strings in the subject
 spam_words = $$$, !!!, XXX, FREE, HGH
 
-# connection ips and hostnames are matched against this glob style list
-# to recognize internal senders
-;internal_connect = 192.168.*.*
-
-# mail that is not an internal_connect and claims to be from an
-# internal domain is rejected.  You should enable SPF instead if you can.
-# SPF is much more comprehensive and flexible.
-;internal_domains = mycorp.com
-
-# connections from a trusted relay can trust the first Received header
-# SPF checks are bypassed for internal connections and trusted relays.
-;trusted_relay = 1.2.3.4, 66.12.34.56
-
-# reject external senders with hello names no legit external sender would use
-# SPF will do this also, but listing your own domain and mailserver here
-# will save some DNS lookups when rejecting certain viruses.
-;hello_blacklist = mycorp.com, 66.12.34.56
-
 # See http://bmsi.com/python/pysrs.html for details
 [srs]
 config=/etc/mail/pysrs.cfg
@@ -118,6 +118,8 @@ blind = 1
 ;dspam_whitelist=getitall@sender.com
 # Reject spam to these domains instead of quarantining it.
 ;dspam_reject=othercorp.com
+# Scan internal mail - often a good source of stats on legit mail.
+;dspam_internal=1
 
 # directory for dspam user quarantine, signature db, and dictionaries
 # defining this activates the dspam application
diff --git a/milter.html b/milter.html
index 3b9a8d5..11d0353 100644
--- a/milter.html
+++ b/milter.html
@@ -24,7 +24,7 @@ ALT="Viewable With Any Browser" BORDER="0"></A>
   Stuart D. Gathman</a><br>
 This web page is written by Stuart D. Gathman<br>and<br>sponsored by
 <a href="http://www.bmsi.com">Business Management Systems, Inc.</a> <br>
-Last updated Nov 24, 2004</h4>
+Last updated Jan 05, 2005</h4>
 
 See the <a href="faq.html">FAQ</a> | <a href="#download">Download now</a> |
 <a href="/mailman/listinfo/pymilter">Subscribe to mailing list</a> |
@@ -60,6 +60,25 @@ recognized, and do not count as a valid PTR.  This does not prevent anyone
 from sending mail from a dynamic IP - they just need to configure a
 valid HELO name or publish an SPF record.
 <p>
+As SPF adoption continues to rise, forged spam is not getting through.  So
+spammers are publishing their SPF records as predicted.  The 0.7.2 RPM
+now provides the <code>rhsbl</code> sendmail hack so that spammer domains
+can be blacklisted.  With the RPM installed, add a line like the following
+to your <code>sendmail.mc</code>.
+<pre>
+HACK(rhsbl,`blackholes.example.com',"550 Rejected: " $&{RHS} " has been spamming our customers.")dnl
+</pre>
+<p>
+Of course, spammers are now starting to register
+throwaway domains.  The next thing we need is a custom DNS server,
+in Python, that
+can recognize patterns.  For instance, one spammer registers ded304.com,
+ded305.com, ded306.com, etc.  We also need the custom DNS server to
+let SPF classic clients check SES (which will be part of pysrs).  
+The <a href="http://twistedmatrix.com/products/twisted">Twisted Python</a>
+framework provides a custom DNS server - but I
+would like a smaller implementation for our use.
+<p>
 The RPM for release 0.7.0 moves the config file and socket locations to
 /etc/mail and /var/run/milter respectively.  We now parse Microsoft CID records
 - but only hotmail.com uses them.  They seem to have applied for a patent on
@@ -226,6 +245,19 @@ will have the Python modules.  The bms milter application will still be named
 <a href="http://bmsi.com/python/milter-0.7.2.tar.gz">
 milter-0.7.2.tar.gz</a> Three strikes and your out policy.  Some SPF fixes.
 Recognizes PTR records for dynamic IPs.
+<br>
+<a href="http://bmsi.com/linux/rh72/milter-0.7.2-2.i386.rpm">
+milter-0.7.2-2.i386.rpm</a> Binary RPM for Redhat 7.x, now requires 
+	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
+	python2.3</a>.
+<br>
+<a href="http://bmsi.com/linux/rh9/milter-0.7.2-2rh9.i386.rpm">
+milter-0.7.2-2rh9.i386.rpm</a> Binary RPM for Redhat 9, now requires 
+	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
+	python2.3</a>.
+<br>
+<a href="http://bmsi.com/linux/rh9/milter-0.7.2-2.src.rpm">
+milter-0.7.2-2.src.rpm</a> Source RPM for Redhat 9,7.x. 
 <p>
 <a href="http://bmsi.com/python/milter-0.7.1.tar.gz">
 milter-0.7.1.tar.gz</a> Support setmlreply, handle some more exceptions
@@ -241,7 +273,6 @@ milter-0.7.1-1.i386.rpm</a> Binary RPM for Redhat 7.x, now requires
 <a href="http://bmsi.com/linux/rh9/milter-0.7.1-1.src.rpm">
 milter-0.7.1-1.src.rpm</a> Source RPM for Redhat 9,7.x. 
 <p>
-<a name="stable"><b>Stable</b></a>
 <a href="http://bmsi.com/python/milter-0.7.0.tar.gz">
 milter-0.7.0.tar.gz</a> Move config file and default socket location.
 Parse M$ CID records.
@@ -535,7 +566,7 @@ The "defang" function of the sample milter was inspired by
 a Perl milter with flexible attachment processing options.  The latest
 version of MIMEDefang uses an apache style process pool to avoid reloading
 the Perl interpreter for each message.  This makes it fast enough for
-production and does not use Perl threading.
+production without using Perl threading.
 <p>
 <a href="http://sourceforge.net/projects/mailchecker">mailchecker</a> is
 a Python project to provide flexible attachment processing for mail.  I
@@ -609,18 +640,10 @@ me if you successfully install milter on a system not mentioned below.
 <td>0.5.4</td><tr>
 <td>RedHat 7.1</td><td>gcc-2.96</td><td>?</td><td>8.12.1</td>
 <td>0.3.5</td><tr>
-<td>RedHat 7.2</td><td>gcc-2.96</td><td>2.1.1</td><td>8.11.6</td>
-<td>0.4.1</td><tr>
-<td>RedHat 7.2</td><td>gcc-2.96</td><td>2.2.1</td><td>8.11.6</td>
-<td>0.4.5</td><tr>
-<td>RedHat 7.2</td><td>gcc-2.96</td><td>2.2.2</td><td>8.11.6</td>
-<td>0.5.5</td><tr>
-<td>RedHat 7.2</td><td>gcc-2.96</td><td>2.3.3</td><td>8.12.10</td>
-<td>0.6.6</td><tr>
 <td>RedHat 7.3</td><td>gcc-2.96</td><td>2.2.2</td><td>8.11.6</td>
 <td>0.5.5</td><tr>
-<td>RedHat 7.3</td><td>gcc-2.96</td><td>2.3.3</td><td>8.12.10</td>
-<td>0.6.6</td><tr>
+<td>RedHat 7.3</td><td>gcc-2.96</td><td>2.3.3</td><td>8.13.1</td>
+<td>0.7.2</td><tr>
 <td>RedHat 8.0</td><td>gcc-3.2</td><td>2.2.1</td><td>8.12.6</td>
 <td>0.5.2</td><tr>
 <td>Debian Linux</td><td>gcc-2.95.2</td><td>2.1.1</td><td>8.12.0</td>
@@ -633,14 +656,14 @@ me if you successfully install milter on a system not mentioned below.
 <td>0.3.4</td><tr>
 <td>AIX-4.1.5</td><td>gcc-2.95.2</td><td>2.1.3</td><td>8.12.3</td>
 <td>0.4.2</td><tr>
-<td>AIX-4.1.5</td><td>gcc-2.95.2</td><td>2.2.2</td><td>8.12.6</td>
-<td>0.5.4</td><tr>
+<td>AIX-4.1.5</td><td>gcc-2.95.2</td><td>2.2.3</td><td>8.13.1</td>
+<td>0.7.1</td><tr>
 <td>Slackware 7.1</td><td>?</td><td>?</td><td>8.12.1</td>
 <td>0.3.8</td><tr>
 <td>Slackware 9.0</td><td>gcc-3.2.2</td><td>2.2.3</td><td>8.12.9</td>
 <td>0.5.4</td><tr>
-<td>OpenBSD</td><td>?</td><td>2.1.1</td><td>8.11.6</td>
-<td>0.3.9</td><tr>
+<td>OpenBSD</td><td>?</td><td>2.3.3?</td><td>8.13.1?</td>
+<td>0.7.2</td><tr>
 <td>SuSE 7.3</td><td>gcc-2.95.3</td><td>2.1.1</td><td>8.12.2</td>
 <td>0.3.9</td><tr>
 <td>FreeBSD</td><td>gcc-2.95.3</td><td>2.2.1</td><td>8.12.3</td>
diff --git a/milter.spec b/milter.spec
index 0bb7fe3..f71ef4a 100644
--- a/milter.spec
+++ b/milter.spec
@@ -1,12 +1,12 @@
 %define name milter
-%define version 0.7.2
-%define release 2
+%define version 0.8.0
+%define release 2.EL3
 # Redhat 7.x and earlier (multiple ps lines per thread)
-%define sysvinit milter.rc7
+#define sysvinit milter.rc7
 # RH9, other systems (single ps line per process)
-#define sysvinit milter.rc
+%define sysvinit milter.rc
 %ifos Linux
-%define python python2.3
+%define python python2.4
 %else
 %define python python
 %endif
@@ -24,7 +24,7 @@ Prefix: %{_prefix}
 Vendor: Stuart D. Gathman <stuart@bmsi.com>
 Packager: Stuart D. Gathman <stuart@bmsi.com>
 Url: http://www.bmsi.com/python/milter.html
-Requires: %{python} >= 2.2.2, sendmail >= 8.12.10
+Requires: %{python} >= 2.4, sendmail >= 8.12.10
 %ifnos aix4.1
 Requires: chkconfig
 %endif
@@ -149,6 +149,8 @@ rm -rf $RPM_BUILD_ROOT
 /usr/share/sendmail-cf/hack/rhsbl.m4
 
 %changelog
+* Tue Feb 08 2005 Stuart Gathman <stuart@bmsi.com> 0.7.3-1.EL3
+- Compile for EL3 and Python4
 * Mon Aug 30 2004 Stuart Gathman <stuart@bmsi.com> 0.7.2-1
 - Fix various SPF bugs
 - Recognize dynamic PTR names, and don't count them as authentication.
diff --git a/mime.py b/mime.py
index 8c1a146..95d8e9a 100644
--- a/mime.py
+++ b/mime.py
@@ -1,4 +1,29 @@
 # $Log$
+# Revision 1.62  2005/02/14 22:31:17  stuart
+# _parseparam replacement not needed for python2.4
+#
+# Revision 1.61  2005/02/12 02:11:11  stuart
+# Pass unit tests with python2.4.
+#
+# Revision 1.60  2005/02/11 18:34:14  stuart
+# Handle garbage after quote in boundary.
+#
+# Revision 1.59  2005/02/10 01:10:59  stuart
+# Fixed MimeMessage.ismodified()
+#
+# Revision 1.58  2005/02/10 00:56:49  stuart
+# Runs with python2.4.  Defang not working correctly - more work needed.
+#
+# Revision 1.57  2004/11/20 16:37:52  stuart
+# fix regex for splitting header and body
+#
+# Revision 1.56  2004/11/09 20:33:51  stuart
+# Recognize more dynamic PTR variations.
+#
+# Revision 1.55  2004/10/06 21:39:20  stuart
+# Handle message attachments with boundary errors by not parsing them
+# until needed.
+#
 # Revision 1.54  2004/08/18 01:59:46  stuart
 # Handle mislabeled multipart messages
 #
@@ -43,24 +68,18 @@
 import StringIO
 import socket
 import Milter
+
 import email
 import email.Message
 from email.Message import Message
 from email.Generator import Generator
 from email.Utils import quote
 from email import Utils
-
-from types import ListType,StringType
-
-# Enhance email.Parser
-# - Fix _parsebody to decode message attachments before parsing
-
 from email.Parser import Parser
-try: from email.Parser import NLCRE
-except: from email.Parser import nlcre as NLCRE
-
 from email import Errors
 
+from types import ListType,StringType
+
 class MimeGenerator(Generator):
     def _dispatch(self, msg):
         # Get the Content-Type: for the message, then try to dispatch to
@@ -73,146 +92,6 @@ class MimeGenerator(Generator):
 	else:
 	  Generator._dispatch(self,msg)
 
-class MimeParser(Parser):
-
-    # This is a copy of _parsebody from email.Parser, with a fix
-    # for message attachments.  I couldn't find a smaller way to patch it
-    # in a subclass.
-
-    def _parsebody(self, container, fp, firstbodyline=None):
-        # Parse the body, but first split the payload on the content-type
-        # boundary if present.
-        boundary = container.get_boundary()
-        isdigest = (container.get_content_type() == 'multipart/digest')
-        # If there's a boundary, split the payload text into its constituent
-        # parts and parse each separately.  Otherwise, just parse the rest of
-        # the body as a single message.  Note: any exceptions raised in the
-        # recursive parse need to have their line numbers coerced.
-        if boundary:
-            preamble = epilogue = None
-            # Split into subparts.  The first boundary we're looking for won't
-            # always have a leading newline since we're at the start of the
-            # body text, and there's not always a preamble before the first
-            # boundary.
-            separator = '--' + boundary
-            payload = fp.read()
-            if firstbodyline is not None:
-                payload = firstbodyline + '\n' + payload
-            # We use an RE here because boundaries can have trailing
-            # whitespace.
-            mo = re.search(
-                r'(?P<sep>' + re.escape(separator) + r')(?P<ws>[ \t]*)',
-                payload)
-            if not mo:
-                if self._strict:
-                    raise Errors.BoundaryError(
-                        "Couldn't find starting boundary: %s" % boundary)
-                container.set_payload(payload)
-                return
-            start = mo.start()
-            if start > 0:
-                # there's some pre-MIME boundary preamble
-                preamble = payload[0:start]
-            # Find out what kind of line endings we're using
-            start += len(mo.group('sep')) + len(mo.group('ws'))
-            mo = NLCRE.search(payload, start)
-            if mo:
-                start += len(mo.group(0))
-            # We create a compiled regexp first because we need to be able to
-            # specify the start position, and the module function doesn't
-            # support this signature. :(
-            cre = re.compile('(?P<sep>\r\n|\r|\n)' +
-                             re.escape(separator) + '--')
-            mo = cre.search(payload, start)
-            if mo:
-                terminator = mo.start()
-                linesep = mo.group('sep')
-                if mo.end() < len(payload):
-                    # There's some post-MIME boundary epilogue
-                    epilogue = payload[mo.end():]
-            elif self._strict:
-                raise Errors.BoundaryError(
-                        "Couldn't find terminating boundary: %s" % boundary)
-            else:
-                # Handle the case of no trailing boundary.  Check that it ends
-                # in a blank line.  Some cases (spamspamspam) don't even have
-                # that!
-                mo = re.search('(?P<sep>\r\n|\r|\n){2}$', payload)
-                if not mo:
-                    mo = re.search('(?P<sep>\r\n|\r|\n)$', payload)
-                    if not mo:
-                        raise Errors.BoundaryError(
-                          'No terminating boundary and no trailing empty line')
-                linesep = mo.group('sep')
-                terminator = len(payload)
-            # We split the textual payload on the boundary separator, which
-            # includes the trailing newline. If the container is a
-            # multipart/digest then the subparts are by default message/rfc822
-            # instead of text/plain.  In that case, they'll have a optional
-            # block of MIME headers, then an empty line followed by the
-            # message headers.
-            parts = re.split(
-                linesep + re.escape(separator) + r'[ \t]*' + linesep,
-                payload[start:terminator])
-            for part in parts:
-                if isdigest:
-                    if part.startswith(linesep):
-                        # There's no header block so create an empty message
-                        # object as the container, and lop off the newline so
-                        # we can parse the sub-subobject
-                        msgobj = self._class()
-                        part = part[len(linesep):]
-                    else:
-                        parthdrs, part = part.split(linesep+linesep, 1)
-                        # msgobj in this case is the "message/rfc822" container
-                        msgobj = self.parsestr(parthdrs, headersonly=1)
-                    # while submsgobj is the message itself
-                    msgobj.set_default_type('message/rfc822')
-                    maintype = msgobj.get_content_maintype()
-                    if maintype in ('message', 'multipart'):
-                        submsgobj = self.parsestr(part)
-                        msgobj.attach(submsgobj)
-                    else:
-                        msgobj.set_payload(part)
-                else:
-                    msgobj = self.parsestr(part)
-                container.preamble = preamble
-                container.epilogue = epilogue
-                container.attach(msgobj)
-        elif container.get_main_type() == 'multipart':
-            # Very bad.  A message is a multipart with no boundary!
-            raise Errors.BoundaryError(
-                'multipart message with no defined boundary')
-        elif container.get_type() == 'message/delivery-status':
-            # This special kind of type contains blocks of headers separated
-            # by a blank line.  We'll represent each header block as a
-            # separate Message object
-            blocks = []
-            while True:
-                blockmsg = self._class()
-                self._parseheaders(blockmsg, fp)
-                if not len(blockmsg):
-                    # No more header blocks left
-                    break
-                blocks.append(blockmsg)
-            container.set_payload(blocks)
-        elif container.get_main_type() == 'message':
-            # Create a container for the payload, but watch out for there not
-            # being any headers left
-            container.set_payload(fp.read())
-	    fp = StringIO.StringIO(container.get_payload(decode=True))
-            try:
-                msg = self.parse(fp)
-            except Errors.HeaderParseError:
-                msg = self._class()
-                self._parsebody(msg, fp)
-            container.set_payload([msg])
-        else:
-            text = fp.read()
-            if firstbodyline is not None:
-                text = firstbodyline + '\n' + text
-            container.set_payload(text)
-
 def unquote(s):
     """Remove quotes from a string."""
     if len(s) > 1:
@@ -221,10 +100,11 @@ def unquote(s):
             s = s[1:-1]
 	  else: # remove garbage after trailing quote
 	    try: s = s[1:s[1:].index('"')+1]
-	    except: return s
+	    except:
+	      return s
 	  return s.replace('\\\\', '\\').replace('\\"', '"')
         if s.startswith('<') and s.endswith('>'):
-            return s[1:-1]
+	  return s[1:-1]
     return s
 
 from types import TupleType
@@ -235,27 +115,11 @@ def _unquotevalue(value):
   else:
       return unquote(value)
 
-email.Message._unquotevalue = _unquotevalue
-
-def _parseparam(s):
-    plist = []
-    while s[:1] == ';':
-	s = s[1:]
-	end = s.find(';')
-	while end > 0 and (s.count('"',0,end) & 1):
-	  end = s.find(';',end + 1)
-	if end < 0: end = len(s)
-	f = s[:end]
-	if '=' in f:
-	    i = f.index('=')
-	    f = f[:i].strip().lower() + \
-		    '=' + f[i+1:].strip()
-	plist.append(f.strip())
-	s = s[end:]
-    return plist
+#email.Message._unquotevalue = _unquotevalue
+
+from email.Message import _parseparam
 
 # Enhance email.Message 
-# - Fix getparam to parse attributes IE style
 # - Provide a headerchange event for integration with Milter
 #   Headerchange attribute can be assigned a function to be called when
 #   changing headers.  The signature is:
@@ -266,64 +130,19 @@ class MimeMessage(Message):
   """Version of email.Message.Message compatible with old mime module
   """
   def __init__(self,fp=None,seekable=1):
+    Message.__init__(self)
     self.headerchange = None
     self.submsg = None
-    Message.__init__(self)
-    self.fp = fp
-    if fp:
-      parser = MimeParser(MimeMessage)
-      self.startofheaders = fp.tell()
-      parser._parseheaders(self,fp)
-      self.startofbody = fp.tell()
-      parser._parsebody(self,fp)
-    for part in self.walk():
-      part.modified = False
-
-  def rewindbody(self):
-    return self.fp.seek(self.startofbody)
-
-  # override param parsing to handle quotes
-  def _get_params_preserve(self,failobj=None,header='content-type'):
-    "Return all parameter names and values. Use parser that handles quotes."
-    missing = []
-    value = self.get(header, missing)
-    if value is missing:
-	return failobj
-    params = []
-    for p in _parseparam(';' + value):
-	  try:
-	      name, val = p.split('=', 1)
-	      name = name.strip()
-	      val = val.strip()
-	  except ValueError:
-	      # Must have been a bare attribute
-	      name = p.strip()
-	      val = ''
-	  params.append((name, val))
-    params = Utils.decode_params(params)
-    return params
-
-  def get_filename(self, failobj=None):
-      """Return the filename associated with the payload if present.
-
-      The filename is extracted from the Content-Disposition header's
-      `filename' parameter, and it is unquoted.
-      """
-      missing = []
-      filename = self.get_param('filename', missing, 'content-disposition')
-      if filename is missing:
-	  return failobj
-      if isinstance(filename, TupleType):
-	  # It's an RFC 2231 encoded parameter
-	  newvalue = _unquotevalue(filename)
-	  if newvalue[0]:
-	    return unicode(newvalue[2], newvalue[0])
-	  return unicode(newvalue[2])
-      else:
-	  newvalue = _unquotevalue(filename.strip())
-	  return newvalue
+    self.modified = False
 
-  getfilename = get_filename
+  def get_param(self, param, failobj=None, header='content-type', unquote=True):
+    val = Message.get_param(self,param,failobj,header,unquote)
+    if val != failobj and param == 'boundary' and unquote:
+      # unquote boundaries an extra time, test case testDefang5
+      return _unquotevalue(val)
+    return val
+
+  getfilename = Message.get_filename
   ismultipart = Message.is_multipart
   getheaders = Message.get_all
   gettype = Message.get_content_type
@@ -338,7 +157,7 @@ class MimeMessage(Message):
     """Return a list of (attr,name) pairs of attributes that IE might
        interpret as a name - and hence decide to execute this message."""
     names = []
-    for attr,val in self.get_params([]):
+    for attr,val in self._get_params_preserve([],'content-type'):
       if isinstance(val, TupleType):
 	  # It's an RFC 2231 encoded parameter
 	  newvalue = _unquotevalue(val)
@@ -354,7 +173,7 @@ class MimeMessage(Message):
   def ismodified(self):
     "True if this message or a subpart has been modified."
     if not self.is_multipart():
-      if self.submsg:
+      if isinstance(self.submsg,Message):
         return self.submsg.ismodified()
       return self.modified
     if self.modified: return True
@@ -401,7 +220,7 @@ class MimeMessage(Message):
 
   def get_payload(self,i=None,decode=False):
     msg = self.submsg
-    if msg and msg.ismodified():
+    if isinstance(msg,Message) and msg.ismodified():
       self.set_payload([msg])
     return Message.get_payload(self,i,decode)
 
@@ -415,18 +234,27 @@ class MimeMessage(Message):
     self.submsg = None
 
   def get_submsg(self):
-    if self.get_content_type().lower() == 'message/rfc822':
+    t = self.get_content_type().lower()
+    if t == 'message/rfc822' or t.startswith('multipart/'):
       if not self.submsg:
         txt = self.get_payload()
 	if type(txt) == str:
 	  txt = self.get_payload(decode=True)
-	  parser = MimeParser(MimeMessage)
-	  self.submsg = parser.parsestr(txt)
+	  self.submsg = email.message_from_string(txt,MimeMessage)
+	  for part in self.submsg.walk():
+	    part.modified = False
 	else:
 	  self.submsg = txt[0]
       return self.submsg
     return None
 
+def message_from_file(fp):
+  msg = email.message_from_file(fp,MimeMessage)
+  for part in msg.walk():
+    part.modified = False
+  assert not msg.ismodified()
+  return msg
+
 extlist = ''.join("""
 ade,adp,asd,asx,asp,bas,bat,chm,cmd,com,cpl,crt,dll,exe,hlp,hta,inf,ins,isp,js,
 jse,lnk,mdb,mde,msc,msi,msp,mst,ocx,pcd,pif,reg,scr,sct,shs,url,vb,vbe,vbs,wsc,
@@ -471,7 +299,7 @@ msg	MimeMessage
 check	function(MimeMessage): int
 	Return CONTINUE, REJECT, ACCEPT
   """
-  if msg.ismultipart() and not msg.get_content_type() == 'message/rfc822':
+  if msg.is_multipart():
     for i in msg.get_payload():
       rc = check_attachments(i,check)
       if rc != Milter.CONTINUE: return rc
@@ -480,28 +308,33 @@ check	function(MimeMessage): int
 
 # save call context for Python without nested_scopes
 class _defang:
-  def __init__(self,savname,check):
-    self._savname = savname
-    self._check = check
-    self.scan_rfc822 = True
+
+  def __init__(self):
     self.scan_html = True
+
   def _chk_name(self,msg):
     rc = check_name(msg,self._savname,self._check)
     if self.scan_html:
       check_html(msg,self._savname)	# remove scripts from HTML
     if self.scan_rfc822:
       msg = msg.get_submsg()
-      if msg: return check_attachments(msg,self._chk_name)
+      if isinstance(msg,Message):
+        return check_attachments(msg,self._chk_name)
     return rc
 
+  def __call__(self,msg,savname=None,check=check_ext,scan_rfc822=True):
+    """Compatible entry point.
+    Replace all attachments with dangerous names."""
+    self._savname = savname
+    self._check = check
+    self.scan_rfc822 = scan_rfc822
+    check_attachments(msg,self._chk_name)
+    if msg.ismodified():
+      return True
+    return False
+
 # emulate old defang function
-def defang(msg,savname=None,check=check_ext):
-  """Compatible entry point.
-Replace all attachments with dangerous names."""
-  check_attachments(msg,_defang(savname,check)._chk_name)
-  if msg.ismodified():
-    return 1;
-  return 0
+defang = _defang()
 
 import sgmllib
 
@@ -631,3 +464,20 @@ def check_html(msg,savname=None):
       del msg["content-transfer-encoding"]
       email.Encoders.encode_quopri(msg)
   return Milter.CONTINUE
+
+if __name__ == '__main__':
+  import sys
+  def _list_attach(msg):
+    t = msg.get_content_type()
+    p = msg.get_payload(decode=True)
+    print msg.get_filename(),msg.get_content_type(),type(p)
+    msg = msg.get_submsg()
+    if isinstance(msg,Message):
+      return check_attachments(msg,_list_attach)
+    return Milter.CONTINUE
+
+  for fname in sys.argv[1:]:
+    fp = open(fname)
+    msg = message_from_file(fp)
+    email.Iterators._structure(msg)
+    check_attachments(msg,_list_attach)
diff --git a/sample.py b/sample.py
index 1b68ce7..5fc8df2 100644
--- a/sample.py
+++ b/sample.py
@@ -126,7 +126,7 @@ class sampleMilter(Milter.Milter):
   def eom(self):
     if not self.fp: return Milter.ACCEPT
     self.fp.seek(0)
-    msg = mime.MimeMessage(self.fp)
+    msg = mime.message_from_file(self.fp)
     msg.headerchange = self._headerChange
     if not mime.defang(msg,self.tempname):
       os.remove(self.tempname)
diff --git a/setup.py b/setup.py
index a1c980c..775cee7 100644
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@ if sys.version < '2.2.3':
   DistributionMetadata.classifiers = None
   DistributionMetadata.download_url = None
 
-setup(name = "milter", version = "0.7.2",
+setup(name = "milter", version = "0.8.0",
 	description="Python interface to sendmail milter API",
 	long_description="""\
 This is a python extension module to enable python scripts to
diff --git a/spf.py b/spf.py
index 8ce7844..450f149 100755
--- a/spf.py
+++ b/spf.py
@@ -45,6 +45,12 @@ For news, bugfixes, etc. visit the home page for this implementation at
 # Terrence is not responding to email.
 #
 # $Log$
+# Revision 1.24  2005/03/16 21:58:39  stuart
+# Change Milter module to package.
+#
+# Revision 1.22  2005/02/09 17:52:59  stuart
+# Report DNS errors as PermError rather than unknown.
+#
 # Revision 1.21  2004/11/20 16:37:03  stuart
 # Handle multi-segment TXT records.
 #
@@ -304,7 +310,7 @@ except NameError:
 DEFAULT_SPF = 'v=spf1 a/24 mx/24 ptr'
 
 # maximum DNS lookups allowed
-MAX_LOOKUP = 50
+MAX_LOOKUP = 100
 MAX_RECURSION = 20
 
 class TempError(Exception):
@@ -312,8 +318,16 @@ class TempError(Exception):
 
 class PermError(Exception):
 	"Permanent SPF error"
-
-def check(i, s, h,local=None):
+	def __init__(self,msg,mech=None):
+	  Exception.__init__(self,msg,mech)
+	  self.msg = msg
+	  self.mech = mech
+	def __str__(self):
+	  if self.mech:
+	    return '%s: %s'%(self.msg,self.mech)
+	  return self.msg
+
+def check(i, s, h,local=None,receiver=None):
 	"""Test an incoming MAIL FROM:<s>, from a client with ip address i.
 	h is the HELO/EHLO domain name.
 
@@ -327,7 +341,7 @@ def check(i, s, h,local=None):
 	#>>> check(i='61.51.192.42', s='liukebing@bcc.com', h='bmsi.com')
 
 	"""
-	return query(i=i, s=s, h=h,local=local).check()
+	return query(i=i, s=s, h=h,local=local,receiver=receiver).check()
 
 class query(object):
 	"""A query object keeps the relevant information about a single SPF
@@ -348,13 +362,17 @@ class query(object):
 
 	Also keeps cache: DNS cache.
 	"""
-	def __init__(self, i, s, h,local=None):
+	def __init__(self, i, s, h,local=None,receiver=None):
 		self.i, self.s, self.h = i, s, h
+		if not s and h:
+		  self.s = 'postmaster@' + h
 		self.l, self.o = split_email(s, h)
 		self.t = str(int(time.time()))
 		self.v = 'in-addr'
 		self.d = self.o
 		self.p = None
+		if receiver:
+		  self.r = receiver
 		self.cache = {}
 		self.exps = dict(EXPLANATIONS)
 		self.local = local	# local policy
@@ -398,7 +416,11 @@ class query(object):
 		except TempError,x:
 			return ('error', 450, 'SPF Temporary Error: ' + str(x))
 		except PermError,x:
-			return ('error', 550, 'SPF Permanent Error: ' + str(x))
+			# Pre-Lentczner draft treats this as an unknown result
+			# and equivalent to no SPF record.
+			self.prob = x.msg
+			self.mech.append(x.mech)
+			return ('unknown', 550, 'SPF Permanent Error: ' + str(x))
 
 	def check1(self, spf, domain, recursion):
 		# spf rfc: 3.7 Processing Limits
@@ -456,87 +478,86 @@ class query(object):
 		# Look for mechanisms
 		#
 		for mech in spf:
-			if RE_MODIFIER.match(mech): continue
-			m, arg, cidrlength = parse_mechanism(mech, self.d)
-
-			# map '?' '+' or '-' to 'unknown' 'pass' or 'fail'
-			if m:
-			  result = RESULTS.get(m[0])
-			  if result:
-				# eat '?' '+' or '-'
-				m = m[1:]
-			  else:
-				# default pass
-				result = 'pass'
-
-			if m in ['a', 'mx', 'ptr', 'prt', 'exists', 'include']:
-				arg = self.expand(arg)
-
-			if m == 'include':
-			  if arg != self.d:
-			    res,code,txt = self.check1(self.dns_spf(arg),
-					      arg, recursion + 1)
-			    if res == 'pass':
-			      break
-			    if res == 'none':
-			      raise PermError(
-			      	'No valid SPF record for included domain')
-			    continue
-			  else:
-			    raise PermError('include mechanism missing domain')
-			elif m == 'all':
-				break
-
-			elif m == 'exists':
-				if len(self.dns_a(arg)) > 0:
-					break
+		    if RE_MODIFIER.match(mech): continue
+		    m, arg, cidrlength = parse_mechanism(mech, self.d)
+
+		    # map '?' '+' or '-' to 'unknown' 'pass' or 'fail'
+		    if m:
+		      result = RESULTS.get(m[0])
+		      if result:
+			    # eat '?' '+' or '-'
+			    m = m[1:]
+		      else:
+			    # default pass
+			    result = 'pass'
+
+		    if m in ['a', 'mx', 'ptr', 'prt', 'exists', 'include']:
+			    arg = self.expand(arg)
+
+		    if m == 'include':
+		      if arg != self.d:
+			res,code,txt = self.check1(self.dns_spf(arg),
+					  arg, recursion + 1)
+			if res == 'pass':
+			  break
+			if res == 'none':
+			  raise PermError(
+			    'No valid SPF record for included domain: %s'%arg,
+			    mech)
+			continue
+		      else:
+			raise PermError('include mechanism missing domain',mech)
+		    elif m == 'all':
+			    break
+
+		    elif m == 'exists':
+			    if len(self.dns_a(arg)) > 0:
+				    break
 
-			elif m == 'a':
-				if cidrmatch(self.i, self.dns_a(arg),
-				             cidrlength):
-					break
+		    elif m == 'a':
+			    if cidrmatch(self.i, self.dns_a(arg),
+					 cidrlength):
+				    break
 
-			elif m == 'mx':
-				if cidrmatch(self.i, self.dns_mx(arg),
-				             cidrlength):
-					break
+		    elif m == 'mx':
+			    if cidrmatch(self.i, self.dns_mx(arg),
+					 cidrlength):
+				    break
 
-			elif m in ('ip4', 'ipv4', 'ip') and arg != self.d:
-			    try:
-				if cidrmatch(self.i, [arg], cidrlength):
+		    elif m in ('ip4', 'ipv4', 'ip') and arg != self.d:
+			try:
+			    if cidrmatch(self.i, [arg], cidrlength):
+				break
+			except socket.error:
+			    raise PermError('syntax error',mech)
+			    
+		    elif m in ('ip6', 'ipv6'):
+		    # Until we support IPV6, we should never
+		    # get an IPv6 connection.  So this mech
+		    # will never match.
+			    pass
+
+		    elif m in ('ptr', 'prt'):
+			    if domainmatch(self.validated_ptrs(self.i),
+					   arg):
 				    break
-			    except socket.error:
-				self.mech.append(mech)
-				self.prob = 'Bad mechanism syntax found'
-				return ('unknown',250,'SPF mechanism syntax error')
-			        
-			elif m in ('ip6', 'ipv6'):
-			# Until we support IPV6, we should never
-			# get an IPv6 connection.  So this mech
-			# will never match.
-				pass
-
-			elif m in ('ptr', 'prt'):
-				if domainmatch(self.validated_ptrs(self.i),
-				               arg):
-					break
 
-			else:
-			  # unknown mechanisms cause immediate unknown
-			  # abort results
-			  raise PermError('Unknown mechanism found: ' + mech)
+		    else:
+		      # unknown mechanisms cause immediate unknown
+		      # abort results
+		      raise PermError('Unknown mechanism found',mech)
 		else:
-			# no matches
-			if redirect:
-				return self.check1(self.dns_spf(redirect),
-				                   redirect, recursion + 1)
-			else:
-				result = default
+		    # no matches
+		    if redirect:
+			return self.check1(self.dns_spf(redirect),
+					       redirect, recursion + 1)
+		    else:
+			result = default
 
 		if result == 'fail':
-			return (result, 550, exps[result])
+		    return (result, 550, exps[result])
 		else:
-			return (result, 250, exps[result])
+		    return (result, 250, exps[result])
 
 	def get_explanation(self, spec):
 		"""Expand an explanation."""
@@ -653,7 +674,10 @@ class query(object):
 		  if not a:
 		    # No SPF record: convert and return CID if present
 		    p = CIDParser(q=self)
-		    return p.spf_txt(domain)
+		    try:
+		      return p.spf_txt(domain)
+		    except xml.sax._exceptions.SAXParseException,x:
+		      raise PermError("Caller-ID parse error",domain)
 
 		if len(a) == 1:
 			return a[0]
@@ -725,7 +749,7 @@ class query(object):
 		return result
 
 	def get_header(self,res,receiver):
-	  if res in ('pass','fail'):
+	  if res in ('pass','fail','softfail'):
 	    return '%s (%s: %s) client-ip=%s; envelope-from=%s; helo=%s;' % (
 	  	res,receiver,self.get_header_comment(res),self.i,
 	        self.l + '@' + self.o, self.h)
@@ -991,13 +1015,15 @@ if __name__ == '__main__':
 		print USAGE
 		_test()
 	elif len(sys.argv) == 2:
-		q = query(i='127.0.0.1', s='localhost', h='unknown')
+		q = query(i='127.0.0.1', s='localhost', h='unknown',
+			receiver=socket.gethostname())
 		print q.dns_spf(sys.argv[1])
 	elif len(sys.argv) == 4:
-		print check(i=sys.argv[1], s=sys.argv[2], h=sys.argv[3])
+		print check(i=sys.argv[1], s=sys.argv[2], h=sys.argv[3],
+			receiver=socket.gethostname())
 	elif len(sys.argv) == 5:
 		i, s, h = sys.argv[2:]
-		q = query(i=i, s=s, h=h)
+		q = query(i=i, s=s, h=h, receiver=socket.gethostname())
 		print q.check(sys.argv[1])
 	else:
 		print USAGE
diff --git a/testbms.py b/testbms.py
index eb70780..56752bd 100644
--- a/testbms.py
+++ b/testbms.py
@@ -4,6 +4,8 @@ import bms
 import mime
 import rfc822
 import StringIO
+import email
+import sys
 #import pdb
 
 class TestMilter(bms.bmsMilter):
@@ -25,7 +27,7 @@ class TestMilter(bms.bmsMilter):
   def replacebody(self,chunk):
     if self._body:
       self._body.write(chunk)
-      self.bodyreplaced = 1
+      self.bodyreplaced = True
     else:
       raise IOError,"replacebody not called from eom()"
 
@@ -39,14 +41,14 @@ class TestMilter(bms.bmsMilter):
       del self._msg[field]
     else:
       self._msg[field] = value
-    self.headerschanged = 1
+    self.headerschanged = True
 
   def addheader(self,field,value):
     if not self._body:
       raise IOError,"addheader not called from eom()"
     self.log('addheader: %s=%s' % (field,value))
     self._msg[field] = value
-    self.headerschanged = 1
+    self.headerschanged = True
 
   def delrcpt(self,rcpt):
     if not self._body:
@@ -63,8 +65,8 @@ class TestMilter(bms.bmsMilter):
 
   def feedFile(self,fp,sender="spam@adv.com",rcpt="victim@lamb.com"):
     self._body = None
-    self.bodyreplaced = 0
-    self.headerschanged = 0
+    self.bodyreplaced = False
+    self.headerschanged = False
     self.reply = None
     msg = rfc822.Message(fp)
     rc = self.envfrom('<%s>'%sender)
@@ -118,7 +120,7 @@ class TestMilter(bms.bmsMilter):
 
   def connect(self,host='localhost'):
     self._body = None
-    self.bodyreplaced = 0
+    self.bodyreplaced = False
     rc =  bms.bmsMilter.connect(self,host,1,('1.2.3.4',1234)) 
     if rc != Milter.CONTINUE and rc != Milter.ACCEPT:
       self.close()
@@ -141,7 +143,7 @@ class BMSMilterTestCase(unittest.TestCase):
     open('test/'+fname+".tstout","w").write(fp.getvalue())
     #self.failUnless(fp.getvalue() == open("test/virus1.out","r").read())
     fp.seek(0)
-    msg = mime.MimeMessage(fp)
+    msg = mime.message_from_file(fp)
     str = msg.get_payload(1).get_payload()
     milter.log(str)
     milter.close()
@@ -218,7 +220,9 @@ class BMSMilterTestCase(unittest.TestCase):
     #pdb.set_trace()
     rc = milter.feedMsg('test8')
     self.assertEqual(rc,Milter.ACCEPT)
-    self.failUnless(milter.bodyreplaced,"Message body not replaced")
+    # python2.4 doesn't scan encoded message attachments
+    if sys.hexversion < 0x02040000:
+      self.failUnless(milter.bodyreplaced,"Message body not replaced")
     #self.failIf(milter.bodyreplaced,"Message body replaced")
     fp = milter._body
     open("test/test8.tstout","w").write(fp.getvalue())
@@ -237,9 +241,12 @@ class BMSMilterTestCase(unittest.TestCase):
     bms.smart_alias[key] = ['ham@eggs.com']
     rc = milter.feedMsg('test8',key[0],key[1])
     self.assertEqual(rc,Milter.ACCEPT)
-    self.failUnless(milter.bodyreplaced,"Message body not replaced")
     self.failUnless(milter._delrcpt == ['<baz@bat.com>'])
     self.failUnless(milter._addrcpt == ['<ham@eggs.com>'])
+    # python2.4 email does not decode message attachments, so script
+    # is not replaced
+    if sys.hexversion < 0x02040000:
+      self.failUnless(milter.bodyreplaced,"Message body not replaced")
 
   def testBadBoundary(self):
     milter = TestMilter()
@@ -247,8 +254,11 @@ class BMSMilterTestCase(unittest.TestCase):
     # test rfc822 attachment with invalid boundaries
     #pdb.set_trace()
     rc = milter.feedMsg('bound')
-    self.assertEqual(rc,Milter.REJECT)
-    self.assertEqual(milter.reply[0],'554')
+    if sys.hexversion < 0x02040000:
+      # python2.4 adds invalid boundaries to decects list and makes
+      # payload a str
+      self.assertEqual(rc,Milter.REJECT)
+      self.assertEqual(milter.reply[0],'554')
     #self.failUnless(milter.bodyreplaced,"Message body not replaced")
     self.failIf(milter.bodyreplaced,"Message body replaced")
     fp = milter._body
@@ -277,7 +287,6 @@ class BMSMilterTestCase(unittest.TestCase):
 def suite(): return unittest.makeSuite(BMSMilterTestCase,'test')
 
 if __name__ == '__main__':
-  import sys
   if len(sys.argv) > 1:
     for fname in sys.argv[1:]:
       milter = TestMilter()
diff --git a/testmime.py b/testmime.py
index 0f6c934..019ec89 100644
--- a/testmime.py
+++ b/testmime.py
@@ -1,8 +1,23 @@
+# $Log$
+# Revision 1.23  2005/02/11 18:34:14  stuart
+# Handle garbage after quote in boundary.
+#
+# Revision 1.22  2005/02/10 01:10:59  stuart
+# Fixed MimeMessage.ismodified()
+#
+# Revision 1.21  2005/02/10 00:56:49  stuart
+# Runs with python2.4.  Defang not working correctly - more work needed.
+#
+# Revision 1.20  2004/11/20 16:38:17  stuart
+# Add rcs log
+#
 import unittest
 import mime
 import socket
 import StringIO
 import email
+import sys
+from email import Errors
 
 samp1_txt1 = """Dear Agent 1
 I hope you can read this.  Whenever you write label it  P.B.S kids.
@@ -24,22 +39,38 @@ class MimeTestCase(unittest.TestCase):
     self.failUnless(plist[0] == 'name="Jim&amp;amp;Girlz.jpg"')
 
   def testParse(self,fname='samp1'):
-    msg = mime.MimeMessage(open('test/'+fname,"r"))
+    msg = mime.message_from_file(open('test/'+fname,"r"))
     self.failUnless(msg.ismultipart())
     parts = msg.get_payload()
     self.failUnless(len(parts) == 2)
     txt1 = parts[0].get_payload()
     self.failUnless(txt1.rstrip() == samp1_txt1,txt1)
+    msg = mime.message_from_file(open('test/missingboundary',"r"))
+    # should get no exception as long as we don't try to parse
+    # message attachments
+    mime.defang(msg,scan_rfc822=False)
+    msg.dump(open('test/missingboundary.out','w'))
+    msg = mime.message_from_file(open('test/missingboundary',"r"))
+    try:
+      mime.defang(msg)
+      # python 2.4 doesn't get exceptions on missing boundaries, and
+      # if message is modified, output is readable by mail clients
+      if sys.hexversion < 0x02040000:
+	self.fail('should get boundary error parsing bad rfc822 attachment')
+    except Errors.BoundaryError:
+      pass
   
   def testDefang(self,vname='virus1',part=1,
   	fname='LOVE-LETTER-FOR-YOU.TXT.vbs'):
-    msg = mime.MimeMessage(open('test/'+vname,"r"))
+    msg = mime.message_from_file(open('test/'+vname,"r"))
     mime.defang(msg)
+    self.failUnless(msg.ismodified(),"virus not removed")
     oname = vname + '.out'
     msg.dump(open('test/'+oname,"w"))
-    msg = mime.MimeMessage(open('test/'+oname,"r"))
-    parts = msg.get_payload()
-    txt2 = parts[part].get_payload()
+    msg = mime.message_from_file(open('test/'+oname,"r"))
+    txt2 = msg.get_payload()
+    if type(txt2) == list:
+      txt2 = txt2[part].get_payload()
     self.failUnless(txt2.rstrip()+'\n' == mime.virus_msg % (fname,hostname,None),txt2)
 
   def testDefang3(self):
@@ -55,11 +86,11 @@ class MimeTestCase(unittest.TestCase):
 
   # virus6 has no parts - the virus is directly inline
   def testDefang6(self,vname="virus6",fname='FAX20.exe'):
-    msg = mime.MimeMessage(open('test/'+vname,"r"))
+    msg = mime.message_from_file(open('test/'+vname,"r"))
     mime.defang(msg)
     oname = vname + '.out'
     msg.dump(open('test/'+oname,"w"))
-    msg = mime.MimeMessage(open('test/'+oname,"r"))
+    msg = mime.message_from_file(open('test/'+oname,"r"))
     self.failIf(msg.ismultipart())
     txt2 = msg.get_payload()
     self.failUnless(txt2 == mime.virus_msg % \
@@ -68,11 +99,11 @@ class MimeTestCase(unittest.TestCase):
   # honey virus has a sneaky ASP payload which is parsed correctly
   # by email package in python-2.2.2, but not by mime.MimeMessage or 2.2.1
   def testDefang7(self,vname="honey",fname='story[1].scr'):
-    msg = mime.MimeMessage(open('test/'+vname,"r"))
+    msg = mime.message_from_file(open('test/'+vname,"r"))
     mime.defang(msg)
     oname = vname + '.out'
     msg.dump(open('test/'+oname,"w"))
-    msg = mime.MimeMessage(open('test/'+oname,"r"))
+    msg = mime.message_from_file(open('test/'+oname,"r"))
     parts = msg.get_payload()
     txt2 = parts[1].get_payload()
     txt3 = parts[2].get_payload()
@@ -83,7 +114,7 @@ class MimeTestCase(unittest.TestCase):
 	  ('story[1].asp',hostname,None),txt3)
 
   def testParse2(self,fname="spam7"):
-    msg = mime.MimeMessage(open('test/'+fname,"r"))
+    msg = mime.message_from_file(open('test/'+fname,"r"))
     self.failUnless(msg.ismultipart())
     parts = msg.get_payload()
     self.failUnless(len(parts) == 2)
@@ -106,10 +137,9 @@ class MimeTestCase(unittest.TestCase):
 def suite(): return unittest.makeSuite(MimeTestCase,'test')
 
 if __name__ == '__main__':
-  import sys
   if len(sys.argv) < 2:
     unittest.main()
   else:
     for fname in sys.argv[1:]:
       fp = open(fname,'r')
-      msg = mime.MimeMessage(fp)
+      msg = mime.message_from_file(fp)
diff --git a/testsample.py b/testsample.py
index 4ed278a..74acdaa 100644
--- a/testsample.py
+++ b/testsample.py
@@ -17,7 +17,7 @@ class TestMilter(sample.sampleMilter):
   def replacebody(self,chunk):
     if self._body:
       self._body.write(chunk)
-      self.bodyreplaced = 1
+      self.bodyreplaced = True
     else:
       raise IOError,"replacebody not called from eom()"
 
@@ -29,16 +29,16 @@ class TestMilter(sample.sampleMilter):
       del self._msg[field]
     else:
       self._msg[field] = value
-    self.headerschanged = 1
+    self.headerschanged = True
 
   def addheader(self,field,value):
     self.log('addheader: %s=%s' % (field,value))
     self._msg[field] = value
-    self.headerschanged = 1
+    self.headerschanged = True
 
   def feedMsg(self,fname):
     self._body = None
-    self.bodyreplaced = 0
+    self.bodyreplaced = False
     self.headerschanged = 0
     fp = open('test/'+fname,'r')
     msg = rfc822.Message(fp)
@@ -85,7 +85,7 @@ class TestMilter(sample.sampleMilter):
 
   def connect(self,host='localhost'):
     self._body = None
-    self.bodyreplaced = 0
+    self.bodyreplaced = False
     rc =  sample.sampleMilter.connect(self,host,1,0) 
     if rc != Milter.CONTINUE and rc != Milter.ACCEPT:
       self.close()
@@ -108,7 +108,7 @@ class BMSMilterTestCase(unittest.TestCase):
     open('test/'+fname+".tstout","w").write(fp.getvalue())
     #self.failUnless(fp.getvalue() == open("test/virus1.out","r").read())
     fp.seek(0)
-    msg = mime.MimeMessage(fp)
+    msg = mime.message_from_file(fp)
     s = msg.get_payload(1).get_payload()
     milter.log(s)
     milter.close()
-- 
GitLab