diff --git a/HOWTO b/HOWTO
index 797c0c1e187e25504b573aab710f184ce39d9c22..2d11e3ce9727c6540db8097fbb344e125fdb74b1 100644
--- a/HOWTO
+++ b/HOWTO
@@ -93,23 +93,38 @@ string blocking.
The sendmail access file, or another readonly database with that
format, can be used for detail spf policy. SPF access policy
record are tagged with "SPF-{Result}:". Results are
-Pass, Neutral, Softfail, Fail, TempError, PermError. Currently supported
-policy keywords are OK, CBV, REJECT, TEMPFAIL, ERROR:"550 description".
+Pass, Neutral, Softfail, Fail, PermError. Currently supported
+policy keywords are OK, CBV, REJECT. Currently, TempError always
+results in TEMPFAIL.
-The default policies are as follows:
+The default policies are set in pymilter.cfg. The defaults
+if none of the config options are set are as follows:
SPF-Fail: REJECT
SPF-Softfail: CBV
SPF-Neutral: OK
SPF-PermError: REJECT
-SPF-TempError: TEMPFAIL
SPF-Pass: OK
The tag may be followed by a specific domain. For instance, to
require a Pass from aol.com:
-SPF-Neutral:aol.com ERROR:"550 AOL mail must get SPF PASS"
-SPF-Softfail:aol.com ERROR:"550 AOL mail must get SPF PASS"
+SPF-Neutral:aol.com REJECT
+SPF-Softfail:aol.com REJECT
+
+The CBV policy requires a valid HELO name. If the EHLO name is
+RFC2822 compliant, then a DSN is sent to the alleged sender. The
+template for the DSN is selected according to the SPF result:
+
+Fail: softfail.txt
+SoftFail: softfail.txt
+Neutral: neutral.txt
+PermError: permerror.txt
+None: strike3.txt
+Pass: strike3.txt
+
+The pass template doesn't make any sense - I assumed that CBV would
+never be used with a Pass result.
To be continued.
diff --git a/MANIFEST.in b/MANIFEST.in
index 7226dda9548d41d43fb9f10f5efb44be83bfbb9f..75bd46bc72e44d6f63d46150eaaccc3a51852d29 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -24,3 +24,4 @@ include milter.rc7
include milter.cfg
include rhsbl.m4
include *.txt
+include *.html
diff --git a/NEWS b/NEWS
index 2647f7398558caac6803882d62727d0c9c42c9b5..c7369eb5c05107d03cb4c2727ce4f60bf5f6feed 100644
--- a/NEWS
+++ b/NEWS
@@ -3,9 +3,12 @@ Here is a history of user visible changes to Python milter.
0.8.3 Keep screened honeypot mail, but optionally discard honeypot only mail.
spf_accept_fail option for braindead SPF senders
(treats fail like softfail)
+ Option to set SPF policy via sendmail access map.
+ Option to supply Sender header from MAIL FROM when missing.
Consider SMTP AUTH connections internal.
Send DSN for SPF errors corrected by extended processing.
Send DSN before SCREENED mail is quarantined
+ Use logging package to keep log lines atomic.
0.8.2 Strict processing limits per SPF RFC
Fixed several parsing bugs under RFC
Support official IANA SPF record (type99)
diff --git a/bms.py b/bms.py
index 0676c4090a8d6687cf2e63fdb3b80bd448960f03..15bc2763b9bdad9d093c92cae700dac7f6d2dd80 100644
--- a/bms.py
+++ b/bms.py
@@ -1,6 +1,9 @@
#!/usr/bin/env python
# A simple milter that has grown quite a bit.
# $Log$
+# Revision 1.29 2005/10/11 22:50:07 customdesigned
+# Always check HELO except for SPF pass, temperror.
+#
# Revision 1.28 2005/10/10 23:50:20 customdesigned
# Use logging module to make logging threadsafe (avoid splitting log lines)
#
@@ -551,9 +554,9 @@ def parse_addr(t):
>>> parse_addr('user@example.com')
['user', 'example.com']
>>> parse_addr('"user@example.com"')
- ['"user@example.com"']
+ ['user@example.com']
>>> parse_addr('"user@bar"@example.com')
- ['"user@bar"','example.com']
+ ['user@bar', 'example.com']
>>> parse_addr('foo')
['foo']
"""
diff --git a/milter.cfg b/milter.cfg
index 89f48a6ffc040aadef1ced51dfb6c3cfc4246763..51299c6a4fcea2e414eb865f2a5f418166b4b029 100644
--- a/milter.cfg
+++ b/milter.cfg
@@ -89,11 +89,11 @@ reject_spoofed = 0
;reject_noptr = 0
# always accept softfail from these domains, or send DSN otherwise
;accept_softfail = bounces.amazon.com
-# treat fail from these domains like softfail: because their SPF record
+# Treat fail from these domains like softfail: because their SPF record
# or an important sender is screwed up. Must have valid HELO, however.
;accept_fail = custhelp.com
-# use sendmail access file or similar format for detailed spf policy
-# This will override any defaults set above
+# Use sendmail access map or similar format for detailed spf policy.
+# SPF entries in the access map will override any defaults set above.
;access_file = /etc/mail/access.db
# Add MAIL FROM as Sender when Sender is missing and From domain
# doesn't match MAIL FROM. Outlook and other email clients will then display
diff --git a/milter.spec b/milter.spec
index c2a8bfb5c6025de33c90f1568502cc0757081759..ae694e45a70ec46c33db64711c299062870d5f55 100644
--- a/milter.spec
+++ b/milter.spec
@@ -146,7 +146,7 @@ rm -rf $RPM_BUILD_ROOT
%files -f INSTALLED_FILES
%defattr(-,root,root)
-%doc README NEWS TODO CREDITS sample.py
+%doc README HOWTO NEWS TODO CREDITS sample.py
/etc/logrotate.d/milter
/etc/cron.daily/milter
%ifos aix4.1
@@ -174,6 +174,9 @@ rm -rf $RPM_BUILD_ROOT
- Consider SMTP AUTH connections internal.
- Send DSN for SPF errors corrected by extended processing.
- Send DSN before SCREENED mail is quarantined
+- Option to set SPF policy via sendmail access map.
+- Option to supply Sender header from MAIL FROM when missing.
+- Use logging package to keep log lines atomic.
* Fri Jul 15 2005 Stuart Gathman <stuart@bmsi.com> 0.8.2-4
- Limit each CNAME chain independently like PTR and MX
* Fri Jul 15 2005 Stuart Gathman <stuart@bmsi.com> 0.8.2-3
diff --git a/policy.html b/policy.html
new file mode 100644
index 0000000000000000000000000000000000000000..a8642d7fc1f120c62e4febe9f3ed7d131aa86ed3
--- /dev/null
+++ b/policy.html
@@ -0,0 +1,237 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<html>
+<head>
+<title>Python Milter Mail Policy </title>
+</head><body>
+
+<h1> Python Milter Mail Policy </h1>
+
+<h3> Classify connection </h3>
+
+When the SMTP client connects, the connection IP address is
+saved for later verification, and the connection
+is classified as INTERNAL or EXTERNAL by matching the ip
+address against the <code>internal_connect</code> configuration.
+IP addresses with no PTR, and PTR names that look like
+the kind assigned to dynamic IPs (as determined by a heuristic
+algorithm) are flagged as DYNAMIC. IPs that match the
+<code>trusted_relay</code> configuration are flagged as TRUSTED.
+<p>
+Examples from the log file (<i>not</i> the SMTP error message returned):
+<pre>
+2005Jul29 13:56:53 [71207] connect from p50863492.dip0.t-ipconnect.de at ('80.134.52.146', 1858) EXTERNAL DYN
+2005Jul29 18:10:15 [74511] connect from foopub at ('1.2.3.4', 46513) EXTERNAL TRUSTED
+2005Jul29 14:41:00 [71805] connect from foobar at ('192.168.0.1', 41205) INTERNAL
+2005Jul29 14:41:15 [71806] connect from cncln.online.ln.cn at ('218.25.240.137', 35992) EXTERNAL
+</pre>
+<p>
+Certain obviously evil PTR names are blocked at this point:
+"localhost" (when IP is not 127.*) and ".".
+<pre>
+2005Jul29 14:49:50 [71918] connect from localhost at ('221.132.0.6', 50507) EXTERNAL
+2005Jul29 14:49:50 [71918] REJECT: PTR is localhost
+</pre>
+
+<h3> HELO Check </h3>
+
+The HELO name provided by the client is saved for later verification
+(for example by SPF). We could validate the HELO at this point
+by verifying that an A record for the HELO name matches the connect ip.
+However, currently we only block certain obvious problems.
+HELO names that look like an IP4 address
+and ones that match the <code>hello_blacklist</code> configuration
+are immediately rejected. The hello_blacklist typically contains
+the current MTAs own HELO name or email domains.
+Clients that attempt to skip HELO are immediately rejected.
+<pre>
+2005Jul29 18:10:15 [74512] hello from example.com
+2005Jul29 18:10:15 [74512] REJECT: spam from self: example.com
+2005Jul29 18:17:09 [74581] hello from 80.191.244.69
+2005Jul29 18:17:09 [74581] REJECT: numeric hello name: 80.191.244.69
+</pre>
+
+<h3> MAIL FROM Check </h3>
+
+Before calling our milter, sendmail checks a DNS blacklist to
+block banned sender domains. We never see a blocked domain.
+<p>
+The MAIL FROM address is saved for possible use by the smart-alias
+feature. First, the <code>internal_domains</code> is used for
+a simple screening if defined. If the MAIL FROM for an INTERNAL connection
+is NOT in <code>internal_domains</code>, then it is rejected (the
+PC is most likely infected and attempting to send out spam).
+If the MAIL FROM for an EXTERNAL connection IS in
+<code>internal_domains</code>, then the message is immediately rejected.
+This is quick and effective for most small company MTAs. For more
+complex mail networks, it is too simplistic, and should not be defined.
+SPF will handle the complex cases.
+
+<h4> wiretap </h4>
+
+The wiretap feature can screen and/or monitor mail to/from certain
+users. If the MAIL FROM is being wiretapped, the recipients are
+altered accordingly.
+
+<h4> SPF check </h4>
+
+Finally, the MAIL FROM, connect IP, and HELO name are checked against
+any SPF records published via DNS for the alleged sender (MAIL FROM).
+If there is no SPF record, we check for a local substitute under the
+domain defined in the <code>[spf]delegate</code> configuration.
+Further checks depend on the result.
+
+<table border=1>
+<tr><th>NONE</th><td>
+If there is no SPF record (official or delegated), then we
+initiate a "three strikes and your out" regime, which looks for
+<b>some</b> form of validated identification.
+<ol>
+<li>We try a "best guess" SPF record of "v=spf1 a/24 mx/24 ptr". If this
+ passes, good.
+<li> We try to validate the HELO name. First check for an SPF record.
+ Otherwise, check whether the connect IP matches any A record for
+ the HELO name, or any A record for any MX name for the HELO name,
+ or is at least in the same /24 subnet as any of the above.
+ (In other words, a HELO SPF "best guess" of "v=spf1 a/24 mx/24".)
+ If so, good. We consider the HELO validated. If the HELO SPF
+ check fails, we reject the email.
+</ol>
+<pre>
+2005Jul30 19:45:16 [93991] connect from [221.200.41.54] at ('221.200.41.54', 3581) EXTERNAL DYN
+2005Jul30 19:45:18 [93991] hello from adelphia.net
+2005Jul30 19:45:19 [93991] mail from <wendy.stubbsua@link-it.com> ()
+2005Jul30 19:45:19 [93991] REJECT: hello SPF: fail 550 access denied
+</pre>
+<ol>
+<li> If there is a validated PTR name, and it doesn't look
+ like a dynamic name, good. We consider the connection validated.
+</ol>
+If any of the above can be validated, we continue on.
+If none of the above can be validated, and the <code>[SPF]reject_noptr</code>
+option is true, we reject the message immediately with the explanation
+that we need some form of valid identification before we accept an email.
+If <code>[SPF]reject_noptr</code> is false, we flag the message as
+needing Call Back Validation.
+The Call Back Valildation sends a DSN to the purported sender informing
+them of the lack of identification. If the message is legitimate, the
+sender needs to know that their email setup is broken and should be corrected.
+If the message is forged, the sender is informed of the forgery,
+and their need to publish an SPF record or at least use a valid HELO name.
+If the purported sender does not accept the DSN,
+then the message is rejected. The CBV status is cached to avoid
+annoying the purported sender with too many DSNs. Currently, the DSN
+is repeated to the same sender once per month.
+<p>
+In this example, although 3com.com has no SPF record, we assume that
+any legitimate mail from them will at least have a valid HELO or PTR.
+<pre>
+2005Jul30 23:52:03 [96777] connect from [222.252.233.200] at ('222.252.233.200', 29934) EXTERNAL DYN
+2005Jul30 23:52:03 [96777] hello from 3mail.3com.com
+2005Jul30 23:52:04 [96777] mail from <etec_nic_family@3mail.3com.com> ()
+2005Jul30 23:52:04 [96777] REJECT: no PTR, HELO or SPF
+</pre>
+</td></tr>
+
+<tr><th>PASS</th><td>
+A pass result normally lets the email continue on, but the domain is
+tracked for reputation (and may be blocked), and may skip content scanning if
+it matches a whitelist.
+<pre>
+2005Jul24 17:44:26 [2104] mail from <gnucash-devel-bounces@gnucash.org> ('SIZE=4410',)
+2005Jul24 17:44:26 [2104] Received-SPF: pass (mail.bmsi.com: domain of gnucash.org
+ designates 204.107.200.65 as permitted sender)
+ client-ip=204.107.200.65; envelope-from=gnucash-devel-bounces@gnucash.org; helo=cvs.gnucash.org;
+</pre>
+</td></tr>
+
+<tr><th>NEUTRAL</th><td>
+A neutral result normally lets the email continue on, but the domain is not
+tracked for reputation or matched against any whitelists.
+Highly forged domains listed in <code>[SPF]reject_neutral</code> are
+rejected.
+<pre>
+2005Jul24 17:41:37 [2070] connect from cp500627-a.dbsch1.nb.home.nl at ('84.27.225.3', 3465) EXTERNAL
+2005Jul24 17:41:37 [2070] hello from cp500627-a.dbsch1.nb.home.nl
+2005Jul24 17:41:38 [2070] mail from <nwarjejkw@yahoo.com> ()
+2005Jul24 17:41:38 [2070] REJECT: SPF neutral for nwarjejkw@yahoo.com
+</pre>
+</td></tr>
+
+<tr><th>SOFTFAIL</th><td>
+A softfail result normally lets the email continue on, but the domain is not
+tracked for reputation or matched against any whitelists. Furthermore,
+the message is flagged as needing Call Back Validation,
+and the highly forged domains listed in <code>[SPF]reject_neutral</code> are
+rejected as well.
+<p>
+At present, we also require a valid HELO or PTR to avoid rejecting
+a softfail. But this should probably change to only require a
+successful CBV.
+<p>
+The Call Back Valildation sends a DSN to the purported sender informing
+them of the softfail. If the message is legitimate, the sender needs
+to know about the softfail so that their email setup can be corrected.
+If the message is forged, the sender is informed of the forgery, confirming
+that SPF is protecting their reputation and encouraging a rapid transition
+to a strict policy. If the purported sender does not accept the DSN,
+then the message is rejected. The CBV status is cached to avoid
+annoying the purported sender with too many DSNs. Currently, the DSN
+is repeated to the same sender once per month.
+<pre>
+2005Jul24 15:41:33 [801] mail from <Aitp@horafeliz.com> ()
+2005Jul24 15:41:33 [801] Received-SPF: softfail (mail.bmsi.com: transitioning domain of horafeliz.com
+ does not designate 221.184.83.185 as permitted sender)
+ client-ip=221.184.83.185; envelope-from=Aitp@horafeliz.com;
+ helo=p8185-ipad30funabasi.chiba.ocn.ne.jp;
+2005Jul24 15:41:33 [801] rcpt to <david@example.com> ()
+2005Jul24 15:41:35 [801] Subject: Microsoft, Adobe, Macromedia, Corel software. Up to 80% discount.
+2005Jul24 15:41:35 [801] X-Mailer: Microsoft Outlook, Build 10.0.2605
+2005Jul24 15:41:35 [801] CBV: Aitp@horafeliz.com
+2005Jul24 15:41:38 [801] REJECT: CBV: 550 <Aitp@horafeliz.com>: User unknown
+</pre>
+</td></tr>
+
+<tr><th>FAIL</th><td>
+The message is rejected with a reference the SPF why page.
+<pre>
+2005Jul30 19:53:27 [94070] connect from [212.70.52.16] at ('212.70.52.16', 3192) EXTERNAL DYN
+2005Jul30 19:53:27 [94070] hello from winzip.com
+2005Jul30 19:53:27 [94070] mail from <dan@winzip.com> ()
+2005Jul30 19:53:27 [94070] REJECT: SPF fail 550 SPF fail:
+ see http://openspf.com/why.html?sender=dan@winzip.com&ip=212.70.52.16
+</pre>
+</td></tr>
+
+<tr><th>PERMERROR</th><td>
+Permanent errors were called "unknown", and are still show that way
+in the log. The message is rejected. Previously, we enabled "lax" parsing
+of the SPF record, but rejecting is better because it informs the
+sender about their problem. The next milter version will
+look for a local substitute SPF record (as for a missing SPF record)
+before rejecting. This will inform the sender of their problem, but
+also let the receiver install a temporary workaround.
+<pre>
+2005Jul24 18:05:37 [2312] mail from <b-mihdbcgaacaa-becibijh-000-@msg.euxiphipops.com> ()
+2005Jul24 18:05:37 [2312] REJECT: SPF unknown 550 SPF Permanent Error:
+ include mechanism missing domain: include
+</pre>
+The SPF record for msg.euxiphipops.com looked like this at the time of the
+above error:
+<pre>
+msg.euxiphipops.com TXT "v=spf1 mx ptr a include"
+</pre>
+</td></tr>
+
+<tr><th>TEMPERROR</th><td>
+Temporary errors result in a 451 "Try again later" response. The sender
+should retry the message at a later time.
+<pre>
+2005Jul24 07:33:13 [29846] mail from <quickenloans@rate.quicken.com> ('SIZE=73775', 'BODY=8BITMIME')
+2005Jul24 07:33:43 [29846] TEMPFAIL: SPF error 450 SPF Temporary Error: DNS Timeout
+</pre>
+</td></tr>
+
+</table>
+
+</body>
+</html>
diff --git a/testbms.py b/testbms.py
index 9cde1f0f928712137136acab8268732264ca4050..b262f67943550144f3e88f0897a43b29b47d9e4d 100644
--- a/testbms.py
+++ b/testbms.py
@@ -300,4 +300,5 @@ if __name__ == '__main__':
fp = milter._body
sys.stdout.write(fp.getvalue())
else:
- unittest.main()
+ #unittest.main()
+ unittest.TextTestRunner().run(suite())