From bob@nas.com Thu May  9 18:00:16 2002
Date: Thu, 25 Apr 2002 11:01:11 -0700
From: Bob Finch <bob@nas.com>
To: info-cyrus@lists.andrew.cmu.edu
Subject: "spam" sieve extension

I've gotten a handful of requests for the "spam" sieve extension I
mentioned yesterday.  The diffs aren't too big, so I'll post them to
the list.

The diffs add a test to sieve that passes the message to SpamAssassin.
If SpamAssassin scores the messages as spam, the test fails, otherwise
the test succeeds.  Here's a simple sieve script using the spam
extension:

  require [ "spam", "fileinto" ];
  if spam {
      fileinto "spamfolder";
  }

The diffs are based on cyrus-imapd-2.1.3 -- I haven't tried them with
other versions.  You'll also need to install SpamAssassin (see
http://www.spamassassin.org/) and have spamd running.  I'm currently
running SpamAssassin 2.11.  I've been running it for about a month on
several mailboxes that typically get about 500 messages a day.

The extension adds three new imapd.conf parameters:

  max_size: 256000
      The spam test will always return false for messages larger than
      max_size.  Messages larger than max_size will not be passed to
      spamd.

  spam_spamd_host: 127.0.0.1
      The IP addresses or hostname of the spamd server.

  spam_spamd_port: 783
      The port number of the spamd server.

When sieve evaluates the spam test, it calls a callback in
imap/lmptd.c that opens a connection to spamd.  It then sends a CHECK
command writes the message to spamd.  It parses the response from
spamd and returns the result of the test to sieve.

Things on my to-do list:

  * Add a configure option and ifdefs to conditionally include the
    spam extension

  * Provide a way for users to get more information about how
    SpamAssassin scores messages.  This will require help from spamd,
    since it currently just returns the score and threshold.

  * Documentation

-- Bob


diff -cr cyrus-imapd-2.1.3-orig/imap/lmtpd.c cyrus-imapd-2.1.3/imap/lmtpd.c
*** cyrus-imapd-2.1.3-orig/imap/lmtpd.c Thu Mar  7 09:55:28 2002
--- cyrus-imapd-2.1.3/imap/lmtpd.c      Thu Apr 18 21:14:10 2002
***************
*** 111,116 ****
--- 111,124 ----

      char *authuser;           /* user who submitted message */
      struct auth_state *authstate;
+
+     /* spam stuff */
+
+     /* This is in script_data, but the spam callback can't get to it */
+     /* so we put a copy here */
+     char *username;           /* Username of mailbox */
+     int spam_result_valid;    /* != 0 iff spam result is valid */
+     int spam_result;          /* != iff message is spam */
  } mydata_t;

  /* data per script */
***************
*** 910,915 ****
--- 918,1087 ----
  static char *markflags[] = { "\\flagged" };
  static sieve_imapflags_t mark = { markflags, 1 };

+ /* spam support */
+
+ static int
+ getline (int s, char *buf, int len)
+ {
+     char *bp = buf;
+     int ret = 1;
+     char ch;
+
+     while ((ret = read (s, &ch, 1)) == 1 && ch != '\n') {
+       if (len > 0) {
+           *bp++ = ch;
+           len--;
+       }
+     }
+     if (len > 0)
+       *bp = '\0';
+     return (buf != bp);
+ }
+
+
+ static int
+ full_write (int s, char *buf, int len)
+ {
+     int total;
+     int ret;
+
+     for (total = 0; total < len; total += ret) {
+       ret = write (s, buf + total, len - total);
+       if (ret < 0)
+           return 0;
+     }
+     return total == len;
+ }
+
+
+ static int
+ read_response (int s, int *result)
+ {
+     char is_spam[6];
+     char buf[1024];
+     int major;
+     int minor;
+     int response;
+     int score;
+     int threshold;
+
+     if (! getline (s, buf, sizeof (buf))) {
+       syslog (LOG_ERR, "read_response: response getline failed");
+       return SIEVE_FAIL;
+     }
+     if (sscanf (buf, "SPAMD/%d.%d %d %*s", &major, &minor, &response) != 3) {
+       syslog (LOG_ERR, "read_response: response sscanf failed, buf: %s",
+               buf);
+       return SIEVE_FAIL;
+     }
+     if (major < 1 || (major == 1 && minor < 1)) {
+       syslog (LOG_ERR, "read_response: bad spamd version: %d.%d",
+               major, minor);
+       return SIEVE_FAIL;
+     }
+     if (! getline (s, buf, sizeof (buf))) {
+       syslog (LOG_ERR, "read_response: header getline failed");
+       return SIEVE_FAIL;
+     }
+     if (sscanf (buf, "Spam: %5s ; %d / %d", is_spam, &score, &threshold) != 3) {
+       syslog (LOG_ERR, "read_response: header sscanf failed, buf: %s",
+               buf);
+       return SIEVE_FAIL;
+     }
+
+     *result = ! strcmp(is_spam, "True");
+     return SIEVE_OK;
+ }
+
+
+ int spam (void *mc, int *is_spam)
+ {
+     mydata_t *d = (mydata_t *) mc;
+     message_data_t *m = d->m;
+     int s;
+     struct sockaddr_in addr;
+     struct hostent *host;
+     char header[128];
+     int max_size = config_getint ("spam_max_size", 250 * 1024);
+     const char *hostname = config_getstring ("spam_spamd_host", "127.0.0.1");
+     int port = config_getint ("spam_spamd_port", 783);
+     char *msg_buf;
+     int ret;
+
+     /* Assume message isn't spam if it is larger than max_size */
+     if (m->size > max_size) {
+       syslog (LOG_INFO, "spam: skipping message bigger than %d", max_size);
+       return SIEVE_FAIL;
+     }
+
+     memset (&addr, 0, sizeof(addr));
+     addr.sin_family = AF_INET;
+     addr.sin_port = htons(port);
+
+     if ((host = gethostbyname (hostname)) == NULL) {
+       syslog (LOG_ERR, "spam: gethostbyname failed");
+       return SIEVE_FAIL;
+     }
+     memcpy (&addr.sin_addr, host->h_addr, sizeof (addr.sin_addr));
+
+     if((s = socket (PF_INET, SOCK_STREAM, 0)) < 0) {
+       syslog (LOG_ERR, "spam: socket failed");
+       return SIEVE_FAIL;
+     }
+
+     if (connect (s, (const struct sockaddr *) &addr, sizeof (addr)) < 0) {
+       syslog (LOG_ERR, "spam: connect failed");
+       close (s);
+       return SIEVE_FAIL;
+     }
+
+     if ((msg_buf = malloc (m->size)) == NULL) {
+       syslog (LOG_ERR, "spam: malloc(%d) failed", m->size);
+       close (s);
+       return SIEVE_FAIL;
+     }
+     rewind (m->f);
+     if (fread (msg_buf, 1, m->size, m->f) != m->size || ferror (m->f)) {
+       syslog (LOG_ERR, "spam: read message failed");
+       free (msg_buf);
+       close (s);
+       return SIEVE_FAIL;
+     }
+
+     if (d->username) {
+       snprintf (header, sizeof (header),
+                 "CHECK SPAMC/1.2\r\nUser: %s\r\nContent-length: %d\r\n\r\n",
+                 d->username, m->size);
+     }
+     else {
+       snprintf (header, sizeof (header),
+                 "CHECK SPAMC/1.2\r\nContent-length: %d\r\n\r\n", m->size);
+     }
+     if (! full_write (s, header, strlen (header))) {
+       syslog (LOG_ERR, "spam: write header failed");
+       free (msg_buf);
+       close (s);
+       return SIEVE_FAIL;
+     }
+     if (! full_write (s, msg_buf, m->size)) {
+       syslog (LOG_ERR, "spam: write message failed");
+       free (msg_buf);
+       close (s);
+       return SIEVE_FAIL;
+     }
+
+     shutdown (s, SHUT_WR);
+     ret = read_response (s, is_spam);
+     shutdown (s, SHUT_RD);
+
+     free (msg_buf);
+     close (s);
+
+     syslog(LOG_DEBUG, "spam result: %d", ret);
+     return ret;
+ }
+
+
  int sieve_parse_error_handler(int lineno, const char *msg, void *ic, void *sc)
  {
      script_data_t *sd = (script_data_t *) sc;
***************
*** 999,1004 ****
--- 1171,1182 ----
        fatal("sieve_register_vacation()", EX_SOFTWARE);
      }

+     res = sieve_register_spam(sieve_interp, &spam);
+     if (res != SIEVE_OK) {
+       syslog(LOG_ERR, "sieve_register_spam() returns %d", res);
+       fatal("sieve_register_spam()", EX_SOFTWARE);
+     }
+
      res = sieve_register_parse_error(sieve_interp, &sieve_parse_error_handler);
      if (res != SIEVE_OK) {
        syslog(LOG_ERR, "sieve_register_parse_error() returns %d", res);
***************
*** 1148,1154 ****
      mydata.notifyheader = generate_notify(msgdata);
      mydata.authuser = authuser;
      mydata.authstate = authstate;
!
      /* loop through each recipient, attempting delivery for each */
      for (n = 0; n < nrcpts; n++) {
        char *rcpt = xstrdup(msg_getrcpt(msgdata, n));
--- 1326,1335 ----
      mydata.notifyheader = generate_notify(msgdata);
      mydata.authuser = authuser;
      mydata.authstate = authstate;
!     mydata.username = NULL;
!     mydata.spam_result = 0;
!     mydata.spam_result_valid = 0;
!
      /* loop through each recipient, attempting delivery for each */
      for (n = 0; n < nrcpts; n++) {
        char *rcpt = xstrdup(msg_getrcpt(msgdata, n));
***************
*** 1187,1192 ****
--- 1368,1376 ----
                sdata->username = rcpt;
                sdata->mailboxname = plus;
                sdata->authstate = auth_newstate(rcpt, (char *)0);
+
+               /* Make a copy of mailbox username for spam stuff */
+               mydata.username = sdata->username;

                /* slap the mailboxname back on so we hash the envelope & id
                   when we figure out whether or not to keep the message */
diff -cr cyrus-imapd-2.1.3-orig/sieve/interp.c cyrus-imapd-2.1.3/sieve/interp.c
*** cyrus-imapd-2.1.3-orig/sieve/interp.c       Tue Oct  2 14:08:13 2001
--- cyrus-imapd-2.1.3/sieve/interp.c    Sun Mar 24 11:39:14 2002
***************
*** 154,159 ****
--- 154,166 ----
      return SIEVE_OK;
  }

+ int sieve_register_spam(sieve_interp_t *interp, sieve_spam *f)
+ {
+     interp->spam = f;
+
+     return SIEVE_OK;
+ }
+
  /* add the callbacks for messages. again, undefined if used after
     sieve_script_parse */
  int sieve_register_size(sieve_interp_t *interp, sieve_get_size *f)
diff -cr cyrus-imapd-2.1.3-orig/sieve/interp.h cyrus-imapd-2.1.3/sieve/interp.h
*** cyrus-imapd-2.1.3-orig/sieve/interp.h       Mon Feb 21 23:56:40 2000
--- cyrus-imapd-2.1.3/sieve/interp.h    Sun Mar 24 11:40:53 2002
***************
*** 35,40 ****
--- 35,41 ----
      sieve_callback *redirect, *discard, *reject, *fileinto, *keep;
      sieve_callback *notify;
      sieve_vacation_t *vacation;
+     sieve_spam *spam;

      sieve_get_size *getsize;
      sieve_get_header *getheader;
diff -cr cyrus-imapd-2.1.3-orig/sieve/script.c cyrus-imapd-2.1.3/sieve/script.c
*** cyrus-imapd-2.1.3-orig/sieve/script.c       Wed Feb 27 13:05:13 2002
--- cyrus-imapd-2.1.3/sieve/script.c    Thu Apr 18 21:02:51 2002
***************
*** 102,107 ****
--- 102,114 ----
        return 1;
      } else if (!strcmp("comparator-i;ascii-casemap", req)) {
        return 1;
+     } else if (!strcmp("spam",req)) {
+       if (s->interp.spam) {
+           s->support.spam = 1;
+           return 1;
+       } else {
+           return 0;
+       }
      }
      return 0;
  }
***************
*** 361,366 ****
--- 368,381 ----
            res = (sz < t->u.sz.n);
        }
        break;
+     }
+     case SPAM:
+     {
+       int is_spam;
+
+       if (i->spam == NULL || i->spam (m, &is_spam) != SIEVE_OK)
+           break;
+       res = is_spam;
      }
      }

diff -cr cyrus-imapd-2.1.3-orig/sieve/script.h cyrus-imapd-2.1.3/sieve/script.h
*** cyrus-imapd-2.1.3-orig/sieve/script.h       Wed Feb  9 16:39:14 2000
--- cyrus-imapd-2.1.3/sieve/script.h    Sun Mar 24 11:39:13 2002
***************
*** 45,50 ****
--- 45,51 ----
        int notify    : 1;
        int regex     : 1;
        int subaddress: 1;
+       int spam      : 1;
      } support;

      void *script_context;
diff -cr cyrus-imapd-2.1.3-orig/sieve/sieve-lex.l cyrus-imapd-2.1.3/sieve/sieve-lex.l
*** cyrus-imapd-2.1.3-orig/sieve/sieve-lex.l    Tue Feb 19 10:09:46 2002
--- cyrus-imapd-2.1.3/sieve/sieve-lex.l Sat Mar 23 18:43:22 2002
***************
*** 90,95 ****
--- 90,96 ----
  <INITIAL>header               return HEADER;
  <INITIAL>not          return NOT;
  <INITIAL>size         return SIZE;
+ <INITIAL>spam         return SPAM;
  <INITIAL>reject               return REJCT;
  <INITIAL>fileinto     return FILEINTO;
  <INITIAL>redirect     return REDIRECT;
diff -cr cyrus-imapd-2.1.3-orig/sieve/sieve.y cyrus-imapd-2.1.3/sieve/sieve.y
*** cyrus-imapd-2.1.3-orig/sieve/sieve.y        Tue Mar  5 08:15:01 2002
--- cyrus-imapd-2.1.3/sieve/sieve.y     Sun Mar 24 11:36:18 2002
***************
*** 141,146 ****
--- 141,147 ----
  %token SETFLAG ADDFLAG REMOVEFLAG MARK UNMARK
  %token NOTIFY DENOTIFY
  %token ANYOF ALLOF EXISTS SFALSE STRUE HEADER NOT SIZE ADDRESS ENVELOPE
+ %token SPAM
  %token COMPARATOR IS CONTAINS MATCHES REGEX OVER UNDER
  %token ALL LOCALPART DOMAIN USER DETAIL
  %token DAYS ADDRESSES SUBJECT MIME
***************
*** 398,403 ****
--- 399,409 ----
        | NOT test               { $$ = new_test(NOT); $$->u.t = $2; }
        | SIZE sizetag NUMBER    { $$ = new_test(SIZE); $$->u.sz.t = $2;
                                   $$->u.sz.n = $3; }
+       | SPAM                   { if (!parse_script->support.spam) {
+                                    yyerror("spam not required");
+                                    YYERROR;
+                                  }
+                                  $$ = new_test(SPAM); }
        | error                  { $$ = NULL; }
        ;

diff -cr cyrus-imapd-2.1.3-orig/sieve/sieve_interface.h cyrus-imapd-2.1.3/sieve/sieve_interface.h
*** cyrus-imapd-2.1.3-orig/sieve/sieve_interface.h      Tue Feb 19 10:09:46 2002
--- cyrus-imapd-2.1.3/sieve/sieve_interface.h   Sun Mar 24 19:46:44 2002
***************
*** 50,55 ****
--- 50,57 ----
  typedef int sieve_get_envelope(void *message_context,
                               const char *field,
                               const char ***contents);
+ typedef int sieve_spam(void *message_context, int *is_spam);
+

  typedef struct sieve_vacation {
      int min_response;         /* 0 -> defaults to 3 */
***************
*** 121,126 ****
--- 123,129 ----
  int sieve_register_vacation(sieve_interp_t *interp, sieve_vacation_t *v);
  int sieve_register_imapflags(sieve_interp_t *interp, sieve_imapflags_t *mark);
  int sieve_register_notify(sieve_interp_t *interp, sieve_callback *f);
+ int sieve_register_spam(sieve_interp_t *interp, sieve_spam *f);

  /* add the callbacks for messages. again, undefined if used after
     sieve_script_parse */
diff -cr cyrus-imapd-2.1.3-orig/timsieved/scripttest.c cyrus-imapd-2.1.3/timsieved/scripttest.c
*** cyrus-imapd-2.1.3-orig/timsieved/scripttest.c       Sun Dec 17 20:53:43 2000
--- cyrus-imapd-2.1.3/timsieved/scripttest.c    Sun Mar 24 13:12:06 2002
***************
*** 171,176 ****
--- 171,182 ----
        return TIMSIEVE_FAIL;
      }

+     res = sieve_register_spam(i, (sieve_spam *) &foo);
+     if (res != SIEVE_OK) {
+       syslog (LOG_ERR, "sieve_register_spam() returns %d", res);
+       return TIMSIEVE_FAIL;
+     }
+
      res = sieve_register_parse_error(i, &mysieve_error);
      if (res != SIEVE_OK) {
        syslog(LOG_ERR, "sieve_register_parse_error() returns %d", res);

