Support --ha-callout for high-availability callouts

diff --git a/ChangeLog b/ChangeLog
index 34be7411fe1370e8d29f32b22af44f53f4303306..38d9671fe6988684ccc5958780ab7d6fc6bce195 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2004-09-06 Paul Clements <paul.clements@steeleye.com>
+       Neil Brown <neilb@cse.unsw.edu.au>
+
+       * utils/mountd/mountd.c(main): support --ha-callout (-H) for
+       specifying a callout program
+       * utils/mountd/rmtab.c: Call ha_callout on mount/unmount
+       * utils/statd/monitor.c: Call ha_callout on add/del client
+       * utils/statd/rmtcall.c: as above
+       * utils/statd/statd.c: handle --ha-callout (-H)
+       * utils/statd/svc_run.c: call notify_hosts is we have received a
+       sighup
+       * support/include/ha-callout.h: define ha_callout function
+        
+       

 2004-08-31 NeilBrown <neilb@cse.unsw.edu.au>
        * utils/mountd/cache.c(cache_process_req): clear fd after
        processing so as not to confused libc/sunrpc into thinking
 2004-08-31 NeilBrown <neilb@cse.unsw.edu.au>
        * utils/mountd/cache.c(cache_process_req): clear fd after
        processing so as not to confused libc/sunrpc into thinking

diff --git a/support/include/ha-callout.h b/support/include/ha-callout.h
new file mode 100644 (file)
index 0000000..707d51b
--- /dev/null
+++ b/support/include/ha-callout.h
@@ -0,0 +1,52 @@
+/*
+ * support/include/ha-callout.h
+ *
+ * High Availability NFS Callout support routines
+ *
+ * Copyright (c) 2004, Paul Clements, SteelEye Technology
+ *
+ * In order to implement HA NFS, we need several callouts at key
+ * points in statd and mountd. These callouts all come to ha_callout(),
+ * which, in turn, calls out to an ha-callout script (not part of nfs-utils;
+ * defined by -H argument to rpc.statd and rpc.mountd).
+ */
+#ifndef HA_CALLOUT_H
+#define HA_CALLOUT_H
+
+#include <sys/wait.h>
+
+extern char *ha_callout_prog;
+
+static inline void
+ha_callout(char *event, char *arg1, char *arg2, int arg3)
+{
+       char buf[16]; /* should be plenty */
+       pid_t pid;
+       int ret = -1;
+
+       if (!ha_callout_prog) /* HA callout is not enabled */
+               return;
+
+       sprintf(buf, "%d", arg3);
+
+       pid = fork();
+       switch (pid) {
+               case 0: execl(ha_callout_prog, ha_callout_prog,
+                               event, arg1, arg2, 
+                             arg3 < 0 ? NULL : buf,
+                             NULL);
+                       perror("execl");
+                       exit(2);
+               case -1: perror("fork");
+                       break;
+               default: ret = waitpid(pid, NULL, 0);
+       }
+
+#ifdef dprintf
+       dprintf(N_DEBUG, "ha callout returned %d\n", WEXITSTATUS(ret));
+#else
+       xlog(D_GENERAL, "ha callout returned %d\n", WEXITSTATUS(ret));
+#endif
+}
+
+#endif

diff --git a/utils/mountd/mountd.c b/utils/mountd/mountd.c
index 8f6f9cb9d85d8ad69754a9db1bdbda2233feab76..7cc1b1bf0e506c5593be3125744bef98d2daaf9a 100644 (file)
--- a/utils/mountd/mountd.c
+++ b/utils/mountd/mountd.c
@@ -36,6 +36,11 @@ static struct nfs_fh_len *get_rootfh(struct svc_req *, dirpath *, int *, int v3)
 
 int new_cache = 0;
 
 
 int new_cache = 0;
 

+/* PRC: a high-availability callout program can be specified with -H
+ * When this is done, the program will receive callouts whenever clients
+ * send mount or unmount requests -- the callout is not needed for 2.6 kernel */
+char *ha_callout_prog = NULL;
+

 static struct option longopts[] =
 {
        { "foreground", 0, 0, 'F' },
 static struct option longopts[] =
 {
        { "foreground", 0, 0, 'F' },


@@ -48,6 +53,7 @@ static struct option longopts[] =
        { "version", 0, 0, 'v' },
        { "port", 1, 0, 'p' },
        { "no-tcp", 0, 0, 'n' },
        { "version", 0, 0, 'v' },
        { "port", 1, 0, 'p' },
        { "no-tcp", 0, 0, 'n' },

+       { "ha-callout", 1, 0, 'H' },

        { NULL, 0, 0, 0 }
 };
 
        { NULL, 0, 0, 0 }
 };
 


@@ -444,7 +450,7 @@ main(int argc, char **argv)
 
        /* Parse the command line options and arguments. */
        opterr = 0;
 
        /* Parse the command line options and arguments. */
        opterr = 0;

-       while ((c = getopt_long(argc, argv, "o:n:Fd:f:p:P:hN:V:v", longopts, NULL)) != EOF)
+       while ((c = getopt_long(argc, argv, "o:n:Fd:f:p:P:hH:N:V:v", longopts, NULL)) != EOF)

                switch (c) {
                case 'o':
                        descriptors = atoi(optarg);
                switch (c) {
                case 'o':
                        descriptors = atoi(optarg);


@@ -463,6 +469,9 @@ main(int argc, char **argv)
                case 'f':
                        export_file = optarg;
                        break;
                case 'f':
                        export_file = optarg;
                        break;

+               case 'H': /* PRC: specify a high-availability callout program */
+                       ha_callout_prog = optarg;
+                       break;

                case 'h':
                        usage(argv [0], 0);
                        break;
                case 'h':
                        usage(argv [0], 0);
                        break;


@@ -596,6 +605,7 @@ usage(const char *prog, int n)
 "Usage: %s [-F|--foreground] [-h|--help] [-v|--version] [-d kind|--debug kind]\n"
 "      [-o num|--descriptors num] [-f exports-file|--exports-file=file]\n"
 "      [-p|--port port] [-V version|--nfs-version version]\n"
 "Usage: %s [-F|--foreground] [-h|--help] [-v|--version] [-d kind|--debug kind]\n"
 "      [-o num|--descriptors num] [-f exports-file|--exports-file=file]\n"
 "      [-p|--port port] [-V version|--nfs-version version]\n"

-"      [-N version|--no-nfs-version version] [-n|--no-tcp]\n", prog);
+"      [-N version|--no-nfs-version version] [-n|--no-tcp]\n"
+"      [-H ha-callout-prog]\n", prog);

        exit(n);
 }
        exit(n);
 }

diff --git a/utils/mountd/mountd.man b/utils/mountd/mountd.man
index 3d2b1a8bbf18845c3fb89cdf2c5756f0af387db3..64e508a26ea2768ad39ca98e4384b9c36162105f 100644 (file)
--- a/utils/mountd/mountd.man
+++ b/utils/mountd/mountd.man
@@ -2,7 +2,8 @@
 .\" mountd(8)
 .\"
 .\" Copyright (C) 1999 Olaf Kirch <okir@monad.swb.de>
 .\" mountd(8)
 .\"
 .\" Copyright (C) 1999 Olaf Kirch <okir@monad.swb.de>

-.TH rpc.mountd 8 "25 Aug 2000"
+.\" Modified by Paul Clements, 2004.
+.TH rpc.mountd 8 "31 Aug 2004"

 .SH NAME
 rpc.mountd \- NFS mount daemon
 .SH SYNOPSIS
 .SH NAME
 rpc.mountd \- NFS mount daemon
 .SH SYNOPSIS


@@ -98,6 +99,26 @@ Force
 to bind to the specified port num, instead of using the random port
 number assigned by the portmapper.
 .TP
 to bind to the specified port num, instead of using the random port
 number assigned by the portmapper.
 .TP

+.B \-H " or " \-\-ha-callout prog
+Specify a high availability callout program, which will receive callouts
+for all client mount and unmount requests. This allows 
+.B rpc.mountd
+to be used in a High Availability NFS (HA-NFS) environment. This callout is not
+needed (and should not be used) with 2.6 and later kernels (instead,
+mount the nfsd filesystem on
+.B /proc/fs/nfsd
+).
+The program will be called with 4 arguments.
+The first will be
+.B mount
+or
+.B unmount
+depending on the reason for the callout.
+The second will be the name of the client performing the mount.
+The third will be the path that the client is mounting.
+The last is the number of concurrent mounts that we believe the client
+has of that path.
+.TP

 .B \-V " or " \-\-nfs-version
 This option can be used to request that
 .B rpc.mountd
 .B \-V " or " \-\-nfs-version
 This option can be used to request that
 .B rpc.mountd

diff --git a/utils/mountd/rmtab.c b/utils/mountd/rmtab.c
index 844de511f48b07d76baac5b0b828b9e5520af33f..90fbef4702eede421b96e0e46ebe35aaf3c8037c 100644 (file)
--- a/utils/mountd/rmtab.c
+++ b/utils/mountd/rmtab.c
@@ -19,6 +19,7 @@
 #include "exportfs.h"
 #include "xio.h"
 #include "mountd.h"
 #include "exportfs.h"
 #include "xio.h"
 #include "mountd.h"

+#include "ha-callout.h"

 
 #include <limits.h> /* PATH_MAX */
 
 
 #include <limits.h> /* PATH_MAX */
 


@@ -61,6 +62,8 @@ mountlist_add(char *host, const char *path)
                            host) == 0
                    && strcmp(rep->r_path, path) == 0) {
                        rep->r_count++;
                            host) == 0
                    && strcmp(rep->r_path, path) == 0) {
                        rep->r_count++;

+                       /* PRC: do the HA callout: */
+                       ha_callout("mount", rep->r_client, rep->r_path, rep->r_count);

                        putrmtabent(rep, &pos);
                        endrmtabent();
                        xfunlock(lockid);
                        putrmtabent(rep, &pos);
                        endrmtabent();
                        xfunlock(lockid);


@@ -75,6 +78,8 @@ mountlist_add(char *host, const char *path)
        xe.r_path [sizeof (xe.r_path) - 1] = '\0';
        xe.r_count = 1;
        if (setrmtabent("a")) {
        xe.r_path [sizeof (xe.r_path) - 1] = '\0';
        xe.r_count = 1;
        if (setrmtabent("a")) {

+               /* PRC: do the HA callout: */
+               ha_callout("mount", xe.r_client, xe.r_path, xe.r_count);

                putrmtabent(&xe, NULL);
                endrmtabent();
        }
                putrmtabent(&xe, NULL);
                endrmtabent();
        }


@@ -103,8 +108,11 @@ mountlist_del(char *hname, const char *path)
        while ((rep = getrmtabent(1, NULL)) != NULL) {
                match = !strcmp (rep->r_client, hname)
                        && !strcmp(rep->r_path, path);
        while ((rep = getrmtabent(1, NULL)) != NULL) {
                match = !strcmp (rep->r_client, hname)
                        && !strcmp(rep->r_path, path);

-               if (match)
+               if (match) {

                        rep->r_count--;
                        rep->r_count--;

+                       /* PRC: do the HA callout: */
+                       ha_callout("unmount", rep->r_client, rep->r_path, rep->r_count);
+               }

                if (!match || rep->r_count)
                        fputrmtabent(fp, rep, NULL);
        }
                if (!match || rep->r_count)
                        fputrmtabent(fp, rep, NULL);
        }

diff --git a/utils/statd/monitor.c b/utils/statd/monitor.c
index 8c1bb50ef98a076aadb00ecc3dbf2f66882e3a28..3b81bdd4af1a6cc829cc9efc0d6a2c8e5ddd3d11 100644 (file)
--- a/utils/statd/monitor.c
+++ b/utils/statd/monitor.c
@@ -19,6 +19,7 @@
 #include "misc.h"
 #include "statd.h"
 #include "notlist.h"
 #include "misc.h"
 #include "statd.h"
 #include "notlist.h"

+#include "ha-callout.h"

 
 notify_list *          rtnl = NULL;    /* Run-time notify list. */
 
 
 notify_list *          rtnl = NULL;    /* Run-time notify list. */
 


@@ -177,6 +178,8 @@ sm_mon_1_svc(struct mon *argp, struct svc_req *rqstp)
                goto failure;
        }
        free(path);
                goto failure;
        }
        free(path);

+       /* PRC: do the HA callout: */
+       ha_callout("add-client", mon_name, my_name, -1);

        nlist_insert(&rtnl, clnt);
        close(fd);
 
        nlist_insert(&rtnl, clnt);
        close(fd);
 


@@ -232,6 +235,10 @@ sm_unmon_1_svc(struct mon_id *argp, struct svc_req *rqstp)
                        /* Match! */
                        dprintf(N_DEBUG, "UNMONITORING %s for %s",
                                        mon_name, my_name);
                        /* Match! */
                        dprintf(N_DEBUG, "UNMONITORING %s for %s",
                                        mon_name, my_name);

+
+                       /* PRC: do the HA callout: */
+                       ha_callout("del-client", mon_name, my_name, -1);
+

                        nlist_free(&rtnl, clnt);
                        xunlink(SM_DIR, mon_name, 1);
 
                        nlist_free(&rtnl, clnt);
                        xunlink(SM_DIR, mon_name, 1);
 


@@ -276,6 +283,8 @@ sm_unmon_all_1_svc(struct my_id *argp, struct svc_req *rqstp)
                                sizeof (mon_name) - 1);
                        mon_name[sizeof (mon_name) - 1] = '\0';
                        temp = NL_NEXT(clnt);
                                sizeof (mon_name) - 1);
                        mon_name[sizeof (mon_name) - 1] = '\0';
                        temp = NL_NEXT(clnt);

+                       /* PRC: do the HA callout: */
+                       ha_callout("del-client", mon_name, argp->my_name, -1);

                        nlist_free(&rtnl, clnt);
                        xunlink(SM_DIR, mon_name, 1);
                        ++count;
                        nlist_free(&rtnl, clnt);
                        xunlink(SM_DIR, mon_name, 1);
                        ++count;

diff --git a/utils/statd/rmtcall.c b/utils/statd/rmtcall.c
index 29972545d7389b9ac7ca54defee529c4cfb18b43..911a93236937245b2f9505faf9e6795c71ce623c 100644 (file)
--- a/utils/statd/rmtcall.c
+++ b/utils/statd/rmtcall.c
@@ -38,6 +38,7 @@
 #include "statd.h"
 #include "notlist.h"
 #include "log.h"
 #include "statd.h"
 #include "notlist.h"
 #include "log.h"

+#include "ha-callout.h"

 
 #define MAXMSGSIZE     (2048 / sizeof(unsigned int))
 
 
 #define MAXMSGSIZE     (2048 / sizeof(unsigned int))
 


@@ -414,6 +415,8 @@ process_notify_list(void)
                        note(N_ERROR,
                                "Can't notify %s, giving up.",
                                        NL_MON_NAME(entry));
                        note(N_ERROR,
                                "Can't notify %s, giving up.",
                                        NL_MON_NAME(entry));

+                       /* PRC: do the HA callout */
+                       ha_callout("del-client", NL_MON_NAME(entry), NL_MY_NAME(entry), -1);

                        xunlink(SM_BAK_DIR, NL_MON_NAME(entry), 0);
                        nlist_free(&notify, entry);
                }
                        xunlink(SM_BAK_DIR, NL_MON_NAME(entry), 0);
                        nlist_free(&notify, entry);
                }

diff --git a/utils/statd/statd.c b/utils/statd/statd.c
index b57f71bb0091bf1bc46096e5dbb996d88cc33107..f666bcd4cba4e1e61247e715d293e06fc281fd88 100644 (file)
--- a/utils/statd/statd.c
+++ b/utils/statd/statd.c
@@ -48,6 +48,11 @@ int  run_mode = 0;           /* foreground logging mode */
 char *name_p = NULL;
 char *version_p = NULL;
 
 char *name_p = NULL;
 char *version_p = NULL;
 

+/* PRC: a high-availability callout program can be specified with -H
+ * When this is done, the program will receive callouts whenever clients
+ * are added or deleted to the notify list */
+char *ha_callout_prog = NULL;
+

 static struct option longopts[] =
 {
        { "foreground", 0, 0, 'F' },
 static struct option longopts[] =
 {
        { "foreground", 0, 0, 'F' },


@@ -59,6 +64,7 @@ static struct option longopts[] =
        { "name", 1, 0, 'n' },
        { "state-directory-path", 1, 0, 'P' },
        { "notify-mode", 0, 0, 'N' },
        { "name", 1, 0, 'n' },
        { "state-directory-path", 1, 0, 'P' },
        { "notify-mode", 0, 0, 'N' },

+       { "ha-callout", 1, 0, 'H' },

        { NULL, 0, 0, 0 }
 };
 
        { NULL, 0, 0, 0 }
 };
 


@@ -102,6 +108,13 @@ killer (int sig)
        exit (0);
 }
 
        exit (0);
 }
 

+static void
+sigusr (int sig)
+{
+       dprintf (N_DEBUG, "Caught signal %d, re-reading notify list.", sig);
+       re_notify = 1;
+}
+

 /*
  * Startup information.
  */
 /*
  * Startup information.
  */


@@ -148,6 +161,7 @@ usage()
        fprintf(stderr,"      -n, --name           Specify a local hostname.\n");
        fprintf(stderr,"      -P                   State directory path.\n");
        fprintf(stderr,"      -N                   Run in notify only mode.\n");
        fprintf(stderr,"      -n, --name           Specify a local hostname.\n");
        fprintf(stderr,"      -P                   State directory path.\n");
        fprintf(stderr,"      -N                   Run in notify only mode.\n");

+       fprintf(stderr,"      -H                   Specify a high-availability callout program.\n");

 }
 
 static const char *pidfile = "/var/run/rpc.statd.pid";
 }
 
 static const char *pidfile = "/var/run/rpc.statd.pid";


@@ -236,7 +250,7 @@ int main (int argc, char **argv)
        MY_NAME = NULL;
 
        /* Process command line switches */
        MY_NAME = NULL;
 
        /* Process command line switches */

-       while ((arg = getopt_long(argc, argv, "h?vVFNdn:p:o:P:", longopts, NULL)) != EOF) {
+       while ((arg = getopt_long(argc, argv, "h?vVFNH:dn:p:o:P:", longopts, NULL)) != EOF) {

                switch (arg) {
                case 'V':       /* Version */
                case 'v':
                switch (arg) {
                case 'V':       /* Version */
                case 'v':


@@ -302,6 +316,13 @@ int main (int argc, char **argv)
                                sprintf(SM_STAT_PATH, "%s/state", DIR_BASE );
                        }
                        break;
                                sprintf(SM_STAT_PATH, "%s/state", DIR_BASE );
                        }
                        break;

+               case 'H': /* PRC: specify the ha-callout program */
+                       if ((ha_callout_prog = xstrdup(optarg)) == NULL) {
+                               fprintf(stderr, "%s: xstrdup(%s) failed!\n",
+                                       argv[0], optarg);
+                               exit(1);
+                       }
+                       break;

                case '?':       /* heeeeeelllllllpppp? heh */
                case 'h':
                        usage();
                case '?':       /* heeeeeelllllllpppp? heh */
                case 'h':
                        usage();


@@ -397,6 +418,8 @@ int main (int argc, char **argv)
        signal (SIGHUP, killer);
        signal (SIGINT, killer);
        signal (SIGTERM, killer);
        signal (SIGHUP, killer);
        signal (SIGINT, killer);
        signal (SIGTERM, killer);

+       /* PRC: trap SIGUSR1 to re-read notify list from disk */
+       signal(SIGUSR1, sigusr);

        /* WARNING: the following works on Linux and SysV, but not BSD! */
        signal(SIGCHLD, SIG_IGN);
 
        /* WARNING: the following works on Linux and SysV, but not BSD! */
        signal(SIGCHLD, SIG_IGN);
 

diff --git a/utils/statd/statd.h b/utils/statd/statd.h
index ef90bdd48380488afc844e3089cdb4a849da7952..e7829722257d75800517bec1b1d1d565a6d93e1e 100644 (file)
--- a/utils/statd/statd.h
+++ b/utils/statd/statd.h
@@ -85,3 +85,5 @@ extern int run_mode;
  */
 extern char *name_p;           /* program basename */
 extern char *version_p;                /* program version */
  */
 extern char *name_p;           /* program basename */
 extern char *version_p;                /* program version */

+
+extern int             re_notify; /* time to re-read notify list */

diff --git a/utils/statd/statd.man b/utils/statd/statd.man
index f35b6c9c015d8a99233e1144d9a2d80b21372970..a22591538cd77d818b611624ed6de15bcac82e38 100644 (file)
--- a/utils/statd/statd.man
+++ b/utils/statd/statd.man
@@ -4,11 +4,12 @@
 .\" Copyright (C) 1999 Olaf Kirch <okir@monad.swb.de>
 .\" Modified by Jeffrey A. Uphoff, 1999, 2002.
 .\" Modified by Lon Hohberger, 2000.
 .\" Copyright (C) 1999 Olaf Kirch <okir@monad.swb.de>
 .\" Modified by Jeffrey A. Uphoff, 1999, 2002.
 .\" Modified by Lon Hohberger, 2000.

-.TH rpc.statd 8 "16 Sep 2002"
+.\" Modified by Paul Clements, 2004.
+.TH rpc.statd 8 "31 Aug 2004"

 .SH NAME
 rpc.statd \- NSM status monitor
 .SH SYNOPSIS
 .SH NAME
 rpc.statd \- NSM status monitor
 .SH SYNOPSIS

-.B "/sbin/rpc.statd [-F] [-d] [-?] [-n " name "] [-o " port "] [-p " port "] [-V]"
+.B "/sbin/rpc.statd [-F] [-d] [-?] [-n " name "] [-o " port "] [-p " port "] [-H " prog "] [-V]"

 .SH DESCRIPTION
 The
 .B rpc.statd
 .SH DESCRIPTION
 The
 .B rpc.statd


@@ -101,6 +102,19 @@ statd program will check its state directory, send notifications to any
 monitored nodes, and exit once the notifications have been sent. This mode is
 used to enable Highly Available NFS implementations (i.e. HA-NFS).
 .TP
 monitored nodes, and exit once the notifications have been sent. This mode is
 used to enable Highly Available NFS implementations (i.e. HA-NFS).
 .TP

+.BI "\-H, " "" " \-\-ha-callout " prog
+Specify a high availability callout program, which will receive callouts
+for all client monitor and unmonitor requests. This allows
+.B rpc.statd
+to be used in a High Availability NFS (HA-NFS) environment. The
+program will be run with 3 arguments:  The first is either
+.B add-client
+or
+.B del-client
+depending on the reason for the callout.
+The second will be the name of the client.
+The third will be the name of the server as known to the client.
+.TP

 .B -?
 Causes
 .B rpc.statd
 .B -?
 Causes
 .B rpc.statd


@@ -135,6 +149,15 @@ and
 .BR hosts_access (5)
 manual pages.
 
 .BR hosts_access (5)
 manual pages.
 

+.SH SIGNALS
+.BR SIGUSR1
+causes
+.B rpc.statd
+to re-read the notify list from disk
+and send notifications to clients. This can be used in High Availability NFS
+(HA-NFS) environments to notify clients to reacquire file locks upon takeover
+of an NFS export from another server.
+

 .SH FILES
 .BR /var/lib/nfs/state
 .br
 .SH FILES
 .BR /var/lib/nfs/state
 .br


@@ -153,3 +176,5 @@ Olaf Kirch <okir@monad.swb.de>
 H.J. Lu <hjl@gnu.org>
 .br
 Lon Hohberger <hohberger@missioncriticallinux.com>
 H.J. Lu <hjl@gnu.org>
 .br
 Lon Hohberger <hohberger@missioncriticallinux.com>

+.br
+Paul Clements <paul.clements@steeleye.com>

diff --git a/utils/statd/svc_run.c b/utils/statd/svc_run.c
index 393a2fd84803ff5ef57ceffa613f767b1d8cecbd..e2d7c975df91a7014e55de09277ac833d0558f76 100644 (file)
--- a/utils/statd/svc_run.c
+++ b/utils/statd/svc_run.c
@@ -60,6 +60,7 @@ static int    svc_stop = 0;
  * requests are put.
  */
 notify_list *  notify = NULL;
  * requests are put.
  */
 notify_list *  notify = NULL;

+int    re_notify = 0;

 
 /*
  * Jump-off function.
 
 /*
  * Jump-off function.


@@ -86,6 +87,10 @@ my_svc_run(void)
        for (;;) {
                if (svc_stop)
                        return;
        for (;;) {
                if (svc_stop)
                        return;

+               if (re_notify) {
+                       notify_hosts();
+                       re_notify = 0;
+               }

 
                /* Ah, there are some notifications to be processed */
                while (notify && NL_WHEN(notify) <= time(&now)) {
 
                /* Ah, there are some notifications to be processed */
                while (notify && NL_WHEN(notify) <= time(&now)) {