From af93e0306d87bba3f703386fd6390e9bf8e76816 Mon Sep 17 00:00:00 2001 From: neilbrown Date: Mon, 6 Sep 2004 02:15:04 +0000 Subject: [PATCH] Support --ha-callout for high-availability callouts --- ChangeLog | 14 ++++++++++ support/include/ha-callout.h | 52 ++++++++++++++++++++++++++++++++++++ utils/mountd/mountd.c | 14 ++++++++-- utils/mountd/mountd.man | 23 +++++++++++++++- utils/mountd/rmtab.c | 10 ++++++- utils/statd/monitor.c | 9 +++++++ utils/statd/rmtcall.c | 3 +++ utils/statd/statd.c | 25 ++++++++++++++++- utils/statd/statd.h | 2 ++ utils/statd/statd.man | 29 ++++++++++++++++++-- utils/statd/svc_run.c | 5 ++++ 11 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 support/include/ha-callout.h diff --git a/ChangeLog b/ChangeLog index 34be741..38d9671 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,17 @@ +2004-09-06 Paul Clements + Neil Brown + + * utils/mountd/mountd.c(main): support --ha-callout (-H) for + specifying a callout program + * utils/mountd/rmtab.c: Call ha_callout on mount/unmount + * utils/statd/monitor.c: Call ha_callout on add/del client + * utils/statd/rmtcall.c: as above + * utils/statd/statd.c: handle --ha-callout (-H) + * utils/statd/svc_run.c: call notify_hosts is we have received a + sighup + * support/include/ha-callout.h: define ha_callout function + + 2004-08-31 NeilBrown * utils/mountd/cache.c(cache_process_req): clear fd after processing so as not to confused libc/sunrpc into thinking diff --git a/support/include/ha-callout.h b/support/include/ha-callout.h new file mode 100644 index 0000000..707d51b --- /dev/null +++ b/support/include/ha-callout.h @@ -0,0 +1,52 @@ +/* + * support/include/ha-callout.h + * + * High Availability NFS Callout support routines + * + * Copyright (c) 2004, Paul Clements, SteelEye Technology + * + * In order to implement HA NFS, we need several callouts at key + * points in statd and mountd. These callouts all come to ha_callout(), + * which, in turn, calls out to an ha-callout script (not part of nfs-utils; + * defined by -H argument to rpc.statd and rpc.mountd). + */ +#ifndef HA_CALLOUT_H +#define HA_CALLOUT_H + +#include + +extern char *ha_callout_prog; + +static inline void +ha_callout(char *event, char *arg1, char *arg2, int arg3) +{ + char buf[16]; /* should be plenty */ + pid_t pid; + int ret = -1; + + if (!ha_callout_prog) /* HA callout is not enabled */ + return; + + sprintf(buf, "%d", arg3); + + pid = fork(); + switch (pid) { + case 0: execl(ha_callout_prog, ha_callout_prog, + event, arg1, arg2, + arg3 < 0 ? NULL : buf, + NULL); + perror("execl"); + exit(2); + case -1: perror("fork"); + break; + default: ret = waitpid(pid, NULL, 0); + } + +#ifdef dprintf + dprintf(N_DEBUG, "ha callout returned %d\n", WEXITSTATUS(ret)); +#else + xlog(D_GENERAL, "ha callout returned %d\n", WEXITSTATUS(ret)); +#endif +} + +#endif diff --git a/utils/mountd/mountd.c b/utils/mountd/mountd.c index 8f6f9cb..7cc1b1b 100644 --- a/utils/mountd/mountd.c +++ b/utils/mountd/mountd.c @@ -36,6 +36,11 @@ static struct nfs_fh_len *get_rootfh(struct svc_req *, dirpath *, int *, int v3) int new_cache = 0; +/* PRC: a high-availability callout program can be specified with -H + * When this is done, the program will receive callouts whenever clients + * send mount or unmount requests -- the callout is not needed for 2.6 kernel */ +char *ha_callout_prog = NULL; + static struct option longopts[] = { { "foreground", 0, 0, 'F' }, @@ -48,6 +53,7 @@ static struct option longopts[] = { "version", 0, 0, 'v' }, { "port", 1, 0, 'p' }, { "no-tcp", 0, 0, 'n' }, + { "ha-callout", 1, 0, 'H' }, { NULL, 0, 0, 0 } }; @@ -444,7 +450,7 @@ main(int argc, char **argv) /* Parse the command line options and arguments. */ opterr = 0; - while ((c = getopt_long(argc, argv, "o:n:Fd:f:p:P:hN:V:v", longopts, NULL)) != EOF) + while ((c = getopt_long(argc, argv, "o:n:Fd:f:p:P:hH:N:V:v", longopts, NULL)) != EOF) switch (c) { case 'o': descriptors = atoi(optarg); @@ -463,6 +469,9 @@ main(int argc, char **argv) case 'f': export_file = optarg; break; + case 'H': /* PRC: specify a high-availability callout program */ + ha_callout_prog = optarg; + break; case 'h': usage(argv [0], 0); break; @@ -596,6 +605,7 @@ usage(const char *prog, int n) "Usage: %s [-F|--foreground] [-h|--help] [-v|--version] [-d kind|--debug kind]\n" " [-o num|--descriptors num] [-f exports-file|--exports-file=file]\n" " [-p|--port port] [-V version|--nfs-version version]\n" -" [-N version|--no-nfs-version version] [-n|--no-tcp]\n", prog); +" [-N version|--no-nfs-version version] [-n|--no-tcp]\n" +" [-H ha-callout-prog]\n", prog); exit(n); } diff --git a/utils/mountd/mountd.man b/utils/mountd/mountd.man index 3d2b1a8..64e508a 100644 --- a/utils/mountd/mountd.man +++ b/utils/mountd/mountd.man @@ -2,7 +2,8 @@ .\" mountd(8) .\" .\" Copyright (C) 1999 Olaf Kirch -.TH rpc.mountd 8 "25 Aug 2000" +.\" Modified by Paul Clements, 2004. +.TH rpc.mountd 8 "31 Aug 2004" .SH NAME rpc.mountd \- NFS mount daemon .SH SYNOPSIS @@ -98,6 +99,26 @@ Force to bind to the specified port num, instead of using the random port number assigned by the portmapper. .TP +.B \-H " or " \-\-ha-callout prog +Specify a high availability callout program, which will receive callouts +for all client mount and unmount requests. This allows +.B rpc.mountd +to be used in a High Availability NFS (HA-NFS) environment. This callout is not +needed (and should not be used) with 2.6 and later kernels (instead, +mount the nfsd filesystem on +.B /proc/fs/nfsd +). +The program will be called with 4 arguments. +The first will be +.B mount +or +.B unmount +depending on the reason for the callout. +The second will be the name of the client performing the mount. +The third will be the path that the client is mounting. +The last is the number of concurrent mounts that we believe the client +has of that path. +.TP .B \-V " or " \-\-nfs-version This option can be used to request that .B rpc.mountd diff --git a/utils/mountd/rmtab.c b/utils/mountd/rmtab.c index 844de51..90fbef4 100644 --- a/utils/mountd/rmtab.c +++ b/utils/mountd/rmtab.c @@ -19,6 +19,7 @@ #include "exportfs.h" #include "xio.h" #include "mountd.h" +#include "ha-callout.h" #include /* PATH_MAX */ @@ -61,6 +62,8 @@ mountlist_add(char *host, const char *path) host) == 0 && strcmp(rep->r_path, path) == 0) { rep->r_count++; + /* PRC: do the HA callout: */ + ha_callout("mount", rep->r_client, rep->r_path, rep->r_count); putrmtabent(rep, &pos); endrmtabent(); xfunlock(lockid); @@ -75,6 +78,8 @@ mountlist_add(char *host, const char *path) xe.r_path [sizeof (xe.r_path) - 1] = '\0'; xe.r_count = 1; if (setrmtabent("a")) { + /* PRC: do the HA callout: */ + ha_callout("mount", xe.r_client, xe.r_path, xe.r_count); putrmtabent(&xe, NULL); endrmtabent(); } @@ -103,8 +108,11 @@ mountlist_del(char *hname, const char *path) while ((rep = getrmtabent(1, NULL)) != NULL) { match = !strcmp (rep->r_client, hname) && !strcmp(rep->r_path, path); - if (match) + if (match) { rep->r_count--; + /* PRC: do the HA callout: */ + ha_callout("unmount", rep->r_client, rep->r_path, rep->r_count); + } if (!match || rep->r_count) fputrmtabent(fp, rep, NULL); } diff --git a/utils/statd/monitor.c b/utils/statd/monitor.c index 8c1bb50..3b81bdd 100644 --- a/utils/statd/monitor.c +++ b/utils/statd/monitor.c @@ -19,6 +19,7 @@ #include "misc.h" #include "statd.h" #include "notlist.h" +#include "ha-callout.h" notify_list * rtnl = NULL; /* Run-time notify list. */ @@ -177,6 +178,8 @@ sm_mon_1_svc(struct mon *argp, struct svc_req *rqstp) goto failure; } free(path); + /* PRC: do the HA callout: */ + ha_callout("add-client", mon_name, my_name, -1); nlist_insert(&rtnl, clnt); close(fd); @@ -232,6 +235,10 @@ sm_unmon_1_svc(struct mon_id *argp, struct svc_req *rqstp) /* Match! */ dprintf(N_DEBUG, "UNMONITORING %s for %s", mon_name, my_name); + + /* PRC: do the HA callout: */ + ha_callout("del-client", mon_name, my_name, -1); + nlist_free(&rtnl, clnt); xunlink(SM_DIR, mon_name, 1); @@ -276,6 +283,8 @@ sm_unmon_all_1_svc(struct my_id *argp, struct svc_req *rqstp) sizeof (mon_name) - 1); mon_name[sizeof (mon_name) - 1] = '\0'; temp = NL_NEXT(clnt); + /* PRC: do the HA callout: */ + ha_callout("del-client", mon_name, argp->my_name, -1); nlist_free(&rtnl, clnt); xunlink(SM_DIR, mon_name, 1); ++count; diff --git a/utils/statd/rmtcall.c b/utils/statd/rmtcall.c index 2997254..911a932 100644 --- a/utils/statd/rmtcall.c +++ b/utils/statd/rmtcall.c @@ -38,6 +38,7 @@ #include "statd.h" #include "notlist.h" #include "log.h" +#include "ha-callout.h" #define MAXMSGSIZE (2048 / sizeof(unsigned int)) @@ -414,6 +415,8 @@ process_notify_list(void) note(N_ERROR, "Can't notify %s, giving up.", NL_MON_NAME(entry)); + /* PRC: do the HA callout */ + ha_callout("del-client", NL_MON_NAME(entry), NL_MY_NAME(entry), -1); xunlink(SM_BAK_DIR, NL_MON_NAME(entry), 0); nlist_free(¬ify, entry); } diff --git a/utils/statd/statd.c b/utils/statd/statd.c index b57f71b..f666bcd 100644 --- a/utils/statd/statd.c +++ b/utils/statd/statd.c @@ -48,6 +48,11 @@ int run_mode = 0; /* foreground logging mode */ char *name_p = NULL; char *version_p = NULL; +/* PRC: a high-availability callout program can be specified with -H + * When this is done, the program will receive callouts whenever clients + * are added or deleted to the notify list */ +char *ha_callout_prog = NULL; + static struct option longopts[] = { { "foreground", 0, 0, 'F' }, @@ -59,6 +64,7 @@ static struct option longopts[] = { "name", 1, 0, 'n' }, { "state-directory-path", 1, 0, 'P' }, { "notify-mode", 0, 0, 'N' }, + { "ha-callout", 1, 0, 'H' }, { NULL, 0, 0, 0 } }; @@ -102,6 +108,13 @@ killer (int sig) exit (0); } +static void +sigusr (int sig) +{ + dprintf (N_DEBUG, "Caught signal %d, re-reading notify list.", sig); + re_notify = 1; +} + /* * Startup information. */ @@ -148,6 +161,7 @@ usage() fprintf(stderr," -n, --name Specify a local hostname.\n"); fprintf(stderr," -P State directory path.\n"); fprintf(stderr," -N Run in notify only mode.\n"); + fprintf(stderr," -H Specify a high-availability callout program.\n"); } static const char *pidfile = "/var/run/rpc.statd.pid"; @@ -236,7 +250,7 @@ int main (int argc, char **argv) MY_NAME = NULL; /* Process command line switches */ - while ((arg = getopt_long(argc, argv, "h?vVFNdn:p:o:P:", longopts, NULL)) != EOF) { + while ((arg = getopt_long(argc, argv, "h?vVFNH:dn:p:o:P:", longopts, NULL)) != EOF) { switch (arg) { case 'V': /* Version */ case 'v': @@ -302,6 +316,13 @@ int main (int argc, char **argv) sprintf(SM_STAT_PATH, "%s/state", DIR_BASE ); } break; + case 'H': /* PRC: specify the ha-callout program */ + if ((ha_callout_prog = xstrdup(optarg)) == NULL) { + fprintf(stderr, "%s: xstrdup(%s) failed!\n", + argv[0], optarg); + exit(1); + } + break; case '?': /* heeeeeelllllllpppp? heh */ case 'h': usage(); @@ -397,6 +418,8 @@ int main (int argc, char **argv) signal (SIGHUP, killer); signal (SIGINT, killer); signal (SIGTERM, killer); + /* PRC: trap SIGUSR1 to re-read notify list from disk */ + signal(SIGUSR1, sigusr); /* WARNING: the following works on Linux and SysV, but not BSD! */ signal(SIGCHLD, SIG_IGN); diff --git a/utils/statd/statd.h b/utils/statd/statd.h index ef90bdd..e782972 100644 --- a/utils/statd/statd.h +++ b/utils/statd/statd.h @@ -85,3 +85,5 @@ extern int run_mode; */ extern char *name_p; /* program basename */ extern char *version_p; /* program version */ + +extern int re_notify; /* time to re-read notify list */ diff --git a/utils/statd/statd.man b/utils/statd/statd.man index f35b6c9..a225915 100644 --- a/utils/statd/statd.man +++ b/utils/statd/statd.man @@ -4,11 +4,12 @@ .\" Copyright (C) 1999 Olaf Kirch .\" Modified by Jeffrey A. Uphoff, 1999, 2002. .\" Modified by Lon Hohberger, 2000. -.TH rpc.statd 8 "16 Sep 2002" +.\" Modified by Paul Clements, 2004. +.TH rpc.statd 8 "31 Aug 2004" .SH NAME rpc.statd \- NSM status monitor .SH SYNOPSIS -.B "/sbin/rpc.statd [-F] [-d] [-?] [-n " name "] [-o " port "] [-p " port "] [-V]" +.B "/sbin/rpc.statd [-F] [-d] [-?] [-n " name "] [-o " port "] [-p " port "] [-H " prog "] [-V]" .SH DESCRIPTION The .B rpc.statd @@ -101,6 +102,19 @@ statd program will check its state directory, send notifications to any monitored nodes, and exit once the notifications have been sent. This mode is used to enable Highly Available NFS implementations (i.e. HA-NFS). .TP +.BI "\-H, " "" " \-\-ha-callout " prog +Specify a high availability callout program, which will receive callouts +for all client monitor and unmonitor requests. This allows +.B rpc.statd +to be used in a High Availability NFS (HA-NFS) environment. The +program will be run with 3 arguments: The first is either +.B add-client +or +.B del-client +depending on the reason for the callout. +The second will be the name of the client. +The third will be the name of the server as known to the client. +.TP .B -? Causes .B rpc.statd @@ -135,6 +149,15 @@ and .BR hosts_access (5) manual pages. +.SH SIGNALS +.BR SIGUSR1 +causes +.B rpc.statd +to re-read the notify list from disk +and send notifications to clients. This can be used in High Availability NFS +(HA-NFS) environments to notify clients to reacquire file locks upon takeover +of an NFS export from another server. + .SH FILES .BR /var/lib/nfs/state .br @@ -153,3 +176,5 @@ Olaf Kirch H.J. Lu .br Lon Hohberger +.br +Paul Clements diff --git a/utils/statd/svc_run.c b/utils/statd/svc_run.c index 393a2fd..e2d7c97 100644 --- a/utils/statd/svc_run.c +++ b/utils/statd/svc_run.c @@ -60,6 +60,7 @@ static int svc_stop = 0; * requests are put. */ notify_list * notify = NULL; +int re_notify = 0; /* * Jump-off function. @@ -86,6 +87,10 @@ my_svc_run(void) for (;;) { if (svc_stop) return; + if (re_notify) { + notify_hosts(); + re_notify = 0; + } /* Ah, there are some notifications to be processed */ while (notify && NL_WHEN(notify) <= time(&now)) { -- 2.39.5