From 7f98c14d38badedd30d2d4a6b1d15e913967bf87 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 14 Jan 2010 12:24:00 -0500 Subject: [PATCH] libnsm.a: Add support for multiple lines in monitor record files To support IPv6, statd must support multi-homed remote peers. For our purposes, "multi-homed peer" means that more than one unique IP address maps to the one canonical host name for that peer. An SM_MON request from the local lockd has a "mon_name" argument that statd reverse maps to a canonical hostname (ie the A record for that host). statd assumes the canonical hostname is unique enough that it stores the callback data for this mon_name in a file named after that canonical hostname. Because lockd can't distinguish between two unique IP addresses that may be from the same physical host, the kernel can hand statd a mon_name that maps to the same canonical hostname as some previous mon_name. So that the kernel can keep this instance of the mon_name unique, it creates a fresh priv cookie for each new address. Note that a mon_name can be a presentation address string, or the caller_name string sent in each NLMPROC_LOCK request. There's nothing that requires the caller_name to be a fully-qualified hostname, thus it's uniqueness is not guaranteed. The current design of statd assumes that canonical hostnames will be unique enough. When a mon_name for a fresh SM_MON request maps to the same canonical hostname as an existing monitored peer, but the priv cookie is new, statd will try to write the information for the fresh request into an existing monitor record file, wiping out the contents of the file. This is because the mon_name/cookie combination won't match any record statd already has. Currently, statd doesn't check if a record file already exists before writing into it. statd's logic assumes that the svc routine has already checked that no matching record exists in the in-core monitor list. And, it doesn't use O_EXCL when opening the record file. Not only is the old data in that file wiped out, but statd's in-core monitor list will no longer match what's in the on-disk monitor list. Note that IPv6 isn't needed to exercise multi-homed peer support. Any IPv4 peer that has multiple addresses that map to its canonical hostname will trigger this behavior. However, this scenario will become quite common when all hosts on a network automatically get both an IPv4 address and an IPv6 address. I can think of a few ways to address this: 1. Replace the current on-disk format with a database that has a uniqueness constraint on the monitor records 2. Create a new file naming scheme; eg. one that uses a truly unique name such as a hash generated from the mon_name, my_name, and priv cookie 3. Support multiple lines in each monitor record file Since statd's on-disk format constitutes a formal API, options 1 and 2 are right out. This patch implements option 3. There are two parts: adding a new line to an existing file; and deleting a line from a file with more than one line. Interestingly, the existing code already supports reading more than one line from these files, so we don't need to add extra code here to do that. One file may contain a line for every unique mon_name / priv cookie where the mon_name reverse maps to the same canonical hostname. We use the atomic write facility added by a previous patch to ensure the on-disk monitor record list is updated atomically. Signed-off-by: Chuck Lever --- support/include/nsm.h | 6 +- support/nsm/file.c | 161 +++++++++++++++++++++++++++++++++++++--- utils/statd/monitor.c | 6 +- utils/statd/sm-notify.c | 5 +- 4 files changed, 162 insertions(+), 16 deletions(-) diff --git a/support/include/nsm.h b/support/include/nsm.h index ce9e294..fb4d823 100644 --- a/support/include/nsm.h +++ b/support/include/nsm.h @@ -58,8 +58,10 @@ extern unsigned int extern _Bool nsm_insert_monitored_host(const char *hostname, const struct sockaddr *sap, const struct mon *m); -extern void nsm_delete_monitored_host(const char *hostname); -extern void nsm_delete_notified_host(const char *hostname); +extern void nsm_delete_monitored_host(const char *hostname, + const char *mon_name, const char *my_name); +extern void nsm_delete_notified_host(const char *hostname, + const char *mon_name, const char *my_name); extern size_t nsm_priv_to_hex(const char *priv, char *buf, const size_t buflen); diff --git a/support/nsm/file.c b/support/nsm/file.c index 10769d9..8796705 100644 --- a/support/nsm/file.c +++ b/support/nsm/file.c @@ -625,6 +625,56 @@ nsm_create_monitor_record(char *buf, const size_t buflen, return buflen - remaining; } +static _Bool +nsm_append_monitored_host(const char *path, const char *line) +{ + _Bool result = false; + char *buf = NULL; + struct stat stb; + size_t buflen; + ssize_t len; + int fd; + + if (stat(path, &stb) == -1) { + xlog(L_ERROR, "Failed to insert: " + "could not stat original file %s: %m", path); + goto out; + } + buflen = (size_t)stb.st_size + strlen(line); + + buf = malloc(buflen + 1); + if (buf == NULL) { + xlog(L_ERROR, "Failed to insert: no memory"); + goto out; + } + memset(buf, 0, buflen + 1); + + fd = open(path, O_RDONLY); + if (fd == -1) { + xlog(L_ERROR, "Failed to insert: " + "could not open original file %s: %m", path); + goto out; + } + + len = read(fd, buf, (size_t)stb.st_size); + if (exact_error_check(len, (size_t)stb.st_size)) { + xlog(L_ERROR, "Failed to insert: " + "could not read original file %s: %m", path); + (void)close(fd); + goto out; + } + (void)close(fd); + + strcat(buf, line); + + if (nsm_atomic_write(path, buf, buflen)) + result = true; + +out: + free(buf); + return result; +} + /** * nsm_insert_monitored_host - write callback data for one host to disk * @hostname: C string containing a hostname @@ -657,9 +707,18 @@ nsm_insert_monitored_host(const char *hostname, const struct sockaddr *sap, goto out; } - fd = open(path, O_WRONLY | O_CREAT | O_SYNC, S_IRUSR | S_IWUSR); + /* + * If exclusive create fails, we're adding a new line to an + * existing file. + */ + fd = open(path, O_WRONLY | O_CREAT | O_EXCL | O_SYNC, S_IRUSR | S_IWUSR); if (fd == -1) { - xlog(L_ERROR, "Failed to insert: creating %s: %m", path); + if (errno != EEXIST) { + xlog(L_ERROR, "Failed to insert: creating %s: %m", path); + goto out; + } + + result = nsm_append_monitored_host(path, buf); goto out; } result = true; @@ -848,9 +907,15 @@ nsm_load_notify_list(nsm_populate_t func) } static void -nsm_delete_host(const char *directory, const char *hostname) +nsm_delete_host(const char *directory, const char *hostname, + const char *mon_name, const char *my_name) { - char *path; + char line[LINELEN + 1 + SM_MAXSTRLEN + 2]; + char *outbuf = NULL; + struct stat stb; + char *path, *next; + size_t remaining; + FILE *f; path = nsm_make_record_pathname(directory, hostname); if (path == NULL) { @@ -858,30 +923,106 @@ nsm_delete_host(const char *directory, const char *hostname) return; } - if (unlink(path) == -1) - xlog(L_ERROR, "Failed to unlink %s: %m", path); + if (stat(path, &stb) == -1) { + xlog(L_ERROR, "Failed to delete: " + "could not stat original file %s: %m", path); + goto out; + } + remaining = (size_t)stb.st_size + 1; + outbuf = malloc(remaining); + if (outbuf == NULL) { + xlog(L_ERROR, "Failed to delete: no memory"); + goto out; + } + + f = fopen(path, "r"); + if (f == NULL) { + xlog(L_ERROR, "Failed to delete: " + "could not open original file %s: %m", path); + goto out; + } + + /* + * Walk the records in the file, and copy the non-matching + * ones to our output buffer. + */ + next = outbuf; + while (fgets(line, (int)sizeof(line), f) != NULL) { + struct sockaddr_in sin; + struct mon m; + size_t len; + + if (!nsm_parse_line(line, &sin, &m)) { + xlog(L_ERROR, "Failed to delete: " + "could not parse original file %s", path); + (void)fclose(f); + goto out; + } + + if (strcmp(mon_name, m.mon_id.mon_name) == 0 && + strcmp(my_name, m.mon_id.my_id.my_name) == 0) + continue; + + /* nsm_parse_line destroys the contents of line[], so + * reconstruct the copy in our output buffer. */ + len = nsm_create_monitor_record(next, remaining, + (struct sockaddr *)(char *)&sin, &m); + if (len == 0) { + xlog(L_ERROR, "Failed to delete: " + "could not construct output record"); + (void)fclose(f); + goto out; + } + next += len; + remaining -= len; + } + + (void)fclose(f); + + /* + * If nothing was copied when we're done, then unlink the file. + * Otherwise, atomically update the contents of the file. + */ + if (next != outbuf) { + if (!nsm_atomic_write(path, outbuf, strlen(outbuf))) + xlog(L_ERROR, "Failed to delete: " + "could not write new file %s: %m", path); + } else { + if (unlink(path) == -1) + xlog(L_ERROR, "Failed to delete: " + "could not unlink file %s: %m", path); + } + +out: + free(outbuf); free(path); } /** * nsm_delete_monitored_host - delete on-disk record for monitored host * @hostname: '\0'-terminated C string containing hostname of record to delete + * @mon_name: '\0'-terminated C string containing monname of record to delete + * @my_name: '\0'-terminated C string containing myname of record to delete * */ void -nsm_delete_monitored_host(const char *hostname) +nsm_delete_monitored_host(const char *hostname, const char *mon_name, + const char *my_name) { - nsm_delete_host(NSM_MONITOR_DIR, hostname); + nsm_delete_host(NSM_MONITOR_DIR, hostname, mon_name, my_name); } /** * nsm_delete_notified_host - delete on-disk host record after notification * @hostname: '\0'-terminated C string containing hostname of record to delete + * @mon_name: '\0'-terminated C string containing monname of record to delete + * @my_name: '\0'-terminated C string containing myname of record to delete * */ void -nsm_delete_notified_host(const char *hostname) +nsm_delete_notified_host(const char *hostname, const char *mon_name, + const char *my_name) { - nsm_delete_host(NSM_NOTIFY_DIR, hostname); + nsm_delete_host(NSM_NOTIFY_DIR, hostname, mon_name, my_name); } diff --git a/utils/statd/monitor.c b/utils/statd/monitor.c index 5bedb3e..fb32196 100644 --- a/utils/statd/monitor.c +++ b/utils/statd/monitor.c @@ -315,7 +315,8 @@ sm_unmon_1_svc(struct mon_id *argp, struct svc_req *rqstp) /* PRC: do the HA callout: */ ha_callout("del-client", mon_name, my_name, -1); - nsm_delete_monitored_host(clnt->dns_name); + nsm_delete_monitored_host(clnt->dns_name, + mon_name, my_name); nlist_free(&rtnl, clnt); return (&result); @@ -369,7 +370,8 @@ sm_unmon_all_1_svc(struct my_id *argp, struct svc_req *rqstp) temp = NL_NEXT(clnt); /* PRC: do the HA callout: */ ha_callout("del-client", mon_name, my_name, -1); - nsm_delete_monitored_host(clnt->dns_name); + nsm_delete_monitored_host(clnt->dns_name, + mon_name, my_name); nlist_free(&rtnl, clnt); ++count; clnt = temp; diff --git a/utils/statd/sm-notify.c b/utils/statd/sm-notify.c index 70d94a8..3259a3e 100644 --- a/utils/statd/sm-notify.c +++ b/utils/statd/sm-notify.c @@ -130,9 +130,10 @@ out_nomem: static void smn_forget_host(struct nsm_host *host) { - xlog(D_CALL, "Removing %s from notify list", host->name); + xlog(D_CALL, "Removing %s (%s, %s) from notify list", + host->name, host->mon_name, host->my_name); - nsm_delete_notified_host(host->name); + nsm_delete_notified_host(host->name, host->mon_name, host->my_name); free(host->my_name); free(host->mon_name); -- 2.39.2