Fix NFSv4.1 client side handling of "soft,retrans=2" mounts.

Normally "soft,retrans=2" cannot be safely used on NFSv4 mounts, since
the RPC can fail and leave the open/lock state in an undefined state.
Doing I/O on a pNFS DS is an exception to this, since no open/lock state
is maintained on the DS server.
It is useful to do "soft,retrans=2" connections to a DS when it is mirrored,
so that the client can detect failure of the DS. As such, mounts from the MDS
to the DSs should use these mount options when mirroring is enabled.
However, the NFSv4.1 client still leaves the session in an undefined state
when this happens.
This patch fixes the problem by setting the session defunct, so it will
no longer be used.
The patch also sets "retries=2" on the connections done by the client to
a DS, which is the internal equivalent of "soft,retrans=2".
The client does not know if the server implements mirroring at connection
time, but always doing this should be safe, since it will fall back on doing
I/O via the MDS as a proxy when there is a failure doing an I/O RPC to the DS.

This patch should not affect non-pNFS client mounts.

MFC after:	2 weeks

1 files changed, 88 insertions, 16 deletions

diff --git a/sys/fs/nfs/nfs_commonkrpc.c b/sys/fs/nfs/nfs_commonkrpc.c
index 48e92426f7a0..cd4bcc180ae6 100644
--- a/sys/fs/nfs/nfs_commonkrpc.c
+++ b/sys/fs/nfs/nfs_commonkrpc.c
@@ -157,6 +157,9 @@ static int nfsv2_procid[NFS_V3NPROCS] = {
 /*
  * Initialize sockets and congestion for a new NFS connection.
  * We do not free the sockaddr if error.
+ * Which arguments are set to NULL indicate what kind of call it is.
+ * cred == NULL --> a call to connect to a pNFS DS
+ * nmp == NULL --> indicates an upcall to userland or a NFSv4 callback
  */
 int
 newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp,
@@ -293,24 +296,38 @@ newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp,
 				retries = nmp->nm_retry;
 		} else
 			retries = INT_MAX;
-		/* cred == NULL for DS connects. */
-		if (NFSHASNFSV4N(nmp) && cred != NULL) {
-			/*
-			 * Make sure the nfscbd_pool doesn't get destroyed
-			 * while doing this.
-			 */
-			NFSD_LOCK();
-			if (nfs_numnfscbd > 0) {
-				nfs_numnfscbd++;
-				NFSD_UNLOCK();
-				xprt = svc_vc_create_backchannel(nfscbd_pool);
-				CLNT_CONTROL(client, CLSET_BACKCHANNEL, xprt);
+		if (NFSHASNFSV4N(nmp)) {
+			if (cred != NULL) {
+				/*
+				 * Make sure the nfscbd_pool doesn't get
+				 * destroyed while doing this.
+				 */
 				NFSD_LOCK();
-				nfs_numnfscbd--;
-				if (nfs_numnfscbd == 0)
-					wakeup(&nfs_numnfscbd);
+				if (nfs_numnfscbd > 0) {
+					nfs_numnfscbd++;
+					NFSD_UNLOCK();
+					xprt = svc_vc_create_backchannel(
+					    nfscbd_pool);
+					CLNT_CONTROL(client, CLSET_BACKCHANNEL,
+					    xprt);
+					NFSD_LOCK();
+					nfs_numnfscbd--;
+					if (nfs_numnfscbd == 0)
+						wakeup(&nfs_numnfscbd);
+				}
+				NFSD_UNLOCK();
+			} else {
+				/*
+				 * cred == NULL for a DS connect.
+				 * For connects to a DS, set a retry limit
+				 * so that failed DSs will be detected.
+				 * This is ok for NFSv4.1, since a DS does
+				 * not maintain open/lock state and is the
+				 * only case where using a "soft" mount is
+				 * recommended for NFSv4.
+				 */
+				retries = 2;
 			}
-			NFSD_UNLOCK();
 		}
 	} else {
 		/*
@@ -762,6 +779,7 @@ tryagain:
 	else
 		stat = CLNT_CALL_MBUF(nrp->nr_client, &ext, procnum,
 		    nd->nd_mreq, &nd->nd_mrep, timo);
+	NFSCL_DEBUG(2, "clnt call=%d\n", stat);
 
 	if (rep != NULL) {
 		/*
@@ -789,6 +807,60 @@ tryagain:
 		error = EPROTONOSUPPORT;
 	} else if (stat == RPC_INTR) {
 		error = EINTR;
+	} else if (stat == RPC_CANTSEND || stat == RPC_CANTRECV ||
+	     stat == RPC_SYSTEMERROR) {
+		if ((nd->nd_flag & ND_NFSV41) != 0 && nmp != NULL &&
+		    nd->nd_procnum != NFSPROC_NULL) {
+			/*
+			 * The nfsess_defunct field is protected by
+			 * the NFSLOCKMNT()/nm_mtx lock and not the
+			 * nfsess_mtx lock to simplify its handling,
+			 * for the MDS session. This lock is also
+			 * sufficient for nfsess_sessionid, since it
+			 * never changes in the structure.
+			 */
+			NFSLOCKCLSTATE();
+			NFSLOCKMNT(nmp);
+			/* The session must be marked defunct. */
+			if (dssep == NULL) {
+				/*
+				 * This is either an MDS proxy operation or
+				 * a client mount with "soft,retrans=N" options.
+				 * Mark the MDS session defunct and initiate
+				 * recovery, as required.
+				 */
+				NFSCL_DEBUG(1, "Failed soft proxy RPC\n");
+				sep = NFSMNT_MDSSESSION(nmp);
+				if (bcmp(sep->nfsess_sessionid, nd->nd_sequence,
+				    NFSX_V4SESSIONID) == 0) {
+					/* Initiate recovery. */
+					sep->nfsess_defunct = 1;
+					NFSCL_DEBUG(1, "Marked defunct\n");
+					if (nmp->nm_clp != NULL) {
+						nmp->nm_clp->nfsc_flags |=
+						    NFSCLFLAGS_RECOVER;
+						wakeup(nmp->nm_clp);
+					}
+				}
+			} else {
+				/*
+				 * This is a client side DS RPC. Just mark
+				 * the session defunct.  A subsequent LayoutGet
+				 * should get a new session.
+				 */
+				NFSCL_DEBUG(1, "Failed client DS RPC\n");
+				if (bcmp(dssep->nfsess_sessionid,
+				    nd->nd_sequence, NFSX_V4SESSIONID) == 0) {
+					/* Mark it defunct. */
+					dssep->nfsess_defunct = 1;
+					NFSCL_DEBUG(1, "Marked defunct\n");
+				}
+			}
+			NFSUNLOCKMNT(nmp);
+			NFSUNLOCKCLSTATE();
+		}
+		NFSINCRGLOBAL(nfsstatsv1.rpcinvalid);
+		error = ENXIO;
 	} else {
 		NFSINCRGLOBAL(nfsstatsv1.rpcinvalid);
 		error = EACCES;