Reimplement gvinum orphanization.

gvinum was the only GEOM class, using consumer nstart/nend fields. Making
it do its own accounting for orphanization purposes allows in perspective
to remove burden of that expensive for SMP accounting from GEOM.

Also the previous implementation spinned in a tight event loop, waiting
for all active BIOs to complete, while the new one knows exactly when it
is possible to close the consumer.

MFC after:	1 month
Sponsored by:	iXsystems, Inc.

6 files changed, 33 insertions, 8 deletions

diff --git a/sys/geom/vinum/geom_vinum.h b/sys/geom/vinum/geom_vinum.h
index f9ddd6859e3b..e61e9dc5f0a9 100644
--- a/sys/geom/vinum/geom_vinum.h
+++ b/sys/geom/vinum/geom_vinum.h
@@ -127,6 +127,7 @@ void	gv_post_event(struct gv_softc *, int, void *, void *, intmax_t,
 void	gv_worker_exit(struct gv_softc *);
 struct gv_event *gv_get_event(struct gv_softc *);
 void	gv_remove_event(struct gv_softc *, struct gv_event *);
+void	gv_drive_done(struct gv_drive *);
 void	gv_drive_tasted(struct gv_softc *, struct g_provider *);
 void	gv_drive_lost(struct gv_softc *, struct gv_drive *);
 void	gv_setup_objects(struct gv_softc *);
diff --git a/sys/geom/vinum/geom_vinum_events.c b/sys/geom/vinum/geom_vinum_events.c
index b31611092576..78aa0adab8cf 100644
--- a/sys/geom/vinum/geom_vinum_events.c
+++ b/sys/geom/vinum/geom_vinum_events.c
@@ -195,6 +195,20 @@ failed:
 }
 
 /*
+ * Count completed BIOs and handle orphanization when all are done.
+ */
+void
+gv_drive_done(struct gv_drive *d)
+{
+
+	KASSERT(d->active >= 0, ("Negative number of BIOs (%d)", d->active));
+	if (--d->active == 0 && (d->flags & GV_DRIVE_ORPHANED)) {
+		d->flags &= ~GV_DRIVE_ORPHANED;
+		gv_post_event(d->vinumconf, GV_EVENT_DRIVE_LOST, d, NULL, 0, 0);
+	}
+}
+
+/*
  * When losing a drive (e.g. hardware failure), we cut down the consumer
  * attached to the underlying device and bring the drive itself to a
  * "referenced" state so that normal tasting could bring it up cleanly if it
@@ -214,10 +228,10 @@ gv_drive_lost(struct gv_softc *sc, struct gv_drive *d)
 	cp = d->consumer;
 
 	if (cp != NULL) {
-		if (cp->nstart != cp->nend) {
-			G_VINUM_DEBUG(0, "dead drive '%s' has still active "
+		if (d->active > 0) {
+			G_VINUM_DEBUG(2, "dead drive '%s' has still active "
 			    "requests, unable to detach consumer", d->name);
-			gv_post_event(sc, GV_EVENT_DRIVE_LOST, d, NULL, 0, 0);
+			d->flags |= GV_DRIVE_ORPHANED;
 			return;
 		}
 		g_topology_lock();
diff --git a/sys/geom/vinum/geom_vinum_plex.c b/sys/geom/vinum/geom_vinum_plex.c
index 179ad19d2fca..e4553817346c 100644
--- a/sys/geom/vinum/geom_vinum_plex.c
+++ b/sys/geom/vinum/geom_vinum_plex.c
@@ -278,6 +278,7 @@ gv_plex_normal_request(struct gv_plex *p, struct bio *bp, off_t boff,
 	cbp->bio_data = addr;
 	cbp->bio_done = gv_done;
 	cbp->bio_caller1 = s;
+	s->drive_sc->active++;
 
 	/* Store the sub-requests now and let others issue them. */
 	bioq_insert_tail(p->bqueue, cbp); 
@@ -580,10 +581,10 @@ gv_sync_request(struct gv_plex *from, struct gv_plex *to, off_t offset,
 		return (ENOMEM);
 	}
 	bp->bio_length = length;
-	bp->bio_done = gv_done;
+	bp->bio_done = NULL;
 	bp->bio_pflags |= GV_BIO_SYNCREQ;
 	bp->bio_offset = offset;
-	bp->bio_caller1 = from;		
+	bp->bio_caller1 = from;
 	bp->bio_caller2 = to;
 	bp->bio_cmd = type;
 	if (data == NULL)
@@ -694,7 +695,7 @@ gv_grow_request(struct gv_plex *p, off_t offset, off_t length, int type,
 	}
 
 	bp->bio_cmd = type;
-	bp->bio_done = gv_done;
+	bp->bio_done = NULL;
 	bp->bio_error = 0;
 	bp->bio_caller1 = p;
 	bp->bio_offset = offset;
@@ -802,7 +803,7 @@ gv_init_request(struct gv_sd *s, off_t start, caddr_t data, off_t length)
 	}
 	bp->bio_cmd = BIO_WRITE;
 	bp->bio_data = data;
-	bp->bio_done = gv_done;
+	bp->bio_done = NULL;
 	bp->bio_error = 0;
 	bp->bio_length = length;
 	bp->bio_pflags |= GV_BIO_INIT;
@@ -819,6 +820,7 @@ gv_init_request(struct gv_sd *s, off_t start, caddr_t data, off_t length)
 	}
 	cbp->bio_done = gv_done;
 	cbp->bio_caller1 = s;
+	d->active++;
 	/* Send it off to the consumer. */
 	g_io_request(cbp, cp);
 }
@@ -905,7 +907,7 @@ gv_parity_request(struct gv_plex *p, int flags, off_t offset)
 	}
 
 	bp->bio_cmd = BIO_WRITE;
-	bp->bio_done = gv_done;
+	bp->bio_done = NULL;
 	bp->bio_error = 0;
 	bp->bio_length = p->stripesize;
 	bp->bio_caller1 = p;
diff --git a/sys/geom/vinum/geom_vinum_raid5.c b/sys/geom/vinum/geom_vinum_raid5.c
index 7f2011d6b198..b466b01346ce 100644
--- a/sys/geom/vinum/geom_vinum_raid5.c
+++ b/sys/geom/vinum/geom_vinum_raid5.c
@@ -93,11 +93,13 @@ gv_raid5_start(struct gv_plex *p, struct bio *bp, caddr_t addr, off_t boff,
 		if (wp->waiting != NULL) {
 			if (wp->waiting->bio_cflags & GV_BIO_MALLOC)
 				g_free(wp->waiting->bio_data);
+			gv_drive_done(wp->waiting->bio_caller1);
 			g_destroy_bio(wp->waiting);
 		}
 		if (wp->parity != NULL) {
 			if (wp->parity->bio_cflags & GV_BIO_MALLOC)
 				g_free(wp->parity->bio_data);
+			gv_drive_done(wp->parity->bio_caller1);
 			g_destroy_bio(wp->parity);
 		}
 		g_free(wp);
@@ -118,6 +120,7 @@ gv_raid5_start(struct gv_plex *p, struct bio *bp, caddr_t addr, off_t boff,
 		while (cbp != NULL) {
 			if (cbp->bio_cflags & GV_BIO_MALLOC)
 				g_free(cbp->bio_data);
+			gv_drive_done(cbp->bio_caller1);
 			g_destroy_bio(cbp);
 			cbp = bioq_takefirst(p->bqueue);
 		}
@@ -657,6 +660,7 @@ gv_raid5_clone_bio(struct bio *bp, struct gv_sd *s, struct gv_raid5_packet *wp,
 	cbp->bio_length = wp->length;
 	cbp->bio_done = gv_done;
 	cbp->bio_caller1 = s;
+	s->drive_sc->active++;
 	if (use_wp)
 		cbp->bio_caller2 = wp;
 
diff --git a/sys/geom/vinum/geom_vinum_var.h b/sys/geom/vinum/geom_vinum_var.h
index ee4e30ba53d1..f6a367db59ed 100644
--- a/sys/geom/vinum/geom_vinum_var.h
+++ b/sys/geom/vinum/geom_vinum_var.h
@@ -260,10 +260,12 @@ struct gv_drive {
 #define	GV_DRIVE_REFERENCED	0x01	/* The drive isn't really existing,
 					   but was referenced by a subdisk
 					   during taste. */
+#define	GV_DRIVE_ORPHANED	0x02	/* The drive was orphaned. */
 
 	struct gv_hdr	*hdr;		/* The drive header. */
 
 	struct g_consumer *consumer;	/* Consumer attached to this drive. */
+	int	active;			/* Number of active requests. */
 
 	int freelist_entries;			/* Count of freelist entries. */
 	LIST_HEAD(,gv_freelist)	freelist;	/* List of freelist entries. */
diff --git a/sys/geom/vinum/geom_vinum_volume.c b/sys/geom/vinum/geom_vinum_volume.c
index bfe3be5585a0..63b1077a26fa 100644
--- a/sys/geom/vinum/geom_vinum_volume.c
+++ b/sys/geom/vinum/geom_vinum_volume.c
@@ -163,4 +163,6 @@ gv_bio_done(struct gv_softc *sc, struct bio *bp)
 		gv_plex_raid5_done(p, bp);
 		break;
 	}
+
+	gv_drive_done(s->drive_sc);
 }