Push down the page queues lock into vm_page_activate().

7 files changed, 53 insertions, 48 deletions

diff --git a/sys/fs/nfsclient/nfs_clbio.c b/sys/fs/nfsclient/nfs_clbio.c
index 0224e157b6e0..7d6fcab68095 100644
--- a/sys/fs/nfsclient/nfs_clbio.c
+++ b/sys/fs/nfsclient/nfs_clbio.c
@@ -196,9 +196,6 @@ ncl_getpages(struct vop_getpages_args *ap)
 		nextoff = toff + PAGE_SIZE;
 		m = pages[i];
 
-		vm_page_lock(m);
-		vm_page_lock_queues();
-
 		if (nextoff <= size) {
 			/*
 			 * Read operation filled an entire page
@@ -236,18 +233,22 @@ ncl_getpages(struct vop_getpages_args *ap)
 			 * now tell them that it is ok to use.
 			 */
 			if (!error) {
-				if (m->oflags & VPO_WANTED)
+				if (m->oflags & VPO_WANTED) {
+					vm_page_lock(m);
 					vm_page_activate(m);
-				else
+					vm_page_unlock(m);
+				} else {
+					vm_page_lock(m);
 					vm_page_deactivate(m);
+					vm_page_unlock(m);
+				}
 				vm_page_wakeup(m);
 			} else {
+				vm_page_lock(m);
 				vm_page_free(m);
+				vm_page_unlock(m);
 			}
 		}
-
-		vm_page_unlock_queues();
-		vm_page_unlock(m);
 	}
 	VM_OBJECT_UNLOCK(object);
 	return (0);
diff --git a/sys/fs/nwfs/nwfs_io.c b/sys/fs/nwfs/nwfs_io.c
index 125201408adb..0bd7dc355ff2 100644
--- a/sys/fs/nwfs/nwfs_io.c
+++ b/sys/fs/nwfs/nwfs_io.c
@@ -446,9 +446,6 @@ nwfs_getpages(ap)
 		nextoff = toff + PAGE_SIZE;
 		m = pages[i];
 
-		vm_page_lock(m);
-		vm_page_lock_queues();
-
 		if (nextoff <= size) {
 			m->valid = VM_PAGE_BITS_ALL;
 			KASSERT(m->dirty == 0,
@@ -474,18 +471,22 @@ nwfs_getpages(ap)
 			 * now tell them that it is ok to use.
 			 */
 			if (!error) {
-				if (m->oflags & VPO_WANTED)
+				if (m->oflags & VPO_WANTED) {
+					vm_page_lock(m);
 					vm_page_activate(m);
-				else
+					vm_page_unlock(m);
+				} else {
+					vm_page_lock(m);
 					vm_page_deactivate(m);
+					vm_page_unlock(m);
+				}
 				vm_page_wakeup(m);
 			} else {
+				vm_page_lock(m);
 				vm_page_free(m);
+				vm_page_unlock(m);
 			}
 		}
-
-		vm_page_unlock_queues();
-		vm_page_unlock(m);
 	}
 	VM_OBJECT_UNLOCK(object);
 	return 0;
diff --git a/sys/fs/smbfs/smbfs_io.c b/sys/fs/smbfs/smbfs_io.c
index 1ffcde9fb803..9becfdda13ea 100644
--- a/sys/fs/smbfs/smbfs_io.c
+++ b/sys/fs/smbfs/smbfs_io.c
@@ -497,9 +497,6 @@ smbfs_getpages(ap)
 		nextoff = toff + PAGE_SIZE;
 		m = pages[i];
 
-		vm_page_lock(m);
-		vm_page_lock_queues();
-
 		if (nextoff <= size) {
 			/*
 			 * Read operation filled an entire page
@@ -538,18 +535,22 @@ smbfs_getpages(ap)
 			 * now tell them that it is ok to use.
 			 */
 			if (!error) {
-				if (m->oflags & VPO_WANTED)
+				if (m->oflags & VPO_WANTED) {
+					vm_page_lock(m);
 					vm_page_activate(m);
-				else
+					vm_page_unlock(m);
+				} else {
+					vm_page_lock(m);
 					vm_page_deactivate(m);
+					vm_page_unlock(m);
+				}
 				vm_page_wakeup(m);
 			} else {
+				vm_page_lock(m);
 				vm_page_free(m);
+				vm_page_unlock(m);
 			}
 		}
-
-		vm_page_unlock_queues();
-		vm_page_unlock(m);
 	}
 	VM_OBJECT_UNLOCK(object);
 	return 0;
diff --git a/sys/nfsclient/nfs_bio.c b/sys/nfsclient/nfs_bio.c
index 37da34b8cce3..e85fab8975d7 100644
--- a/sys/nfsclient/nfs_bio.c
+++ b/sys/nfsclient/nfs_bio.c
@@ -194,9 +194,6 @@ nfs_getpages(struct vop_getpages_args *ap)
 		nextoff = toff + PAGE_SIZE;
 		m = pages[i];
 
-		vm_page_lock(m);
-		vm_page_lock_queues();
-
 		if (nextoff <= size) {
 			/*
 			 * Read operation filled an entire page
@@ -234,18 +231,22 @@ nfs_getpages(struct vop_getpages_args *ap)
 			 * now tell them that it is ok to use.
 			 */
 			if (!error) {
-				if (m->oflags & VPO_WANTED)
+				if (m->oflags & VPO_WANTED) {
+					vm_page_lock(m);
 					vm_page_activate(m);
-				else
+					vm_page_unlock(m);
+				} else {
+					vm_page_lock(m);
 					vm_page_deactivate(m);
+					vm_page_unlock(m);
+				}
 				vm_page_wakeup(m);
 			} else {
+				vm_page_lock(m);
 				vm_page_free(m);
+				vm_page_unlock(m);
 			}
 		}
-
-		vm_page_unlock_queues();
-		vm_page_unlock(m);
 	}
 	VM_OBJECT_UNLOCK(object);
 	return (0);
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 0088248142de..05c3228643e0 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -937,7 +937,6 @@ vnode_locked:
 		vm_fault_prefault(fs.map->pmap, vaddr, fs.entry);
 	VM_OBJECT_LOCK(fs.object);
 	vm_page_lock(fs.m);
-	vm_page_lock_queues();
 
 	/*
 	 * If the page is not wired down, then put it where the pageout daemon
@@ -948,10 +947,8 @@ vnode_locked:
 			vm_page_wire(fs.m);
 		else
 			vm_page_unwire(fs.m, 1);
-	} else {
+	} else
 		vm_page_activate(fs.m);
-	}
-	vm_page_unlock_queues();
 	vm_page_unlock(fs.m);
 	vm_page_wakeup(fs.m);
 
@@ -1267,9 +1264,7 @@ vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
 			vm_page_unlock(dst_m);
 		} else {
 			vm_page_lock(dst_m);
-			vm_page_lock_queues();
 			vm_page_activate(dst_m);
-			vm_page_unlock_queues();
 			vm_page_unlock(dst_m);
 		}
 		vm_page_wakeup(dst_m);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index afa5cfb76671..455c99029454 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -1374,22 +1374,25 @@ vm_page_enqueue(int queue, vm_page_t m)
  *	Ensure that act_count is at least ACT_INIT but do not otherwise
  *	mess with it.
  *
- *	The page queues must be locked.
+ *	The page must be locked.
  *	This routine may not block.
  */
 void
 vm_page_activate(vm_page_t m)
 {
 
-	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	vm_page_lock_assert(m, MA_OWNED);
 	if (VM_PAGE_GETKNOWNQUEUE2(m) != PQ_ACTIVE) {
-		vm_pageq_remove(m);
 		if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
 			if (m->act_count < ACT_INIT)
 				m->act_count = ACT_INIT;
+			vm_page_lock_queues();
+			vm_pageq_remove(m);
 			vm_page_enqueue(PQ_ACTIVE, m);
-		}
+			vm_page_unlock_queues();
+		} else
+			KASSERT(m->queue == PQ_NONE,
+			    ("vm_page_activate: wired page %p is queued", m));
 	} else {
 		if (m->act_count < ACT_INIT)
 			m->act_count = ACT_INIT;
diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index 69fd5d101c71..cfe9f994e0e3 100644
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -948,8 +948,6 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage)
 		nextoff = tfoff + PAGE_SIZE;
 		mt = m[i];
 
-		vm_page_lock(mt);
-		vm_page_lock_queues();
 		if (nextoff <= object->un_pager.vnp.vnp_size) {
 			/*
 			 * Read filled up entire page.
@@ -992,17 +990,22 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage)
 			 * now tell them that it is ok to use
 			 */
 			if (!error) {
-				if (mt->oflags & VPO_WANTED)
+				if (mt->oflags & VPO_WANTED) {
+					vm_page_lock(mt);
 					vm_page_activate(mt);
-				else
+					vm_page_unlock(mt);
+				} else {
+					vm_page_lock(mt);
 					vm_page_deactivate(mt);
+					vm_page_unlock(mt);
+				}
 				vm_page_wakeup(mt);
 			} else {
+				vm_page_lock(mt);
 				vm_page_free(mt);
+				vm_page_unlock(mt);
 			}
 		}
-		vm_page_unlock_queues();
-		vm_page_unlock(mt);
 	}
 	VM_OBJECT_UNLOCK(object);
 	if (error) {