From 3d2e54c3176f9344a9f60bfd92862b1c799d55c1 Mon Sep 17 00:00:00 2001 From: Alan Cox Date: Thu, 15 Jul 2004 18:00:43 +0000 Subject: Push down the acquisition and release of the page queues lock into pmap_protect() and pmap_remove(). In general, they require the lock in order to modify a page's pv list or flags. In some cases, however, pmap_protect() can avoid acquiring the lock. --- sys/alpha/alpha/pmap.c | 5 +++++ sys/amd64/amd64/pmap.c | 9 +++++++-- sys/i386/acpica/acpi_wakeup.c | 2 -- sys/i386/i386/pmap.c | 9 +++++++-- sys/ia64/ia64/pmap.c | 11 +++++++---- sys/powerpc/aim/mmu_oea.c | 4 ++++ sys/powerpc/powerpc/mmu_oea.c | 4 ++++ sys/powerpc/powerpc/pmap.c | 4 ++++ sys/sparc64/sparc64/pmap.c | 5 ++++- sys/vm/vm_map.c | 8 -------- sys/vm/vm_pageout.c | 2 -- 11 files changed, 42 insertions(+), 21 deletions(-) (limited to 'sys') diff --git a/sys/alpha/alpha/pmap.c b/sys/alpha/alpha/pmap.c index fdd2e959a2dc..6b5f25203606 100644 --- a/sys/alpha/alpha/pmap.c +++ b/sys/alpha/alpha/pmap.c @@ -1573,6 +1573,8 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) */ if (pmap->pm_stats.resident_count == 0) return; + + vm_page_lock_queues(); PMAP_LOCK(pmap); /* @@ -1600,6 +1602,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) nva = va + PAGE_SIZE; } out: + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } @@ -1700,6 +1703,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) if ((sva & PAGE_MASK) || (eva & PAGE_MASK)) panic("pmap_protect: unaligned addresses"); + vm_page_lock_queues(); PMAP_LOCK(pmap); while (sva < eva) { @@ -1751,6 +1755,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) sva += PAGE_SIZE; } + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 9dd4f67cd4dc..39436b9e0a7c 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -1602,6 +1602,8 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) */ if (pmap->pm_stats.resident_count == 0) return; + + vm_page_lock_queues(); PMAP_LOCK(pmap); /* @@ -1613,8 +1615,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) pde = pmap_pde(pmap, sva); if (pde && (*pde & PG_PS) == 0) { pmap_remove_page(pmap, sva); - PMAP_UNLOCK(pmap); - return; + goto out; } } @@ -1684,6 +1685,8 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) if (anyvalid) pmap_invalidate_all(pmap); +out: + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } @@ -1778,6 +1781,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) anychanged = 0; + vm_page_lock_queues(); PMAP_LOCK(pmap); for (; sva < eva; sva = va_next) { @@ -1856,6 +1860,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) } if (anychanged) pmap_invalidate_all(pmap); + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } diff --git a/sys/i386/acpica/acpi_wakeup.c b/sys/i386/acpica/acpi_wakeup.c index fb06c16c4ca7..57e3a22c5ee6 100644 --- a/sys/i386/acpica/acpi_wakeup.c +++ b/sys/i386/acpica/acpi_wakeup.c @@ -273,9 +273,7 @@ acpi_sleep_machdep(struct acpi_softc *sc, int state) } out: - vm_page_lock_queues(); pmap_remove(pm, sc->acpi_wakephys, sc->acpi_wakephys + PAGE_SIZE); - vm_page_unlock_queues(); if (opage) { pmap_enter(pm, sc->acpi_wakephys, page, VM_PROT_READ | VM_PROT_WRITE, 0); diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index 52eb344bdc1f..6b516b5c6a75 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -1637,6 +1637,8 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) */ if (pmap->pm_stats.resident_count == 0) return; + + vm_page_lock_queues(); PMAP_LOCK(pmap); /* @@ -1647,8 +1649,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) if ((sva + PAGE_SIZE == eva) && ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) { pmap_remove_page(pmap, sva); - PMAP_UNLOCK(pmap); - return; + goto out; } anyvalid = 0; @@ -1703,6 +1704,8 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) if (anyvalid) pmap_invalidate_all(pmap); +out: + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } @@ -1796,6 +1799,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) anychanged = 0; + vm_page_lock_queues(); PMAP_LOCK(pmap); for (; sva < eva; sva = pdnxt) { unsigned pdirindex; @@ -1859,6 +1863,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) } if (anychanged) pmap_invalidate_all(pmap); + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 4e869f0e37f1..f557390dfde7 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -1340,6 +1340,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) if (pmap->pm_stats.resident_count == 0) return; + vm_page_lock_queues(); oldpmap = pmap_install(pmap); /* @@ -1349,8 +1350,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) */ if (sva + PAGE_SIZE == eva) { pmap_remove_page(pmap, sva); - pmap_install(oldpmap); - return; + goto out; } if (pmap->pm_stats.resident_count < ((eva - sva) >> PAGE_SHIFT)) { @@ -1374,7 +1374,9 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) } } +out: pmap_install(oldpmap); + vm_page_unlock_queues(); } /* @@ -1452,13 +1454,13 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) if (prot & VM_PROT_WRITE) return; - oldpmap = pmap_install(pmap); - newprot = pte_prot(pmap, prot); if ((sva & PAGE_MASK) || (eva & PAGE_MASK)) panic("pmap_protect: unaligned addresses"); + vm_page_lock_queues(); + oldpmap = pmap_install(pmap); while (sva < eva) { /* * If page is invalid, skip this page @@ -1491,6 +1493,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) sva += PAGE_SIZE; } pmap_install(oldpmap); + vm_page_unlock_queues(); } /* diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c index 01ce77198879..d9ad9661cf74 100644 --- a/sys/powerpc/aim/mmu_oea.c +++ b/sys/powerpc/aim/mmu_oea.c @@ -1460,6 +1460,7 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) return; } + vm_page_lock_queues(); for (; sva < eva; sva += PAGE_SIZE) { pvo = pmap_pvo_find_va(pm, sva, &pteidx); if (pvo == NULL) @@ -1485,6 +1486,7 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) if (pt != NULL) pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); } + vm_page_unlock_queues(); } /* @@ -1547,12 +1549,14 @@ pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) struct pvo_entry *pvo; int pteidx; + vm_page_lock_queues(); for (; sva < eva; sva += PAGE_SIZE) { pvo = pmap_pvo_find_va(pm, sva, &pteidx); if (pvo != NULL) { pmap_pvo_remove(pvo, pteidx); } } + vm_page_unlock_queues(); } /* diff --git a/sys/powerpc/powerpc/mmu_oea.c b/sys/powerpc/powerpc/mmu_oea.c index 01ce77198879..d9ad9661cf74 100644 --- a/sys/powerpc/powerpc/mmu_oea.c +++ b/sys/powerpc/powerpc/mmu_oea.c @@ -1460,6 +1460,7 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) return; } + vm_page_lock_queues(); for (; sva < eva; sva += PAGE_SIZE) { pvo = pmap_pvo_find_va(pm, sva, &pteidx); if (pvo == NULL) @@ -1485,6 +1486,7 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) if (pt != NULL) pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); } + vm_page_unlock_queues(); } /* @@ -1547,12 +1549,14 @@ pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) struct pvo_entry *pvo; int pteidx; + vm_page_lock_queues(); for (; sva < eva; sva += PAGE_SIZE) { pvo = pmap_pvo_find_va(pm, sva, &pteidx); if (pvo != NULL) { pmap_pvo_remove(pvo, pteidx); } } + vm_page_unlock_queues(); } /* diff --git a/sys/powerpc/powerpc/pmap.c b/sys/powerpc/powerpc/pmap.c index 01ce77198879..d9ad9661cf74 100644 --- a/sys/powerpc/powerpc/pmap.c +++ b/sys/powerpc/powerpc/pmap.c @@ -1460,6 +1460,7 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) return; } + vm_page_lock_queues(); for (; sva < eva; sva += PAGE_SIZE) { pvo = pmap_pvo_find_va(pm, sva, &pteidx); if (pvo == NULL) @@ -1485,6 +1486,7 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) if (pt != NULL) pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); } + vm_page_unlock_queues(); } /* @@ -1547,12 +1549,14 @@ pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) struct pvo_entry *pvo; int pteidx; + vm_page_lock_queues(); for (; sva < eva; sva += PAGE_SIZE) { pvo = pmap_pvo_find_va(pm, sva, &pteidx); if (pvo != NULL) { pmap_pvo_remove(pvo, pteidx); } } + vm_page_unlock_queues(); } /* diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c index 432f523a45e7..60dfc8cb03ee 100644 --- a/sys/sparc64/sparc64/pmap.c +++ b/sys/sparc64/sparc64/pmap.c @@ -1125,11 +1125,11 @@ pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) struct tte *tp; vm_offset_t va; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", pm->pm_context[PCPU_GET(cpuid)], start, end); if (PMAP_REMOVE_DONE(pm)) return; + vm_page_lock_queues(); if (end - start > PMAP_TSB_THRESH) { tsb_foreach(pm, NULL, start, end, pmap_remove_tte); tlb_context_demap(pm); @@ -1142,6 +1142,7 @@ pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) } tlb_range_demap(pm, start, end - 1); } + vm_page_unlock_queues(); } void @@ -1214,6 +1215,7 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) if (prot & VM_PROT_WRITE) return; + vm_page_lock_queues(); if (eva - sva > PMAP_TSB_THRESH) { tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); tlb_context_demap(pm); @@ -1224,6 +1226,7 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) } tlb_range_demap(pm, sva, eva - 1); } + vm_page_unlock_queues(); } /* diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index 55726f5a7dc6..ba077963ef6c 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -1391,14 +1391,12 @@ vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end, */ if (current->protection != old_prot) { mtx_lock(&Giant); - vm_page_lock_queues(); #define MASK(entry) (((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \ VM_PROT_ALL) pmap_protect(map->pmap, current->start, current->end, current->protection & MASK(current)); #undef MASK - vm_page_unlock_queues(); mtx_unlock(&Giant); } vm_map_simplify_entry(map, current); @@ -2007,9 +2005,7 @@ vm_map_sync( if (invalidate) { mtx_lock(&Giant); - vm_page_lock_queues(); pmap_remove(map->pmap, start, end); - vm_page_unlock_queues(); mtx_unlock(&Giant); } /* @@ -2182,9 +2178,7 @@ vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) if (!map->system_map) mtx_lock(&Giant); - vm_page_lock_queues(); pmap_remove(map->pmap, entry->start, entry->end); - vm_page_unlock_queues(); if (!map->system_map) mtx_unlock(&Giant); @@ -2295,12 +2289,10 @@ vm_map_copy_entry( * write-protected. */ if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) { - vm_page_lock_queues(); pmap_protect(src_map->pmap, src_entry->start, src_entry->end, src_entry->protection & ~VM_PROT_WRITE); - vm_page_unlock_queues(); } /* diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index bb448a9150cf..f1e6ffb8211e 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -609,10 +609,8 @@ vm_pageout_map_deactivate_pages(map, desired) */ if (desired == 0 && nothingwired) { GIANT_REQUIRED; - vm_page_lock_queues(); pmap_remove(vm_map_pmap(map), vm_map_min(map), vm_map_max(map)); - vm_page_unlock_queues(); } vm_map_unlock(map); } -- cgit v1.2.3