diff options
author | Jason Evans <jasone@FreeBSD.org> | 2016-12-04 21:13:26 +0000 |
---|---|---|
committer | Jason Evans <jasone@FreeBSD.org> | 2016-12-04 21:13:26 +0000 |
commit | 7fa7f12ff8259b667d0a4d272a575a2d547496d6 (patch) | |
tree | 47a59fdd79369c5c55a5a98e36f7de8c18628c20 /contrib/jemalloc | |
parent | 435da9856468d7d43d6cedb1bb4e5bf7289516e2 (diff) | |
download | src-7fa7f12ff8259b667d0a4d272a575a2d547496d6.tar.gz src-7fa7f12ff8259b667d0a4d272a575a2d547496d6.zip |
Update jemalloc to 4.4.0.
Notes
Notes:
svn path=/head/; revision=309540
Diffstat (limited to 'contrib/jemalloc')
26 files changed, 542 insertions, 263 deletions
diff --git a/contrib/jemalloc/ChangeLog b/contrib/jemalloc/ChangeLog index 587685d02971..f75edd933ad3 100644 --- a/contrib/jemalloc/ChangeLog +++ b/contrib/jemalloc/ChangeLog @@ -4,6 +4,33 @@ brevity. Much more detail can be found in the git revision history: https://github.com/jemalloc/jemalloc +* 4.4.0 (December 3, 2016) + + New features: + - Add configure support for *-*-linux-android. (@cferris1000, @jasone) + - Add the --disable-syscall configure option, for use on systems that place + security-motivated limitations on syscall(2). (@jasone) + - Add support for Debian GNU/kFreeBSD. (@thesam) + + Optimizations: + - Add extent serial numbers and use them where appropriate as a sort key that + is higher priority than address, so that the allocation policy prefers older + extents. This tends to improve locality (decrease fragmentation) when + memory grows downward. (@jasone) + - Refactor madvise(2) configuration so that MADV_FREE is detected and utilized + on Linux 4.5 and newer. (@jasone) + - Mark partially purged arena chunks as non-huge-page. This improves + interaction with Linux's transparent huge page functionality. (@jasone) + + Bug fixes: + - Fix size class computations for edge conditions involving extremely large + allocations. This regression was first released in 4.0.0. (@jasone, + @ingvarha) + - Remove overly restrictive assertions related to the cactive statistic. This + regression was first released in 4.1.0. (@jasone) + - Implement a more reliable detection scheme for os_unfair_lock on macOS. + (@jszakmeister) + * 4.3.1 (November 7, 2016) Bug fixes: diff --git a/contrib/jemalloc/FREEBSD-diffs b/contrib/jemalloc/FREEBSD-diffs index 4b2a527b279d..fce089d83d65 100644 --- a/contrib/jemalloc/FREEBSD-diffs +++ b/contrib/jemalloc/FREEBSD-diffs @@ -1,5 +1,5 @@ diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in -index 3d2e721..b361db2 100644 +index d9c8345..9898c3c 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -53,11 +53,23 @@ @@ -47,10 +47,10 @@ index 3d2e721..b361db2 100644 + </refsect1> </refentry> diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h -index f39ce54..a3ba55d 100644 +index ce4e602..35360b6 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h -@@ -719,8 +719,13 @@ arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind) +@@ -730,8 +730,13 @@ arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind) JEMALLOC_ALWAYS_INLINE const arena_chunk_map_misc_t * arena_miscelm_get_const(const arena_chunk_t *chunk, size_t pageind) { @@ -64,7 +64,7 @@ index f39ce54..a3ba55d 100644 } JEMALLOC_ALWAYS_INLINE size_t -@@ -779,8 +784,13 @@ arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind) +@@ -790,8 +795,13 @@ arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind) JEMALLOC_ALWAYS_INLINE const size_t * arena_mapbitsp_get_const(const arena_chunk_t *chunk, size_t pageind) { @@ -79,7 +79,7 @@ index f39ce54..a3ba55d 100644 JEMALLOC_ALWAYS_INLINE size_t diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in -index fdc8fef..56a35a4 100644 +index e7ace7d..d86c61d 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -8,6 +8,9 @@ @@ -144,10 +144,10 @@ index b442d2d..76518db 100644 #endif /* JEMALLOC_H_EXTERNS */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt -index 87c8c9b..df576f6 100644 +index c1c6c40..c6395fd 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt -@@ -307,7 +307,6 @@ iralloct_realign +@@ -310,7 +310,6 @@ iralloct_realign isalloc isdalloct isqalloc @@ -335,7 +335,7 @@ index f943891..47d032c 100755 +#include "jemalloc_FreeBSD.h" EOF diff --git a/src/jemalloc.c b/src/jemalloc.c -index 38650ff..f659b55 100644 +index baead66..8a49f26 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -4,6 +4,10 @@ @@ -349,7 +349,7 @@ index 38650ff..f659b55 100644 /* Runtime configuration options. */ const char *je_malloc_conf #ifndef _WIN32 -@@ -2756,6 +2760,107 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) +@@ -2775,6 +2779,107 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) */ /******************************************************************************/ /* @@ -457,7 +457,7 @@ index 38650ff..f659b55 100644 * The following functions are used by threading libraries for protection of * malloc during fork(). */ -@@ -2894,4 +2999,11 @@ jemalloc_postfork_child(void) +@@ -2913,4 +3018,11 @@ jemalloc_postfork_child(void) ctl_postfork_child(tsd_tsdn(tsd)); } @@ -516,7 +516,7 @@ index 6333e73..13f8d79 100644 +#endif +} diff --git a/src/util.c b/src/util.c -index 7905267..bee1c77 100644 +index dd8c236..a4ff287 100755 --- a/src/util.c +++ b/src/util.c @@ -67,6 +67,22 @@ wrtmessage(void *cbopaque, const char *s) diff --git a/contrib/jemalloc/VERSION b/contrib/jemalloc/VERSION index c108ecb84aad..810bd6d4c9a7 100644 --- a/contrib/jemalloc/VERSION +++ b/contrib/jemalloc/VERSION @@ -1 +1 @@ -4.3.1-0-g0110fa8451af905affd77c3bea0d545fee2251b2 +4.4.0-0-gf1f76357313e7dcad7262f17a48ff0a2e005fcdc diff --git a/contrib/jemalloc/doc/jemalloc.3 b/contrib/jemalloc/doc/jemalloc.3 index 5787e6ea446b..cd3f197172ef 100644 --- a/contrib/jemalloc/doc/jemalloc.3 +++ b/contrib/jemalloc/doc/jemalloc.3 @@ -2,12 +2,12 @@ .\" Title: JEMALLOC .\" Author: Jason Evans .\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/> -.\" Date: 11/08/2016 +.\" Date: 12/04/2016 .\" Manual: User Manual -.\" Source: jemalloc 4.3.1-0-g0110fa8451af905affd77c3bea0d545fee2251b2 +.\" Source: jemalloc 4.4.0-0-gf1f76357313e7dcad7262f17a48ff0a2e005fcdc .\" Language: English .\" -.TH "JEMALLOC" "3" "11/08/2016" "jemalloc 4.3.1-0-g0110fa8451af" "User Manual" +.TH "JEMALLOC" "3" "12/04/2016" "jemalloc 4.4.0-0-gf1f76357313e" "User Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- @@ -31,7 +31,7 @@ jemalloc \- general purpose memory allocation functions .SH "LIBRARY" .PP -This manual describes jemalloc 4\&.3\&.1\-0\-g0110fa8451af905affd77c3bea0d545fee2251b2\&. More information can be found at the +This manual describes jemalloc 4\&.4\&.0\-0\-gf1f76357313e7dcad7262f17a48ff0a2e005fcdc\&. More information can be found at the \m[blue]\fBjemalloc website\fR\m[]\&\s-2\u[1]\d\s+2\&. .PP The following configuration options are enabled in libc\*(Aqs built\-in jemalloc: @@ -365,7 +365,7 @@ for (i = 0; i < nbins; i++) { mib[2] = i; len = sizeof(bin_size); - mallctlbymib(mib, miblen, &bin_size, &len, NULL, 0); + mallctlbymib(mib, miblen, (void *)&bin_size, &len, NULL, 0); /* Do something with bin_size\&.\&.\&. */ } .fi diff --git a/contrib/jemalloc/include/jemalloc/internal/arena.h b/contrib/jemalloc/include/jemalloc/internal/arena.h index a3ba55d88552..35360b64ad11 100644 --- a/contrib/jemalloc/include/jemalloc/internal/arena.h +++ b/contrib/jemalloc/include/jemalloc/internal/arena.h @@ -191,6 +191,14 @@ struct arena_chunk_s { extent_node_t node; /* + * True if memory could be backed by transparent huge pages. This is + * only directly relevant to Linux, since it is the only supported + * platform on which jemalloc interacts with explicit transparent huge + * page controls. + */ + bool hugepage; + + /* * Map of pages within chunk that keeps track of free/large/small. The * first map_bias entries are omitted, since the chunk header does not * need to be tracked in the map. This omission saves a header page @@ -374,10 +382,12 @@ struct arena_s { dss_prec_t dss_prec; - /* Extant arena chunks. */ ql_head(extent_node_t) achunks; + /* Extent serial number generator state. */ + size_t extent_sn_next; + /* * In order to avoid rapid chunk allocation/deallocation when an arena * oscillates right on the cusp of needing a new chunk, cache the most @@ -453,9 +463,9 @@ struct arena_s { * orderings are needed, which is why there are two trees with the same * contents. */ - extent_tree_t chunks_szad_cached; + extent_tree_t chunks_szsnad_cached; extent_tree_t chunks_ad_cached; - extent_tree_t chunks_szad_retained; + extent_tree_t chunks_szsnad_retained; extent_tree_t chunks_ad_retained; malloc_mutex_t chunks_mtx; @@ -522,13 +532,13 @@ void arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, extent_node_t *arena_node_alloc(tsdn_t *tsdn, arena_t *arena); void arena_node_dalloc(tsdn_t *tsdn, arena_t *arena, extent_node_t *node); void *arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, - size_t alignment, bool *zero); + size_t alignment, size_t *sn, bool *zero); void arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, - size_t usize); + size_t usize, size_t sn); void arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t oldsize, size_t usize); void arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, - void *chunk, size_t oldsize, size_t usize); + void *chunk, size_t oldsize, size_t usize, size_t sn); bool arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t oldsize, size_t usize, bool *zero); ssize_t arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena); @@ -601,6 +611,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, unsigned arena_nthreads_get(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); +size_t arena_extent_sn_next(arena_t *arena); arena_t *arena_new(tsdn_t *tsdn, unsigned ind); void arena_boot(void); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); diff --git a/contrib/jemalloc/include/jemalloc/internal/chunk.h b/contrib/jemalloc/include/jemalloc/internal/chunk.h index 38c9a012da11..50b9904b04ec 100644 --- a/contrib/jemalloc/include/jemalloc/internal/chunk.h +++ b/contrib/jemalloc/include/jemalloc/internal/chunk.h @@ -58,15 +58,16 @@ void chunk_deregister(const void *chunk, const extent_node_t *node); void *chunk_alloc_base(size_t size); void *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, - bool *zero, bool *commit, bool dalloc_node); + size_t *sn, bool *zero, bool *commit, bool dalloc_node); void *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, - bool *zero, bool *commit); + size_t *sn, bool *zero, bool *commit); void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool committed); -void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool zeroed, + chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t sn, bool committed); +void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, + chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t sn, + bool zeroed, bool committed); bool chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t offset, size_t length); diff --git a/contrib/jemalloc/include/jemalloc/internal/extent.h b/contrib/jemalloc/include/jemalloc/internal/extent.h index 49d76a57f503..168ffe64389d 100644 --- a/contrib/jemalloc/include/jemalloc/internal/extent.h +++ b/contrib/jemalloc/include/jemalloc/internal/extent.h @@ -19,6 +19,20 @@ struct extent_node_s { size_t en_size; /* + * Serial number (potentially non-unique). + * + * In principle serial numbers can wrap around on 32-bit systems if + * JEMALLOC_MUNMAP is defined, but as long as comparison functions fall + * back on address comparison for equal serial numbers, stable (if + * imperfect) ordering is maintained. + * + * Serial numbers may not be unique even in the absence of wrap-around, + * e.g. when splitting an extent and assigning the same serial number to + * both resulting adjacent extents. + */ + size_t en_sn; + + /* * The zeroed flag is used by chunk recycling code to track whether * memory is zero-filled. */ @@ -45,8 +59,8 @@ struct extent_node_s { qr(extent_node_t) cc_link; union { - /* Linkage for the size/address-ordered tree. */ - rb_node(extent_node_t) szad_link; + /* Linkage for the size/sn/address-ordered tree. */ + rb_node(extent_node_t) szsnad_link; /* Linkage for arena's achunks, huge, and node_cache lists. */ ql_elm(extent_node_t) ql_link; @@ -61,7 +75,7 @@ typedef rb_tree(extent_node_t) extent_tree_t; /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t) +rb_proto(, extent_tree_szsnad_, extent_tree_t, extent_node_t) rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) @@ -73,6 +87,7 @@ rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) arena_t *extent_node_arena_get(const extent_node_t *node); void *extent_node_addr_get(const extent_node_t *node); size_t extent_node_size_get(const extent_node_t *node); +size_t extent_node_sn_get(const extent_node_t *node); bool extent_node_zeroed_get(const extent_node_t *node); bool extent_node_committed_get(const extent_node_t *node); bool extent_node_achunk_get(const extent_node_t *node); @@ -80,12 +95,13 @@ prof_tctx_t *extent_node_prof_tctx_get(const extent_node_t *node); void extent_node_arena_set(extent_node_t *node, arena_t *arena); void extent_node_addr_set(extent_node_t *node, void *addr); void extent_node_size_set(extent_node_t *node, size_t size); +void extent_node_sn_set(extent_node_t *node, size_t sn); void extent_node_zeroed_set(extent_node_t *node, bool zeroed); void extent_node_committed_set(extent_node_t *node, bool committed); void extent_node_achunk_set(extent_node_t *node, bool achunk); void extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx); void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, - size_t size, bool zeroed, bool committed); + size_t size, size_t sn, bool zeroed, bool committed); void extent_node_dirty_linkage_init(extent_node_t *node); void extent_node_dirty_insert(extent_node_t *node, arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty); @@ -114,6 +130,13 @@ extent_node_size_get(const extent_node_t *node) return (node->en_size); } +JEMALLOC_INLINE size_t +extent_node_sn_get(const extent_node_t *node) +{ + + return (node->en_sn); +} + JEMALLOC_INLINE bool extent_node_zeroed_get(const extent_node_t *node) { @@ -165,6 +188,13 @@ extent_node_size_set(extent_node_t *node, size_t size) } JEMALLOC_INLINE void +extent_node_sn_set(extent_node_t *node, size_t sn) +{ + + node->en_sn = sn; +} + +JEMALLOC_INLINE void extent_node_zeroed_set(extent_node_t *node, bool zeroed) { @@ -194,12 +224,13 @@ extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx) JEMALLOC_INLINE void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size, - bool zeroed, bool committed) + size_t sn, bool zeroed, bool committed) { extent_node_arena_set(node, arena); extent_node_addr_set(node, addr); extent_node_size_set(node, size); + extent_node_sn_set(node, sn); extent_node_zeroed_set(node, zeroed); extent_node_committed_set(node, committed); extent_node_achunk_set(node, false); diff --git a/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal.h b/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal.h index 15483e271dd6..bf489970ecdf 100644 --- a/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal.h +++ b/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal.h @@ -334,7 +334,7 @@ typedef unsigned szind_t; /* Return the nearest aligned address at or below a. */ #define ALIGNMENT_ADDR2BASE(a, alignment) \ - ((void *)((uintptr_t)(a) & (-(alignment)))) + ((void *)((uintptr_t)(a) & ((~(alignment)) + 1))) /* Return the offset between a and the nearest aligned address at or below a. */ #define ALIGNMENT_ADDR2OFFSET(a, alignment) \ @@ -342,7 +342,7 @@ typedef unsigned szind_t; /* Return the smallest alignment multiple that is >= s. */ #define ALIGNMENT_CEILING(s, alignment) \ - (((s) + (alignment - 1)) & (-(alignment))) + (((s) + (alignment - 1)) & ((~(alignment)) + 1)) /* Declare a variable-length array. */ #if __STDC_VERSION__ < 199901L diff --git a/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h b/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h index 7636ac49d745..79ce117f0d9a 100644 --- a/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h +++ b/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h @@ -57,11 +57,6 @@ #define JEMALLOC_HAVE_BUILTIN_CLZ /* - * Defined if madvise(2) is available. - */ -#define JEMALLOC_HAVE_MADVISE - -/* * Defined if os_unfair_lock_*() functions are available, as provided by Darwin. */ /* #undef JEMALLOC_OS_UNFAIR_LOCK */ @@ -72,8 +67,8 @@ */ /* #undef JEMALLOC_OSSPIN */ -/* Defined if syscall(2) is available. */ -#define JEMALLOC_HAVE_SYSCALL +/* Defined if syscall(2) is usable. */ +#define JEMALLOC_USE_SYSCALL /* * Defined if secure_getenv(3) is available. @@ -85,6 +80,9 @@ */ #define JEMALLOC_HAVE_ISSETUGID +/* Defined if pthread_atfork(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_ATFORK + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ @@ -253,18 +251,26 @@ #define JEMALLOC_SYSCTL_VM_OVERCOMMIT /* #undef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY */ +/* Defined if madvise(2) is available. */ +#define JEMALLOC_HAVE_MADVISE + /* * Methods for purging unused pages differ between operating systems. * - * madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages, - * such that new pages will be demand-zeroed if - * the address region is later touched. - * madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being - * unused, such that they will be discarded rather - * than swapped out. - */ -/* #undef JEMALLOC_PURGE_MADVISE_DONTNEED */ + * madvise(..., MADV_FREE) : This marks pages as being unused, such that they + * will be discarded rather than swapped out. + * madvise(..., MADV_DONTNEED) : This immediately discards pages, such that + * new pages will be demand-zeroed if the + * address region is later touched. + */ #define JEMALLOC_PURGE_MADVISE_FREE +#define JEMALLOC_PURGE_MADVISE_DONTNEED + +/* + * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE + * arguments to madvise(2). + */ +/* #undef JEMALLOC_THP */ /* Define if operating system has alloca.h header. */ /* #undef JEMALLOC_HAS_ALLOCA_H */ diff --git a/contrib/jemalloc/include/jemalloc/internal/pages.h b/contrib/jemalloc/include/jemalloc/internal/pages.h index e21effd14f77..4ae9f156a7b3 100644 --- a/contrib/jemalloc/include/jemalloc/internal/pages.h +++ b/contrib/jemalloc/include/jemalloc/internal/pages.h @@ -16,6 +16,8 @@ void *pages_trim(void *addr, size_t alloc_size, size_t leadsize, bool pages_commit(void *addr, size_t size); bool pages_decommit(void *addr, size_t size); bool pages_purge(void *addr, size_t size); +bool pages_huge(void *addr, size_t size); +bool pages_nohuge(void *addr, size_t size); void pages_boot(void); #endif /* JEMALLOC_H_EXTERNS */ diff --git a/contrib/jemalloc/include/jemalloc/internal/private_namespace.h b/contrib/jemalloc/include/jemalloc/internal/private_namespace.h index 350835cbe641..d68d99d8df39 100644 --- a/contrib/jemalloc/include/jemalloc/internal/private_namespace.h +++ b/contrib/jemalloc/include/jemalloc/internal/private_namespace.h @@ -36,6 +36,7 @@ #define arena_decay_time_set JEMALLOC_N(arena_decay_time_set) #define arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get) #define arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set) +#define arena_extent_sn_next JEMALLOC_N(arena_extent_sn_next) #define arena_get JEMALLOC_N(arena_get) #define arena_ichoose JEMALLOC_N(arena_ichoose) #define arena_init JEMALLOC_N(arena_init) @@ -218,6 +219,8 @@ #define extent_node_prof_tctx_set JEMALLOC_N(extent_node_prof_tctx_set) #define extent_node_size_get JEMALLOC_N(extent_node_size_get) #define extent_node_size_set JEMALLOC_N(extent_node_size_set) +#define extent_node_sn_get JEMALLOC_N(extent_node_sn_get) +#define extent_node_sn_set JEMALLOC_N(extent_node_sn_set) #define extent_node_zeroed_get JEMALLOC_N(extent_node_zeroed_get) #define extent_node_zeroed_set JEMALLOC_N(extent_node_zeroed_set) #define extent_tree_ad_destroy JEMALLOC_N(extent_tree_ad_destroy) @@ -239,25 +242,25 @@ #define extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse) #define extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start) #define extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search) -#define extent_tree_szad_destroy JEMALLOC_N(extent_tree_szad_destroy) -#define extent_tree_szad_destroy_recurse JEMALLOC_N(extent_tree_szad_destroy_recurse) -#define extent_tree_szad_empty JEMALLOC_N(extent_tree_szad_empty) -#define extent_tree_szad_first JEMALLOC_N(extent_tree_szad_first) -#define extent_tree_szad_insert JEMALLOC_N(extent_tree_szad_insert) -#define extent_tree_szad_iter JEMALLOC_N(extent_tree_szad_iter) -#define extent_tree_szad_iter_recurse JEMALLOC_N(extent_tree_szad_iter_recurse) -#define extent_tree_szad_iter_start JEMALLOC_N(extent_tree_szad_iter_start) -#define extent_tree_szad_last JEMALLOC_N(extent_tree_szad_last) -#define extent_tree_szad_new JEMALLOC_N(extent_tree_szad_new) -#define extent_tree_szad_next JEMALLOC_N(extent_tree_szad_next) -#define extent_tree_szad_nsearch JEMALLOC_N(extent_tree_szad_nsearch) -#define extent_tree_szad_prev JEMALLOC_N(extent_tree_szad_prev) -#define extent_tree_szad_psearch JEMALLOC_N(extent_tree_szad_psearch) -#define extent_tree_szad_remove JEMALLOC_N(extent_tree_szad_remove) -#define extent_tree_szad_reverse_iter JEMALLOC_N(extent_tree_szad_reverse_iter) -#define extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse) -#define extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start) -#define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search) +#define extent_tree_szsnad_destroy JEMALLOC_N(extent_tree_szsnad_destroy) +#define extent_tree_szsnad_destroy_recurse JEMALLOC_N(extent_tree_szsnad_destroy_recurse) +#define extent_tree_szsnad_empty JEMALLOC_N(extent_tree_szsnad_empty) +#define extent_tree_szsnad_first JEMALLOC_N(extent_tree_szsnad_first) +#define extent_tree_szsnad_insert JEMALLOC_N(extent_tree_szsnad_insert) +#define extent_tree_szsnad_iter JEMALLOC_N(extent_tree_szsnad_iter) +#define extent_tree_szsnad_iter_recurse JEMALLOC_N(extent_tree_szsnad_iter_recurse) +#define extent_tree_szsnad_iter_start JEMALLOC_N(extent_tree_szsnad_iter_start) +#define extent_tree_szsnad_last JEMALLOC_N(extent_tree_szsnad_last) +#define extent_tree_szsnad_new JEMALLOC_N(extent_tree_szsnad_new) +#define extent_tree_szsnad_next JEMALLOC_N(extent_tree_szsnad_next) +#define extent_tree_szsnad_nsearch JEMALLOC_N(extent_tree_szsnad_nsearch) +#define extent_tree_szsnad_prev JEMALLOC_N(extent_tree_szsnad_prev) +#define extent_tree_szsnad_psearch JEMALLOC_N(extent_tree_szsnad_psearch) +#define extent_tree_szsnad_remove JEMALLOC_N(extent_tree_szsnad_remove) +#define extent_tree_szsnad_reverse_iter JEMALLOC_N(extent_tree_szsnad_reverse_iter) +#define extent_tree_szsnad_reverse_iter_recurse JEMALLOC_N(extent_tree_szsnad_reverse_iter_recurse) +#define extent_tree_szsnad_reverse_iter_start JEMALLOC_N(extent_tree_szsnad_reverse_iter_start) +#define extent_tree_szsnad_search JEMALLOC_N(extent_tree_szsnad_search) #define ffs_llu JEMALLOC_N(ffs_llu) #define ffs_lu JEMALLOC_N(ffs_lu) #define ffs_u JEMALLOC_N(ffs_u) @@ -393,7 +396,9 @@ #define pages_boot JEMALLOC_N(pages_boot) #define pages_commit JEMALLOC_N(pages_commit) #define pages_decommit JEMALLOC_N(pages_decommit) +#define pages_huge JEMALLOC_N(pages_huge) #define pages_map JEMALLOC_N(pages_map) +#define pages_nohuge JEMALLOC_N(pages_nohuge) #define pages_purge JEMALLOC_N(pages_purge) #define pages_trim JEMALLOC_N(pages_trim) #define pages_unmap JEMALLOC_N(pages_unmap) diff --git a/contrib/jemalloc/include/jemalloc/internal/stats.h b/contrib/jemalloc/include/jemalloc/internal/stats.h index b62181783eab..04e7dae14c7e 100644 --- a/contrib/jemalloc/include/jemalloc/internal/stats.h +++ b/contrib/jemalloc/include/jemalloc/internal/stats.h @@ -175,25 +175,21 @@ stats_cactive_get(void) JEMALLOC_INLINE void stats_cactive_add(size_t size) { - UNUSED size_t cactive; assert(size > 0); assert((size & chunksize_mask) == 0); - cactive = atomic_add_z(&stats_cactive, size); - assert(cactive - size < cactive); + atomic_add_z(&stats_cactive, size); } JEMALLOC_INLINE void stats_cactive_sub(size_t size) { - UNUSED size_t cactive; assert(size > 0); assert((size & chunksize_mask) == 0); - cactive = atomic_sub_z(&stats_cactive, size); - assert(cactive + size > cactive); + atomic_sub_z(&stats_cactive, size); } #endif diff --git a/contrib/jemalloc/include/jemalloc/internal/util.h b/contrib/jemalloc/include/jemalloc/internal/util.h index aee00d6d9a47..4b56d652ed31 100644 --- a/contrib/jemalloc/include/jemalloc/internal/util.h +++ b/contrib/jemalloc/include/jemalloc/internal/util.h @@ -41,8 +41,12 @@ #define MALLOC_PRINTF_BUFSIZE 4096 /* Junk fill patterns. */ -#define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) -#define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) +#ifndef JEMALLOC_ALLOC_JUNK +# define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) +#endif +#ifndef JEMALLOC_FREE_JUNK +# define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) +#endif /* * Wrap a cpp argument that contains commas such that it isn't broken up into diff --git a/contrib/jemalloc/include/jemalloc/internal/valgrind.h b/contrib/jemalloc/include/jemalloc/internal/valgrind.h index 1a8680828acf..877a142b62d9 100644 --- a/contrib/jemalloc/include/jemalloc/internal/valgrind.h +++ b/contrib/jemalloc/include/jemalloc/internal/valgrind.h @@ -36,13 +36,25 @@ zero); \ } \ } while (0) -#define JEMALLOC_VALGRIND_REALLOC(maybe_moved, tsdn, ptr, usize, \ - ptr_maybe_null, old_ptr, old_usize, old_rzsize, old_ptr_maybe_null, \ - zero) do { \ +#define JEMALLOC_VALGRIND_REALLOC_MOVED_no(ptr, old_ptr) \ + (false) +#define JEMALLOC_VALGRIND_REALLOC_MOVED_maybe(ptr, old_ptr) \ + ((ptr) != (old_ptr)) +#define JEMALLOC_VALGRIND_REALLOC_PTR_NULL_no(ptr) \ + (false) +#define JEMALLOC_VALGRIND_REALLOC_PTR_NULL_maybe(ptr) \ + (ptr == NULL) +#define JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_no(old_ptr) \ + (false) +#define JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_maybe(old_ptr) \ + (old_ptr == NULL) +#define JEMALLOC_VALGRIND_REALLOC(moved, tsdn, ptr, usize, ptr_null, \ + old_ptr, old_usize, old_rzsize, old_ptr_null, zero) do { \ if (unlikely(in_valgrind)) { \ size_t rzsize = p2rz(tsdn, ptr); \ \ - if (!maybe_moved || ptr == old_ptr) { \ + if (!JEMALLOC_VALGRIND_REALLOC_MOVED_##moved(ptr, \ + old_ptr)) { \ VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \ usize, rzsize); \ if (zero && old_usize < usize) { \ @@ -51,11 +63,13 @@ old_usize), usize - old_usize); \ } \ } else { \ - if (!old_ptr_maybe_null || old_ptr != NULL) { \ + if (!JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_## \ + old_ptr_null(old_ptr)) { \ valgrind_freelike_block(old_ptr, \ old_rzsize); \ } \ - if (!ptr_maybe_null || ptr != NULL) { \ + if (!JEMALLOC_VALGRIND_REALLOC_PTR_NULL_## \ + ptr_null(ptr)) { \ size_t copy_size = (old_usize < usize) \ ? old_usize : usize; \ size_t tail_size = usize - copy_size; \ diff --git a/contrib/jemalloc/include/jemalloc/jemalloc.h b/contrib/jemalloc/include/jemalloc/jemalloc.h index a8b2a6c33998..d3b175dd0a24 100644 --- a/contrib/jemalloc/include/jemalloc/jemalloc.h +++ b/contrib/jemalloc/include/jemalloc/jemalloc.h @@ -87,12 +87,12 @@ extern "C" { #include <limits.h> #include <strings.h> -#define JEMALLOC_VERSION "4.3.1-0-g0110fa8451af905affd77c3bea0d545fee2251b2" +#define JEMALLOC_VERSION "4.4.0-0-gf1f76357313e7dcad7262f17a48ff0a2e005fcdc" #define JEMALLOC_VERSION_MAJOR 4 -#define JEMALLOC_VERSION_MINOR 3 -#define JEMALLOC_VERSION_BUGFIX 1 +#define JEMALLOC_VERSION_MINOR 4 +#define JEMALLOC_VERSION_BUGFIX 0 #define JEMALLOC_VERSION_NREV 0 -#define JEMALLOC_VERSION_GID "0110fa8451af905affd77c3bea0d545fee2251b2" +#define JEMALLOC_VERSION_GID "f1f76357313e7dcad7262f17a48ff0a2e005fcdc" # define MALLOCX_LG_ALIGN(la) ((int)(la)) # if LG_SIZEOF_PTR == 2 diff --git a/contrib/jemalloc/src/arena.c b/contrib/jemalloc/src/arena.c index e196b13378e6..648a8da3ab4b 100644 --- a/contrib/jemalloc/src/arena.c +++ b/contrib/jemalloc/src/arena.c @@ -38,8 +38,8 @@ static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, bool dirty, bool cleaned, bool decommitted); static void arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin); -static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, - arena_run_t *run, arena_bin_t *bin); +static void arena_bin_lower_run(arena_t *arena, arena_run_t *run, + arena_bin_t *bin); /******************************************************************************/ @@ -55,8 +55,31 @@ arena_miscelm_size_get(const arena_chunk_map_misc_t *miscelm) return (arena_mapbits_size_decode(mapbits)); } +JEMALLOC_INLINE_C const extent_node_t * +arena_miscelm_extent_get(const arena_chunk_map_misc_t *miscelm) +{ + arena_chunk_t *chunk; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); + return (&chunk->node); +} + +JEMALLOC_INLINE_C int +arena_sn_comp(const arena_chunk_map_misc_t *a, const arena_chunk_map_misc_t *b) +{ + size_t a_sn, b_sn; + + assert(a != NULL); + assert(b != NULL); + + a_sn = extent_node_sn_get(arena_miscelm_extent_get(a)); + b_sn = extent_node_sn_get(arena_miscelm_extent_get(b)); + + return ((a_sn > b_sn) - (a_sn < b_sn)); +} + JEMALLOC_INLINE_C int -arena_run_addr_comp(const arena_chunk_map_misc_t *a, +arena_ad_comp(const arena_chunk_map_misc_t *a, const arena_chunk_map_misc_t *b) { uintptr_t a_miscelm = (uintptr_t)a; @@ -68,9 +91,26 @@ arena_run_addr_comp(const arena_chunk_map_misc_t *a, return ((a_miscelm > b_miscelm) - (a_miscelm < b_miscelm)); } +JEMALLOC_INLINE_C int +arena_snad_comp(const arena_chunk_map_misc_t *a, + const arena_chunk_map_misc_t *b) +{ + int ret; + + assert(a != NULL); + assert(b != NULL); + + ret = arena_sn_comp(a, b); + if (ret != 0) + return (ret); + + ret = arena_ad_comp(a, b); + return (ret); +} + /* Generate pairing heap functions. */ ph_gen(static UNUSED, arena_run_heap_, arena_run_heap_t, arena_chunk_map_misc_t, - ph_link, arena_run_addr_comp) + ph_link, arena_snad_comp) #ifdef JEMALLOC_JET #undef run_quantize_floor @@ -529,7 +569,7 @@ arena_chunk_init_spare(arena_t *arena) static bool arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, - bool zero) + size_t sn, bool zero) { /* @@ -538,7 +578,7 @@ arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, * of runs is tracked individually, and upon chunk deallocation the * entire chunk is in a consistent commit state. */ - extent_node_init(&chunk->node, arena, chunk, chunksize, zero, true); + extent_node_init(&chunk->node, arena, chunk, chunksize, sn, zero, true); extent_node_achunk_set(&chunk->node, true); return (chunk_register(tsdn, chunk, &chunk->node)); } @@ -548,28 +588,30 @@ arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, bool *zero, bool *commit) { arena_chunk_t *chunk; + size_t sn; malloc_mutex_unlock(tsdn, &arena->lock); chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks, - NULL, chunksize, chunksize, zero, commit); + NULL, chunksize, chunksize, &sn, zero, commit); if (chunk != NULL && !*commit) { /* Commit header. */ if (chunk_hooks->commit(chunk, chunksize, 0, map_bias << LG_PAGE, arena->ind)) { chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, - (void *)chunk, chunksize, *zero, *commit); + (void *)chunk, chunksize, sn, *zero, *commit); chunk = NULL; } } - if (chunk != NULL && arena_chunk_register(tsdn, arena, chunk, *zero)) { + if (chunk != NULL && arena_chunk_register(tsdn, arena, chunk, sn, + *zero)) { if (!*commit) { /* Undo commit of header. */ chunk_hooks->decommit(chunk, chunksize, 0, map_bias << LG_PAGE, arena->ind); } chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, (void *)chunk, - chunksize, *zero, *commit); + chunksize, sn, *zero, *commit); chunk = NULL; } @@ -583,13 +625,14 @@ arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero, { arena_chunk_t *chunk; chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; + size_t sn; chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize, - chunksize, zero, commit, true); + chunksize, &sn, zero, commit, true); if (chunk != NULL) { - if (arena_chunk_register(tsdn, arena, chunk, *zero)) { + if (arena_chunk_register(tsdn, arena, chunk, sn, *zero)) { chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, - chunksize, true); + chunksize, sn, true); return (NULL); } } @@ -621,6 +664,8 @@ arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena) if (chunk == NULL) return (NULL); + chunk->hugepage = true; + /* * Initialize the map to contain one maximal free untouched run. Mark * the pages as zeroed if arena_chunk_alloc_internal() returned a zeroed @@ -684,11 +729,14 @@ arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena) static void arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) { + size_t sn, hugepage; bool committed; chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; chunk_deregister(chunk, &chunk->node); + sn = extent_node_sn_get(&chunk->node); + hugepage = chunk->hugepage; committed = (arena_mapbits_decommitted_get(chunk, map_bias) == 0); if (!committed) { /* @@ -701,9 +749,17 @@ arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) chunk_hooks.decommit(chunk, chunksize, 0, map_bias << LG_PAGE, arena->ind); } + if (!hugepage) { + /* + * Convert chunk back to the default state, so that all + * subsequent chunk allocations start out with chunks that can + * be backed by transparent huge pages. + */ + pages_huge(chunk, chunksize); + } chunk_dalloc_cache(tsdn, arena, &chunk_hooks, (void *)chunk, chunksize, - committed); + sn, committed); if (config_stats) { arena->stats.mapped -= chunksize; @@ -859,14 +915,14 @@ arena_node_dalloc(tsdn_t *tsdn, arena_t *arena, extent_node_t *node) static void * arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero, - size_t csize) + chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, size_t *sn, + bool *zero, size_t csize) { void *ret; bool commit = true; ret = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, csize, - alignment, zero, &commit); + alignment, sn, zero, &commit); if (ret == NULL) { /* Revert optimistic stats updates. */ malloc_mutex_lock(tsdn, &arena->lock); @@ -883,7 +939,7 @@ arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena, void * arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, - size_t alignment, bool *zero) + size_t alignment, size_t *sn, bool *zero) { void *ret; chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; @@ -900,18 +956,19 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, arena_nactive_add(arena, usize >> LG_PAGE); ret = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, csize, - alignment, zero, &commit, true); + alignment, sn, zero, &commit, true); malloc_mutex_unlock(tsdn, &arena->lock); if (ret == NULL) { ret = arena_chunk_alloc_huge_hard(tsdn, arena, &chunk_hooks, - usize, alignment, zero, csize); + usize, alignment, sn, zero, csize); } return (ret); } void -arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize) +arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize, + size_t sn) { chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; size_t csize; @@ -924,7 +981,7 @@ arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize) } arena_nactive_sub(arena, usize >> LG_PAGE); - chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, csize, true); + chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, csize, sn, true); malloc_mutex_unlock(tsdn, &arena->lock); } @@ -948,7 +1005,7 @@ arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, void *chunk, void arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk, - size_t oldsize, size_t usize) + size_t oldsize, size_t usize, size_t sn) { size_t udiff = oldsize - usize; size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); @@ -967,7 +1024,7 @@ arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk, CHUNK_CEILING(usize)); chunk_dalloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff, - true); + sn, true); } malloc_mutex_unlock(tsdn, &arena->lock); } @@ -975,13 +1032,13 @@ arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk, static bool arena_chunk_ralloc_huge_expand_hard(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk, size_t oldsize, size_t usize, - bool *zero, void *nchunk, size_t udiff, size_t cdiff) + size_t *sn, bool *zero, void *nchunk, size_t udiff, size_t cdiff) { bool err; bool commit = true; err = (chunk_alloc_wrapper(tsdn, arena, chunk_hooks, nchunk, cdiff, - chunksize, zero, &commit) == NULL); + chunksize, sn, zero, &commit) == NULL); if (err) { /* Revert optimistic stats updates. */ malloc_mutex_lock(tsdn, &arena->lock); @@ -995,7 +1052,7 @@ arena_chunk_ralloc_huge_expand_hard(tsdn_t *tsdn, arena_t *arena, } else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk, cdiff, true, arena->ind)) { chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, nchunk, cdiff, - *zero, true); + *sn, *zero, true); err = true; } return (err); @@ -1010,6 +1067,7 @@ arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk, void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)); size_t udiff = usize - oldsize; size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); + size_t sn; bool commit = true; malloc_mutex_lock(tsdn, &arena->lock); @@ -1022,16 +1080,16 @@ arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk, arena_nactive_add(arena, udiff >> LG_PAGE); err = (chunk_alloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff, - chunksize, zero, &commit, true) == NULL); + chunksize, &sn, zero, &commit, true) == NULL); malloc_mutex_unlock(tsdn, &arena->lock); if (err) { err = arena_chunk_ralloc_huge_expand_hard(tsdn, arena, - &chunk_hooks, chunk, oldsize, usize, zero, nchunk, udiff, - cdiff); + &chunk_hooks, chunk, oldsize, usize, &sn, zero, nchunk, + udiff, cdiff); } else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk, cdiff, true, arena->ind)) { chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, nchunk, cdiff, - *zero, true); + sn, *zero, true); err = true; } @@ -1519,6 +1577,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, if (rdelm == &chunkselm->rd) { extent_node_t *chunkselm_next; + size_t sn; bool zero, commit; UNUSED void *chunk; @@ -1536,8 +1595,8 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, commit = false; chunk = chunk_alloc_cache(tsdn, arena, chunk_hooks, extent_node_addr_get(chunkselm), - extent_node_size_get(chunkselm), chunksize, &zero, - &commit, false); + extent_node_size_get(chunkselm), chunksize, &sn, + &zero, &commit, false); assert(chunk == extent_node_addr_get(chunkselm)); assert(zero == extent_node_zeroed_get(chunkselm)); extent_node_dirty_insert(chunkselm, purge_runs_sentinel, @@ -1634,6 +1693,17 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, run_size = arena_mapbits_large_size_get(chunk, pageind); npages = run_size >> LG_PAGE; + /* + * If this is the first run purged within chunk, mark + * the chunk as non-huge. This will prevent all use of + * transparent huge pages for this chunk until the chunk + * as a whole is deallocated. + */ + if (chunk->hugepage) { + pages_nohuge(chunk, chunksize); + chunk->hugepage = false; + } + assert(pageind + npages <= chunk_npages); assert(!arena_mapbits_decommitted_get(chunk, pageind)); assert(!arena_mapbits_decommitted_get(chunk, @@ -1703,13 +1773,14 @@ arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, cc_link); void *addr = extent_node_addr_get(chunkselm); size_t size = extent_node_size_get(chunkselm); + size_t sn = extent_node_sn_get(chunkselm); bool zeroed = extent_node_zeroed_get(chunkselm); bool committed = extent_node_committed_get(chunkselm); extent_node_dirty_remove(chunkselm); arena_node_dalloc(tsdn, arena, chunkselm); chunkselm = chunkselm_next; chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, addr, - size, zeroed, committed); + size, sn, zeroed, committed); } else { arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); @@ -2315,7 +2386,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin) arena_dalloc_bin_run(tsdn, arena, chunk, run, bin); } else - arena_bin_lower_run(arena, chunk, run, bin); + arena_bin_lower_run(arena, run, bin); } return (ret); } @@ -2820,16 +2891,18 @@ arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, } static void -arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, - arena_bin_t *bin) +arena_bin_lower_run(arena_t *arena, arena_run_t *run, arena_bin_t *bin) { /* - * Make sure that if bin->runcur is non-NULL, it refers to the lowest - * non-full run. It is okay to NULL runcur out rather than proactively - * keeping it pointing at the lowest non-full run. + * Make sure that if bin->runcur is non-NULL, it refers to the + * oldest/lowest non-full run. It is okay to NULL runcur out rather + * than proactively keeping it pointing at the oldest/lowest non-full + * run. */ - if ((uintptr_t)run < (uintptr_t)bin->runcur) { + if (bin->runcur != NULL && + arena_snad_comp(arena_run_to_miscelm(bin->runcur), + arena_run_to_miscelm(run)) > 0) { /* Switch runcur. */ if (bin->runcur->nfree > 0) arena_bin_runs_insert(bin, bin->runcur); @@ -2865,7 +2938,7 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, arena_dissociate_bin_run(chunk, run, bin); arena_dalloc_bin_run(tsdn, arena, chunk, run, bin); } else if (run->nfree == 1 && run != bin->runcur) - arena_bin_lower_run(arena, chunk, run, bin); + arena_bin_lower_run(arena, run, bin); if (config_stats) { bin->stats.ndalloc++; @@ -3452,6 +3525,13 @@ arena_nthreads_dec(arena_t *arena, bool internal) atomic_sub_u(&arena->nthreads[internal], 1); } +size_t +arena_extent_sn_next(arena_t *arena) +{ + + return (atomic_add_z(&arena->extent_sn_next, 1) - 1); +} + arena_t * arena_new(tsdn_t *tsdn, unsigned ind) { @@ -3511,6 +3591,8 @@ arena_new(tsdn_t *tsdn, unsigned ind) ql_new(&arena->achunks); + arena->extent_sn_next = 0; + arena->spare = NULL; arena->lg_dirty_mult = arena_lg_dirty_mult_default_get(); @@ -3532,9 +3614,9 @@ arena_new(tsdn_t *tsdn, unsigned ind) WITNESS_RANK_ARENA_HUGE)) return (NULL); - extent_tree_szad_new(&arena->chunks_szad_cached); + extent_tree_szsnad_new(&arena->chunks_szsnad_cached); extent_tree_ad_new(&arena->chunks_ad_cached); - extent_tree_szad_new(&arena->chunks_szad_retained); + extent_tree_szsnad_new(&arena->chunks_szsnad_retained); extent_tree_ad_new(&arena->chunks_ad_retained); if (malloc_mutex_init(&arena->chunks_mtx, "arena_chunks", WITNESS_RANK_ARENA_CHUNKS)) diff --git a/contrib/jemalloc/src/base.c b/contrib/jemalloc/src/base.c index 81b0801fdefc..5681a3f36d40 100644 --- a/contrib/jemalloc/src/base.c +++ b/contrib/jemalloc/src/base.c @@ -5,7 +5,8 @@ /* Data. */ static malloc_mutex_t base_mtx; -static extent_tree_t base_avail_szad; +static size_t base_extent_sn_next; +static extent_tree_t base_avail_szsnad; static extent_node_t *base_nodes; static size_t base_allocated; static size_t base_resident; @@ -39,6 +40,14 @@ base_node_dalloc(tsdn_t *tsdn, extent_node_t *node) base_nodes = node; } +static void +base_extent_node_init(extent_node_t *node, void *addr, size_t size) +{ + size_t sn = atomic_add_z(&base_extent_sn_next, 1) - 1; + + extent_node_init(node, NULL, addr, size, sn, true, true); +} + static extent_node_t * base_chunk_alloc(tsdn_t *tsdn, size_t minsize) { @@ -68,7 +77,7 @@ base_chunk_alloc(tsdn_t *tsdn, size_t minsize) base_resident += PAGE_CEILING(nsize); } } - extent_node_init(node, NULL, addr, csize, true, true); + base_extent_node_init(node, addr, csize); return (node); } @@ -92,12 +101,12 @@ base_alloc(tsdn_t *tsdn, size_t size) csize = CACHELINE_CEILING(size); usize = s2u(csize); - extent_node_init(&key, NULL, NULL, usize, false, false); + extent_node_init(&key, NULL, NULL, usize, 0, false, false); malloc_mutex_lock(tsdn, &base_mtx); - node = extent_tree_szad_nsearch(&base_avail_szad, &key); + node = extent_tree_szsnad_nsearch(&base_avail_szsnad, &key); if (node != NULL) { /* Use existing space. */ - extent_tree_szad_remove(&base_avail_szad, node); + extent_tree_szsnad_remove(&base_avail_szsnad, node); } else { /* Try to allocate more space. */ node = base_chunk_alloc(tsdn, csize); @@ -111,7 +120,7 @@ base_alloc(tsdn_t *tsdn, size_t size) if (extent_node_size_get(node) > csize) { extent_node_addr_set(node, (void *)((uintptr_t)ret + csize)); extent_node_size_set(node, extent_node_size_get(node) - csize); - extent_tree_szad_insert(&base_avail_szad, node); + extent_tree_szsnad_insert(&base_avail_szsnad, node); } else base_node_dalloc(tsdn, node); if (config_stats) { @@ -149,7 +158,8 @@ base_boot(void) if (malloc_mutex_init(&base_mtx, "base", WITNESS_RANK_BASE)) return (true); - extent_tree_szad_new(&base_avail_szad); + base_extent_sn_next = 0; + extent_tree_szsnad_new(&base_avail_szsnad); base_nodes = NULL; return (false); diff --git a/contrib/jemalloc/src/chunk.c b/contrib/jemalloc/src/chunk.c index 07e26f77c9f3..c1c514a860fa 100644 --- a/contrib/jemalloc/src/chunk.c +++ b/contrib/jemalloc/src/chunk.c @@ -50,9 +50,9 @@ const chunk_hooks_t chunk_hooks_default = { */ static void chunk_record(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size, bool zeroed, - bool committed); + chunk_hooks_t *chunk_hooks, extent_tree_t *chunks_szsnad, + extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size, size_t sn, + bool zeroed, bool committed); /******************************************************************************/ @@ -183,33 +183,35 @@ chunk_deregister(const void *chunk, const extent_node_t *node) } /* - * Do first-best-fit chunk selection, i.e. select the lowest chunk that best - * fits. + * Do first-best-fit chunk selection, i.e. select the oldest/lowest chunk that + * best fits. */ static extent_node_t * -chunk_first_best_fit(arena_t *arena, extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, size_t size) +chunk_first_best_fit(arena_t *arena, extent_tree_t *chunks_szsnad, size_t size) { extent_node_t key; assert(size == CHUNK_CEILING(size)); - extent_node_init(&key, arena, NULL, size, false, false); - return (extent_tree_szad_nsearch(chunks_szad, &key)); + extent_node_init(&key, arena, NULL, size, 0, false, false); + return (extent_tree_szsnad_nsearch(chunks_szsnad, &key)); } static void * chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache, - void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, - bool dalloc_node) + extent_tree_t *chunks_szsnad, extent_tree_t *chunks_ad, bool cache, + void *new_addr, size_t size, size_t alignment, size_t *sn, bool *zero, + bool *commit, bool dalloc_node) { void *ret; extent_node_t *node; size_t alloc_size, leadsize, trailsize; bool zeroed, committed; + assert(CHUNK_CEILING(size) == size); + assert(alignment > 0); assert(new_addr == NULL || alignment == chunksize); + assert(CHUNK_ADDR2BASE(new_addr) == new_addr); /* * Cached chunks use the node linkage embedded in their headers, in * which case dalloc_node is true, and new_addr is non-NULL because @@ -217,7 +219,7 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, */ assert(dalloc_node || new_addr != NULL); - alloc_size = CHUNK_CEILING(s2u(size + alignment - chunksize)); + alloc_size = size + CHUNK_CEILING(alignment) - chunksize; /* Beware size_t wrap-around. */ if (alloc_size < size) return (NULL); @@ -225,12 +227,11 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks); if (new_addr != NULL) { extent_node_t key; - extent_node_init(&key, arena, new_addr, alloc_size, false, + extent_node_init(&key, arena, new_addr, alloc_size, 0, false, false); node = extent_tree_ad_search(chunks_ad, &key); } else { - node = chunk_first_best_fit(arena, chunks_szad, chunks_ad, - alloc_size); + node = chunk_first_best_fit(arena, chunks_szsnad, alloc_size); } if (node == NULL || (new_addr != NULL && extent_node_size_get(node) < size)) { @@ -243,6 +244,7 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, assert(extent_node_size_get(node) >= leadsize + size); trailsize = extent_node_size_get(node) - leadsize - size; ret = (void *)((uintptr_t)extent_node_addr_get(node) + leadsize); + *sn = extent_node_sn_get(node); zeroed = extent_node_zeroed_get(node); if (zeroed) *zero = true; @@ -257,13 +259,13 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, return (NULL); } /* Remove node from the tree. */ - extent_tree_szad_remove(chunks_szad, node); + extent_tree_szsnad_remove(chunks_szsnad, node); extent_tree_ad_remove(chunks_ad, node); arena_chunk_cache_maybe_remove(arena, node, cache); if (leadsize != 0) { /* Insert the leading space as a smaller chunk. */ extent_node_size_set(node, leadsize); - extent_tree_szad_insert(chunks_szad, node); + extent_tree_szsnad_insert(chunks_szsnad, node); extent_tree_ad_insert(chunks_ad, node); arena_chunk_cache_maybe_insert(arena, node, cache); node = NULL; @@ -275,9 +277,9 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, if (dalloc_node && node != NULL) arena_node_dalloc(tsdn, arena, node); malloc_mutex_unlock(tsdn, &arena->chunks_mtx); - chunk_record(tsdn, arena, chunk_hooks, chunks_szad, - chunks_ad, cache, ret, size + trailsize, zeroed, - committed); + chunk_record(tsdn, arena, chunk_hooks, chunks_szsnad, + chunks_ad, cache, ret, size + trailsize, *sn, + zeroed, committed); return (NULL); } /* Insert the trailing space as a smaller chunk. */ @@ -286,22 +288,22 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, if (node == NULL) { malloc_mutex_unlock(tsdn, &arena->chunks_mtx); chunk_record(tsdn, arena, chunk_hooks, - chunks_szad, chunks_ad, cache, ret, size + - trailsize, zeroed, committed); + chunks_szsnad, chunks_ad, cache, ret, size + + trailsize, *sn, zeroed, committed); return (NULL); } } extent_node_init(node, arena, (void *)((uintptr_t)(ret) + size), - trailsize, zeroed, committed); - extent_tree_szad_insert(chunks_szad, node); + trailsize, *sn, zeroed, committed); + extent_tree_szsnad_insert(chunks_szsnad, node); extent_tree_ad_insert(chunks_ad, node); arena_chunk_cache_maybe_insert(arena, node, cache); node = NULL; } if (!committed && chunk_hooks->commit(ret, size, 0, size, arena->ind)) { malloc_mutex_unlock(tsdn, &arena->chunks_mtx); - chunk_record(tsdn, arena, chunk_hooks, chunks_szad, chunks_ad, - cache, ret, size, zeroed, committed); + chunk_record(tsdn, arena, chunk_hooks, chunks_szsnad, chunks_ad, + cache, ret, size, *sn, zeroed, committed); return (NULL); } malloc_mutex_unlock(tsdn, &arena->chunks_mtx); @@ -385,8 +387,8 @@ chunk_alloc_base(size_t size) void * chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, - bool dalloc_node) + void *new_addr, size_t size, size_t alignment, size_t *sn, bool *zero, + bool *commit, bool dalloc_node) { void *ret; @@ -396,8 +398,8 @@ chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, assert((alignment & chunksize_mask) == 0); ret = chunk_recycle(tsdn, arena, chunk_hooks, - &arena->chunks_szad_cached, &arena->chunks_ad_cached, true, - new_addr, size, alignment, zero, commit, dalloc_node); + &arena->chunks_szsnad_cached, &arena->chunks_ad_cached, true, + new_addr, size, alignment, sn, zero, commit, dalloc_node); if (ret == NULL) return (NULL); if (config_valgrind) @@ -451,7 +453,8 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, static void * chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) + void *new_addr, size_t size, size_t alignment, size_t *sn, bool *zero, + bool *commit) { void *ret; @@ -461,8 +464,8 @@ chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, assert((alignment & chunksize_mask) == 0); ret = chunk_recycle(tsdn, arena, chunk_hooks, - &arena->chunks_szad_retained, &arena->chunks_ad_retained, false, - new_addr, size, alignment, zero, commit, true); + &arena->chunks_szsnad_retained, &arena->chunks_ad_retained, false, + new_addr, size, alignment, sn, zero, commit, true); if (config_stats && ret != NULL) arena->stats.retained -= size; @@ -472,14 +475,15 @@ chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void * chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) + void *new_addr, size_t size, size_t alignment, size_t *sn, bool *zero, + bool *commit) { void *ret; chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); ret = chunk_alloc_retained(tsdn, arena, chunk_hooks, new_addr, size, - alignment, zero, commit); + alignment, sn, zero, commit); if (ret == NULL) { if (chunk_hooks->alloc == chunk_alloc_default) { /* Call directly to propagate tsdn. */ @@ -493,6 +497,8 @@ chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, if (ret == NULL) return (NULL); + *sn = arena_extent_sn_next(arena); + if (config_valgrind && chunk_hooks->alloc != chunk_alloc_default) JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, chunksize); @@ -503,8 +509,8 @@ chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, static void chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache, - void *chunk, size_t size, bool zeroed, bool committed) + extent_tree_t *chunks_szsnad, extent_tree_t *chunks_ad, bool cache, + void *chunk, size_t size, size_t sn, bool zeroed, bool committed) { bool unzeroed; extent_node_t *node, *prev; @@ -516,7 +522,7 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, malloc_mutex_lock(tsdn, &arena->chunks_mtx); chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks); - extent_node_init(&key, arena, (void *)((uintptr_t)chunk + size), 0, + extent_node_init(&key, arena, (void *)((uintptr_t)chunk + size), 0, 0, false, false); node = extent_tree_ad_nsearch(chunks_ad, &key); /* Try to coalesce forward. */ @@ -528,15 +534,17 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, /* * Coalesce chunk with the following address range. This does * not change the position within chunks_ad, so only - * remove/insert from/into chunks_szad. + * remove/insert from/into chunks_szsnad. */ - extent_tree_szad_remove(chunks_szad, node); + extent_tree_szsnad_remove(chunks_szsnad, node); arena_chunk_cache_maybe_remove(arena, node, cache); extent_node_addr_set(node, chunk); extent_node_size_set(node, size + extent_node_size_get(node)); + if (sn < extent_node_sn_get(node)) + extent_node_sn_set(node, sn); extent_node_zeroed_set(node, extent_node_zeroed_get(node) && !unzeroed); - extent_tree_szad_insert(chunks_szad, node); + extent_tree_szsnad_insert(chunks_szsnad, node); arena_chunk_cache_maybe_insert(arena, node, cache); } else { /* Coalescing forward failed, so insert a new node. */ @@ -554,10 +562,10 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, } goto label_return; } - extent_node_init(node, arena, chunk, size, !unzeroed, + extent_node_init(node, arena, chunk, size, sn, !unzeroed, committed); extent_tree_ad_insert(chunks_ad, node); - extent_tree_szad_insert(chunks_szad, node); + extent_tree_szsnad_insert(chunks_szsnad, node); arena_chunk_cache_maybe_insert(arena, node, cache); } @@ -571,19 +579,21 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, /* * Coalesce chunk with the previous address range. This does * not change the position within chunks_ad, so only - * remove/insert node from/into chunks_szad. + * remove/insert node from/into chunks_szsnad. */ - extent_tree_szad_remove(chunks_szad, prev); + extent_tree_szsnad_remove(chunks_szsnad, prev); extent_tree_ad_remove(chunks_ad, prev); arena_chunk_cache_maybe_remove(arena, prev, cache); - extent_tree_szad_remove(chunks_szad, node); + extent_tree_szsnad_remove(chunks_szsnad, node); arena_chunk_cache_maybe_remove(arena, node, cache); extent_node_addr_set(node, extent_node_addr_get(prev)); extent_node_size_set(node, extent_node_size_get(prev) + extent_node_size_get(node)); + if (extent_node_sn_get(prev) < extent_node_sn_get(node)) + extent_node_sn_set(node, extent_node_sn_get(prev)); extent_node_zeroed_set(node, extent_node_zeroed_get(prev) && extent_node_zeroed_get(node)); - extent_tree_szad_insert(chunks_szad, node); + extent_tree_szsnad_insert(chunks_szsnad, node); arena_chunk_cache_maybe_insert(arena, node, cache); arena_node_dalloc(tsdn, arena, prev); @@ -595,7 +605,7 @@ label_return: void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - void *chunk, size_t size, bool committed) + void *chunk, size_t size, size_t sn, bool committed) { assert(chunk != NULL); @@ -603,8 +613,9 @@ chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, assert(size != 0); assert((size & chunksize_mask) == 0); - chunk_record(tsdn, arena, chunk_hooks, &arena->chunks_szad_cached, - &arena->chunks_ad_cached, true, chunk, size, false, committed); + chunk_record(tsdn, arena, chunk_hooks, &arena->chunks_szsnad_cached, + &arena->chunks_ad_cached, true, chunk, size, sn, false, + committed); arena_maybe_purge(tsdn, arena); } @@ -627,7 +638,7 @@ chunk_dalloc_default(void *chunk, size_t size, bool committed, void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - void *chunk, size_t size, bool zeroed, bool committed) + void *chunk, size_t size, size_t sn, bool zeroed, bool committed) { bool err; @@ -653,8 +664,9 @@ chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, } zeroed = !committed || !chunk_hooks->purge(chunk, size, 0, size, arena->ind); - chunk_record(tsdn, arena, chunk_hooks, &arena->chunks_szad_retained, - &arena->chunks_ad_retained, false, chunk, size, zeroed, committed); + chunk_record(tsdn, arena, chunk_hooks, &arena->chunks_szsnad_retained, + &arena->chunks_ad_retained, false, chunk, size, sn, zeroed, + committed); if (config_stats) arena->stats.retained += size; diff --git a/contrib/jemalloc/src/chunk_dss.c b/contrib/jemalloc/src/chunk_dss.c index 85a13548f403..ee3f83888e0f 100644 --- a/contrib/jemalloc/src/chunk_dss.c +++ b/contrib/jemalloc/src/chunk_dss.c @@ -162,7 +162,8 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, CHUNK_HOOKS_INITIALIZER; chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, cpad, cpad_size, - false, true); + arena_extent_sn_next(arena), false, + true); } if (*zero) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( diff --git a/contrib/jemalloc/src/extent.c b/contrib/jemalloc/src/extent.c index 9f5146e5ff5c..218156c608c1 100644 --- a/contrib/jemalloc/src/extent.c +++ b/contrib/jemalloc/src/extent.c @@ -3,43 +3,46 @@ /******************************************************************************/ +/* + * Round down to the nearest chunk size that can actually be requested during + * normal huge allocation. + */ JEMALLOC_INLINE_C size_t extent_quantize(size_t size) { + size_t ret; + szind_t ind; - /* - * Round down to the nearest chunk size that can actually be requested - * during normal huge allocation. - */ - return (index2size(size2index(size + 1) - 1)); + assert(size > 0); + + ind = size2index(size + 1); + if (ind == 0) { + /* Avoid underflow. */ + return (index2size(0)); + } + ret = index2size(ind - 1); + assert(ret <= size); + return (ret); } JEMALLOC_INLINE_C int -extent_szad_comp(const extent_node_t *a, const extent_node_t *b) +extent_sz_comp(const extent_node_t *a, const extent_node_t *b) { - int ret; size_t a_qsize = extent_quantize(extent_node_size_get(a)); size_t b_qsize = extent_quantize(extent_node_size_get(b)); - /* - * Compare based on quantized size rather than size, in order to sort - * equally useful extents only by address. - */ - ret = (a_qsize > b_qsize) - (a_qsize < b_qsize); - if (ret == 0) { - uintptr_t a_addr = (uintptr_t)extent_node_addr_get(a); - uintptr_t b_addr = (uintptr_t)extent_node_addr_get(b); + return ((a_qsize > b_qsize) - (a_qsize < b_qsize)); +} - ret = (a_addr > b_addr) - (a_addr < b_addr); - } +JEMALLOC_INLINE_C int +extent_sn_comp(const extent_node_t *a, const extent_node_t *b) +{ + size_t a_sn = extent_node_sn_get(a); + size_t b_sn = extent_node_sn_get(b); - return (ret); + return ((a_sn > b_sn) - (a_sn < b_sn)); } -/* Generate red-black tree functions. */ -rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, szad_link, - extent_szad_comp) - JEMALLOC_INLINE_C int extent_ad_comp(const extent_node_t *a, const extent_node_t *b) { @@ -49,5 +52,26 @@ extent_ad_comp(const extent_node_t *a, const extent_node_t *b) return ((a_addr > b_addr) - (a_addr < b_addr)); } +JEMALLOC_INLINE_C int +extent_szsnad_comp(const extent_node_t *a, const extent_node_t *b) +{ + int ret; + + ret = extent_sz_comp(a, b); + if (ret != 0) + return (ret); + + ret = extent_sn_comp(a, b); + if (ret != 0) + return (ret); + + ret = extent_ad_comp(a, b); + return (ret); +} + +/* Generate red-black tree functions. */ +rb_gen(, extent_tree_szsnad_, extent_tree_t, extent_node_t, szsnad_link, + extent_szsnad_comp) + /* Generate red-black tree functions. */ rb_gen(, extent_tree_ad_, extent_tree_t, extent_node_t, ad_link, extent_ad_comp) diff --git a/contrib/jemalloc/src/huge.c b/contrib/jemalloc/src/huge.c index 62e6932b7f00..8abd8c00caa1 100644 --- a/contrib/jemalloc/src/huge.c +++ b/contrib/jemalloc/src/huge.c @@ -56,6 +56,7 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, size_t ausize; arena_t *iarena; extent_node_t *node; + size_t sn; bool is_zeroed; /* Allocate one or more contiguous chunks for this request. */ @@ -68,7 +69,8 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, assert(ausize >= chunksize); /* Allocate an extent node with which to track the chunk. */ - iarena = (!tsdn_null(tsdn)) ? arena_ichoose(tsdn_tsd(tsdn), NULL) : a0get(); + iarena = (!tsdn_null(tsdn)) ? arena_ichoose(tsdn_tsd(tsdn), NULL) : + a0get(); node = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_node_t)), CACHELINE, false, NULL, true, iarena); if (node == NULL) @@ -82,15 +84,15 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, if (likely(!tsdn_null(tsdn))) arena = arena_choose(tsdn_tsd(tsdn), arena); if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn, - arena, usize, alignment, &is_zeroed)) == NULL) { + arena, usize, alignment, &sn, &is_zeroed)) == NULL) { idalloctm(tsdn, node, NULL, true, true); return (NULL); } - extent_node_init(node, arena, ret, usize, is_zeroed, true); + extent_node_init(node, arena, ret, usize, sn, is_zeroed, true); if (huge_node_set(tsdn, ret, node)) { - arena_chunk_dalloc_huge(tsdn, arena, ret, usize); + arena_chunk_dalloc_huge(tsdn, arena, ret, usize, sn); idalloctm(tsdn, node, NULL, true, true); return (NULL); } @@ -245,7 +247,8 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize, malloc_mutex_unlock(tsdn, &arena->huge_mtx); /* Zap the excess chunks. */ - arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize); + arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize, + extent_node_sn_get(node)); return (false); } @@ -407,7 +410,8 @@ huge_dalloc(tsdn_t *tsdn, void *ptr) huge_dalloc_junk(extent_node_addr_get(node), extent_node_size_get(node)); arena_chunk_dalloc_huge(tsdn, extent_node_arena_get(node), - extent_node_addr_get(node), extent_node_size_get(node)); + extent_node_addr_get(node), extent_node_size_get(node), + extent_node_sn_get(node)); idalloctm(tsdn, node, NULL, true, true); arena_decay_tick(tsdn, arena); diff --git a/contrib/jemalloc/src/jemalloc.c b/contrib/jemalloc/src/jemalloc.c index f659b55c395a..8a49f267183e 100644 --- a/contrib/jemalloc/src/jemalloc.c +++ b/contrib/jemalloc/src/jemalloc.c @@ -1060,7 +1060,11 @@ malloc_conf_init(void) if (cont) \ continue; \ } -#define CONF_HANDLE_T_U(t, o, n, min, max, clip) \ +#define CONF_MIN_no(um, min) false +#define CONF_MIN_yes(um, min) ((um) < (min)) +#define CONF_MAX_no(um, max) false +#define CONF_MAX_yes(um, max) ((um) > (max)) +#define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \ if (CONF_MATCH(n)) { \ uintmax_t um; \ char *end; \ @@ -1073,15 +1077,19 @@ malloc_conf_init(void) "Invalid conf value", \ k, klen, v, vlen); \ } else if (clip) { \ - if ((min) != 0 && um < (min)) \ + if (CONF_MIN_##check_min(um, \ + (min))) \ o = (t)(min); \ - else if (um > (max)) \ + else if (CONF_MAX_##check_max( \ + um, (max))) \ o = (t)(max); \ else \ o = (t)um; \ } else { \ - if (((min) != 0 && um < (min)) \ - || um > (max)) { \ + if (CONF_MIN_##check_min(um, \ + (min)) || \ + CONF_MAX_##check_max(um, \ + (max))) { \ malloc_conf_error( \ "Out-of-range " \ "conf value", \ @@ -1091,10 +1099,13 @@ malloc_conf_init(void) } \ continue; \ } -#define CONF_HANDLE_UNSIGNED(o, n, min, max, clip) \ - CONF_HANDLE_T_U(unsigned, o, n, min, max, clip) -#define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ - CONF_HANDLE_T_U(size_t, o, n, min, max, clip) +#define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \ + clip) \ + CONF_HANDLE_T_U(unsigned, o, n, min, max, \ + check_min, check_max, clip) +#define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \ + CONF_HANDLE_T_U(size_t, o, n, min, max, \ + check_min, check_max, clip) #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ if (CONF_MATCH(n)) { \ long l; \ @@ -1137,7 +1148,7 @@ malloc_conf_init(void) */ CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1), - (sizeof(size_t) << 3) - 1, true) + (sizeof(size_t) << 3) - 1, yes, yes, true) if (strncmp("dss", k, klen) == 0) { int i; bool match = false; @@ -1163,7 +1174,7 @@ malloc_conf_init(void) continue; } CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, - UINT_MAX, false) + UINT_MAX, yes, no, false) if (strncmp("purge", k, klen) == 0) { int i; bool match = false; @@ -1234,7 +1245,7 @@ malloc_conf_init(void) continue; } CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", - 0, SIZE_T_MAX, false) + 0, SIZE_T_MAX, no, no, false) CONF_HANDLE_BOOL(opt_redzone, "redzone", true) CONF_HANDLE_BOOL(opt_zero, "zero", true) } @@ -1271,8 +1282,8 @@ malloc_conf_init(void) CONF_HANDLE_BOOL(opt_prof_thread_active_init, "prof_thread_active_init", true) CONF_HANDLE_SIZE_T(opt_lg_prof_sample, - "lg_prof_sample", 0, - (sizeof(uint64_t) << 3) - 1, true) + "lg_prof_sample", 0, (sizeof(uint64_t) << 3) + - 1, no, yes, true) CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", true) CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, @@ -1288,7 +1299,14 @@ malloc_conf_init(void) malloc_conf_error("Invalid conf pair", k, klen, v, vlen); #undef CONF_MATCH +#undef CONF_MATCH_VALUE #undef CONF_HANDLE_BOOL +#undef CONF_MIN_no +#undef CONF_MIN_yes +#undef CONF_MAX_no +#undef CONF_MAX_yes +#undef CONF_HANDLE_T_U +#undef CONF_HANDLE_UNSIGNED #undef CONF_HANDLE_SIZE_T #undef CONF_HANDLE_SSIZE_T #undef CONF_HANDLE_CHAR_P @@ -1397,8 +1415,9 @@ malloc_init_hard_recursible(void) ncpus = malloc_ncpus(); -#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ - && !defined(_WIN32) && !defined(__native_client__)) +#if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \ + && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \ + !defined(__native_client__)) /* LinuxThreads' pthread_atfork() allocates. */ if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, jemalloc_postfork_child) != 0) { @@ -1977,8 +1996,8 @@ je_realloc(void *ptr, size_t size) *tsd_thread_deallocatedp_get(tsd) += old_usize; } UTRACE(ptr, size, ret); - JEMALLOC_VALGRIND_REALLOC(true, tsdn, ret, usize, true, ptr, old_usize, - old_rzsize, true, false); + JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr, + old_usize, old_rzsize, maybe, false); witness_assert_lockless(tsdn); return (ret); } @@ -2404,8 +2423,8 @@ je_rallocx(void *ptr, size_t size, int flags) *tsd_thread_deallocatedp_get(tsd) += old_usize; } UTRACE(ptr, size, p); - JEMALLOC_VALGRIND_REALLOC(true, tsd_tsdn(tsd), p, usize, false, ptr, - old_usize, old_rzsize, false, zero); + JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr, + old_usize, old_rzsize, no, zero); witness_assert_lockless(tsd_tsdn(tsd)); return (p); label_oom: @@ -2547,8 +2566,8 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) *tsd_thread_allocatedp_get(tsd) += usize; *tsd_thread_deallocatedp_get(tsd) += old_usize; } - JEMALLOC_VALGRIND_REALLOC(false, tsd_tsdn(tsd), ptr, usize, false, ptr, - old_usize, old_rzsize, false, zero); + JEMALLOC_VALGRIND_REALLOC(no, tsd_tsdn(tsd), ptr, usize, no, ptr, + old_usize, old_rzsize, no, zero); label_not_resized: UTRACE(ptr, size, ptr); witness_assert_lockless(tsd_tsdn(tsd)); diff --git a/contrib/jemalloc/src/pages.c b/contrib/jemalloc/src/pages.c index 647952ac3ef6..5f0c9669d2b8 100644 --- a/contrib/jemalloc/src/pages.c +++ b/contrib/jemalloc/src/pages.c @@ -170,15 +170,16 @@ pages_purge(void *addr, size_t size) #ifdef _WIN32 VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); unzeroed = true; -#elif defined(JEMALLOC_HAVE_MADVISE) -# ifdef JEMALLOC_PURGE_MADVISE_DONTNEED -# define JEMALLOC_MADV_PURGE MADV_DONTNEED -# define JEMALLOC_MADV_ZEROS true -# elif defined(JEMALLOC_PURGE_MADVISE_FREE) +#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \ + defined(JEMALLOC_PURGE_MADVISE_DONTNEED)) +# if defined(JEMALLOC_PURGE_MADVISE_FREE) # define JEMALLOC_MADV_PURGE MADV_FREE # define JEMALLOC_MADV_ZEROS false +# elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) +# define JEMALLOC_MADV_PURGE MADV_DONTNEED +# define JEMALLOC_MADV_ZEROS true # else -# error "No madvise(2) flag defined for purging unused dirty pages." +# error No madvise(2) flag defined for purging unused dirty pages # endif int err = madvise(addr, size, JEMALLOC_MADV_PURGE); unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0); @@ -191,6 +192,34 @@ pages_purge(void *addr, size_t size) return (unzeroed); } +bool +pages_huge(void *addr, size_t size) +{ + + assert(PAGE_ADDR2BASE(addr) == addr); + assert(PAGE_CEILING(size) == size); + +#ifdef JEMALLOC_THP + return (madvise(addr, size, MADV_HUGEPAGE) != 0); +#else + return (false); +#endif +} + +bool +pages_nohuge(void *addr, size_t size) +{ + + assert(PAGE_ADDR2BASE(addr) == addr); + assert(PAGE_CEILING(size) == size); + +#ifdef JEMALLOC_THP + return (madvise(addr, size, MADV_NOHUGEPAGE) != 0); +#else + return (false); +#endif +} + #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT static bool os_overcommits_sysctl(void) @@ -219,7 +248,7 @@ os_overcommits_proc(void) char buf[1]; ssize_t nread; -#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_open) +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open) fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY); #else fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); @@ -227,13 +256,13 @@ os_overcommits_proc(void) if (fd == -1) return (false); /* Error. */ -#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_read) +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read) nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf)); #else nread = read(fd, &buf, sizeof(buf)); #endif -#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_close) +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close) syscall(SYS_close, fd); #else close(fd); diff --git a/contrib/jemalloc/src/stats.c b/contrib/jemalloc/src/stats.c index bd8af3999ba9..1360f3bd0012 100644 --- a/contrib/jemalloc/src/stats.c +++ b/contrib/jemalloc/src/stats.c @@ -3,7 +3,7 @@ #define CTL_GET(n, v, t) do { \ size_t sz = sizeof(t); \ - xmallctl(n, v, &sz, NULL, 0); \ + xmallctl(n, (void *)v, &sz, NULL, 0); \ } while (0) #define CTL_M2_GET(n, i, v, t) do { \ @@ -12,7 +12,7 @@ size_t sz = sizeof(t); \ xmallctlnametomib(n, mib, &miblen); \ mib[2] = (i); \ - xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ + xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \ } while (0) #define CTL_M2_M4_GET(n, i, j, v, t) do { \ @@ -22,7 +22,7 @@ xmallctlnametomib(n, mib, &miblen); \ mib[2] = (i); \ mib[4] = (j); \ - xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ + xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \ } while (0) /******************************************************************************/ @@ -647,7 +647,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, #define OPT_WRITE_BOOL_MUTABLE(n, m, c) { \ bool bv2; \ if (je_mallctl("opt."#n, (void *)&bv, &bsz, NULL, 0) == 0 && \ - je_mallctl(#m, &bv2, &bsz, NULL, 0) == 0) { \ + je_mallctl(#m, &bv2, (void *)&bsz, NULL, 0) == 0) { \ if (json) { \ malloc_cprintf(write_cb, cbopaque, \ "\t\t\t\""#n"\": %s%s\n", bv ? "true" : \ @@ -692,7 +692,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, #define OPT_WRITE_SSIZE_T_MUTABLE(n, m, c) { \ ssize_t ssv2; \ if (je_mallctl("opt."#n, (void *)&ssv, &sssz, NULL, 0) == 0 && \ - je_mallctl(#m, &ssv2, &sssz, NULL, 0) == 0) { \ + je_mallctl(#m, (void *)&ssv2, &sssz, NULL, 0) == 0) { \ if (json) { \ malloc_cprintf(write_cb, cbopaque, \ "\t\t\t\""#n"\": %zd%s\n", ssv, (c)); \ @@ -1084,7 +1084,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, * */ epoch = 1; u64sz = sizeof(uint64_t); - err = je_mallctl("epoch", &epoch, &u64sz, &epoch, sizeof(uint64_t)); + err = je_mallctl("epoch", (void *)&epoch, &u64sz, (void *)&epoch, + sizeof(uint64_t)); if (err != 0) { if (err == EAGAIN) { malloc_write("<jemalloc>: Memory allocation failure in " diff --git a/contrib/jemalloc/src/tcache.c b/contrib/jemalloc/src/tcache.c index f97aa420ca49..21540ff46e75 100644 --- a/contrib/jemalloc/src/tcache.c +++ b/contrib/jemalloc/src/tcache.c @@ -517,12 +517,12 @@ tcache_boot(tsdn_t *tsdn) * If necessary, clamp opt_lg_tcache_max, now that large_maxclass is * known. */ - if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) < SMALL_MAXCLASS) + if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) < SMALL_MAXCLASS) tcache_maxclass = SMALL_MAXCLASS; - else if ((1U << opt_lg_tcache_max) > large_maxclass) + else if ((ZU(1) << opt_lg_tcache_max) > large_maxclass) tcache_maxclass = large_maxclass; else - tcache_maxclass = (1U << opt_lg_tcache_max); + tcache_maxclass = (ZU(1) << opt_lg_tcache_max); nhbins = size2index(tcache_maxclass) + 1; diff --git a/contrib/jemalloc/src/util.c b/contrib/jemalloc/src/util.c index bee1c77e7ece..a4ff287c1206 100644 --- a/contrib/jemalloc/src/util.c +++ b/contrib/jemalloc/src/util.c @@ -49,7 +49,7 @@ static void wrtmessage(void *cbopaque, const char *s) { -#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_write) +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write) /* * Use syscall(2) rather than write(2) when possible in order to avoid * the possibility of memory allocation within libc. This is necessary @@ -216,7 +216,7 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) p++; } if (neg) - ret = -ret; + ret = (uintmax_t)(-((intmax_t)ret)); if (p == ns) { /* No conversion performed. */ |