diff options
Diffstat (limited to 'lib/libthr')
| -rw-r--r-- | lib/libthr/Makefile | 2 | ||||
| -rw-r--r-- | lib/libthr/thread/thr_fork.c | 6 | ||||
| -rw-r--r-- | lib/libthr/thread/thr_init.c | 26 | ||||
| -rw-r--r-- | lib/libthr/thread/thr_mutex.c | 75 | ||||
| -rw-r--r-- | lib/libthr/thread/thr_private.h | 11 | ||||
| -rw-r--r-- | lib/libthr/thread/thr_pshared.c | 14 | ||||
| -rw-r--r-- | lib/libthr/thread/thr_pspinlock.c | 107 | ||||
| -rw-r--r-- | lib/libthr/thread/thr_rtld.c | 1 |
8 files changed, 161 insertions, 81 deletions
diff --git a/lib/libthr/Makefile b/lib/libthr/Makefile index 7a080e0f2580..517a30f21ee8 100644 --- a/lib/libthr/Makefile +++ b/lib/libthr/Makefile @@ -39,7 +39,7 @@ SYMBOL_MAPS=${.CURDIR}/pthread.map MAN= libthr.3 -# enable extra internal consistancy checks +# enable extra internal consistency checks CFLAGS+=-D_PTHREADS_INVARIANTS PRECIOUSLIB= diff --git a/lib/libthr/thread/thr_fork.c b/lib/libthr/thread/thr_fork.c index 7256b68b7a3f..531e09cef5f1 100644 --- a/lib/libthr/thread/thr_fork.c +++ b/lib/libthr/thread/thr_fork.c @@ -168,6 +168,7 @@ __thr_fork(void) if (_thr_isthreaded() != 0) { was_threaded = 1; _malloc_prefork(); + __thr_pshared_atfork_pre(); _rtld_atfork_pre(rtld_locks); } else { was_threaded = 0; @@ -202,8 +203,10 @@ __thr_fork(void) _thr_signal_postfork_child(); - if (was_threaded) + if (was_threaded) { _rtld_atfork_post(rtld_locks); + __thr_pshared_atfork_post(); + } _thr_setthreaded(0); /* reinitalize library. */ @@ -236,6 +239,7 @@ __thr_fork(void) if (was_threaded) { _rtld_atfork_post(rtld_locks); + __thr_pshared_atfork_post(); _malloc_postfork(); } diff --git a/lib/libthr/thread/thr_init.c b/lib/libthr/thread/thr_init.c index 3c81299937f2..c9f30ab02387 100644 --- a/lib/libthr/thread/thr_init.c +++ b/lib/libthr/thread/thr_init.c @@ -108,7 +108,6 @@ struct pthread_cond_attr _pthread_condattr_default = { .c_clockid = CLOCK_REALTIME }; -pid_t _thr_pid; int _thr_is_smp = 0; size_t _thr_guard_default; size_t _thr_stack_default = THR_STACK_DEFAULT; @@ -305,7 +304,7 @@ _thread_init_hack(void) void _libpthread_init(struct pthread *curthread) { - int fd, first, dlopened; + int first, dlopened; /* Check if this function has already been called: */ if ((_thr_initial != NULL) && (curthread == NULL)) @@ -321,27 +320,6 @@ _libpthread_init(struct pthread *curthread) memcpy(__thr_jtable, jmp_table, sizeof(jmp_table)); __thr_interpose_libc(); - /* - * Check for the special case of this process running as - * or in place of init as pid = 1: - */ - if ((_thr_pid = getpid()) == 1) { - /* - * Setup a new session for this process which is - * assumed to be running as root. - */ - if (setsid() == -1) - PANIC("Can't set session ID"); - if (revoke(_PATH_CONSOLE) != 0) - PANIC("Can't revoke console"); - if ((fd = __sys_openat(AT_FDCWD, _PATH_CONSOLE, O_RDWR)) < 0) - PANIC("Can't open console"); - if (setlogin("root") == -1) - PANIC("Can't set login to root"); - if (_ioctl(fd, TIOCSCTTY, (char *) NULL) == -1) - PANIC("Can't set controlling terminal"); - } - /* Initialize pthread private data. */ init_private(); @@ -466,7 +444,6 @@ init_private(void) _thr_once_init(); _thr_spinlock_init(); _thr_list_init(); - __thr_pshared_init(); _thr_wake_addr_init(); _sleepq_init(); _single_thread = NULL; @@ -477,6 +454,7 @@ init_private(void) * e.g. after a fork(). */ if (init_once == 0) { + __thr_pshared_init(); /* Find the stack top */ mib[0] = CTL_KERN; mib[1] = KERN_USRSTACK; diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c index 30a8be2767f5..865e4cfbb29f 100644 --- a/lib/libthr/thread/thr_mutex.c +++ b/lib/libthr/thread/thr_mutex.c @@ -38,6 +38,7 @@ * $FreeBSD$ */ +#include <stdbool.h> #include "namespace.h" #include <stdlib.h> #include <errno.h> @@ -124,8 +125,14 @@ mutex_assert_is_owned(struct pthread_mutex *m) { #if defined(_PTHREADS_INVARIANTS) - if (__predict_false(m->m_qe.tqe_prev == NULL)) - PANIC("mutex is not on list"); + if (__predict_false(m->m_qe.tqe_prev == NULL)) { + char msg[128]; + snprintf(msg, sizeof(msg), + "mutex %p own %#x %#x is not on list %p %p", + m, m->m_lock.m_owner, m->m_owner, m->m_qe.tqe_prev, + m->m_qe.tqe_next); + PANIC(msg); + } #endif } @@ -135,8 +142,14 @@ mutex_assert_not_owned(struct pthread_mutex *m) #if defined(_PTHREADS_INVARIANTS) if (__predict_false(m->m_qe.tqe_prev != NULL || - m->m_qe.tqe_next != NULL)) - PANIC("mutex is on list"); + m->m_qe.tqe_next != NULL)) { + char msg[128]; + snprintf(msg, sizeof(msg), + "mutex %p own %#x %#x is on list %p %p", + m, m->m_lock.m_owner, m->m_owner, m->m_qe.tqe_prev, + m->m_qe.tqe_next); + PANIC(msg); + } #endif } @@ -252,6 +265,51 @@ set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m) m->m_lock.m_ceilings[1] = -1; } +static void +shared_mutex_init(struct pthread_mutex *pmtx, const struct + pthread_mutex_attr *mutex_attr) +{ + static const struct pthread_mutex_attr foobar_mutex_attr = { + .m_type = PTHREAD_MUTEX_DEFAULT, + .m_protocol = PTHREAD_PRIO_NONE, + .m_ceiling = 0, + .m_pshared = PTHREAD_PROCESS_SHARED + }; + bool done; + + /* + * Hack to allow multiple pthread_mutex_init() calls on the + * same process-shared mutex. We rely on kernel allocating + * zeroed offpage for the mutex, i.e. the + * PMUTEX_INITSTAGE_ALLOC value must be zero. + */ + for (done = false; !done;) { + switch (pmtx->m_ps) { + case PMUTEX_INITSTAGE_DONE: + atomic_thread_fence_acq(); + done = true; + break; + case PMUTEX_INITSTAGE_ALLOC: + if (atomic_cmpset_int(&pmtx->m_ps, + PMUTEX_INITSTAGE_ALLOC, PMUTEX_INITSTAGE_BUSY)) { + if (mutex_attr == NULL) + mutex_attr = &foobar_mutex_attr; + mutex_init_body(pmtx, mutex_attr); + atomic_store_rel_int(&pmtx->m_ps, + PMUTEX_INITSTAGE_DONE); + done = true; + } + break; + case PMUTEX_INITSTAGE_BUSY: + _pthread_yield(); + break; + default: + PANIC("corrupted offpage"); + break; + } + } +} + int __pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *mutex_attr) @@ -273,7 +331,7 @@ __pthread_mutex_init(pthread_mutex_t *mutex, if (pmtx == NULL) return (EFAULT); *mutex = THR_PSHARED_PTR; - mutex_init_body(pmtx, *mutex_attr); + shared_mutex_init(pmtx, *mutex_attr); return (0); } @@ -414,6 +472,7 @@ check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m) *m = __thr_pshared_offpage(mutex, 0); if (*m == NULL) ret = EINVAL; + shared_mutex_init(*m, NULL); } else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) { if (*m == THR_MUTEX_DESTROYED) { ret = EINVAL; @@ -576,6 +635,7 @@ _pthread_mutex_unlock(pthread_mutex_t *mutex) mp = __thr_pshared_offpage(mutex, 0); if (mp == NULL) return (EINVAL); + shared_mutex_init(mp, NULL); } else { mp = *mutex; } @@ -803,6 +863,7 @@ _pthread_mutex_getprioceiling(pthread_mutex_t *mutex, m = __thr_pshared_offpage(mutex, 0); if (m == NULL) return (EINVAL); + shared_mutex_init(m, NULL); } else { m = *mutex; if (m <= THR_MUTEX_DESTROYED) @@ -827,6 +888,7 @@ _pthread_mutex_setprioceiling(pthread_mutex_t *mutex, m = __thr_pshared_offpage(mutex, 0); if (m == NULL) return (EINVAL); + shared_mutex_init(m, NULL); } else { m = *mutex; if (m <= THR_MUTEX_DESTROYED) @@ -930,12 +992,13 @@ __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count) int _pthread_mutex_isowned_np(pthread_mutex_t *mutex) { - struct pthread_mutex *m; + struct pthread_mutex *m; if (*mutex == THR_PSHARED_PTR) { m = __thr_pshared_offpage(mutex, 0); if (m == NULL) return (0); + shared_mutex_init(m, NULL); } else { m = *mutex; if (m <= THR_MUTEX_DESTROYED) diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h index 31f8e6c863da..f35d3cd3c357 100644 --- a/lib/libthr/thread/thr_private.h +++ b/lib/libthr/thread/thr_private.h @@ -146,6 +146,13 @@ TAILQ_HEAD(mutex_queue, pthread_mutex); #define MAX_DEFER_WAITERS 50 +/* + * Values for pthread_mutex m_ps indicator. + */ +#define PMUTEX_INITSTAGE_ALLOC 0 +#define PMUTEX_INITSTAGE_BUSY 1 +#define PMUTEX_INITSTAGE_DONE 2 + struct pthread_mutex { /* * Lock for accesses to this structure. @@ -156,6 +163,7 @@ struct pthread_mutex { int m_count; int m_spinloops; int m_yieldloops; + int m_ps; /* pshared init stage */ /* * Link for all mutexes a thread currently owns, of the same * prio type. @@ -717,7 +725,6 @@ extern struct pthread_cond_attr _pthread_condattr_default __hidden; extern struct pthread_prio _thr_priorities[] __hidden; -extern pid_t _thr_pid __hidden; extern int _thr_is_smp __hidden; extern size_t _thr_guard_default __hidden; @@ -952,6 +959,8 @@ void _tcb_dtor(struct tcb *); void __thr_pshared_init(void) __hidden; void *__thr_pshared_offpage(void *key, int doalloc) __hidden; void __thr_pshared_destroy(void *key) __hidden; +void __thr_pshared_atfork_pre(void) __hidden; +void __thr_pshared_atfork_post(void) __hidden; __END_DECLS diff --git a/lib/libthr/thread/thr_pshared.c b/lib/libthr/thread/thr_pshared.c index e8ccf1cebb10..83714785f9b1 100644 --- a/lib/libthr/thread/thr_pshared.c +++ b/lib/libthr/thread/thr_pshared.c @@ -252,3 +252,17 @@ __thr_pshared_destroy(void *key) pshared_clean(key, val); pshared_gc(curthread); } + +void +__thr_pshared_atfork_pre(void) +{ + + _thr_rwl_rdlock(&pshared_lock); +} + +void +__thr_pshared_atfork_post(void) +{ + + _thr_rwl_unlock(&pshared_lock); +} diff --git a/lib/libthr/thread/thr_pspinlock.c b/lib/libthr/thread/thr_pspinlock.c index 9e1f96eb39cd..1c9b41285599 100644 --- a/lib/libthr/thread/thr_pspinlock.c +++ b/lib/libthr/thread/thr_pspinlock.c @@ -1,7 +1,11 @@ /*- * Copyright (c) 2003 David Xu <davidxu@freebsd.org> + * Copyright (c) 2016 The FreeBSD Foundation * All rights reserved. * + * Portions of this software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -46,93 +50,100 @@ int _pthread_spin_init(pthread_spinlock_t *lock, int pshared) { struct pthread_spinlock *lck; - int ret; - if (lock == NULL || pshared != PTHREAD_PROCESS_PRIVATE) - ret = EINVAL; - else if ((lck = malloc(sizeof(struct pthread_spinlock))) == NULL) - ret = ENOMEM; - else { - _thr_umutex_init(&lck->s_lock); + if (lock == NULL) + return (EINVAL); + if (pshared == PTHREAD_PROCESS_PRIVATE) { + lck = malloc(sizeof(struct pthread_spinlock)); + if (lck == NULL) + return (ENOMEM); *lock = lck; - ret = 0; + } else if (pshared == PTHREAD_PROCESS_SHARED) { + lck = __thr_pshared_offpage(lock, 1); + if (lck == NULL) + return (EFAULT); + *lock = THR_PSHARED_PTR; + } else { + return (EINVAL); } - - return (ret); + _thr_umutex_init(&lck->s_lock); + return (0); } int _pthread_spin_destroy(pthread_spinlock_t *lock) { + void *l; int ret; - if (lock == NULL || *lock == NULL) + if (lock == NULL || *lock == NULL) { ret = EINVAL; - else { + } else if (*lock == THR_PSHARED_PTR) { + l = __thr_pshared_offpage(lock, 0); + if (l != NULL) + __thr_pshared_destroy(l); + ret = 0; + } else { free(*lock); *lock = NULL; ret = 0; } - return (ret); } int _pthread_spin_trylock(pthread_spinlock_t *lock) { - struct pthread *curthread = _get_curthread(); struct pthread_spinlock *lck; - int ret; - if (lock == NULL || (lck = *lock) == NULL) - ret = EINVAL; - else - ret = THR_UMUTEX_TRYLOCK(curthread, &lck->s_lock); - return (ret); + if (lock == NULL || *lock == NULL) + return (EINVAL); + lck = *lock == THR_PSHARED_PTR ? __thr_pshared_offpage(lock, 0) : *lock; + if (lck == NULL) + return (EINVAL); + return (THR_UMUTEX_TRYLOCK(_get_curthread(), &lck->s_lock)); } int _pthread_spin_lock(pthread_spinlock_t *lock) { - struct pthread *curthread = _get_curthread(); + struct pthread *curthread; struct pthread_spinlock *lck; - int ret, count; - - if (lock == NULL || (lck = *lock) == NULL) - ret = EINVAL; - else { - count = SPIN_COUNT; - while ((ret = THR_UMUTEX_TRYLOCK(curthread, &lck->s_lock)) != 0) { - while (lck->s_lock.m_owner) { - if (!_thr_is_smp) { + int count; + + if (lock == NULL) + return (EINVAL); + lck = *lock == THR_PSHARED_PTR ? __thr_pshared_offpage(lock, 0) : *lock; + if (lck == NULL) + return (EINVAL); + + curthread = _get_curthread(); + count = SPIN_COUNT; + while (THR_UMUTEX_TRYLOCK(curthread, &lck->s_lock) != 0) { + while (lck->s_lock.m_owner) { + if (!_thr_is_smp) { + _pthread_yield(); + } else { + CPU_SPINWAIT; + if (--count <= 0) { + count = SPIN_COUNT; _pthread_yield(); - } else { - CPU_SPINWAIT; - - if (--count <= 0) { - count = SPIN_COUNT; - _pthread_yield(); - } } } } - ret = 0; } - - return (ret); + return (0); } int _pthread_spin_unlock(pthread_spinlock_t *lock) { - struct pthread *curthread = _get_curthread(); struct pthread_spinlock *lck; - int ret; - if (lock == NULL || (lck = *lock) == NULL) - ret = EINVAL; - else { - ret = THR_UMUTEX_UNLOCK(curthread, &lck->s_lock); - } - return (ret); + if (lock == NULL) + return (EINVAL); + lck = *lock == THR_PSHARED_PTR ? __thr_pshared_offpage(lock, 0) : *lock; + if (lck == NULL) + return (EINVAL); + return (THR_UMUTEX_UNLOCK(_get_curthread(), &lck->s_lock)); } diff --git a/lib/libthr/thread/thr_rtld.c b/lib/libthr/thread/thr_rtld.c index cb200980cb26..6c407d1ca866 100644 --- a/lib/libthr/thread/thr_rtld.c +++ b/lib/libthr/thread/thr_rtld.c @@ -227,6 +227,7 @@ _thr_rtld_init(void) _rtld_atfork_post(NULL); _malloc_prefork(); _malloc_postfork(); + getpid(); syscall(SYS_getpid); /* mask signals, also force to resolve __sys_sigprocmask PLT */ |