As comments in _mutex_lock_backout state, only current thread

can clear the pointer to mutex, not the thread doing mutex
handoff. Because _mutex_lock_backout does not hold scheduler
lock while testing THR_FLAGS_IN_SYNCQ and then reading mutex
pointer, it is possible mutex owner begin to unlock and
handoff the mutex to the current thread, and mutex pointer
will be cleared to NULL before current thread reading it, so
current thread will end up with deferencing a NULL pointer,
Fix the race by making mutex waiters to clear their mutex pointers.
While I am here, also save inherited priority in mutex for
PTHREAD_PRIO_INERIT mutex in mutex_trylock_common just like what
we did in mutex_lock_common.

2 files changed, 8 insertions, 12 deletions

diff --git a/lib/libkse/thread/thr_mutex.c b/lib/libkse/thread/thr_mutex.c
index 177a296688f8..0717768fc0b0 100644
--- a/lib/libkse/thread/thr_mutex.c
+++ b/lib/libkse/thread/thr_mutex.c
@@ -346,6 +346,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
 			(*mutex)->m_prio = curthread->active_priority;
 			(*mutex)->m_saved_prio =
 			    curthread->inherited_priority;
+			curthread->inherited_priority = (*mutex)->m_prio;
 			THR_SCHED_UNLOCK(curthread, curthread);
 
 			/* Add to the list of owned mutexes: */
@@ -540,10 +541,10 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m,
 				/* Schedule the next thread: */
 				_thr_sched_switch(curthread);
 
+				curthread->data.mutex = NULL;
 				if (THR_IN_MUTEXQ(curthread)) {
 					THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
 					mutex_queue_remove(*m, curthread);
-					curthread->data.mutex = NULL;
 					THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
 				}
 			}
@@ -619,10 +620,10 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m,
 				/* Schedule the next thread: */
 				_thr_sched_switch(curthread);
 
+				curthread->data.mutex = NULL;
 				if (THR_IN_MUTEXQ(curthread)) {
 					THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
 					mutex_queue_remove(*m, curthread);
-					curthread->data.mutex = NULL;
 					THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
 				}
 			}
@@ -708,10 +709,10 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m,
 				/* Schedule the next thread: */
 				_thr_sched_switch(curthread);
 
+				curthread->data.mutex = NULL;
 				if (THR_IN_MUTEXQ(curthread)) {
 					THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
 					mutex_queue_remove(*m, curthread);
-					curthread->data.mutex = NULL;
 					THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
 				}
 
@@ -1590,9 +1591,6 @@ mutex_handoff(struct pthread *curthread, struct pthread_mutex *mutex)
 		TAILQ_REMOVE(&mutex->m_queue, pthread, sqe);
 		pthread->sflags &= ~THR_FLAGS_IN_SYNCQ;
 
-		/* This thread is no longer waiting for this mutex. */
-		pthread->data.mutex = NULL;
-
 		/*
 		 * Only exit the loop if the thread hasn't been
 		 * cancelled.
diff --git a/lib/libpthread/thread/thr_mutex.c b/lib/libpthread/thread/thr_mutex.c
index 177a296688f8..0717768fc0b0 100644
--- a/lib/libpthread/thread/thr_mutex.c
+++ b/lib/libpthread/thread/thr_mutex.c
@@ -346,6 +346,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
 			(*mutex)->m_prio = curthread->active_priority;
 			(*mutex)->m_saved_prio =
 			    curthread->inherited_priority;
+			curthread->inherited_priority = (*mutex)->m_prio;
 			THR_SCHED_UNLOCK(curthread, curthread);
 
 			/* Add to the list of owned mutexes: */
@@ -540,10 +541,10 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m,
 				/* Schedule the next thread: */
 				_thr_sched_switch(curthread);
 
+				curthread->data.mutex = NULL;
 				if (THR_IN_MUTEXQ(curthread)) {
 					THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
 					mutex_queue_remove(*m, curthread);
-					curthread->data.mutex = NULL;
 					THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
 				}
 			}
@@ -619,10 +620,10 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m,
 				/* Schedule the next thread: */
 				_thr_sched_switch(curthread);
 
+				curthread->data.mutex = NULL;
 				if (THR_IN_MUTEXQ(curthread)) {
 					THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
 					mutex_queue_remove(*m, curthread);
-					curthread->data.mutex = NULL;
 					THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
 				}
 			}
@@ -708,10 +709,10 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m,
 				/* Schedule the next thread: */
 				_thr_sched_switch(curthread);
 
+				curthread->data.mutex = NULL;
 				if (THR_IN_MUTEXQ(curthread)) {
 					THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
 					mutex_queue_remove(*m, curthread);
-					curthread->data.mutex = NULL;
 					THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
 				}
 
@@ -1590,9 +1591,6 @@ mutex_handoff(struct pthread *curthread, struct pthread_mutex *mutex)
 		TAILQ_REMOVE(&mutex->m_queue, pthread, sqe);
 		pthread->sflags &= ~THR_FLAGS_IN_SYNCQ;
 
-		/* This thread is no longer waiting for this mutex. */
-		pthread->data.mutex = NULL;
-
 		/*
 		 * Only exit the loop if the thread hasn't been
 		 * cancelled.