From 98928d02d2473ceb9f81b1c3bc527f8b0a0039e6 Mon Sep 17 00:00:00 2001 From: Graham Leggett Date: Fri, 21 Sep 2018 13:30:15 +0000 Subject: [PATCH 188/504] MPMs: Initialize all runtime/asynchronous objects on a dedicated pool and before signals handling to avoid lifetime issues on restart or shutdown. PR 62658. trunk patch: http://svn.apache.org/r1835845 http://svn.apache.org/r1835846 http://svn.apache.org/r1837354 http://svn.apache.org/r1837356 http://svn.apache.org/r1839571 http://svn.apache.org/r1839583 2.4.x patch: http://home.apache.org/~ylavic/patches/2.4.x-mpms_async_objects_lifetime.patch +1: ylavic, jim (but not for 2.4.35), minfrin git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x@1841586 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 4 + STATUS | 11 -- server/mpm/event/event.c | 203 +++++++++++++++------------ server/mpm/mpmt_os2/mpmt_os2_child.c | 1 + server/mpm/netware/mpm_netware.c | 1 + server/mpm/prefork/prefork.c | 6 +- server/mpm/winnt/child.c | 3 + server/mpm/winnt/mpm_winnt.c | 1 + server/mpm/worker/worker.c | 148 +++++++++++-------- 9 files changed, 215 insertions(+), 163 deletions(-) diff --git a/server/mpm/event/event.c b/server/mpm/event/event.c index f07b757ab9..ffe8a23cbd 100644 --- a/server/mpm/event/event.c +++ b/server/mpm/event/event.c @@ -436,6 +436,7 @@ int raise_sigstop_flags; static apr_pool_t *pconf; /* Pool for config stuff */ static apr_pool_t *pchild; /* Pool for httpd child stuff */ +static apr_pool_t *pruntime; /* Pool for MPM threads stuff */ static pid_t ap_my_pid; /* Linux getpid() doesn't work except in main thread. Use this instead */ @@ -709,10 +710,10 @@ static void event_note_child_killed(int childnum, pid_t pid, ap_generation_t gen static void event_note_child_started(int slot, pid_t pid) { + ap_generation_t gen = retained->mpm->my_generation; ap_scoreboard_image->parent[slot].pid = pid; - ap_run_child_status(ap_server_conf, - ap_scoreboard_image->parent[slot].pid, - retained->mpm->my_generation, slot, MPM_CHILD_STARTED); + ap_scoreboard_image->parent[slot].generation = gen; + ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED); } static const char *event_get_name(void) @@ -1270,36 +1271,6 @@ static void dummy_signal_handler(int sig) } -static apr_status_t init_pollset(apr_pool_t *p) -{ - ap_listen_rec *lr; - listener_poll_type *pt; - int i = 0; - - listener_pollfd = apr_palloc(p, sizeof(apr_pollfd_t) * num_listensocks); - for (lr = my_bucket->listeners; lr != NULL; lr = lr->next, i++) { - apr_pollfd_t *pfd; - AP_DEBUG_ASSERT(i < num_listensocks); - pfd = &listener_pollfd[i]; - pt = apr_pcalloc(p, sizeof(*pt)); - pfd->desc_type = APR_POLL_SOCKET; - pfd->desc.s = lr->sd; - pfd->reqevents = APR_POLLIN; - - pt->type = PT_ACCEPT; - pt->baton = lr; - - pfd->client_data = pt; - - apr_socket_opt_set(pfd->desc.s, APR_SO_NONBLOCK, 1); - apr_pollset_add(event_pollset, pfd); - - lr->accept_func = ap_unixd_accept; - } - - return APR_SUCCESS; -} - static apr_status_t push_timer2worker(timer_event_t* te) { return ap_queue_push_timer(worker_queue, te); @@ -1611,7 +1582,6 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) proc_info *ti = dummy; int process_slot = ti->pslot; struct process_score *ps = ap_get_scoreboard_process(process_slot); - apr_pool_t *tpool = apr_thread_pool_get(thd); int closed = 0; int have_idle_worker = 0; apr_time_t last_log; @@ -1619,16 +1589,6 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) last_log = apr_time_now(); free(ti); - rc = init_pollset(tpool); - if (rc != APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_ERR, rc, ap_server_conf, - "failed to initialize pollset, " - "shutdown process now"); - resource_shortage = 1; - signal_threads(ST_UNGRACEFUL); - return NULL; - } - /* Unblock the signal used to wake this thread up, and set a handler for * it. */ @@ -2168,8 +2128,6 @@ static int check_signal(int signum) return 0; } - - static void create_listener_thread(thread_starter * ts) { int my_child_num = ts->child_num_arg; @@ -2181,7 +2139,7 @@ static void create_listener_thread(thread_starter * ts) my_info->pslot = my_child_num; my_info->tslot = -1; /* listener thread doesn't have a thread slot */ rv = apr_thread_create(&ts->listener, thread_attr, listener_thread, - my_info, pchild); + my_info, pruntime); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00474) "apr_thread_create: unable to create listener thread"); @@ -2191,25 +2149,12 @@ static void create_listener_thread(thread_starter * ts) apr_os_thread_get(&listener_os_thread, ts->listener); } -/* XXX under some circumstances not understood, children can get stuck - * in start_threads forever trying to take over slots which will - * never be cleaned up; for now there is an APLOG_DEBUG message issued - * every so often when this condition occurs - */ -static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) +static void setup_threads_runtime(void) { - thread_starter *ts = dummy; - apr_thread_t **threads = ts->threads; - apr_threadattr_t *thread_attr = ts->threadattr; - int my_child_num = ts->child_num_arg; - proc_info *my_info; apr_status_t rv; - int i; - int threads_created = 0; - int listener_started = 0; - int loops; - int prev_threads_created; - int max_recycled_pools = -1; + ap_listen_rec *lr; + apr_pool_t *pskip = NULL; + int max_recycled_pools = -1, i; const int good_methods[] = { APR_POLLSET_KQUEUE, APR_POLLSET_PORT, APR_POLLSET_EPOLL }; @@ -2218,10 +2163,39 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) const apr_uint32_t pollset_size = (apr_uint32_t)num_listensocks + (apr_uint32_t)threads_per_child * (async_factor > 2 ? async_factor : 2); + int pollset_flags; + + /* Event's skiplist operations will happen concurrently with other modules' + * runtime so they need their own pool for allocations, and its lifetime + * should be at least the one of the connections (ptrans). Thus pskip is + * created as a subpool of pconf like/before ptrans (before so that it's + * destroyed after). In forked mode pconf is never destroyed so we are good + * anyway, but in ONE_PROCESS mode this ensures that the skiplist works + * from connection/ptrans cleanups (even after pchild is destroyed). + */ + apr_pool_create(&pskip, pconf); + apr_pool_tag(pskip, "mpm_skiplist"); + apr_thread_mutex_create(&g_timer_skiplist_mtx, APR_THREAD_MUTEX_DEFAULT, pskip); + APR_RING_INIT(&timer_free_ring, timer_event_t, link); + apr_skiplist_init(&timer_skiplist, pskip); + apr_skiplist_set_compare(timer_skiplist, timer_comp, timer_comp); + + /* All threads (listener, workers) and synchronization objects (queues, + * pollset, mutexes...) created here should have at least the lifetime of + * the connections they handle (i.e. ptrans). We can't use this thread's + * self pool because all these objects survive it, nor use pchild or pconf + * directly because this starter thread races with other modules' runtime, + * nor finally pchild (or subpool thereof) because it is killed explicitely + * before pconf (thus connections/ptrans can live longer, which matters in + * ONE_PROCESS mode). So this leaves us with a subpool of pconf, created + * before any ptrans hence destroyed after. + */ + apr_pool_create(&pruntime, pconf); + apr_pool_tag(pruntime, "mpm_runtime"); /* We must create the fd queues before we start up the listener * and worker threads. */ - rv = ap_queue_create(&worker_queue, threads_per_child, pchild); + rv = ap_queue_create(&worker_queue, threads_per_child, pruntime); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03100) "ap_queue_create() failed"); @@ -2235,7 +2209,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) */ max_recycled_pools = threads_per_child * 3 / 4 ; } - rv = ap_queue_info_create(&worker_queue_info, pchild, + rv = ap_queue_info_create(&worker_queue_info, pruntime, threads_per_child, max_recycled_pools); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03101) @@ -2247,7 +2221,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) * thread starts. */ rv = apr_thread_mutex_create(&timeout_mutex, APR_THREAD_MUTEX_DEFAULT, - pchild); + pruntime); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(03102) "creation of the timeout mutex failed."); @@ -2255,25 +2229,30 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) } /* Create the main pollset */ + pollset_flags = APR_POLLSET_THREADSAFE | APR_POLLSET_NOCOPY | + APR_POLLSET_NODEFAULT | APR_POLLSET_WAKEABLE; for (i = 0; i < sizeof(good_methods) / sizeof(good_methods[0]); i++) { - apr_uint32_t flags = APR_POLLSET_THREADSAFE | APR_POLLSET_NOCOPY | - APR_POLLSET_NODEFAULT | APR_POLLSET_WAKEABLE; - rv = apr_pollset_create_ex(&event_pollset, pollset_size, pchild, flags, - good_methods[i]); + rv = apr_pollset_create_ex(&event_pollset, pollset_size, pruntime, + pollset_flags, good_methods[i]); if (rv == APR_SUCCESS) { listener_is_wakeable = 1; break; } - flags &= ~APR_POLLSET_WAKEABLE; - rv = apr_pollset_create_ex(&event_pollset, pollset_size, pchild, flags, - good_methods[i]); - if (rv == APR_SUCCESS) { - break; + } + if (rv != APR_SUCCESS) { + pollset_flags &= ~APR_POLLSET_WAKEABLE; + for (i = 0; i < sizeof(good_methods) / sizeof(good_methods[0]); i++) { + rv = apr_pollset_create_ex(&event_pollset, pollset_size, pruntime, + pollset_flags, good_methods[i]); + if (rv == APR_SUCCESS) { + break; + } } } if (rv != APR_SUCCESS) { - rv = apr_pollset_create(&event_pollset, pollset_size, pchild, - APR_POLLSET_THREADSAFE | APR_POLLSET_NOCOPY); + pollset_flags &= ~APR_POLLSET_NODEFAULT; + rv = apr_pollset_create(&event_pollset, pollset_size, pruntime, + pollset_flags); } if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(03103) @@ -2281,12 +2260,57 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) clean_child_exit(APEXIT_CHILDFATAL); } + /* Add listeners to the main pollset */ + listener_pollfd = apr_pcalloc(pruntime, num_listensocks * + sizeof(apr_pollfd_t)); + for (i = 0, lr = my_bucket->listeners; lr; lr = lr->next, i++) { + apr_pollfd_t *pfd; + listener_poll_type *pt; + + AP_DEBUG_ASSERT(i < num_listensocks); + pfd = &listener_pollfd[i]; + + pfd->reqevents = APR_POLLIN; + pfd->desc_type = APR_POLL_SOCKET; + pfd->desc.s = lr->sd; + + pt = apr_pcalloc(pruntime, sizeof(*pt)); + pfd->client_data = pt; + pt->type = PT_ACCEPT; + pt->baton = lr; + + apr_socket_opt_set(pfd->desc.s, APR_SO_NONBLOCK, 1); + apr_pollset_add(event_pollset, pfd); + + lr->accept_func = ap_unixd_accept; + } + + worker_sockets = apr_pcalloc(pruntime, threads_per_child * + sizeof(apr_socket_t *)); +} + +/* XXX under some circumstances not understood, children can get stuck + * in start_threads forever trying to take over slots which will + * never be cleaned up; for now there is an APLOG_DEBUG message issued + * every so often when this condition occurs + */ +static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) +{ + thread_starter *ts = dummy; + apr_thread_t **threads = ts->threads; + apr_threadattr_t *thread_attr = ts->threadattr; + int my_child_num = ts->child_num_arg; + proc_info *my_info; + apr_status_t rv; + int threads_created = 0; + int listener_started = 0; + int prev_threads_created; + int loops, i; + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(02471) "start_threads: Using %s (%swakeable)", apr_pollset_method_name(event_pollset), listener_is_wakeable ? "" : "not "); - worker_sockets = apr_pcalloc(pchild, threads_per_child - * sizeof(apr_socket_t *)); loops = prev_threads_created = 0; while (1) { @@ -2310,7 +2334,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) * done because it lets us deal with tid better. */ rv = apr_thread_create(&threads[i], thread_attr, - worker_thread, my_info, pchild); + worker_thread, my_info, pruntime); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03104) @@ -2431,7 +2455,6 @@ static void child_main(int child_num_arg, int child_bucket) apr_threadattr_t *thread_attr; apr_thread_t *start_thread_id; int i; - apr_pool_t *pskip; /* for benefit of any hooks that run as this child initializes */ retained->mpm->mpm_state = AP_MPMQ_STARTING; @@ -2439,7 +2462,12 @@ static void child_main(int child_num_arg, int child_bucket) ap_my_pid = getpid(); ap_child_slot = child_num_arg; ap_fatal_signal_child_setup(ap_server_conf); + + /* Get a sub context for global allocations in this child, so that + * we can have cleanups occur when the child exits. + */ apr_pool_create(&pchild, pconf); + apr_pool_tag(pchild, "pchild"); /* close unused listeners and pods */ for (i = 0; i < retained->mpm->num_buckets; i++) { @@ -2457,12 +2485,6 @@ static void child_main(int child_num_arg, int child_bucket) clean_child_exit(APEXIT_CHILDFATAL); } - apr_thread_mutex_create(&g_timer_skiplist_mtx, APR_THREAD_MUTEX_DEFAULT, pchild); - APR_RING_INIT(&timer_free_ring, timer_event_t, link); - apr_pool_create(&pskip, pchild); - apr_skiplist_init(&timer_skiplist, pskip); - apr_skiplist_set_compare(timer_skiplist, timer_comp, timer_comp); - /* Just use the standard apr_setup_signal_thread to block all signals * from being received. The child processes no longer use signals for * any communication with the parent process. Let's also do this before @@ -2486,7 +2508,10 @@ static void child_main(int child_num_arg, int child_bucket) conns_this_child = APR_INT32_MAX; } - /* Setup worker threads */ + /* Setup threads */ + + /* Globals used by signal_threads() so to be initialized before */ + setup_threads_runtime(); /* clear the storage; we may not create all our threads immediately, * and we want a 0 entry to indicate a thread which was not created diff --git a/server/mpm/mpmt_os2/mpmt_os2_child.c b/server/mpm/mpmt_os2/mpmt_os2_child.c index ca9f594754..bb7e1369ea 100644 --- a/server/mpm/mpmt_os2/mpmt_os2_child.c +++ b/server/mpm/mpmt_os2/mpmt_os2_child.c @@ -110,6 +110,7 @@ void ap_mpm_child_main(apr_pool_t *pconf) /* Create pool for child */ apr_pool_create(&pchild, pconf); + apr_pool_tag(pchild, "pchild"); ap_run_child_init(pchild, ap_server_conf); diff --git a/server/mpm/netware/mpm_netware.c b/server/mpm/netware/mpm_netware.c index 2fab52f598..82480334b8 100644 --- a/server/mpm/netware/mpm_netware.c +++ b/server/mpm/netware/mpm_netware.c @@ -886,6 +886,7 @@ static int netware_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s) /* Only set slot 0 since that is all NetWare will ever have. */ ap_scoreboard_image->parent[0].pid = getpid(); + ap_scoreboard_image->parent[0].generation = ap_my_generation; ap_run_child_status(ap_server_conf, ap_scoreboard_image->parent[0].pid, ap_my_generation, diff --git a/server/mpm/prefork/prefork.c b/server/mpm/prefork/prefork.c index 3fb328467d..8efda72ee1 100644 --- a/server/mpm/prefork/prefork.c +++ b/server/mpm/prefork/prefork.c @@ -208,10 +208,10 @@ static void prefork_note_child_killed(int childnum, pid_t pid, static void prefork_note_child_started(int slot, pid_t pid) { + ap_generation_t gen = retained->mpm->my_generation; ap_scoreboard_image->parent[slot].pid = pid; - ap_run_child_status(ap_server_conf, - ap_scoreboard_image->parent[slot].pid, - retained->mpm->my_generation, slot, MPM_CHILD_STARTED); + ap_scoreboard_image->parent[slot].generation = gen; + ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED); } /* a clean exit from a child with proper cleanup */ diff --git a/server/mpm/winnt/child.c b/server/mpm/winnt/child.c index 9ddba18f80..21755f398c 100644 --- a/server/mpm/winnt/child.c +++ b/server/mpm/winnt/child.c @@ -917,6 +917,9 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid) int i; int num_events; + /* Get a sub context for global allocations in this child, so that + * we can have cleanups occur when the child exits. + */ apr_pool_create(&pchild, pconf); apr_pool_tag(pchild, "pchild"); diff --git a/server/mpm/winnt/mpm_winnt.c b/server/mpm/winnt/mpm_winnt.c index 2487e45be2..3d71f80e72 100644 --- a/server/mpm/winnt/mpm_winnt.c +++ b/server/mpm/winnt/mpm_winnt.c @@ -139,6 +139,7 @@ AP_INIT_TAKE1("ThreadLimit", set_thread_limit, NULL, RSRC_CONF, static void winnt_note_child_started(int slot, pid_t pid) { ap_scoreboard_image->parent[slot].pid = pid; + ap_scoreboard_image->parent[slot].generation = my_generation; ap_run_child_status(ap_server_conf, ap_scoreboard_image->parent[slot].pid, my_generation, slot, MPM_CHILD_STARTED); diff --git a/server/mpm/worker/worker.c b/server/mpm/worker/worker.c index 7804efc457..8012fe29d8 100644 --- a/server/mpm/worker/worker.c +++ b/server/mpm/worker/worker.c @@ -134,6 +134,8 @@ static int num_listensocks = 0; static int resource_shortage = 0; static fd_queue_t *worker_queue; static fd_queue_info_t *worker_queue_info; +static apr_pollset_t *worker_pollset; + /* data retained by worker across load/unload of the module * allocated on first call to pre-config hook; located on @@ -218,6 +220,7 @@ int raise_sigstop_flags; static apr_pool_t *pconf; /* Pool for config stuff */ static apr_pool_t *pchild; /* Pool for httpd child stuff */ +static apr_pool_t *pruntime; /* Pool for MPM threads stuff */ static pid_t ap_my_pid; /* Linux getpid() doesn't work except in main thread. Use this instead */ @@ -392,10 +395,10 @@ static void worker_note_child_killed(int childnum, pid_t pid, ap_generation_t ge static void worker_note_child_started(int slot, pid_t pid) { + ap_generation_t gen = retained->mpm->my_generation; ap_scoreboard_image->parent[slot].pid = pid; - ap_run_child_status(ap_server_conf, - ap_scoreboard_image->parent[slot].pid, - retained->mpm->my_generation, slot, MPM_CHILD_STARTED); + ap_scoreboard_image->parent[slot].generation = gen; + ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED); } static void worker_note_child_lost_slot(int slot, pid_t newpid) @@ -538,47 +541,15 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t *thd, void * dummy) { proc_info * ti = dummy; int process_slot = ti->pid; - apr_pool_t *tpool = apr_thread_pool_get(thd); void *csd = NULL; apr_pool_t *ptrans = NULL; /* Pool for per-transaction stuff */ - apr_pollset_t *pollset; apr_status_t rv; - ap_listen_rec *lr; + ap_listen_rec *lr = NULL; int have_idle_worker = 0; int last_poll_idx = 0; free(ti); - rv = apr_pollset_create(&pollset, num_listensocks, tpool, - APR_POLLSET_NOCOPY); - if (rv != APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, - "Couldn't create pollset in thread;" - " check system or user limits"); - /* let the parent decide how bad this really is */ - clean_child_exit(APEXIT_CHILDSICK); - } - - for (lr = my_bucket->listeners; lr != NULL; lr = lr->next) { - apr_pollfd_t *pfd = apr_pcalloc(tpool, sizeof *pfd); - - pfd->desc_type = APR_POLL_SOCKET; - pfd->desc.s = lr->sd; - pfd->reqevents = APR_POLLIN; - pfd->client_data = lr; - - rv = apr_pollset_add(pollset, pfd); - if (rv != APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, - "Couldn't create add listener to pollset;" - " check system or user limits"); - /* let the parent decide how bad this really is */ - clean_child_exit(APEXIT_CHILDSICK); - } - - lr->accept_func = ap_unixd_accept; - } - /* Unblock the signal used to wake this thread up, and set a handler for * it. */ @@ -630,7 +601,7 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t *thd, void * dummy) apr_int32_t numdesc; const apr_pollfd_t *pdesc; - rv = apr_pollset_poll(pollset, -1, &numdesc, &pdesc); + rv = apr_pollset_poll(worker_pollset, -1, &numdesc, &pdesc); if (rv != APR_SUCCESS) { if (APR_STATUS_IS_EINTR(rv)) { continue; @@ -871,7 +842,7 @@ static void create_listener_thread(thread_starter *ts) my_info->tid = -1; /* listener thread doesn't have a thread slot */ my_info->sd = 0; rv = apr_thread_create(&ts->listener, thread_attr, listener_thread, - my_info, pchild); + my_info, pruntime); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00275) "apr_thread_create: unable to create listener thread"); @@ -881,35 +852,34 @@ static void create_listener_thread(thread_starter *ts) apr_os_thread_get(&listener_os_thread, ts->listener); } -/* XXX under some circumstances not understood, children can get stuck - * in start_threads forever trying to take over slots which will - * never be cleaned up; for now there is an APLOG_DEBUG message issued - * every so often when this condition occurs - */ -static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) +static void setup_threads_runtime(void) { - thread_starter *ts = dummy; - apr_thread_t **threads = ts->threads; - apr_threadattr_t *thread_attr = ts->threadattr; - int my_child_num = ts->child_num_arg; - proc_info *my_info; + ap_listen_rec *lr; apr_status_t rv; - int i; - int threads_created = 0; - int listener_started = 0; - int loops; - int prev_threads_created; + + /* All threads (listener, workers) and synchronization objects (queues, + * pollset, mutexes...) created here should have at least the lifetime of + * the connections they handle (i.e. ptrans). We can't use this thread's + * self pool because all these objects survive it, nor use pchild or pconf + * directly because this starter thread races with other modules' runtime, + * nor finally pchild (or subpool thereof) because it is killed explicitely + * before pconf (thus connections/ptrans can live longer, which matters in + * ONE_PROCESS mode). So this leaves us with a subpool of pconf, created + * before any ptrans hence destroyed after. + */ + apr_pool_create(&pruntime, pconf); + apr_pool_tag(pruntime, "mpm_runtime"); /* We must create the fd queues before we start up the listener * and worker threads. */ - rv = ap_queue_create(&worker_queue, threads_per_child, pchild); + rv = ap_queue_create(&worker_queue, threads_per_child, pruntime); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03140) "ap_queue_create() failed"); clean_child_exit(APEXIT_CHILDFATAL); } - rv = ap_queue_info_create(&worker_queue_info, pchild, + rv = ap_queue_info_create(&worker_queue_info, pruntime, threads_per_child, -1); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03141) @@ -917,8 +887,58 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) clean_child_exit(APEXIT_CHILDFATAL); } - worker_sockets = apr_pcalloc(pchild, threads_per_child - * sizeof(apr_socket_t *)); + /* Create the main pollset */ + rv = apr_pollset_create(&worker_pollset, num_listensocks, pruntime, + APR_POLLSET_NOCOPY); + if (rv != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03285) + "Couldn't create pollset in thread;" + " check system or user limits"); + /* let the parent decide how bad this really is */ + clean_child_exit(APEXIT_CHILDSICK); + } + + for (lr = my_bucket->listeners; lr != NULL; lr = lr->next) { + apr_pollfd_t *pfd = apr_pcalloc(pruntime, sizeof *pfd); + + pfd->desc_type = APR_POLL_SOCKET; + pfd->desc.s = lr->sd; + pfd->reqevents = APR_POLLIN; + pfd->client_data = lr; + + rv = apr_pollset_add(worker_pollset, pfd); + if (rv != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03286) + "Couldn't create add listener to pollset;" + " check system or user limits"); + /* let the parent decide how bad this really is */ + clean_child_exit(APEXIT_CHILDSICK); + } + + lr->accept_func = ap_unixd_accept; + } + + worker_sockets = apr_pcalloc(pruntime, threads_per_child * + sizeof(apr_socket_t *)); +} + +/* XXX under some circumstances not understood, children can get stuck + * in start_threads forever trying to take over slots which will + * never be cleaned up; for now there is an APLOG_DEBUG message issued + * every so often when this condition occurs + */ +static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) +{ + thread_starter *ts = dummy; + apr_thread_t **threads = ts->threads; + apr_threadattr_t *thread_attr = ts->threadattr; + int my_child_num = ts->child_num_arg; + proc_info *my_info; + apr_status_t rv; + int threads_created = 0; + int listener_started = 0; + int prev_threads_created; + int loops, i; loops = prev_threads_created = 0; while (1) { @@ -942,7 +962,7 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) * done because it lets us deal with tid better. */ rv = apr_thread_create(&threads[i], thread_attr, - worker_thread, my_info, pchild); + worker_thread, my_info, pruntime); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03142) "apr_thread_create: unable to create worker thread"); @@ -1082,7 +1102,12 @@ static void child_main(int child_num_arg, int child_bucket) ap_my_pid = getpid(); ap_fatal_signal_child_setup(ap_server_conf); + + /* Get a sub context for global allocations in this child, so that + * we can have cleanups occur when the child exits. + */ apr_pool_create(&pchild, pconf); + apr_pool_tag(pchild, "pchild"); /* close unused listeners and pods */ for (i = 0; i < retained->mpm->num_buckets; i++) { @@ -1132,7 +1157,10 @@ static void child_main(int child_num_arg, int child_bucket) requests_this_child = INT_MAX; } - /* Setup worker threads */ + /* Setup threads */ + + /* Globals used by signal_threads() so to be initialized before */ + setup_threads_runtime(); /* clear the storage; we may not create all our threads immediately, * and we want a 0 entry to indicate a thread which was not created -- 2.19.1