From cca279886b3bbb4d3af5768880565c9f7b372c08 Mon Sep 17 00:00:00 2001 From: ray Date: Mon, 23 Mar 2020 15:27:37 -0700 Subject: [PATCH] Fix Bug 702181: SEGV when BufferSpace is at or above -K alloc limit The issue with this file is that one of the pages is done in page mode because it was so small, but the 'resize' call when doing page 2 was not checking the allocation request against the limit, so the malloc would succeed, but the very next (and all subsequent) allocations would fail. The gdev_prn_allocate would capture the ecode, but would still call the clist_enable_multi_thread_render resulting in the SEGV. Add the check in the gs_heap_resize_object function, and make sure and leave the 'monitor' after failure, then don't call clist_enable_multi_thread if the resize failed. --- base/gdevprn.c | 14 ++++++++------ base/gsmalloc.c | 11 ++++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/base/gdevprn.c b/base/gdevprn.c index c365509..ed89cd7 100644 --- a/base/gdevprn.c +++ b/base/gdevprn.c @@ -520,13 +520,15 @@ gdev_prn_allocate(gx_device *pdev, gdev_prn_space_params *new_space_params, if (ecode == 0) ecode = code; - if ( code >= 0 || (reallocate && pass > 1) ) + if (code >= 0 || (reallocate && pass > 1)) ppdev->procs = gs_clist_device_procs; - /* - * Now the device is a clist device, we enable multi-threaded rendering. - * It will remain enabled, but that doesn't really cause any problems. - */ - clist_enable_multi_thread_render(pdev); + if (code > 0) { + /* + * Now the device is a clist device, we enable multi-threaded rendering. + * It will remain enabled, but that doesn't really cause any problems. + */ + clist_enable_multi_thread_render(pdev); + } } else { /* Render entirely in memory. */ gx_device *bdev = (gx_device *)pmemdev; diff --git a/base/gsmalloc.c b/base/gsmalloc.c index 14979e8..2fbccab 100644 --- a/base/gsmalloc.c +++ b/base/gsmalloc.c @@ -279,9 +279,18 @@ gs_heap_resize_object(gs_memory_t * mem, void *obj, size_t new_num_elements, return obj; if (mmem->monitor) gx_monitor_enter(mmem->monitor); /* Exclusive access */ + if (new_size > mmem->limit - sizeof(gs_malloc_block_t)) { + /* too large to allocate; also avoids overflow. */ + if (mmem->monitor) + gx_monitor_leave(mmem->monitor); /* Done with exclusive access */ + return 0; + } new_ptr = (gs_malloc_block_t *) gs_realloc(ptr, old_size, new_size); - if (new_ptr == 0) + if (new_ptr == 0) { + if (mmem->monitor) + gx_monitor_leave(mmem->monitor); /* Done with exclusive access */ return 0; + } if (new_ptr->prev) new_ptr->prev->next = new_ptr; else -- 1.8.3.1