/**
|
* \file
|
* The copy/mark and gray stack draining functions of the M&S major collector.
|
*
|
* Copyright (C) 2014 Xamarin Inc
|
*
|
* Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
*/
|
|
/*
|
* COPY_OR_MARK_FUNCTION_NAME must be defined to be the function name of the copy/mark
|
* function.
|
*
|
* SCAN_OBJECT_FUNCTION_NAME must be defined to be the function name of the object scanning
|
* function.
|
*
|
* DRAIN_GRAY_STACK_FUNCTION_NAME must be defined to be the function name of the gray stack
|
* draining function.
|
*
|
* Define COPY_OR_MARK_WITH_EVACUATION to support evacuation.
|
*/
|
|
/* Returns whether the object is still in the nursery. */
|
static inline MONO_ALWAYS_INLINE gboolean
|
COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
|
{
|
MSBlockInfo *block;
|
|
#ifdef HEAVY_STATISTICS
|
++stat_optimized_copy;
|
{
|
GCObject *forwarded;
|
SgenDescriptor desc;
|
if ((forwarded = SGEN_OBJECT_IS_FORWARDED (obj)))
|
desc = sgen_obj_get_descriptor_safe (forwarded);
|
else
|
desc = sgen_obj_get_descriptor_safe (obj);
|
|
sgen_descriptor_count_copied_object (desc);
|
}
|
#endif
|
|
SGEN_ASSERT (9, obj, "null object from pointer %p", ptr);
|
#if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
|
SGEN_ASSERT (9, current_collection_generation == GENERATION_OLD, "old gen parallel allocator called from a %d collection", current_collection_generation);
|
#endif
|
|
if (sgen_ptr_in_nursery (obj)) {
|
#if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
|
int word, bit;
|
gboolean first = TRUE;
|
GCObject *forwarded, *old_obj;
|
mword vtable_word = *(mword*)obj;
|
|
HEAVY_STAT (++stat_optimized_copy_nursery);
|
|
#if SGEN_MAX_DEBUG_LEVEL >= 9
|
if (sgen_nursery_is_to_space (obj))
|
SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word) && !SGEN_VTABLE_IS_FORWARDED (vtable_word), "To-space object can't be pinned or forwarded.");
|
#endif
|
|
if (SGEN_VTABLE_IS_PINNED (vtable_word)) {
|
SGEN_ASSERT (9, !SGEN_VTABLE_IS_FORWARDED (vtable_word), "Cannot be both pinned and forwarded.");
|
HEAVY_STAT (++stat_optimized_copy_nursery_pinned);
|
return TRUE;
|
}
|
if ((forwarded = (GCObject *)SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
|
HEAVY_STAT (++stat_optimized_copy_nursery_forwarded);
|
SGEN_UPDATE_REFERENCE (ptr, forwarded);
|
return sgen_ptr_in_nursery (forwarded);
|
}
|
|
/* An object in the nursery To Space has already been copied and grayed. Nothing to do. */
|
if (sgen_nursery_is_to_space (obj))
|
return TRUE;
|
|
#ifdef COPY_OR_MARK_WITH_EVACUATION
|
do_copy_object:
|
#endif
|
old_obj = obj;
|
#ifdef COPY_OR_MARK_PARALLEL
|
obj = copy_object_no_checks_par (obj, queue);
|
#else
|
obj = copy_object_no_checks (obj, queue);
|
#endif
|
if (G_UNLIKELY (old_obj == obj)) {
|
/*
|
* If we fail to evacuate an object we just stop doing it for a
|
* given block size as all other will surely fail too.
|
*/
|
/* FIXME: test this case somehow. */
|
if (!sgen_ptr_in_nursery (obj)) {
|
int size_index;
|
block = MS_BLOCK_FOR_OBJ (obj);
|
size_index = block->obj_size_index;
|
evacuate_block_obj_sizes [size_index] = FALSE;
|
MS_MARK_OBJECT_AND_ENQUEUE (obj, sgen_obj_get_descriptor (obj), block, queue);
|
return FALSE;
|
}
|
return TRUE;
|
}
|
HEAVY_STAT (++stat_objects_copied_major);
|
SGEN_UPDATE_REFERENCE (ptr, obj);
|
|
if (sgen_ptr_in_nursery (obj))
|
return TRUE;
|
|
/*
|
* FIXME: See comment for copy_object_no_checks(). If
|
* we have that, we can let the allocation function
|
* give us the block info, too, and we won't have to
|
* re-fetch it.
|
*
|
* FIXME (2): We should rework this to avoid all those nursery checks.
|
*/
|
/*
|
* For the split nursery allocator the object might
|
* still be in the nursery despite having being
|
* promoted, in which case we can't mark it.
|
*/
|
block = MS_BLOCK_FOR_OBJ (obj);
|
MS_CALC_MARK_BIT (word, bit, obj);
|
SGEN_ASSERT (9, !MS_MARK_BIT (block, word, bit), "object %p already marked", obj);
|
#ifdef COPY_OR_MARK_PARALLEL
|
MS_SET_MARK_BIT_PAR (block, word, bit, first);
|
#else
|
MS_SET_MARK_BIT (block, word, bit);
|
#endif
|
if (first)
|
binary_protocol_mark (obj, (gpointer)SGEN_LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
|
|
return FALSE;
|
#endif
|
} else {
|
mword vtable_word = *(mword*)obj;
|
SgenDescriptor desc;
|
int type;
|
|
HEAVY_STAT (++stat_optimized_copy_major);
|
|
#ifdef COPY_OR_MARK_WITH_EVACUATION
|
{
|
GCObject *forwarded;
|
if ((forwarded = (GCObject *)SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
|
HEAVY_STAT (++stat_optimized_copy_major_forwarded);
|
SGEN_UPDATE_REFERENCE (ptr, forwarded);
|
SGEN_ASSERT (9, !sgen_ptr_in_nursery (forwarded), "Cannot be forwarded to nursery.");
|
return FALSE;
|
}
|
}
|
#endif
|
|
SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word), "Pinned object in non-pinned block?");
|
|
/* We untag the vtable for concurrent M&S, in case bridge is running and it tagged it */
|
desc = sgen_vtable_get_descriptor ((GCVTable)SGEN_POINTER_UNTAG_VTABLE (vtable_word));
|
type = desc & DESC_TYPE_MASK;
|
|
if (sgen_safe_object_is_small (obj, type)) {
|
#ifdef HEAVY_STATISTICS
|
if (type <= DESC_TYPE_MAX_SMALL_OBJ)
|
++stat_optimized_copy_major_small_fast;
|
else
|
++stat_optimized_copy_major_small_slow;
|
#endif
|
|
block = MS_BLOCK_FOR_OBJ (obj);
|
|
#ifdef COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
|
if (G_UNLIKELY (major_block_is_evacuating (block))) {
|
/*
|
* We don't copy within the concurrent phase. These objects will
|
* be handled below in the finishing pause, by scanning the mod-union
|
* card table.
|
*/
|
return FALSE;
|
}
|
#endif
|
|
#ifdef COPY_OR_MARK_WITH_EVACUATION
|
if (major_block_is_evacuating (block)) {
|
HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
|
goto do_copy_object;
|
}
|
#endif
|
|
#ifdef COPY_OR_MARK_PARALLEL
|
MS_MARK_OBJECT_AND_ENQUEUE_PAR (obj, desc, block, queue);
|
#else
|
MS_MARK_OBJECT_AND_ENQUEUE (obj, desc, block, queue);
|
#endif
|
} else {
|
gboolean first = TRUE;
|
HEAVY_STAT (++stat_optimized_copy_major_large);
|
#ifdef COPY_OR_MARK_PARALLEL
|
first = sgen_los_pin_object_par (obj);
|
#else
|
if (sgen_los_object_is_pinned (obj))
|
first = FALSE;
|
else
|
sgen_los_pin_object (obj);
|
#endif
|
|
if (first) {
|
binary_protocol_pin (obj, (gpointer)SGEN_LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
|
if (SGEN_OBJECT_HAS_REFERENCES (obj))
|
#ifdef COPY_OR_MARK_PARALLEL
|
GRAY_OBJECT_ENQUEUE_PARALLEL (queue, obj, desc);
|
#else
|
GRAY_OBJECT_ENQUEUE_SERIAL (queue, obj, desc);
|
#endif
|
}
|
}
|
return FALSE;
|
}
|
|
return TRUE;
|
}
|
|
static void
|
SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQueue *queue)
|
{
|
char *start = (char*)full_object;
|
|
#ifdef HEAVY_STATISTICS
|
++stat_optimized_major_scan;
|
if (!sgen_gc_descr_has_references (desc))
|
++stat_optimized_major_scan_no_refs;
|
sgen_descriptor_count_scanned_object (desc);
|
#endif
|
#ifdef SGEN_HEAVY_BINARY_PROTOCOL
|
add_scanned_object (start);
|
#endif
|
|
/* Now scan the object. */
|
|
#undef HANDLE_PTR
|
#if defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
|
#define HANDLE_PTR(ptr,obj) do { \
|
GCObject *__old = *(ptr); \
|
binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
|
if (__old && !sgen_ptr_in_nursery (__old)) { \
|
if (G_UNLIKELY (full_object && !sgen_ptr_in_nursery (ptr) && \
|
sgen_safe_object_is_small (__old, sgen_obj_get_descriptor (__old) & DESC_TYPE_MASK) && \
|
major_block_is_evacuating (MS_BLOCK_FOR_OBJ (__old)))) { \
|
mark_mod_union_card ((full_object), (void**)(ptr), __old); \
|
} else { \
|
PREFETCH_READ (__old); \
|
COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
|
} \
|
} else { \
|
if (G_UNLIKELY (full_object && sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)) && !sgen_cement_is_forced (__old))) \
|
mark_mod_union_card ((full_object), (void**)(ptr), __old); \
|
} \
|
} while (0)
|
#elif defined(COPY_OR_MARK_CONCURRENT)
|
#define HANDLE_PTR(ptr,obj) do { \
|
GCObject *__old = *(ptr); \
|
binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
|
if (__old && !sgen_ptr_in_nursery (__old)) { \
|
PREFETCH_READ (__old); \
|
COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
|
} else { \
|
if (G_UNLIKELY (full_object && sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)) && !sgen_cement_is_forced (__old))) \
|
mark_mod_union_card ((full_object), (void**)(ptr), __old); \
|
} \
|
} while (0)
|
#else
|
#define HANDLE_PTR(ptr,obj) do { \
|
GCObject *__old = *(ptr); \
|
binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
|
if (__old) { \
|
gboolean __still_in_nursery = COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
|
if (G_UNLIKELY (__still_in_nursery && !sgen_ptr_in_nursery ((ptr)) && !SGEN_OBJECT_IS_CEMENTED (*(ptr)))) { \
|
GCObject *__copy = *(ptr); \
|
sgen_add_to_global_remset ((ptr), __copy); \
|
} \
|
} \
|
} while (0)
|
#endif
|
|
#define SCAN_OBJECT_PROTOCOL
|
#include "sgen-scan-object.h"
|
}
|
|
#ifdef SCAN_VTYPE_FUNCTION_NAME
|
static void
|
SCAN_VTYPE_FUNCTION_NAME (GCObject *full_object, char *start, SgenDescriptor desc, SgenGrayQueue *queue BINARY_PROTOCOL_ARG (size_t size))
|
{
|
SGEN_OBJECT_LAYOUT_STATISTICS_DECLARE_BITMAP;
|
|
#ifdef HEAVY_STATISTICS
|
/* FIXME: We're half scanning this object. How do we account for that? */
|
//add_scanned_object (start);
|
#endif
|
|
/* The descriptors include info about the object header as well */
|
start -= SGEN_CLIENT_OBJECT_HEADER_SIZE;
|
|
/* We use the same HANDLE_PTR from the obj scan function */
|
#define SCAN_OBJECT_NOVTABLE
|
#define SCAN_OBJECT_PROTOCOL
|
#include "sgen-scan-object.h"
|
|
SGEN_OBJECT_LAYOUT_STATISTICS_COMMIT_BITMAP;
|
}
|
#endif
|
|
#ifdef SCAN_PTR_FIELD_FUNCTION_NAME
|
static void
|
SCAN_PTR_FIELD_FUNCTION_NAME (GCObject *full_object, GCObject **ptr, SgenGrayQueue *queue)
|
{
|
/*
|
* full_object is NULL if we scan unmanaged memory. This means we can't mark
|
* mod unions for it, so these types of roots currently don't have support
|
* for the concurrent collector (aka they need to be scanned as normal roots
|
* both in the start and finishing pause)
|
*/
|
HANDLE_PTR (ptr, NULL);
|
}
|
#endif
|
|
static gboolean
|
DRAIN_GRAY_STACK_FUNCTION_NAME (SgenGrayQueue *queue)
|
{
|
#if defined(COPY_OR_MARK_CONCURRENT) || defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION) || defined(COPY_OR_MARK_PARALLEL)
|
int i;
|
for (i = 0; i < 32; i++) {
|
#else
|
for (;;) {
|
#endif
|
GCObject *obj;
|
SgenDescriptor desc;
|
|
HEAVY_STAT (++stat_drain_loops);
|
|
#if defined(COPY_OR_MARK_PARALLEL)
|
GRAY_OBJECT_DEQUEUE_PARALLEL (queue, &obj, &desc);
|
#else
|
GRAY_OBJECT_DEQUEUE_SERIAL (queue, &obj, &desc);
|
#endif
|
if (!obj)
|
return TRUE;
|
|
SCAN_OBJECT_FUNCTION_NAME (obj, desc, queue);
|
}
|
return FALSE;
|
}
|
|
#undef COPY_OR_MARK_PARALLEL
|
#undef COPY_OR_MARK_FUNCTION_NAME
|
#undef COPY_OR_MARK_WITH_EVACUATION
|
#undef COPY_OR_MARK_CONCURRENT
|
#undef COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
|
#undef SCAN_OBJECT_FUNCTION_NAME
|
#undef SCAN_VTYPE_FUNCTION_NAME
|
#undef SCAN_PTR_FIELD_FUNCTION_NAME
|
#undef DRAIN_GRAY_STACK_FUNCTION_NAME
|
