Gitly

v / thirdparty / libgc
Raw file | 31367 loc (31366 sloc) | 968.61 KB | Latest commit hash 775c4c34b

1 /*
2  * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
3  * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
4  * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
5  * Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
6  * All rights reserved.
7  * Copyright (c) 2009-2018 Ivan Maidanski
8  *
9  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
10  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
11  *
12  * Permission is hereby granted to use or copy this program
13  * for any purpose,  provided the above notices are retained on all copies.
14  * Permission to modify the code and to distribute modified code is granted,
15  * provided the above notices are retained, and a notice that the code was
16  * modified is included with the above copyright notice.
17  */
18 
19 #ifndef __cplusplus
20 #define GC_INNER STATIC
21 #define GC_EXTERN GC_INNER
22 #endif
23 #ifndef GC_DBG_MLC_H
24 #define GC_DBG_MLC_H
25 #ifndef GC_PRIVATE_H
26 #define GC_PRIVATE_H
27 #ifdef HAVE_CONFIG_H
28 #include "config.h"
29 #endif
30 #if !defined(GC_BUILD) && !defined(NOT_GCBUILD)
31 #define GC_BUILD
32 #endif
33 #if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__) \
34      || (defined(__CYGWIN__) && (defined(GC_THREADS) || !defined(USE_MMAP)))) \
35     && !defined(_GNU_SOURCE)
36 #define _GNU_SOURCE 1
37 #endif
38 #if defined(__INTERIX) && !defined(_ALL_SOURCE)
39 #define _ALL_SOURCE 1
40 #endif
41 #if (defined(DGUX) && defined(GC_THREADS) || defined(DGUX386_THREADS) \
42      || defined(GC_DGUX386_THREADS)) && !defined(_USING_POSIX4A_DRAFT10)
43 #define _USING_POSIX4A_DRAFT10 1
44 #endif
45 #if defined(__MINGW32__) && !defined(__MINGW_EXCPT_DEFINE_PSDK) \
46     && defined(__i386__) && defined(GC_EXTERN)
47 #define __MINGW_EXCPT_DEFINE_PSDK 1
48 #endif
49 #if defined(NO_DEBUGGING) && !defined(GC_ASSERTIONS) && !defined(NDEBUG)
50 #define NDEBUG 1
51 #endif
52 #ifndef GC_H
53 #ifndef GC_H
54 #define GC_H
55 #if (defined(WIN64) && !defined(_WIN64)) && defined(_MSC_VER)
56 #pragma message("Warning: Expecting _WIN64 for x64 targets! Notice the leading underscore!")
57 #endif
58 #if defined(GC_H)
59 #define GC_TMP_VERSION_MAJOR 8
60 #define GC_TMP_VERSION_MINOR 2
61 #define GC_TMP_VERSION_MICRO 0
62 #ifdef GC_VERSION_MAJOR
63 #if GC_TMP_VERSION_MAJOR != GC_VERSION_MAJOR \
64      || GC_TMP_VERSION_MINOR != GC_VERSION_MINOR \
65      || GC_TMP_VERSION_MICRO != GC_VERSION_MICRO
66 #error Inconsistent version info.  Check README.md, include/gc_version.h and configure.ac.
67 #endif
68 #else
69 #define GC_VERSION_MAJOR GC_TMP_VERSION_MAJOR
70 #define GC_VERSION_MINOR GC_TMP_VERSION_MINOR
71 #define GC_VERSION_MICRO GC_TMP_VERSION_MICRO
72 #endif
73 #endif
74 #if defined(GC_H)
75 #if defined(__GNUC__) && defined(__GNUC_MINOR__)
76 #define GC_GNUC_PREREQ(major, minor) \
77             ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((major) << 16) + (minor))
78 #else
79 #define GC_GNUC_PREREQ(major, minor) 0
80 #endif
81 #if defined(SOLARIS_THREADS) || defined(_SOLARIS_THREADS) \
82     || defined(_SOLARIS_PTHREADS) || defined(GC_SOLARIS_PTHREADS)
83 #ifndef GC_SOLARIS_THREADS
84 #define GC_SOLARIS_THREADS
85 #endif
86 #endif
87 #if defined(IRIX_THREADS)
88 #define GC_IRIX_THREADS
89 #endif
90 #if defined(DGUX_THREADS) && !defined(GC_DGUX386_THREADS)
91 #define GC_DGUX386_THREADS
92 #endif
93 #if defined(AIX_THREADS)
94 #define GC_AIX_THREADS
95 #endif
96 #if defined(HPUX_THREADS)
97 #define GC_HPUX_THREADS
98 #endif
99 #if defined(OSF1_THREADS)
100 #define GC_OSF1_THREADS
101 #endif
102 #if defined(LINUX_THREADS)
103 #define GC_LINUX_THREADS
104 #endif
105 #if defined(WIN32_THREADS)
106 #define GC_WIN32_THREADS
107 #endif
108 #if defined(RTEMS_THREADS)
109 #define GC_RTEMS_PTHREADS
110 #endif
111 #if defined(USE_LD_WRAP)
112 #define GC_USE_LD_WRAP
113 #endif
114 #if defined(GC_WIN32_PTHREADS) && !defined(GC_WIN32_THREADS)
115 #define GC_WIN32_THREADS
116 #endif
117 #if defined(GC_AIX_THREADS) || defined(GC_DARWIN_THREADS) \
118     || defined(GC_DGUX386_THREADS) || defined(GC_FREEBSD_THREADS) \
119     || defined(GC_HPUX_THREADS) \
120     || defined(GC_IRIX_THREADS) || defined(GC_LINUX_THREADS) \
121     || defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS) \
122     || defined(GC_OSF1_THREADS) || defined(GC_SOLARIS_THREADS) \
123     || defined(GC_WIN32_THREADS) || defined(GC_RTEMS_PTHREADS)
124 #ifndef GC_THREADS
125 #define GC_THREADS
126 #endif
127 #elif defined(GC_THREADS)
128 #if defined(__linux__)
129 #define GC_LINUX_THREADS
130 #elif defined(__OpenBSD__)
131 #define GC_OPENBSD_THREADS
132 #elif defined(_PA_RISC1_1) || defined(_PA_RISC2_0) || defined(hppa) \
133        || defined(__HPPA) || (defined(__ia64) && defined(_HPUX_SOURCE))
134 #define GC_HPUX_THREADS
135 #elif defined(__HAIKU__)
136 #define GC_HAIKU_THREADS
137 #elif (defined(__DragonFly__) || defined(__FreeBSD_kernel__) \
138         || defined(__FreeBSD__)) && !defined(GC_NO_FREEBSD)
139 #define GC_FREEBSD_THREADS
140 #elif defined(__NetBSD__)
141 #define GC_NETBSD_THREADS
142 #elif defined(__alpha) || defined(__alpha__)
143 #define GC_OSF1_THREADS
144 #elif (defined(mips) || defined(__mips) || defined(_mips)) \
145         && !(defined(nec_ews) || defined(_nec_ews) \
146              || defined(ultrix) || defined(__ultrix))
147 #define GC_IRIX_THREADS
148 #elif defined(__sparc)  \
149        || ((defined(sun) || defined(__sun)) \
150            && (defined(i386) || defined(__i386__) \
151                || defined(__amd64) || defined(__amd64__)))
152 #define GC_SOLARIS_THREADS
153 #elif defined(__APPLE__) && defined(__MACH__)
154 #define GC_DARWIN_THREADS
155 #endif
156 #if defined(DGUX) && (defined(i386) || defined(__i386__))
157 #define GC_DGUX386_THREADS
158 #endif
159 #if defined(_AIX)
160 #define GC_AIX_THREADS
161 #endif
162 #if (defined(_WIN32) || defined(_MSC_VER) || defined(__BORLANDC__) \
163       || defined(__CYGWIN32__) || defined(__CYGWIN__) || defined(__CEGCC__) \
164       || defined(_WIN32_WCE) || defined(__MINGW32__)) \
165      && !defined(GC_WIN32_THREADS)
166 #define GC_WIN32_THREADS
167 #endif
168 #if defined(__rtems__) && (defined(i386) || defined(__i386__))
169 #define GC_RTEMS_PTHREADS
170 #endif
171 #endif
172 #undef GC_PTHREADS
173 #if (!defined(GC_WIN32_THREADS) || defined(GC_WIN32_PTHREADS) \
174      || defined(__CYGWIN32__) || defined(__CYGWIN__)) && defined(GC_THREADS) \
175     && !defined(NN_PLATFORM_CTR) && !defined(NN_BUILD_TARGET_PLATFORM_NX)
176 #define GC_PTHREADS
177 #endif
178 #if !defined(_PTHREADS) && defined(GC_NETBSD_THREADS)
179 #define _PTHREADS
180 #endif
181 #if defined(GC_DGUX386_THREADS) && !defined(_POSIX4A_DRAFT10_SOURCE)
182 #define _POSIX4A_DRAFT10_SOURCE 1
183 #endif
184 #if !defined(_REENTRANT) && defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)
185 #define _REENTRANT 1
186 #endif
187 #define __GC
188 #if !defined(_WIN32_WCE) || defined(__GNUC__)
189 #include <stddef.h>
190 #if defined(__MINGW32__) && !defined(_WIN32_WCE)
191 #include <stdint.h>
192 #endif
193 #else
194 #include <stdlib.h>
195 #ifndef _PTRDIFF_T_DEFINED
196 #define _PTRDIFF_T_DEFINED
197     typedef long ptrdiff_t;
198 #endif
199 #endif
200 #if !defined(GC_NOT_DLL) && !defined(GC_DLL) \
201     && ((defined(_DLL) && !defined(__GNUC__)) \
202         || (defined(DLL_EXPORT) && defined(GC_BUILD)))
203 #define GC_DLL
204 #endif
205 #if defined(GC_DLL) && !defined(GC_API)
206 #if defined(__CEGCC__)
207 #if defined(GC_BUILD)
208 #define GC_API __declspec(dllexport)
209 #else
210 #define GC_API __declspec(dllimport)
211 #endif
212 #elif defined(__MINGW32__)
213 #if defined(__cplusplus) && defined(GC_BUILD)
214 #define GC_API extern __declspec(dllexport)
215 #elif defined(GC_BUILD) || defined(__MINGW32_DELAY_LOAD__)
216 #define GC_API __declspec(dllexport)
217 #else
218 #define GC_API extern __declspec(dllimport)
219 #endif
220 #elif defined(_MSC_VER) || defined(__DMC__) || defined(__BORLANDC__) \
221         || defined(__CYGWIN__)
222 #ifdef GC_BUILD
223 #define GC_API extern __declspec(dllexport)
224 #else
225 #define GC_API __declspec(dllimport)
226 #endif
227 #elif defined(__WATCOMC__)
228 #ifdef GC_BUILD
229 #define GC_API extern __declspec(dllexport)
230 #else
231 #define GC_API extern __declspec(dllimport)
232 #endif
233 #elif defined(__SYMBIAN32__)
234 #ifdef GC_BUILD
235 #define GC_API extern EXPORT_C
236 #else
237 #define GC_API extern IMPORT_C
238 #endif
239 #elif defined(__GNUC__)
240 #if defined(GC_BUILD) && !defined(GC_NO_VISIBILITY) \
241             && (GC_GNUC_PREREQ(4, 0) || defined(GC_VISIBILITY_HIDDEN_SET))
242 #define GC_API extern __attribute__((__visibility__("default")))
243 #endif
244 #endif
245 #endif
246 #ifndef GC_API
247 #define GC_API extern
248 #endif
249 #ifndef GC_CALL
250 #define GC_CALL
251 #endif
252 #ifndef GC_CALLBACK
253 #define GC_CALLBACK GC_CALL
254 #endif
255 #ifndef GC_ATTR_MALLOC
256 #ifdef GC_OOM_FUNC_RETURNS_ALIAS
257 #define GC_ATTR_MALLOC
258 #elif GC_GNUC_PREREQ(3, 1)
259 #define GC_ATTR_MALLOC __attribute__((__malloc__))
260 #elif defined(_MSC_VER) && (_MSC_VER >= 1900) && !defined(__EDG__)
261 #define GC_ATTR_MALLOC \
262                 __declspec(allocator) __declspec(noalias) __declspec(restrict)
263 #elif defined(_MSC_VER) && _MSC_VER >= 1400
264 #define GC_ATTR_MALLOC __declspec(noalias) __declspec(restrict)
265 #else
266 #define GC_ATTR_MALLOC
267 #endif
268 #endif
269 #ifndef GC_ATTR_ALLOC_SIZE
270 #undef GC_ATTR_CALLOC_SIZE
271 #ifdef __clang__
272 #if __has_attribute(__alloc_size__)
273 #define GC_ATTR_ALLOC_SIZE(argnum) __attribute__((__alloc_size__(argnum)))
274 #define GC_ATTR_CALLOC_SIZE(n, s) __attribute__((__alloc_size__(n, s)))
275 #else
276 #define GC_ATTR_ALLOC_SIZE(argnum)
277 #endif
278 #elif GC_GNUC_PREREQ(4, 3) && !defined(__ICC)
279 #define GC_ATTR_ALLOC_SIZE(argnum) __attribute__((__alloc_size__(argnum)))
280 #define GC_ATTR_CALLOC_SIZE(n, s) __attribute__((__alloc_size__(n, s)))
281 #else
282 #define GC_ATTR_ALLOC_SIZE(argnum)
283 #endif
284 #endif
285 #ifndef GC_ATTR_CALLOC_SIZE
286 #define GC_ATTR_CALLOC_SIZE(n, s)
287 #endif
288 #ifndef GC_ATTR_NONNULL
289 #if GC_GNUC_PREREQ(4, 0)
290 #define GC_ATTR_NONNULL(argnum) __attribute__((__nonnull__(argnum)))
291 #else
292 #define GC_ATTR_NONNULL(argnum)
293 #endif
294 #endif
295 #ifndef GC_ATTR_CONST
296 #if GC_GNUC_PREREQ(4, 0)
297 #define GC_ATTR_CONST __attribute__((__const__))
298 #else
299 #define GC_ATTR_CONST
300 #endif
301 #endif
302 #ifndef GC_ATTR_DEPRECATED
303 #ifdef GC_BUILD
304 #undef GC_ATTR_DEPRECATED
305 #define GC_ATTR_DEPRECATED
306 #elif GC_GNUC_PREREQ(4, 0)
307 #define GC_ATTR_DEPRECATED __attribute__((__deprecated__))
308 #elif defined(_MSC_VER) && _MSC_VER >= 1200
309 #define GC_ATTR_DEPRECATED __declspec(deprecated)
310 #else
311 #define GC_ATTR_DEPRECATED
312 #endif
313 #endif
314 #if defined(__sgi) && !defined(__GNUC__) && _COMPILER_VERSION >= 720
315 #define GC_ADD_CALLER
316 #define GC_RETURN_ADDR (GC_word)__return_address
317 #endif
318 #if defined(__linux__) || defined(__GLIBC__)
319 #if !defined(__native_client__)
320 #include <features.h>
321 #endif
322 #if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1 || __GLIBC__ > 2) \
323         && !defined(__ia64__) \
324         && !defined(GC_MISSING_EXECINFO_H) \
325         && !defined(GC_HAVE_BUILTIN_BACKTRACE)
326 #define GC_HAVE_BUILTIN_BACKTRACE
327 #endif
328 #if defined(__i386__) || defined(__amd64__) || defined(__x86_64__)
329 #define GC_CAN_SAVE_CALL_STACKS
330 #endif
331 #endif
332 #if defined(_MSC_VER) && _MSC_VER >= 1200  \
333         && !defined(_AMD64_) && !defined(_M_X64) && !defined(_WIN32_WCE) \
334         && !defined(GC_HAVE_NO_BUILTIN_BACKTRACE) \
335         && !defined(GC_HAVE_BUILTIN_BACKTRACE)
336 #define GC_HAVE_BUILTIN_BACKTRACE
337 #endif
338 #if defined(GC_HAVE_BUILTIN_BACKTRACE) && !defined(GC_CAN_SAVE_CALL_STACKS)
339 #define GC_CAN_SAVE_CALL_STACKS
340 #endif
341 #if defined(__sparc__)
342 #define GC_CAN_SAVE_CALL_STACKS
343 #endif
344 #if (defined(__linux__) || defined(__DragonFly__) || defined(__FreeBSD__) \
345      || defined(__FreeBSD_kernel__) || defined(__HAIKU__) \
346      || defined(__NetBSD__) || defined(__OpenBSD__) \
347      || defined(HOST_ANDROID) || defined(__ANDROID__)) \
348     && !defined(GC_CAN_SAVE_CALL_STACKS)
349 #define GC_ADD_CALLER
350 #if GC_GNUC_PREREQ(2, 95)
351 #define GC_RETURN_ADDR (GC_word)__builtin_return_address(0)
352 #if GC_GNUC_PREREQ(4, 0) && (defined(__i386__) || defined(__amd64__) \
353                         || defined(__x86_64__) ) \
354        && !defined(GC_NO_RETURN_ADDR_PARENT)
355 #define GC_HAVE_RETURN_ADDR_PARENT
356 #define GC_RETURN_ADDR_PARENT \
357         (GC_word)__builtin_extract_return_addr(__builtin_return_address(1))
358 #endif
359 #else
360 #define GC_RETURN_ADDR 0
361 #endif
362 #endif
363 #ifdef GC_PTHREADS
364 #if (defined(GC_DARWIN_THREADS) || defined(GC_WIN32_PTHREADS) \
365       || defined(__native_client__) || defined(GC_RTEMS_PTHREADS)) \
366       && !defined(GC_NO_DLOPEN)
367 #define GC_NO_DLOPEN
368 #endif
369 #if (defined(GC_DARWIN_THREADS) || defined(GC_WIN32_PTHREADS) \
370       || defined(GC_OPENBSD_THREADS) || defined(__native_client__)) \
371      && !defined(GC_NO_PTHREAD_SIGMASK)
372 #define GC_NO_PTHREAD_SIGMASK
373 #endif
374 #if defined(__native_client__)
375 #ifndef GC_PTHREAD_CREATE_CONST
376 #define GC_PTHREAD_CREATE_CONST
377 #endif
378 #ifndef GC_HAVE_PTHREAD_EXIT
379 #define GC_HAVE_PTHREAD_EXIT
380 #define GC_PTHREAD_EXIT_ATTRIBUTE
381 #endif
382 #endif
383 #if !defined(GC_HAVE_PTHREAD_EXIT) \
384      && !defined(HOST_ANDROID) && !defined(__ANDROID__) \
385      && (defined(GC_LINUX_THREADS) || defined(GC_SOLARIS_THREADS))
386 #define GC_HAVE_PTHREAD_EXIT
387 #if GC_GNUC_PREREQ(2, 7)
388 #define GC_PTHREAD_EXIT_ATTRIBUTE __attribute__((__noreturn__))
389 #elif defined(__NORETURN)
390 #define GC_PTHREAD_EXIT_ATTRIBUTE __NORETURN
391 #else
392 #define GC_PTHREAD_EXIT_ATTRIBUTE
393 #endif
394 #endif
395 #if (!defined(GC_HAVE_PTHREAD_EXIT) || defined(__native_client__)) \
396      && !defined(GC_NO_PTHREAD_CANCEL)
397 #define GC_NO_PTHREAD_CANCEL
398 #endif
399 #endif
400 #ifdef __cplusplus
401 #ifndef GC_ATTR_EXPLICIT
402 #if __cplusplus >= 201103L && !defined(__clang__) || _MSVC_LANG >= 201103L \
403      || defined(CPPCHECK)
404 #define GC_ATTR_EXPLICIT explicit
405 #else
406 #define GC_ATTR_EXPLICIT
407 #endif
408 #endif
409 #ifndef GC_NOEXCEPT
410 #if defined(__DMC__) || (defined(__BORLANDC__) \
411         && (defined(_RWSTD_NO_EXCEPTIONS) || defined(_RWSTD_NO_EX_SPEC))) \
412      || (defined(_MSC_VER) && defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) \
413      || (defined(__WATCOMC__) && !defined(_CPPUNWIND))
414 #define GC_NOEXCEPT
415 #ifndef GC_NEW_ABORTS_ON_OOM
416 #define GC_NEW_ABORTS_ON_OOM
417 #endif
418 #elif __cplusplus >= 201103L || _MSVC_LANG >= 201103L
419 #define GC_NOEXCEPT noexcept
420 #else
421 #define GC_NOEXCEPT throw()
422 #endif
423 #endif
424 #endif
425 #endif
426 #ifdef __cplusplus
427   extern "C" {
428 #endif
429 typedef void * GC_PTR;
430 #ifdef _WIN64
431 #if defined(__int64) && !defined(CPPCHECK)
432     typedef unsigned __int64 GC_word;
433     typedef __int64 GC_signed_word;
434 #else
435     typedef unsigned long long GC_word;
436     typedef long long GC_signed_word;
437 #endif
438 #else
439   typedef unsigned long GC_word;
440   typedef long GC_signed_word;
441 #endif
442 GC_API unsigned GC_CALL GC_get_version(void);
443 GC_API GC_ATTR_DEPRECATED GC_word GC_gc_no;
444 GC_API GC_word GC_CALL GC_get_gc_no(void);
445 #ifdef GC_THREADS
446   GC_API GC_ATTR_DEPRECATED int GC_parallel;
447   GC_API int GC_CALL GC_get_parallel(void);
448   GC_API void GC_CALL GC_set_markers_count(unsigned);
449 #endif
450 typedef void * (GC_CALLBACK * GC_oom_func)(size_t );
451 GC_API GC_ATTR_DEPRECATED GC_oom_func GC_oom_fn;
452 GC_API void GC_CALL GC_set_oom_fn(GC_oom_func) GC_ATTR_NONNULL(1);
453 GC_API GC_oom_func GC_CALL GC_get_oom_fn(void);
454 typedef void (GC_CALLBACK * GC_on_heap_resize_proc)(GC_word );
455 GC_API GC_ATTR_DEPRECATED GC_on_heap_resize_proc GC_on_heap_resize;
456 GC_API void GC_CALL GC_set_on_heap_resize(GC_on_heap_resize_proc);
457 GC_API GC_on_heap_resize_proc GC_CALL GC_get_on_heap_resize(void);
458 typedef enum {
459     GC_EVENT_START ,
460     GC_EVENT_MARK_START,
461     GC_EVENT_MARK_END,
462     GC_EVENT_RECLAIM_START,
463     GC_EVENT_RECLAIM_END,
464     GC_EVENT_END ,
465     GC_EVENT_PRE_STOP_WORLD ,
466     GC_EVENT_POST_STOP_WORLD ,
467     GC_EVENT_PRE_START_WORLD ,
468     GC_EVENT_POST_START_WORLD ,
469     GC_EVENT_THREAD_SUSPENDED,
470     GC_EVENT_THREAD_UNSUSPENDED
471 } GC_EventType;
472 typedef void (GC_CALLBACK * GC_on_collection_event_proc)(GC_EventType);
473 GC_API void GC_CALL GC_set_on_collection_event(GC_on_collection_event_proc);
474 GC_API GC_on_collection_event_proc GC_CALL GC_get_on_collection_event(void);
475 #if defined(GC_THREADS) || (defined(GC_BUILD) && defined(NN_PLATFORM_CTR))
476   typedef void (GC_CALLBACK * GC_on_thread_event_proc)(GC_EventType,
477                                                 void * );
478   GC_API void GC_CALL GC_set_on_thread_event(GC_on_thread_event_proc);
479   GC_API GC_on_thread_event_proc GC_CALL GC_get_on_thread_event(void);
480 #endif
481 GC_API GC_ATTR_DEPRECATED int GC_find_leak;
482 GC_API void GC_CALL GC_set_find_leak(int);
483 GC_API int GC_CALL GC_get_find_leak(void);
484 GC_API GC_ATTR_DEPRECATED int GC_all_interior_pointers;
485 GC_API void GC_CALL GC_set_all_interior_pointers(int);
486 GC_API int GC_CALL GC_get_all_interior_pointers(void);
487 GC_API GC_ATTR_DEPRECATED int GC_finalize_on_demand;
488 GC_API void GC_CALL GC_set_finalize_on_demand(int);
489 GC_API int GC_CALL GC_get_finalize_on_demand(void);
490 GC_API GC_ATTR_DEPRECATED int GC_java_finalization;
491 GC_API void GC_CALL GC_set_java_finalization(int);
492 GC_API int GC_CALL GC_get_java_finalization(void);
493 typedef void (GC_CALLBACK * GC_finalizer_notifier_proc)(void);
494 GC_API GC_ATTR_DEPRECATED GC_finalizer_notifier_proc GC_finalizer_notifier;
495 GC_API void GC_CALL GC_set_finalizer_notifier(GC_finalizer_notifier_proc);
496 GC_API GC_finalizer_notifier_proc GC_CALL GC_get_finalizer_notifier(void);
497 GC_API
498 #ifndef GC_DONT_GC
499     GC_ATTR_DEPRECATED
500 #endif
501   int GC_dont_gc;
502 GC_API GC_ATTR_DEPRECATED int GC_dont_expand;
503 GC_API void GC_CALL GC_set_dont_expand(int);
504 GC_API int GC_CALL GC_get_dont_expand(void);
505 GC_API GC_ATTR_DEPRECATED int GC_use_entire_heap;
506 GC_API GC_ATTR_DEPRECATED int GC_full_freq;
507 GC_API void GC_CALL GC_set_full_freq(int);
508 GC_API int GC_CALL GC_get_full_freq(void);
509 GC_API GC_ATTR_DEPRECATED GC_word GC_non_gc_bytes;
510 GC_API void GC_CALL GC_set_non_gc_bytes(GC_word);
511 GC_API GC_word GC_CALL GC_get_non_gc_bytes(void);
512 GC_API GC_ATTR_DEPRECATED int GC_no_dls;
513 GC_API void GC_CALL GC_set_no_dls(int);
514 GC_API int GC_CALL GC_get_no_dls(void);
515 GC_API GC_ATTR_DEPRECATED GC_word GC_free_space_divisor;
516 GC_API void GC_CALL GC_set_free_space_divisor(GC_word);
517 GC_API GC_word GC_CALL GC_get_free_space_divisor(void);
518 GC_API GC_ATTR_DEPRECATED GC_word GC_max_retries;
519 GC_API void GC_CALL GC_set_max_retries(GC_word);
520 GC_API GC_word GC_CALL GC_get_max_retries(void);
521 GC_API GC_ATTR_DEPRECATED char *GC_stackbottom;
522 GC_API GC_ATTR_DEPRECATED int GC_dont_precollect;
523 GC_API void GC_CALL GC_set_dont_precollect(int);
524 GC_API int GC_CALL GC_get_dont_precollect(void);
525 GC_API GC_ATTR_DEPRECATED unsigned long GC_time_limit;
526 #define GC_TIME_UNLIMITED 999999
527 GC_API void GC_CALL GC_set_time_limit(unsigned long);
528 GC_API unsigned long GC_CALL GC_get_time_limit(void);
529 struct GC_timeval_s {
530   unsigned long tv_ms;
531   unsigned long tv_nsec;
532 };
533 GC_API void GC_CALL GC_set_time_limit_tv(struct GC_timeval_s);
534 GC_API struct GC_timeval_s GC_CALL GC_get_time_limit_tv(void);
535 GC_API void GC_CALL GC_set_allocd_bytes_per_finalizer(GC_word);
536 GC_API GC_word GC_CALL GC_get_allocd_bytes_per_finalizer(void);
537 GC_API void GC_CALL GC_start_performance_measurement(void);
538 GC_API unsigned long GC_CALL GC_get_full_gc_total_time(void);
539 GC_API void GC_CALL GC_set_pages_executable(int);
540 GC_API int GC_CALL GC_get_pages_executable(void);
541 GC_API void GC_CALL GC_set_min_bytes_allocd(size_t);
542 GC_API size_t GC_CALL GC_get_min_bytes_allocd(void);
543 GC_API void GC_CALL GC_set_rate(int);
544 GC_API int GC_CALL GC_get_rate(void);
545 GC_API void GC_CALL GC_set_max_prior_attempts(int);
546 GC_API int GC_CALL GC_get_max_prior_attempts(void);
547 GC_API void GC_CALL GC_set_disable_automatic_collection(int);
548 GC_API int GC_CALL GC_get_disable_automatic_collection(void);
549 GC_API void GC_CALL GC_set_handle_fork(int);
550 GC_API void GC_CALL GC_atfork_prepare(void);
551 GC_API void GC_CALL GC_atfork_parent(void);
552 GC_API void GC_CALL GC_atfork_child(void);
553 GC_API void GC_CALL GC_init(void);
554 GC_API int GC_CALL GC_is_init_called(void);
555 GC_API void GC_CALL GC_deinit(void);
556 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
557         GC_malloc(size_t );
558 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
559         GC_malloc_atomic(size_t );
560 GC_API GC_ATTR_MALLOC char * GC_CALL GC_strdup(const char *);
561 GC_API GC_ATTR_MALLOC char * GC_CALL
562         GC_strndup(const char *, size_t) GC_ATTR_NONNULL(1);
563 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
564         GC_malloc_uncollectable(size_t );
565 GC_API GC_ATTR_DEPRECATED void * GC_CALL GC_malloc_stubborn(size_t);
566 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(2) void * GC_CALL
567         GC_memalign(size_t , size_t );
568 GC_API int GC_CALL GC_posix_memalign(void ** , size_t ,
569                         size_t ) GC_ATTR_NONNULL(1);
570 GC_API void GC_CALL GC_free(void *);
571 #define GC_MALLOC_STUBBORN(sz)  GC_MALLOC(sz)
572 #define GC_NEW_STUBBORN(t)      GC_NEW(t)
573 #define GC_CHANGE_STUBBORN(p)   GC_change_stubborn(p)
574 GC_API GC_ATTR_DEPRECATED void GC_CALL GC_change_stubborn(const void *);
575 GC_API void GC_CALL GC_end_stubborn_change(const void *) GC_ATTR_NONNULL(1);
576 GC_API void * GC_CALL GC_base(void * );
577 GC_API int GC_CALL GC_is_heap_ptr(const void *);
578 GC_API size_t GC_CALL GC_size(const void * ) GC_ATTR_NONNULL(1);
579 GC_API void * GC_CALL GC_realloc(void * ,
580                                  size_t )
581                          GC_ATTR_ALLOC_SIZE(2);
582 GC_API int GC_CALL GC_expand_hp(size_t );
583 GC_API void GC_CALL GC_set_max_heap_size(GC_word );
584 GC_API void GC_CALL GC_exclude_static_roots(void * ,
585                                             void * );
586 GC_API void GC_CALL GC_clear_exclusion_table(void);
587 GC_API void GC_CALL GC_clear_roots(void);
588 GC_API void GC_CALL GC_add_roots(void * ,
589                                  void * );
590 GC_API void GC_CALL GC_remove_roots(void * ,
591                                     void * );
592 GC_API void GC_CALL GC_register_displacement(size_t );
593 GC_API void GC_CALL GC_debug_register_displacement(size_t );
594 GC_API void GC_CALL GC_gcollect(void);
595 GC_API void GC_CALL GC_gcollect_and_unmap(void);
596 typedef int (GC_CALLBACK * GC_stop_func)(void);
597 GC_API int GC_CALL GC_try_to_collect(GC_stop_func )
598                                                         GC_ATTR_NONNULL(1);
599 GC_API void GC_CALL GC_set_stop_func(GC_stop_func )
600                                                         GC_ATTR_NONNULL(1);
601 GC_API GC_stop_func GC_CALL GC_get_stop_func(void);
602 GC_API size_t GC_CALL GC_get_heap_size(void);
603 GC_API size_t GC_CALL GC_get_free_bytes(void);
604 GC_API size_t GC_CALL GC_get_unmapped_bytes(void);
605 GC_API size_t GC_CALL GC_get_bytes_since_gc(void);
606 GC_API size_t GC_CALL GC_get_expl_freed_bytes_since_gc(void);
607 GC_API size_t GC_CALL GC_get_total_bytes(void);
608 GC_API size_t GC_CALL GC_get_obtained_from_os_bytes(void);
609 GC_API void GC_CALL GC_get_heap_usage_safe(GC_word * ,
610                                            GC_word * ,
611                                            GC_word * ,
612                                            GC_word * ,
613                                            GC_word * );
614 struct GC_prof_stats_s {
615   GC_word heapsize_full;
616   GC_word free_bytes_full;
617   GC_word unmapped_bytes;
618   GC_word bytes_allocd_since_gc;
619   GC_word allocd_bytes_before_gc;
620   GC_word non_gc_bytes;
621   GC_word gc_no;
622   GC_word markers_m1;
623   GC_word bytes_reclaimed_since_gc;
624   GC_word reclaimed_bytes_before_gc;
625   GC_word expl_freed_bytes_since_gc;
626   GC_word obtained_from_os_bytes;
627 };
628 GC_API size_t GC_CALL GC_get_prof_stats(struct GC_prof_stats_s *,
629                                         size_t );
630 #ifdef GC_THREADS
631   GC_API size_t GC_CALL GC_get_prof_stats_unsafe(struct GC_prof_stats_s *,
632                                                  size_t );
633 #endif
634 GC_API size_t GC_CALL GC_get_size_map_at(int i);
635 GC_API size_t GC_CALL GC_get_memory_use(void);
636 GC_API void GC_CALL GC_disable(void);
637 GC_API int GC_CALL GC_is_disabled(void);
638 GC_API void GC_CALL GC_enable(void);
639 GC_API void GC_CALL GC_set_manual_vdb_allowed(int);
640 GC_API int GC_CALL GC_get_manual_vdb_allowed(void);
641 GC_API void GC_CALL GC_enable_incremental(void);
642 GC_API int GC_CALL GC_is_incremental_mode(void);
643 #define GC_PROTECTS_POINTER_HEAP  1
644 #define GC_PROTECTS_PTRFREE_HEAP  2
645 #define GC_PROTECTS_STATIC_DATA   4
646 #define GC_PROTECTS_STACK         8
647 #define GC_PROTECTS_NONE 0
648 GC_API int GC_CALL GC_incremental_protection_needs(void);
649 GC_API void GC_CALL GC_start_incremental_collection(void);
650 GC_API int GC_CALL GC_collect_a_little(void);
651 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
652         GC_malloc_ignore_off_page(size_t );
653 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
654         GC_malloc_atomic_ignore_off_page(size_t );
655 #ifdef GC_ADD_CALLER
656 #define GC_EXTRAS GC_RETURN_ADDR, __FILE__, __LINE__
657 #define GC_EXTRA_PARAMS GC_word ra, const char * s, int i
658 #else
659 #define GC_EXTRAS __FILE__, __LINE__
660 #define GC_EXTRA_PARAMS const char * s, int i
661 #endif
662 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
663         GC_malloc_atomic_uncollectable(size_t );
664 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
665         GC_debug_malloc_atomic_uncollectable(size_t, GC_EXTRA_PARAMS);
666 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
667         GC_debug_malloc(size_t , GC_EXTRA_PARAMS);
668 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
669         GC_debug_malloc_atomic(size_t , GC_EXTRA_PARAMS);
670 GC_API GC_ATTR_MALLOC char * GC_CALL
671         GC_debug_strdup(const char *, GC_EXTRA_PARAMS);
672 GC_API GC_ATTR_MALLOC char * GC_CALL
673         GC_debug_strndup(const char *, size_t, GC_EXTRA_PARAMS)
674                                                         GC_ATTR_NONNULL(1);
675 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
676         GC_debug_malloc_uncollectable(size_t ,
677                                       GC_EXTRA_PARAMS);
678 GC_API GC_ATTR_DEPRECATED void * GC_CALL
679         GC_debug_malloc_stubborn(size_t , GC_EXTRA_PARAMS);
680 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
681         GC_debug_malloc_ignore_off_page(size_t ,
682                                         GC_EXTRA_PARAMS);
683 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
684         GC_debug_malloc_atomic_ignore_off_page(size_t ,
685                                         GC_EXTRA_PARAMS);
686 GC_API void GC_CALL GC_debug_free(void *);
687 GC_API void * GC_CALL GC_debug_realloc(void * ,
688                         size_t , GC_EXTRA_PARAMS)
689                          GC_ATTR_ALLOC_SIZE(2);
690 GC_API GC_ATTR_DEPRECATED void GC_CALL GC_debug_change_stubborn(const void *);
691 GC_API void GC_CALL GC_debug_end_stubborn_change(const void *)
692                                                         GC_ATTR_NONNULL(1);
693 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
694         GC_debug_malloc_replacement(size_t );
695 GC_API  GC_ATTR_ALLOC_SIZE(2) void * GC_CALL
696         GC_debug_realloc_replacement(void * ,
697                                      size_t );
698 #ifdef GC_DEBUG_REPLACEMENT
699 #define GC_MALLOC(sz) GC_debug_malloc_replacement(sz)
700 #define GC_REALLOC(old, sz) GC_debug_realloc_replacement(old, sz)
701 #elif defined(GC_DEBUG)
702 #define GC_MALLOC(sz) GC_debug_malloc(sz, GC_EXTRAS)
703 #define GC_REALLOC(old, sz) GC_debug_realloc(old, sz, GC_EXTRAS)
704 #else
705 #define GC_MALLOC(sz) GC_malloc(sz)
706 #define GC_REALLOC(old, sz) GC_realloc(old, sz)
707 #endif
708 #ifdef GC_DEBUG
709 #define GC_MALLOC_ATOMIC(sz) GC_debug_malloc_atomic(sz, GC_EXTRAS)
710 #define GC_STRDUP(s) GC_debug_strdup(s, GC_EXTRAS)
711 #define GC_STRNDUP(s, sz) GC_debug_strndup(s, sz, GC_EXTRAS)
712 #define GC_MALLOC_ATOMIC_UNCOLLECTABLE(sz) \
713                         GC_debug_malloc_atomic_uncollectable(sz, GC_EXTRAS)
714 #define GC_MALLOC_UNCOLLECTABLE(sz) \
715                         GC_debug_malloc_uncollectable(sz, GC_EXTRAS)
716 #define GC_MALLOC_IGNORE_OFF_PAGE(sz) \
717                         GC_debug_malloc_ignore_off_page(sz, GC_EXTRAS)
718 #define GC_MALLOC_ATOMIC_IGNORE_OFF_PAGE(sz) \
719                         GC_debug_malloc_atomic_ignore_off_page(sz, GC_EXTRAS)
720 #define GC_FREE(p) GC_debug_free(p)
721 #define GC_REGISTER_FINALIZER(p, f, d, of, od) \
722       GC_debug_register_finalizer(p, f, d, of, od)
723 #define GC_REGISTER_FINALIZER_IGNORE_SELF(p, f, d, of, od) \
724       GC_debug_register_finalizer_ignore_self(p, f, d, of, od)
725 #define GC_REGISTER_FINALIZER_NO_ORDER(p, f, d, of, od) \
726       GC_debug_register_finalizer_no_order(p, f, d, of, od)
727 #define GC_REGISTER_FINALIZER_UNREACHABLE(p, f, d, of, od) \
728       GC_debug_register_finalizer_unreachable(p, f, d, of, od)
729 #define GC_END_STUBBORN_CHANGE(p) GC_debug_end_stubborn_change(p)
730 #define GC_PTR_STORE_AND_DIRTY(p, q) GC_debug_ptr_store_and_dirty(p, q)
731 #define GC_GENERAL_REGISTER_DISAPPEARING_LINK(link, obj) \
732       GC_general_register_disappearing_link(link, \
733                                         GC_base(( void *)(obj)))
734 #define GC_REGISTER_LONG_LINK(link, obj) \
735       GC_register_long_link(link, GC_base(( void *)(obj)))
736 #define GC_REGISTER_DISPLACEMENT(n) GC_debug_register_displacement(n)
737 #else
738 #define GC_MALLOC_ATOMIC(sz) GC_malloc_atomic(sz)
739 #define GC_STRDUP(s) GC_strdup(s)
740 #define GC_STRNDUP(s, sz) GC_strndup(s, sz)
741 #define GC_MALLOC_ATOMIC_UNCOLLECTABLE(sz) GC_malloc_atomic_uncollectable(sz)
742 #define GC_MALLOC_UNCOLLECTABLE(sz) GC_malloc_uncollectable(sz)
743 #define GC_MALLOC_IGNORE_OFF_PAGE(sz) \
744                         GC_malloc_ignore_off_page(sz)
745 #define GC_MALLOC_ATOMIC_IGNORE_OFF_PAGE(sz) \
746                         GC_malloc_atomic_ignore_off_page(sz)
747 #define GC_FREE(p) GC_free(p)
748 #define GC_REGISTER_FINALIZER(p, f, d, of, od) \
749       GC_register_finalizer(p, f, d, of, od)
750 #define GC_REGISTER_FINALIZER_IGNORE_SELF(p, f, d, of, od) \
751       GC_register_finalizer_ignore_self(p, f, d, of, od)
752 #define GC_REGISTER_FINALIZER_NO_ORDER(p, f, d, of, od) \
753       GC_register_finalizer_no_order(p, f, d, of, od)
754 #define GC_REGISTER_FINALIZER_UNREACHABLE(p, f, d, of, od) \
755       GC_register_finalizer_unreachable(p, f, d, of, od)
756 #define GC_END_STUBBORN_CHANGE(p) GC_end_stubborn_change(p)
757 #define GC_PTR_STORE_AND_DIRTY(p, q) GC_ptr_store_and_dirty(p, q)
758 #define GC_GENERAL_REGISTER_DISAPPEARING_LINK(link, obj) \
759       GC_general_register_disappearing_link(link, obj)
760 #define GC_REGISTER_LONG_LINK(link, obj) \
761       GC_register_long_link(link, obj)
762 #define GC_REGISTER_DISPLACEMENT(n) GC_register_displacement(n)
763 #endif
764 #define GC_NEW(t)               ((t*)GC_MALLOC(sizeof(t)))
765 #define GC_NEW_ATOMIC(t)        ((t*)GC_MALLOC_ATOMIC(sizeof(t)))
766 #define GC_NEW_UNCOLLECTABLE(t) ((t*)GC_MALLOC_UNCOLLECTABLE(sizeof(t)))
767 #ifdef GC_REQUIRE_WCSDUP
768   GC_API GC_ATTR_MALLOC wchar_t * GC_CALL
769         GC_wcsdup(const wchar_t *) GC_ATTR_NONNULL(1);
770   GC_API GC_ATTR_MALLOC wchar_t * GC_CALL
771         GC_debug_wcsdup(const wchar_t *, GC_EXTRA_PARAMS) GC_ATTR_NONNULL(1);
772 #ifdef GC_DEBUG
773 #define GC_WCSDUP(s) GC_debug_wcsdup(s, GC_EXTRAS)
774 #else
775 #define GC_WCSDUP(s) GC_wcsdup(s)
776 #endif
777 #endif
778 typedef void (GC_CALLBACK * GC_finalization_proc)(void * ,
779                                                   void * );
780 GC_API void GC_CALL GC_register_finalizer(void * ,
781                         GC_finalization_proc , void * ,
782                         GC_finalization_proc * , void ** )
783                                                 GC_ATTR_NONNULL(1);
784 GC_API void GC_CALL GC_debug_register_finalizer(void * ,
785                         GC_finalization_proc , void * ,
786                         GC_finalization_proc * , void ** )
787                                                 GC_ATTR_NONNULL(1);
788 GC_API void GC_CALL GC_register_finalizer_ignore_self(void * ,
789                         GC_finalization_proc , void * ,
790                         GC_finalization_proc * , void ** )
791                                                 GC_ATTR_NONNULL(1);
792 GC_API void GC_CALL GC_debug_register_finalizer_ignore_self(void * ,
793                         GC_finalization_proc , void * ,
794                         GC_finalization_proc * , void ** )
795                                                 GC_ATTR_NONNULL(1);
796 GC_API void GC_CALL GC_register_finalizer_no_order(void * ,
797                         GC_finalization_proc , void * ,
798                         GC_finalization_proc * , void ** )
799                                                 GC_ATTR_NONNULL(1);
800 GC_API void GC_CALL GC_debug_register_finalizer_no_order(void * ,
801                         GC_finalization_proc , void * ,
802                         GC_finalization_proc * , void ** )
803                                                 GC_ATTR_NONNULL(1);
804 GC_API void GC_CALL GC_register_finalizer_unreachable(void * ,
805                         GC_finalization_proc , void * ,
806                         GC_finalization_proc * , void ** )
807                                                 GC_ATTR_NONNULL(1);
808 GC_API void GC_CALL GC_debug_register_finalizer_unreachable(void * ,
809                         GC_finalization_proc , void * ,
810                         GC_finalization_proc * , void ** )
811                                                 GC_ATTR_NONNULL(1);
812 #define GC_NO_MEMORY 2
813 GC_API int GC_CALL GC_register_disappearing_link(void ** )
814                                                 GC_ATTR_NONNULL(1);
815 GC_API int GC_CALL GC_general_register_disappearing_link(void ** ,
816                                                     const void * )
817                         GC_ATTR_NONNULL(1) GC_ATTR_NONNULL(2);
818 GC_API int GC_CALL GC_move_disappearing_link(void ** ,
819                                              void ** )
820                         GC_ATTR_NONNULL(2);
821 GC_API int GC_CALL GC_unregister_disappearing_link(void ** );
822 GC_API int GC_CALL GC_register_long_link(void ** ,
823                                     const void * )
824                         GC_ATTR_NONNULL(1) GC_ATTR_NONNULL(2);
825 GC_API int GC_CALL GC_move_long_link(void ** ,
826                                      void ** )
827                         GC_ATTR_NONNULL(2);
828 GC_API int GC_CALL GC_unregister_long_link(void ** );
829 typedef enum {
830    GC_TOGGLE_REF_DROP,
831    GC_TOGGLE_REF_STRONG,
832    GC_TOGGLE_REF_WEAK
833 } GC_ToggleRefStatus;
834 typedef GC_ToggleRefStatus (GC_CALLBACK *GC_toggleref_func)(void * );
835 GC_API void GC_CALL GC_set_toggleref_func(GC_toggleref_func);
836 GC_API GC_toggleref_func GC_CALL GC_get_toggleref_func(void);
837 GC_API int GC_CALL GC_toggleref_add(void * , int )
838                                                 GC_ATTR_NONNULL(1);
839 typedef void (GC_CALLBACK * GC_await_finalize_proc)(void * );
840 GC_API void GC_CALL GC_set_await_finalize_proc(GC_await_finalize_proc);
841 GC_API GC_await_finalize_proc GC_CALL GC_get_await_finalize_proc(void);
842 GC_API int GC_CALL GC_should_invoke_finalizers(void);
843 GC_API int GC_CALL GC_invoke_finalizers(void);
844 #if defined(__GNUC__) && !defined(__INTEL_COMPILER)
845 #define GC_reachable_here(ptr) \
846                 __asm__ __volatile__(" " : : "X"(ptr) : "memory")
847 #else
848   GC_API void GC_CALL GC_noop1(GC_word);
849 #ifdef LINT2
850 #define GC_reachable_here(ptr) GC_noop1(~(GC_word)(ptr)^(~(GC_word)0))
851 #else
852 #define GC_reachable_here(ptr) GC_noop1((GC_word)(ptr))
853 #endif
854 #endif
855 typedef void (GC_CALLBACK * GC_warn_proc)(char * ,
856                                           GC_word );
857 GC_API void GC_CALL GC_set_warn_proc(GC_warn_proc ) GC_ATTR_NONNULL(1);
858 GC_API GC_warn_proc GC_CALL GC_get_warn_proc(void);
859 GC_API void GC_CALLBACK GC_ignore_warn_proc(char *, GC_word);
860 GC_API void GC_CALL GC_set_log_fd(int);
861 typedef void (GC_CALLBACK * GC_abort_func)(const char * );
862 GC_API void GC_CALL GC_set_abort_func(GC_abort_func) GC_ATTR_NONNULL(1);
863 GC_API GC_abort_func GC_CALL GC_get_abort_func(void);
864 GC_API void GC_CALL GC_abort_on_oom(void);
865 typedef GC_word GC_hidden_pointer;
866 #define GC_HIDE_POINTER(p) (~(GC_hidden_pointer)(p))
867 #define GC_REVEAL_POINTER(p) ((void *)GC_HIDE_POINTER(p))
868 #if defined(I_HIDE_POINTERS) || defined(GC_I_HIDE_POINTERS)
869 #define HIDE_POINTER(p) GC_HIDE_POINTER(p)
870 #define REVEAL_POINTER(p) GC_REVEAL_POINTER(p)
871 #endif
872 #ifdef GC_THREADS
873   GC_API void GC_CALL GC_alloc_lock(void);
874   GC_API void GC_CALL GC_alloc_unlock(void);
875 #else
876 #define GC_alloc_lock() (void)0
877 #define GC_alloc_unlock() (void)0
878 #endif
879 typedef void * (GC_CALLBACK * GC_fn_type)(void * );
880 GC_API void * GC_CALL GC_call_with_alloc_lock(GC_fn_type ,
881                                 void * ) GC_ATTR_NONNULL(1);
882 struct GC_stack_base {
883   void * mem_base;
884 #if defined(__ia64) || defined(__ia64__) || defined(_M_IA64)
885     void * reg_base;
886 #endif
887 };
888 typedef void * (GC_CALLBACK * GC_stack_base_func)(
889                 struct GC_stack_base * , void * );
890 GC_API void * GC_CALL GC_call_with_stack_base(GC_stack_base_func ,
891                                         void * ) GC_ATTR_NONNULL(1);
892 #define GC_SUCCESS 0
893 #define GC_DUPLICATE 1
894 #define GC_NO_THREADS 2
895 #define GC_UNIMPLEMENTED 3
896 #define GC_NOT_FOUND 4
897 #if defined(GC_DARWIN_THREADS) || defined(GC_WIN32_THREADS)
898   GC_API void GC_CALL GC_use_threads_discovery(void);
899 #endif
900 #ifdef GC_THREADS
901   GC_API void GC_CALL GC_set_suspend_signal(int);
902   GC_API void GC_CALL GC_set_thr_restart_signal(int);
903   GC_API int GC_CALL GC_get_suspend_signal(void);
904   GC_API int GC_CALL GC_get_thr_restart_signal(void);
905   GC_API void GC_CALL GC_start_mark_threads(void);
906   GC_API void GC_CALL GC_allow_register_threads(void);
907   GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base *)
908                                                         GC_ATTR_NONNULL(1);
909   GC_API int GC_CALL GC_thread_is_registered(void);
910   GC_API void GC_CALL GC_register_altstack(void * ,
911                                            GC_word ,
912                                            void * ,
913                                            GC_word );
914   GC_API int GC_CALL GC_unregister_my_thread(void);
915   GC_API void GC_CALL GC_stop_world_external(void);
916   GC_API void GC_CALL GC_start_world_external(void);
917 #endif
918 GC_API void * GC_CALL GC_do_blocking(GC_fn_type ,
919                                 void * ) GC_ATTR_NONNULL(1);
920 GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type ,
921                                 void * ) GC_ATTR_NONNULL(1);
922 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *)
923                                                         GC_ATTR_NONNULL(1);
924 GC_API void * GC_CALL GC_get_my_stackbottom(struct GC_stack_base *)
925                                                         GC_ATTR_NONNULL(1);
926 GC_API void GC_CALL GC_set_stackbottom(void * ,
927                                        const struct GC_stack_base *)
928                                                         GC_ATTR_NONNULL(2);
929 GC_API void * GC_CALL GC_same_obj(void * , void * );
930 GC_API void * GC_CALL GC_pre_incr(void **, ptrdiff_t )
931                                                         GC_ATTR_NONNULL(1);
932 GC_API void * GC_CALL GC_post_incr(void **, ptrdiff_t )
933                                                         GC_ATTR_NONNULL(1);
934 GC_API void * GC_CALL GC_is_visible(void * );
935 GC_API void * GC_CALL GC_is_valid_displacement(void * );
936 GC_API void GC_CALL GC_dump(void);
937 GC_API void GC_CALL GC_dump_named(const char * );
938 GC_API void GC_CALL GC_dump_regions(void);
939 GC_API void GC_CALL GC_dump_finalization(void);
940 #if defined(GC_DEBUG) && defined(__GNUC__)
941 #define GC_PTR_ADD3(x, n, type_of_result) \
942         ((type_of_result)GC_same_obj((x)+(n), (x)))
943 #define GC_PRE_INCR3(x, n, type_of_result) \
944         ((type_of_result)GC_pre_incr((void **)(&(x)), (n)*sizeof(*x)))
945 #define GC_POST_INCR3(x, n, type_of_result) \
946         ((type_of_result)GC_post_incr((void **)(&(x)), (n)*sizeof(*x)))
947 #define GC_PTR_ADD(x, n) GC_PTR_ADD3(x, n, __typeof__(x))
948 #define GC_PRE_INCR(x, n) GC_PRE_INCR3(x, n, __typeof__(x))
949 #define GC_POST_INCR(x) GC_POST_INCR3(x, 1, __typeof__(x))
950 #define GC_POST_DECR(x) GC_POST_INCR3(x, -1, __typeof__(x))
951 #else
952 #define GC_PTR_ADD(x, n) ((x)+(n))
953 #define GC_PRE_INCR(x, n) ((x) += (n))
954 #define GC_POST_INCR(x) ((x)++)
955 #define GC_POST_DECR(x) ((x)--)
956 #endif
957 #ifdef GC_DEBUG
958 #define GC_PTR_STORE(p, q) \
959         (*(void **)GC_is_visible((void *)(p)) = \
960                     GC_is_valid_displacement((void *)(q)))
961 #else
962 #define GC_PTR_STORE(p, q) (*(void **)(p) = (void *)(q))
963 #endif
964 GC_API void GC_CALL GC_ptr_store_and_dirty(void * ,
965                                            const void * );
966 GC_API void GC_CALL GC_debug_ptr_store_and_dirty(void * ,
967                                                  const void * );
968 GC_API void (GC_CALLBACK * GC_same_obj_print_proc)(void * ,
969                                                    void * );
970 GC_API void (GC_CALLBACK * GC_is_valid_displacement_print_proc)(void *);
971 GC_API void (GC_CALLBACK * GC_is_visible_print_proc)(void *);
972 #ifdef GC_PTHREADS
973 #ifdef __cplusplus
974     }
975 #endif
976 #ifndef GC_PTHREAD_REDIRECTS_H
977 #define GC_PTHREAD_REDIRECTS_H
978 #if defined(GC_H) && defined(GC_PTHREADS)
979 #ifndef GC_PTHREAD_REDIRECTS_ONLY
980 #include <pthread.h>
981 #ifndef GC_NO_DLOPEN
982 #include <dlfcn.h>
983 #endif
984 #ifndef GC_NO_PTHREAD_SIGMASK
985 #include <signal.h>
986 #endif
987 #ifdef __cplusplus
988     extern "C" {
989 #endif
990 #ifndef GC_SUSPEND_THREAD_ID
991 #define GC_SUSPEND_THREAD_ID pthread_t
992 #endif
993 #ifndef GC_NO_DLOPEN
994     GC_API void *GC_dlopen(const char * , int );
995 #endif
996 #ifndef GC_NO_PTHREAD_SIGMASK
997 #if defined(GC_PTHREAD_SIGMASK_NEEDED) \
998         || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) \
999         || (_POSIX_C_SOURCE >= 199506L) || (_XOPEN_SOURCE >= 500)
1000       GC_API int GC_pthread_sigmask(int , const sigset_t *,
1001                                     sigset_t * );
1002 #endif
1003 #endif
1004 #ifndef GC_PTHREAD_CREATE_CONST
1005 #define GC_PTHREAD_CREATE_CONST const
1006 #endif
1007   GC_API int GC_pthread_create(pthread_t *,
1008                                GC_PTHREAD_CREATE_CONST pthread_attr_t *,
1009                                void *(*)(void *), void * );
1010   GC_API int GC_pthread_join(pthread_t, void ** );
1011   GC_API int GC_pthread_detach(pthread_t);
1012 #ifndef GC_NO_PTHREAD_CANCEL
1013     GC_API int GC_pthread_cancel(pthread_t);
1014 #endif
1015 #if defined(GC_HAVE_PTHREAD_EXIT) && !defined(GC_PTHREAD_EXIT_DECLARED)
1016 #define GC_PTHREAD_EXIT_DECLARED
1017     GC_API void GC_pthread_exit(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
1018 #endif
1019 #ifdef __cplusplus
1020     }
1021 #endif
1022 #endif
1023 #if !defined(GC_NO_THREAD_REDIRECTS) && !defined(GC_USE_LD_WRAP)
1024 #undef pthread_create
1025 #undef pthread_join
1026 #undef pthread_detach
1027 #define pthread_create GC_pthread_create
1028 #define pthread_join GC_pthread_join
1029 #define pthread_detach GC_pthread_detach
1030 #ifndef GC_NO_PTHREAD_SIGMASK
1031 #undef pthread_sigmask
1032 #define pthread_sigmask GC_pthread_sigmask
1033 #endif
1034 #ifndef GC_NO_DLOPEN
1035 #undef dlopen
1036 #define dlopen GC_dlopen
1037 #endif
1038 #ifndef GC_NO_PTHREAD_CANCEL
1039 #undef pthread_cancel
1040 #define pthread_cancel GC_pthread_cancel
1041 #endif
1042 #ifdef GC_HAVE_PTHREAD_EXIT
1043 #undef pthread_exit
1044 #define pthread_exit GC_pthread_exit
1045 #endif
1046 #endif
1047 #endif
1048 #endif
1049 #ifdef __cplusplus
1050     extern "C" {
1051 #endif
1052 #endif
1053 GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_many(size_t );
1054 #define GC_NEXT(p) (*(void * *)(p))
1055 typedef int (GC_CALLBACK * GC_has_static_roots_func)(
1056                                         const char * ,
1057                                         void * ,
1058                                         size_t );
1059 GC_API void GC_CALL GC_register_has_static_roots_callback(
1060                                         GC_has_static_roots_func);
1061 #if !defined(CPPCHECK) && !defined(GC_WINDOWS_H_INCLUDED) && defined(WINAPI)
1062 #define GC_WINDOWS_H_INCLUDED
1063 #endif
1064 #if defined(GC_WIN32_THREADS) \
1065     && (!defined(GC_PTHREADS) || defined(GC_BUILD) \
1066         || defined(GC_WINDOWS_H_INCLUDED))
1067 #if (!defined(GC_NO_THREAD_DECLS) || defined(GC_BUILD)) \
1068      && !defined(GC_DONT_INCL_WINDOWS_H)
1069 #ifdef __cplusplus
1070       }
1071 #endif
1072 #if !defined(_WIN32_WCE) && !defined(__CEGCC__)
1073 #include <process.h>
1074 #endif
1075 #if defined(GC_BUILD) || !defined(GC_DONT_INCLUDE_WINDOWS_H)
1076 #include <windows.h>
1077 #define GC_WINDOWS_H_INCLUDED
1078 #endif
1079 #ifdef __cplusplus
1080       extern "C" {
1081 #endif
1082 #ifdef GC_UNDERSCORE_STDCALL
1083 #define GC_CreateThread _GC_CreateThread
1084 #define GC_ExitThread _GC_ExitThread
1085 #endif
1086 #ifndef DECLSPEC_NORETURN
1087 #ifdef GC_WINDOWS_H_INCLUDED
1088 #define DECLSPEC_NORETURN
1089 #else
1090 #define DECLSPEC_NORETURN __declspec(noreturn)
1091 #endif
1092 #endif
1093 #if !defined(_UINTPTR_T) && !defined(_UINTPTR_T_DEFINED) \
1094        && !defined(UINTPTR_MAX)
1095       typedef GC_word GC_uintptr_t;
1096 #else
1097       typedef uintptr_t GC_uintptr_t;
1098 #endif
1099 #ifdef _WIN64
1100 #define GC_WIN32_SIZE_T GC_uintptr_t
1101 #elif defined(GC_WINDOWS_H_INCLUDED)
1102 #define GC_WIN32_SIZE_T DWORD
1103 #else
1104 #define GC_WIN32_SIZE_T unsigned long
1105 #endif
1106 #ifdef GC_INSIDE_DLL
1107 #ifdef GC_UNDERSCORE_STDCALL
1108 #define GC_DllMain _GC_DllMain
1109 #endif
1110 #ifdef GC_WINDOWS_H_INCLUDED
1111         GC_API BOOL WINAPI GC_DllMain(HINSTANCE ,
1112                                       ULONG ,
1113                                       LPVOID );
1114 #else
1115         GC_API int __stdcall GC_DllMain(void *, unsigned long, void *);
1116 #endif
1117 #endif
1118 #ifdef GC_WINDOWS_H_INCLUDED
1119       GC_API HANDLE WINAPI GC_CreateThread(
1120                 LPSECURITY_ATTRIBUTES ,
1121                 GC_WIN32_SIZE_T ,
1122                 LPTHREAD_START_ROUTINE ,
1123                 LPVOID , DWORD ,
1124                 LPDWORD );
1125       GC_API DECLSPEC_NORETURN void WINAPI GC_ExitThread(
1126                                                 DWORD );
1127 #else
1128       struct _SECURITY_ATTRIBUTES;
1129       GC_API void *__stdcall GC_CreateThread(struct _SECURITY_ATTRIBUTES *,
1130                                 GC_WIN32_SIZE_T,
1131                                 unsigned long (__stdcall *)(void *),
1132                                 void *, unsigned long, unsigned long *);
1133       GC_API DECLSPEC_NORETURN void __stdcall GC_ExitThread(unsigned long);
1134 #endif
1135 #if !defined(_WIN32_WCE) && !defined(__CEGCC__)
1136       GC_API GC_uintptr_t GC_CALL GC_beginthreadex(
1137                         void * , unsigned ,
1138                         unsigned (__stdcall *)(void *),
1139                         void * , unsigned ,
1140                         unsigned * );
1141       GC_API void GC_CALL GC_endthreadex(unsigned );
1142 #endif
1143 #endif
1144 #ifdef GC_WINMAIN_REDIRECT
1145 #define WinMain GC_WinMain
1146 #endif
1147 #define GC_use_DllMain GC_use_threads_discovery
1148 #ifndef GC_NO_THREAD_REDIRECTS
1149 #define CreateThread GC_CreateThread
1150 #define ExitThread GC_ExitThread
1151 #undef _beginthreadex
1152 #define _beginthreadex GC_beginthreadex
1153 #undef _endthreadex
1154 #define _endthreadex GC_endthreadex
1155 #endif
1156 #endif
1157 GC_API void GC_CALL GC_set_force_unmap_on_gcollect(int);
1158 GC_API int GC_CALL GC_get_force_unmap_on_gcollect(void);
1159 #if defined(__CYGWIN32__) || defined(__CYGWIN__)
1160 #ifdef __x86_64__
1161     extern int __data_start__[], __data_end__[];
1162     extern int __bss_start__[], __bss_end__[];
1163 #define GC_DATASTART ((GC_word)__data_start__ < (GC_word)__bss_start__ \
1164                          ? (void *)__data_start__ : (void *)__bss_start__)
1165 #define GC_DATAEND ((GC_word)__data_end__ > (GC_word)__bss_end__ \
1166                        ? (void *)__data_end__ : (void *)__bss_end__)
1167 #else
1168     extern int _data_start__[], _data_end__[], _bss_start__[], _bss_end__[];
1169 #define GC_DATASTART ((GC_word)_data_start__ < (GC_word)_bss_start__ \
1170                          ? (void *)_data_start__ : (void *)_bss_start__)
1171 #define GC_DATAEND ((GC_word)_data_end__ > (GC_word)_bss_end__ \
1172                       ? (void *)_data_end__ : (void *)_bss_end__)
1173 #endif
1174 #define GC_INIT_CONF_ROOTS GC_add_roots(GC_DATASTART, GC_DATAEND); \
1175                                  GC_gcollect()
1176 #elif defined(_AIX)
1177   extern int _data[], _end[];
1178 #define GC_DATASTART ((void *)_data)
1179 #define GC_DATAEND ((void *)_end)
1180 #define GC_INIT_CONF_ROOTS GC_add_roots(GC_DATASTART, GC_DATAEND)
1181 #elif (defined(HOST_ANDROID) || defined(__ANDROID__)) \
1182       && defined(IGNORE_DYNAMIC_LOADING)
1183 #pragma weak __dso_handle
1184   extern int __dso_handle[];
1185   GC_API void * GC_CALL GC_find_limit(void * , int );
1186 #define GC_INIT_CONF_ROOTS (void)(__dso_handle != 0 \
1187                                    ? (GC_add_roots(__dso_handle, \
1188                                             GC_find_limit(__dso_handle, \
1189                                                           1 )), 0) : 0)
1190 #else
1191 #define GC_INIT_CONF_ROOTS
1192 #endif
1193 #ifdef GC_DONT_EXPAND
1194 #define GC_INIT_CONF_DONT_EXPAND GC_set_dont_expand(1)
1195 #else
1196 #define GC_INIT_CONF_DONT_EXPAND
1197 #endif
1198 #ifdef GC_FORCE_UNMAP_ON_GCOLLECT
1199 #define GC_INIT_CONF_FORCE_UNMAP_ON_GCOLLECT \
1200                 GC_set_force_unmap_on_gcollect(1)
1201 #else
1202 #define GC_INIT_CONF_FORCE_UNMAP_ON_GCOLLECT
1203 #endif
1204 #ifdef GC_DONT_GC
1205 #define GC_INIT_CONF_MAX_RETRIES (void)(GC_dont_gc = 1)
1206 #elif defined(GC_MAX_RETRIES) && !defined(CPPCHECK)
1207 #define GC_INIT_CONF_MAX_RETRIES GC_set_max_retries(GC_MAX_RETRIES)
1208 #else
1209 #define GC_INIT_CONF_MAX_RETRIES
1210 #endif
1211 #if defined(GC_ALLOCD_BYTES_PER_FINALIZER) && !defined(CPPCHECK)
1212 #define GC_INIT_CONF_ALLOCD_BYTES_PER_FINALIZER \
1213         GC_set_allocd_bytes_per_finalizer(GC_ALLOCD_BYTES_PER_FINALIZER)
1214 #else
1215 #define GC_INIT_CONF_ALLOCD_BYTES_PER_FINALIZER
1216 #endif
1217 #if defined(GC_FREE_SPACE_DIVISOR) && !defined(CPPCHECK)
1218 #define GC_INIT_CONF_FREE_SPACE_DIVISOR \
1219                 GC_set_free_space_divisor(GC_FREE_SPACE_DIVISOR)
1220 #else
1221 #define GC_INIT_CONF_FREE_SPACE_DIVISOR
1222 #endif
1223 #if defined(GC_FULL_FREQ) && !defined(CPPCHECK)
1224 #define GC_INIT_CONF_FULL_FREQ GC_set_full_freq(GC_FULL_FREQ)
1225 #else
1226 #define GC_INIT_CONF_FULL_FREQ
1227 #endif
1228 #if defined(GC_TIME_LIMIT) && !defined(CPPCHECK)
1229 #define GC_INIT_CONF_TIME_LIMIT GC_set_time_limit(GC_TIME_LIMIT)
1230 #else
1231 #define GC_INIT_CONF_TIME_LIMIT
1232 #endif
1233 #if defined(GC_MARKERS) && defined(GC_THREADS) && !defined(CPPCHECK)
1234 #define GC_INIT_CONF_MARKERS GC_set_markers_count(GC_MARKERS)
1235 #else
1236 #define GC_INIT_CONF_MARKERS
1237 #endif
1238 #if defined(GC_SIG_SUSPEND) && defined(GC_THREADS) && !defined(CPPCHECK)
1239 #define GC_INIT_CONF_SUSPEND_SIGNAL GC_set_suspend_signal(GC_SIG_SUSPEND)
1240 #else
1241 #define GC_INIT_CONF_SUSPEND_SIGNAL
1242 #endif
1243 #if defined(GC_SIG_THR_RESTART) && defined(GC_THREADS) && !defined(CPPCHECK)
1244 #define GC_INIT_CONF_THR_RESTART_SIGNAL \
1245                 GC_set_thr_restart_signal(GC_SIG_THR_RESTART)
1246 #else
1247 #define GC_INIT_CONF_THR_RESTART_SIGNAL
1248 #endif
1249 #if defined(GC_MAXIMUM_HEAP_SIZE) && !defined(CPPCHECK)
1250 #define GC_INIT_CONF_MAXIMUM_HEAP_SIZE \
1251                 GC_set_max_heap_size(GC_MAXIMUM_HEAP_SIZE)
1252 #else
1253 #define GC_INIT_CONF_MAXIMUM_HEAP_SIZE
1254 #endif
1255 #ifdef GC_IGNORE_WARN
1256 #define GC_INIT_CONF_IGNORE_WARN GC_set_warn_proc(GC_ignore_warn_proc)
1257 #else
1258 #define GC_INIT_CONF_IGNORE_WARN
1259 #endif
1260 #if defined(GC_INITIAL_HEAP_SIZE) && !defined(CPPCHECK)
1261 #define GC_INIT_CONF_INITIAL_HEAP_SIZE \
1262                 { size_t heap_size = GC_get_heap_size(); \
1263                   if (heap_size < (GC_INITIAL_HEAP_SIZE)) \
1264                     (void)GC_expand_hp((GC_INITIAL_HEAP_SIZE) - heap_size); }
1265 #else
1266 #define GC_INIT_CONF_INITIAL_HEAP_SIZE
1267 #endif
1268 #define GC_INIT() { GC_INIT_CONF_DONT_EXPAND;  \
1269                     GC_INIT_CONF_FORCE_UNMAP_ON_GCOLLECT; \
1270                     GC_INIT_CONF_MAX_RETRIES; \
1271                     GC_INIT_CONF_ALLOCD_BYTES_PER_FINALIZER; \
1272                     GC_INIT_CONF_FREE_SPACE_DIVISOR; \
1273                     GC_INIT_CONF_FULL_FREQ; \
1274                     GC_INIT_CONF_TIME_LIMIT; \
1275                     GC_INIT_CONF_MARKERS; \
1276                     GC_INIT_CONF_SUSPEND_SIGNAL; \
1277                     GC_INIT_CONF_THR_RESTART_SIGNAL; \
1278                     GC_INIT_CONF_MAXIMUM_HEAP_SIZE; \
1279                     GC_init();  \
1280                     GC_INIT_CONF_ROOTS;  \
1281                     GC_INIT_CONF_IGNORE_WARN; \
1282                     GC_INIT_CONF_INITIAL_HEAP_SIZE; }
1283 GC_API void GC_CALL GC_win32_free_heap(void);
1284 #if defined(__SYMBIAN32__)
1285   void GC_init_global_static_roots(void);
1286 #endif
1287 #if defined(_AMIGA) && !defined(GC_AMIGA_MAKINGLIB)
1288   void *GC_amiga_realloc(void *, size_t);
1289 #define GC_realloc(a,b) GC_amiga_realloc(a,b)
1290   void GC_amiga_set_toany(void (*)(void));
1291   extern int GC_amiga_free_space_divisor_inc;
1292   extern void *(*GC_amiga_allocwrapper_do)(size_t, void *(GC_CALL *)(size_t));
1293 #define GC_malloc(a) \
1294         (*GC_amiga_allocwrapper_do)(a,GC_malloc)
1295 #define GC_malloc_atomic(a) \
1296         (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic)
1297 #define GC_malloc_uncollectable(a) \
1298         (*GC_amiga_allocwrapper_do)(a,GC_malloc_uncollectable)
1299 #define GC_malloc_atomic_uncollectable(a) \
1300         (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_uncollectable)
1301 #define GC_malloc_ignore_off_page(a) \
1302         (*GC_amiga_allocwrapper_do)(a,GC_malloc_ignore_off_page)
1303 #define GC_malloc_atomic_ignore_off_page(a) \
1304         (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page)
1305 #endif
1306 #ifdef __cplusplus
1307   }
1308 #endif
1309 #endif
1310 #endif
1311 #include <stdlib.h>
1312 #if !defined(sony_news)
1313 #include <stddef.h>
1314 #endif
1315 #ifdef DGUX
1316 #include <sys/types.h>
1317 #include <sys/time.h>
1318 #include <sys/resource.h>
1319 #endif
1320 #ifdef BSD_TIME
1321 #include <sys/types.h>
1322 #include <sys/time.h>
1323 #include <sys/resource.h>
1324 #endif
1325 #ifdef PARALLEL_MARK
1326 #define AO_REQUIRE_CAS
1327 #if !defined(__GNUC__) && !defined(AO_ASSUME_WINDOWS98)
1328 #define AO_ASSUME_WINDOWS98
1329 #endif
1330 #endif
1331 #ifndef GC_TINY_FL_H
1332 #define GC_TINY_FL_H
1333 #ifndef GC_GRANULE_BYTES
1334 #if defined(__LP64__) || defined (_LP64) || defined(_WIN64) \
1335         || defined(__s390x__) \
1336         || (defined(__x86_64__) && !defined(__ILP32__)) \
1337         || defined(__alpha__) || defined(__powerpc64__) \
1338         || defined(__arch64__)
1339 #define GC_GRANULE_BYTES 16
1340 #define GC_GRANULE_WORDS 2
1341 #else
1342 #define GC_GRANULE_BYTES 8
1343 #define GC_GRANULE_WORDS 2
1344 #endif
1345 #endif
1346 #if GC_GRANULE_WORDS == 2
1347 #define GC_WORDS_TO_GRANULES(n) ((n)>>1)
1348 #else
1349 #define GC_WORDS_TO_GRANULES(n) ((n)*sizeof(void *)/GC_GRANULE_BYTES)
1350 #endif
1351 #ifndef GC_TINY_FREELISTS
1352 #if GC_GRANULE_BYTES == 16
1353 #define GC_TINY_FREELISTS 25
1354 #else
1355 #define GC_TINY_FREELISTS 33
1356 #endif
1357 #endif
1358 #define GC_RAW_BYTES_FROM_INDEX(i) ((i) * GC_GRANULE_BYTES)
1359 #endif
1360 #ifndef GC_MARK_H
1361 #define GC_MARK_H
1362 #ifndef GC_H
1363 #endif
1364 #ifdef __cplusplus
1365   extern "C" {
1366 #endif
1367 #define GC_PROC_BYTES 100
1368 #if defined(GC_BUILD) || defined(NOT_GCBUILD)
1369   struct GC_ms_entry;
1370 #else
1371   struct GC_ms_entry { void *opaque; };
1372 #endif
1373 typedef struct GC_ms_entry * (*GC_mark_proc)(GC_word * ,
1374                                 struct GC_ms_entry * ,
1375                                 struct GC_ms_entry * ,
1376                                 GC_word );
1377 #define GC_LOG_MAX_MARK_PROCS 6
1378 #define GC_MAX_MARK_PROCS (1 << GC_LOG_MAX_MARK_PROCS)
1379 #define GC_RESERVED_MARK_PROCS 8
1380 #define GC_GCJ_RESERVED_MARK_PROC_INDEX 0
1381 #define GC_DS_TAG_BITS 2
1382 #define GC_DS_TAGS   ((1 << GC_DS_TAG_BITS) - 1)
1383 #define GC_DS_LENGTH 0
1384 #define GC_DS_BITMAP 1
1385 #define GC_DS_PROC   2
1386 #define GC_MAKE_PROC(proc_index, env) \
1387             (((((env) << GC_LOG_MAX_MARK_PROCS) \
1388                | (proc_index)) << GC_DS_TAG_BITS) | GC_DS_PROC)
1389 #define GC_DS_PER_OBJECT 3
1390 #define GC_INDIR_PER_OBJ_BIAS 0x10
1391 GC_API void * GC_least_plausible_heap_addr;
1392 GC_API void * GC_greatest_plausible_heap_addr;
1393 GC_API struct GC_ms_entry * GC_CALL GC_mark_and_push(void * ,
1394                                 struct GC_ms_entry * ,
1395                                 struct GC_ms_entry * ,
1396                                 void ** );
1397 #define GC_MARK_AND_PUSH(obj, msp, lim, src) \
1398           ((GC_word)(obj) >= (GC_word)GC_least_plausible_heap_addr && \
1399            (GC_word)(obj) <= (GC_word)GC_greatest_plausible_heap_addr ? \
1400            GC_mark_and_push(obj, msp, lim, src) : (msp))
1401 GC_API GC_ATTR_CONST size_t GC_CALL GC_get_debug_header_size(void);
1402 #define GC_USR_PTR_FROM_BASE(p) \
1403                 ((void *)((char *)(p) + GC_get_debug_header_size()))
1404 GC_API GC_ATTR_DEPRECATED
1405 #ifdef GC_BUILD
1406     const
1407 #endif
1408   size_t GC_debug_header_size;
1409 GC_API void ** GC_CALL GC_new_free_list(void);
1410 GC_API void ** GC_CALL GC_new_free_list_inner(void);
1411 GC_API unsigned GC_CALL GC_new_kind(void ** ,
1412                             GC_word ,
1413                             int ,
1414                             int ) GC_ATTR_NONNULL(1);
1415 GC_API unsigned GC_CALL GC_new_kind_inner(void ** ,
1416                             GC_word ,
1417                             int ,
1418                             int ) GC_ATTR_NONNULL(1);
1419 GC_API unsigned GC_CALL GC_new_proc(GC_mark_proc);
1420 GC_API unsigned GC_CALL GC_new_proc_inner(GC_mark_proc);
1421 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL GC_generic_malloc(
1422                                                             size_t ,
1423                                                             int );
1424 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
1425                                         GC_generic_malloc_ignore_off_page(
1426                                             size_t , int );
1427 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
1428                                         GC_generic_malloc_uncollectable(
1429                                             size_t , int );
1430 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
1431                                         GC_generic_or_special_malloc(
1432                                             size_t , int );
1433 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
1434                                         GC_debug_generic_or_special_malloc(
1435                                             size_t , int ,
1436                                             GC_EXTRA_PARAMS);
1437 #ifdef GC_DEBUG
1438 #define GC_GENERIC_OR_SPECIAL_MALLOC(sz, knd) \
1439                 GC_debug_generic_or_special_malloc(sz, knd, GC_EXTRAS)
1440 #else
1441 #define GC_GENERIC_OR_SPECIAL_MALLOC(sz, knd) \
1442                 GC_generic_or_special_malloc(sz, knd)
1443 #endif
1444 GC_API int GC_CALL GC_get_kind_and_size(const void *, size_t * )
1445                                                         GC_ATTR_NONNULL(1);
1446 typedef void (GC_CALLBACK * GC_describe_type_fn)(void * ,
1447                                                  char * );
1448 #define GC_TYPE_DESCR_LEN 40
1449 GC_API void GC_CALL GC_register_describe_type_fn(int ,
1450                                                  GC_describe_type_fn);
1451 GC_API void * GC_CALL GC_clear_stack(void *);
1452 typedef void (GC_CALLBACK * GC_start_callback_proc)(void);
1453 GC_API void GC_CALL GC_set_start_callback(GC_start_callback_proc);
1454 GC_API GC_start_callback_proc GC_CALL GC_get_start_callback(void);
1455 GC_API int GC_CALL GC_is_marked(const void *) GC_ATTR_NONNULL(1);
1456 GC_API void GC_CALL GC_clear_mark_bit(const void *) GC_ATTR_NONNULL(1);
1457 GC_API void GC_CALL GC_set_mark_bit(const void *) GC_ATTR_NONNULL(1);
1458 GC_API void GC_CALL GC_push_all(void * , void * );
1459 GC_API void GC_CALL GC_push_all_eager(void * , void * );
1460 GC_API void GC_CALL GC_push_conditional(void * , void * ,
1461                                         int );
1462 GC_API void GC_CALL GC_push_finalizer_structures(void);
1463 typedef void (GC_CALLBACK * GC_push_other_roots_proc)(void);
1464 GC_API void GC_CALL GC_set_push_other_roots(GC_push_other_roots_proc);
1465 GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void);
1466 typedef void (GC_CALLBACK *GC_reachable_object_proc)(void * ,
1467                                                 size_t ,
1468                                                 void * );
1469 GC_API void GC_CALL GC_enumerate_reachable_objects_inner(
1470                                 GC_reachable_object_proc,
1471                                 void * ) GC_ATTR_NONNULL(1);
1472 GC_API int GC_CALL GC_is_tmp_root(void *);
1473 GC_API void GC_CALL GC_print_trace(GC_word );
1474 GC_API void GC_CALL GC_print_trace_inner(GC_word );
1475 #ifdef __cplusplus
1476   }
1477 #endif
1478 #endif
1479 typedef GC_word word;
1480 typedef GC_signed_word signed_word;
1481 typedef unsigned int unsigned32;
1482 typedef int GC_bool;
1483 #define TRUE 1
1484 #define FALSE 0
1485 #ifndef PTR_T_DEFINED
1486   typedef char * ptr_t;
1487 #define PTR_T_DEFINED
1488 #endif
1489 #ifndef SIZE_MAX
1490 #include <limits.h>
1491 #endif
1492 #if defined(SIZE_MAX) && !defined(CPPCHECK)
1493 #define GC_SIZE_MAX ((size_t)SIZE_MAX)
1494 #else
1495 #define GC_SIZE_MAX (~(size_t)0)
1496 #endif
1497 #if GC_GNUC_PREREQ(3, 0) && !defined(LINT2)
1498 #define EXPECT(expr, outcome) __builtin_expect(expr,outcome)
1499 #else
1500 #define EXPECT(expr, outcome) (expr)
1501 #endif
1502 #define SIZET_SAT_ADD(a, b) \
1503             (EXPECT((a) < GC_SIZE_MAX - (b), TRUE) ? (a) + (b) : GC_SIZE_MAX)
1504 #ifndef GCCONFIG_H
1505 #define GCCONFIG_H
1506 #ifdef CPPCHECK
1507 #undef CLOCKS_PER_SEC
1508 #undef FIXUP_POINTER
1509 #undef POINTER_MASK
1510 #undef POINTER_SHIFT
1511 #undef REDIRECT_REALLOC
1512 #undef _MAX_PATH
1513 #endif
1514 #ifndef PTR_T_DEFINED
1515   typedef char * ptr_t;
1516 #define PTR_T_DEFINED
1517 #endif
1518 #if !defined(sony_news)
1519 #include <stddef.h>
1520 #endif
1521 #ifdef __cplusplus
1522 #define EXTERN_C_BEGIN extern "C" {
1523 #define EXTERN_C_END }
1524 #else
1525 #define EXTERN_C_BEGIN
1526 #define EXTERN_C_END
1527 #endif
1528 EXTERN_C_BEGIN
1529 #if defined(__clang__) && defined(__clang_major__)
1530 #define GC_CLANG_PREREQ(major, minor) \
1531     ((__clang_major__ << 16) + __clang_minor__ >= ((major) << 16) + (minor))
1532 #define GC_CLANG_PREREQ_FULL(major, minor, patchlevel) \
1533             (GC_CLANG_PREREQ(major, (minor) + 1) \
1534                 || (__clang_major__ == (major) && __clang_minor__ == (minor) \
1535                     && __clang_patchlevel__ >= (patchlevel)))
1536 #else
1537 #define GC_CLANG_PREREQ(major, minor) 0
1538 #define GC_CLANG_PREREQ_FULL(major, minor, patchlevel) 0
1539 #endif
1540 #ifdef LINT2
1541 #define COVERT_DATAFLOW(w) (~(GC_word)(w)^(~(GC_word)0))
1542 #else
1543 #define COVERT_DATAFLOW(w) ((GC_word)(w))
1544 #endif
1545 #if defined(__ANDROID__) && !defined(HOST_ANDROID)
1546 #define HOST_ANDROID 1
1547 #endif
1548 #if defined(TIZEN) && !defined(HOST_TIZEN)
1549 #define HOST_TIZEN 1
1550 #endif
1551 #if defined(__SYMBIAN32__) && !defined(SYMBIAN)
1552 #define SYMBIAN
1553 #ifdef __WINS__
1554 #pragma data_seg(".data2")
1555 #endif
1556 #endif
1557 #if (defined(linux) || defined(__linux__) || defined(HOST_ANDROID)) \
1558      && !defined(LINUX) && !defined(__native_client__)
1559 #define LINUX
1560 #endif
1561 #if defined(__NetBSD__)
1562 #define NETBSD
1563 #endif
1564 #if defined(__OpenBSD__)
1565 #define OPENBSD
1566 #endif
1567 #if (defined(__FreeBSD__) || defined(__DragonFly__) \
1568       || defined(__FreeBSD_kernel__)) && !defined(FREEBSD) \
1569      && !defined(GC_NO_FREEBSD)
1570 #define FREEBSD
1571 #endif
1572 #if defined(macosx) || (defined(__APPLE__) && defined(__MACH__))
1573 #define DARWIN
1574     EXTERN_C_END
1575 #include <TargetConditionals.h>
1576     EXTERN_C_BEGIN
1577 #endif
1578 #if defined(__native_client__)
1579 #define NACL
1580 #if !defined(__portable_native_client__) && !defined(__arm__)
1581 #define I386
1582 #define mach_type_known
1583 #else
1584 #endif
1585 #endif
1586 #if defined(__aarch64__)
1587 #define AARCH64
1588 #if !defined(LINUX) && !defined(DARWIN) && !defined(FREEBSD) \
1589         && !defined(NETBSD) && !defined(NN_BUILD_TARGET_PLATFORM_NX) \
1590         && !defined(OPENBSD)
1591 #define NOSYS
1592 #define mach_type_known
1593 #endif
1594 #endif
1595 #if defined(__arm) || defined(__arm__) || defined(__thumb__)
1596 #define ARM32
1597 #if defined(NACL)
1598 #define mach_type_known
1599 #elif !defined(LINUX) && !defined(NETBSD) && !defined(FREEBSD) \
1600           && !defined(OPENBSD) && !defined(DARWIN) && !defined(_WIN32) \
1601           && !defined(__CEGCC__) && !defined(NN_PLATFORM_CTR) \
1602           && !defined(GC_NO_NOSYS) && !defined(SN_TARGET_PSP2) \
1603           && !defined(SYMBIAN)
1604 #define NOSYS
1605 #define mach_type_known
1606 #endif
1607 #endif
1608 #if defined(sun) && defined(mc68000) && !defined(CPPCHECK)
1609 #error SUNOS4 no longer supported
1610 #endif
1611 #if defined(hp9000s300) && !defined(CPPCHECK)
1612 #error M68K based HP machines no longer supported
1613 #endif
1614 #if defined(OPENBSD) && defined(m68k)
1615 #define M68K
1616 #define mach_type_known
1617 #endif
1618 #if defined(OPENBSD) && defined(__sparc__)
1619 #define SPARC
1620 #define mach_type_known
1621 #endif
1622 #if defined(OPENBSD) && defined(__arm__)
1623 #define ARM32
1624 #define mach_type_known
1625 #endif
1626 #if defined(OPENBSD) && defined(__aarch64__)
1627 #define AARCH64
1628 #define mach_type_known
1629 #endif
1630 #if defined(OPENBSD) && defined(__sh__)
1631 #define SH
1632 #define mach_type_known
1633 #endif
1634 #if defined(NETBSD) && (defined(m68k) || defined(__m68k__))
1635 #define M68K
1636 #define mach_type_known
1637 #endif
1638 #if defined(NETBSD) && defined(__powerpc__)
1639 #define POWERPC
1640 #define mach_type_known
1641 #endif
1642 #if defined(NETBSD) && (defined(__arm32__) || defined(__arm__))
1643 #define ARM32
1644 #define mach_type_known
1645 #endif
1646 #if defined(NETBSD) && defined(__aarch64__)
1647 #define AARCH64
1648 #define mach_type_known
1649 #endif
1650 #if defined(NETBSD) && defined(__sh__)
1651 #define SH
1652 #define mach_type_known
1653 #endif
1654 #if defined(vax) || defined(__vax__)
1655 #define VAX
1656 #ifdef ultrix
1657 #define ULTRIX
1658 #else
1659 #define BSD
1660 #endif
1661 #define mach_type_known
1662 #endif
1663 #if defined(NETBSD) && defined(__vax__)
1664 #define VAX
1665 #define mach_type_known
1666 #endif
1667 #if defined(mips) || defined(__mips) || defined(_mips)
1668 #define MIPS
1669 #if defined(nec_ews) || defined(_nec_ews)
1670 #define EWS4800
1671 #endif
1672 #if !defined(LINUX) && !defined(EWS4800) && !defined(NETBSD) \
1673         && !defined(OPENBSD)
1674 #if defined(ultrix) || defined(__ultrix)
1675 #define ULTRIX
1676 #else
1677 #define IRIX5
1678 #endif
1679 #endif
1680 #if defined(NETBSD) && defined(__MIPSEL__)
1681 #undef ULTRIX
1682 #endif
1683 #define mach_type_known
1684 #endif
1685 #if defined(__QNX__)
1686 #define I386
1687 #define mach_type_known
1688 #endif
1689 #if defined(__NIOS2__) || defined(__NIOS2) || defined(__nios2__)
1690 #define NIOS2
1691 #define mach_type_known
1692 #endif
1693 #if defined(__or1k__)
1694 #define OR1K
1695 #define mach_type_known
1696 #endif
1697 #if defined(DGUX) && (defined(i386) || defined(__i386__))
1698 #define I386
1699 #ifndef _USING_DGUX
1700 #define _USING_DGUX
1701 #endif
1702 #define mach_type_known
1703 #endif
1704 #if defined(sequent) && (defined(i386) || defined(__i386__))
1705 #define I386
1706 #define SEQUENT
1707 #define mach_type_known
1708 #endif
1709 #if (defined(sun) || defined(__sun)) && (defined(i386) || defined(__i386__))
1710 #define I386
1711 #define SOLARIS
1712 #define mach_type_known
1713 #endif
1714 #if (defined(sun) || defined(__sun)) && defined(__amd64)
1715 #define X86_64
1716 #define SOLARIS
1717 #define mach_type_known
1718 #endif
1719 #if (defined(__OS2__) || defined(__EMX__)) && defined(__32BIT__)
1720 #define I386
1721 #define OS2
1722 #define mach_type_known
1723 #endif
1724 #if defined(ibm032) && !defined(CPPCHECK)
1725 #error IBM PC/RT no longer supported
1726 #endif
1727 #if (defined(sun) || defined(__sun)) && (defined(sparc) || defined(__sparc))
1728     EXTERN_C_END
1729 #include <errno.h>
1730     EXTERN_C_BEGIN
1731 #define SPARC
1732 #define SOLARIS
1733 #define mach_type_known
1734 #elif defined(sparc) && defined(unix) && !defined(sun) && !defined(linux) \
1735        && !defined(FREEBSD) && !defined(NETBSD) && !defined(OPENBSD)
1736 #define SPARC
1737 #define DRSNX
1738 #define mach_type_known
1739 #endif
1740 #if defined(_IBMR2)
1741 #define POWERPC
1742 #define AIX
1743 #define mach_type_known
1744 #endif
1745 #if defined(NETBSD) && defined(__sparc__)
1746 #define SPARC
1747 #define mach_type_known
1748 #endif
1749 #if defined(_M_XENIX) && defined(_M_SYSV) && defined(_M_I386)
1750 #define I386
1751 #if defined(_SCO_ELF)
1752 #define SCO_ELF
1753 #else
1754 #define SCO
1755 #endif
1756 #define mach_type_known
1757 #endif
1758 #if defined(_AUX_SOURCE) && !defined(CPPCHECK)
1759 #error A/UX no longer supported
1760 #endif
1761 #if defined(_PA_RISC1_0) || defined(_PA_RISC1_1) || defined(_PA_RISC2_0) \
1762      || defined(hppa) || defined(__hppa__)
1763 #define HP_PA
1764 #if !defined(LINUX) && !defined(HPUX) && !defined(OPENBSD)
1765 #define HPUX
1766 #endif
1767 #define mach_type_known
1768 #endif
1769 #if defined(__ia64) && (defined(_HPUX_SOURCE) || defined(__HP_aCC))
1770 #define IA64
1771 #ifndef HPUX
1772 #define HPUX
1773 #endif
1774 #define mach_type_known
1775 #endif
1776 #if (defined(__BEOS__) || defined(__HAIKU__)) && defined(_X86_)
1777 #define I386
1778 #define HAIKU
1779 #define mach_type_known
1780 #endif
1781 #if defined(__HAIKU__) && (defined(__amd64__) || defined(__x86_64__))
1782 #define X86_64
1783 #define HAIKU
1784 #define mach_type_known
1785 #endif
1786 #if defined(OPENBSD) && defined(__amd64__)
1787 #define X86_64
1788 #define mach_type_known
1789 #endif
1790 #if defined(LINUX) && (defined(i386) || defined(__i386__))
1791 #define I386
1792 #define mach_type_known
1793 #endif
1794 #if defined(LINUX) && defined(__x86_64__)
1795 #define X86_64
1796 #define mach_type_known
1797 #endif
1798 #if defined(LINUX) && (defined(__ia64__) || defined(__ia64))
1799 #define IA64
1800 #define mach_type_known
1801 #endif
1802 #if defined(LINUX) && defined(__aarch64__)
1803 #define AARCH64
1804 #define mach_type_known
1805 #endif
1806 #if defined(LINUX) && (defined(__arm) || defined(__arm__))
1807 #define ARM32
1808 #define mach_type_known
1809 #endif
1810 #if defined(LINUX) && defined(__cris__)
1811 #ifndef CRIS
1812 #define CRIS
1813 #endif
1814 #define mach_type_known
1815 #endif
1816 #if defined(LINUX) && defined(__loongarch__)
1817 #define LOONGARCH
1818 #define mach_type_known
1819 #endif
1820 #if defined(LINUX) && (defined(powerpc) || defined(__powerpc__) \
1821                         || defined(powerpc64) || defined(__powerpc64__))
1822 #define POWERPC
1823 #define mach_type_known
1824 #endif
1825 #if defined(LINUX) && defined(__mc68000__)
1826 #define M68K
1827 #define mach_type_known
1828 #endif
1829 #if defined(LINUX) && (defined(sparc) || defined(__sparc__))
1830 #define SPARC
1831 #define mach_type_known
1832 #endif
1833 #if defined(LINUX) && defined(__sh__)
1834 #define SH
1835 #define mach_type_known
1836 #endif
1837 #if defined(LINUX) && defined(__avr32__)
1838 #define AVR32
1839 #define mach_type_known
1840 #endif
1841 #if defined(LINUX) && defined(__m32r__)
1842 #define M32R
1843 #define mach_type_known
1844 #endif
1845 #if defined(__alpha) || defined(__alpha__)
1846 #define ALPHA
1847 #if !defined(LINUX) && !defined(NETBSD) && !defined(OPENBSD) \
1848        && !defined(FREEBSD)
1849 #define OSF1
1850 #endif
1851 #define mach_type_known
1852 #endif
1853 #if defined(_AMIGA) && !defined(AMIGA)
1854 #define AMIGA
1855 #endif
1856 #ifdef AMIGA
1857 #define M68K
1858 #define mach_type_known
1859 #endif
1860 #if defined(THINK_C) \
1861      || (defined(__MWERKS__) && !defined(__powerc) && !defined(SYMBIAN))
1862 #define M68K
1863 #define MACOS
1864 #define mach_type_known
1865 #endif
1866 #if defined(__MWERKS__) && defined(__powerc) && !defined(__MACH__) \
1867      && !defined(SYMBIAN)
1868 #define POWERPC
1869 #define MACOS
1870 #define mach_type_known
1871 #endif
1872 #if defined(OPENBSD) && defined(__powerpc__)
1873 #define POWERPC
1874 #define mach_type_known
1875 #endif
1876 #if defined(DARWIN)
1877 #if defined(__ppc__)  || defined(__ppc64__)
1878 #define POWERPC
1879 #define mach_type_known
1880 #elif defined(__x86_64__) || defined(__x86_64)
1881 #define X86_64
1882 #define mach_type_known
1883 #elif defined(__i386__)
1884 #define I386
1885 #define mach_type_known
1886 #elif defined(__arm__)
1887 #define ARM32
1888 #define mach_type_known
1889 #elif defined(__aarch64__)
1890 #define AARCH64
1891 #define mach_type_known
1892 #endif
1893 #endif
1894 #if defined(__rtems__) && (defined(i386) || defined(__i386__))
1895 #define I386
1896 #define RTEMS
1897 #define mach_type_known
1898 #endif
1899 #if defined(NeXT) && defined(mc68000)
1900 #define M68K
1901 #define NEXT
1902 #define mach_type_known
1903 #endif
1904 #if defined(NeXT) && (defined(i386) || defined(__i386__))
1905 #define I386
1906 #define NEXT
1907 #define mach_type_known
1908 #endif
1909 #if defined(OPENBSD) && (defined(i386) || defined(__i386__))
1910 #define I386
1911 #define mach_type_known
1912 #endif
1913 #if defined(NETBSD) && (defined(i386) || defined(__i386__))
1914 #define I386
1915 #define mach_type_known
1916 #endif
1917 #if defined(NETBSD) && defined(__x86_64__)
1918 #define X86_64
1919 #define mach_type_known
1920 #endif
1921 #if defined(FREEBSD) && (defined(i386) || defined(__i386__))
1922 #define I386
1923 #define mach_type_known
1924 #endif
1925 #if defined(FREEBSD) && (defined(__amd64__) || defined(__x86_64__))
1926 #define X86_64
1927 #define mach_type_known
1928 #endif
1929 #if defined(FREEBSD) && defined(__sparc__)
1930 #define SPARC
1931 #define mach_type_known
1932 #endif
1933 #if defined(FREEBSD) && (defined(powerpc) || defined(__powerpc__))
1934 #define POWERPC
1935 #define mach_type_known
1936 #endif
1937 #if defined(FREEBSD) && defined(__arm__)
1938 #define ARM32
1939 #define mach_type_known
1940 #endif
1941 #if defined(FREEBSD) && defined(__aarch64__)
1942 #define AARCH64
1943 #define mach_type_known
1944 #endif
1945 #if defined(FREEBSD) && (defined(mips) || defined(__mips) || defined(_mips))
1946 #define MIPS
1947 #define mach_type_known
1948 #endif
1949 #if defined(bsdi) && (defined(i386) || defined(__i386__))
1950 #define I386
1951 #define BSDI
1952 #define mach_type_known
1953 #endif
1954 #if !defined(mach_type_known) && defined(__386BSD__)
1955 #define I386
1956 #define THREE86BSD
1957 #define mach_type_known
1958 #endif
1959 #if defined(_CX_UX) && defined(_M88K)
1960 #define M88K
1961 #define CX_UX
1962 #define mach_type_known
1963 #endif
1964 #if defined(DGUX) && defined(m88k)
1965 #define M88K
1966 #define mach_type_known
1967 #endif
1968 #if defined(_WIN32_WCE) || defined(__CEGCC__) || defined(__MINGW32CE__)
1969 #if defined(SH3) || defined(SH4)
1970 #define SH
1971 #endif
1972 #if defined(x86) || defined(__i386__)
1973 #define I386
1974 #endif
1975 #if defined(_M_ARM) || defined(ARM) || defined(_ARM_)
1976 #define ARM32
1977 #endif
1978 #define MSWINCE
1979 #define mach_type_known
1980 #else
1981 #if ((defined(_MSDOS) || defined(_MSC_VER)) && (_M_IX86 >= 300)) \
1982        || (defined(_WIN32) && !defined(__CYGWIN32__) && !defined(__CYGWIN__) \
1983            && !defined(__INTERIX) && !defined(SYMBIAN))
1984 #if defined(__LP64__) || defined(_M_X64)
1985 #define X86_64
1986 #elif defined(_M_ARM)
1987 #define ARM32
1988 #elif defined(_M_ARM64)
1989 #define AARCH64
1990 #else
1991 #define I386
1992 #endif
1993 #ifdef _XBOX_ONE
1994 #define MSWIN_XBOX1
1995 #else
1996 #ifndef MSWIN32
1997 #define MSWIN32
1998 #endif
1999 #if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP)
2000 #define MSWINRT_FLAVOR
2001 #endif
2002 #endif
2003 #define mach_type_known
2004 #endif
2005 #if defined(_MSC_VER) && defined(_M_IA64)
2006 #define IA64
2007 #define MSWIN32
2008 #endif
2009 #endif
2010 #if defined(__DJGPP__)
2011 #define I386
2012 #ifndef DJGPP
2013 #define DJGPP
2014 #endif
2015 #define mach_type_known
2016 #endif
2017 #if defined(__CYGWIN32__) || defined(__CYGWIN__)
2018 #if defined(__LP64__)
2019 #define X86_64
2020 #else
2021 #define I386
2022 #endif
2023 #define CYGWIN32
2024 #define mach_type_known
2025 #endif
2026 #if defined(__INTERIX)
2027 #define I386
2028 #define INTERIX
2029 #define mach_type_known
2030 #endif
2031 #if defined(__MINGW32__) && !defined(mach_type_known)
2032 #define I386
2033 #define MSWIN32
2034 #define mach_type_known
2035 #endif
2036 #if defined(__BORLANDC__)
2037 #define I386
2038 #define MSWIN32
2039 #define mach_type_known
2040 #endif
2041 #if defined(_UTS) && !defined(mach_type_known)
2042 #define S370
2043 #define UTS4
2044 #define mach_type_known
2045 #endif
2046 #if defined(__pj__) && !defined(CPPCHECK)
2047 #error PicoJava no longer supported
2048 #endif
2049 #if defined(__embedded__) && defined(PPC)
2050 #define POWERPC
2051 #define NOSYS
2052 #define mach_type_known
2053 #endif
2054 #if defined(__WATCOMC__) && defined(__386__)
2055 #define I386
2056 #if !defined(OS2) && !defined(MSWIN32) && !defined(DOS4GW)
2057 #if defined(__OS2__)
2058 #define OS2
2059 #else
2060 #if defined(__WINDOWS_386__) || defined(__NT__)
2061 #define MSWIN32
2062 #else
2063 #define DOS4GW
2064 #endif
2065 #endif
2066 #endif
2067 #define mach_type_known
2068 #endif
2069 #if defined(__s390__) && defined(LINUX)
2070 #define S390
2071 #define mach_type_known
2072 #endif
2073 #if defined(__GNU__)
2074 #if defined(__i386__)
2075 #define  HURD
2076 #define  I386
2077 #define  mach_type_known
2078 #endif
2079 #endif
2080 #if defined(__TANDEM)
2081 #define MIPS
2082 #define NONSTOP
2083 #define mach_type_known
2084 #endif
2085 #if defined(__arc__) && defined(LINUX)
2086 #define ARC
2087 #define mach_type_known
2088 #endif
2089 #if defined(__hexagon__) && defined(LINUX)
2090 #define HEXAGON
2091 #define mach_type_known
2092 #endif
2093 #if defined(__tile__) && defined(LINUX)
2094 #ifdef __tilegx__
2095 #define TILEGX
2096 #else
2097 #define TILEPRO
2098 #endif
2099 #define mach_type_known
2100 #endif
2101 #if defined(__riscv) && (defined(FREEBSD) || defined(LINUX))
2102 #define RISCV
2103 #define mach_type_known
2104 #endif
2105 #if defined(SN_TARGET_PSP2)
2106 #define mach_type_known
2107 #endif
2108 #if defined(NN_PLATFORM_CTR)
2109 #define mach_type_known
2110 #endif
2111 #if defined(NN_BUILD_TARGET_PLATFORM_NX)
2112 #define NINTENDO_SWITCH
2113 #define mach_type_known
2114 #endif
2115 #if defined(SYMBIAN)
2116 #define mach_type_known
2117 #endif
2118 #if defined(__EMSCRIPTEN__)
2119 #define EMSCRIPTEN
2120 #define I386
2121 #define mach_type_known
2122 #endif
2123 #if !defined(mach_type_known) && !defined(CPPCHECK)
2124 #error The collector has not been ported to this machine/OS combination
2125 #endif
2126 #if GC_GNUC_PREREQ(2, 8) \
2127      && !GC_GNUC_PREREQ(11, 0)  \
2128      && !defined(__INTEL_COMPILER) && !defined(__PATHCC__) \
2129      && !defined(__FUJITSU)  \
2130      && !(defined(POWERPC) && defined(DARWIN))  \
2131      && !defined(RTEMS) \
2132      && !defined(__ARMCC_VERSION)  \
2133      && (!defined(__clang__) \
2134          || GC_CLANG_PREREQ(8, 0) )
2135 #define HAVE_BUILTIN_UNWIND_INIT
2136 #endif
2137 #ifdef CYGWIN32
2138 #define OS_TYPE "CYGWIN32"
2139 #define RETRY_GET_THREAD_CONTEXT
2140 #ifdef USE_WINALLOC
2141 #define GWW_VDB
2142 #elif defined(USE_MMAP)
2143 #define USE_MMAP_ANON
2144 #endif
2145 #endif
2146 #ifdef DARWIN
2147 #define OS_TYPE "DARWIN"
2148 #define DYNAMIC_LOADING
2149 #define DATASTART ((ptr_t)get_etext())
2150 #define DATAEND   ((ptr_t)get_end())
2151 #define USE_MMAP_ANON
2152     EXTERN_C_END
2153 #include <unistd.h>
2154     EXTERN_C_BEGIN
2155 #define GETPAGESIZE() (unsigned)getpagesize()
2156 #define NO_PTHREAD_TRYLOCK
2157 #endif
2158 #ifdef FREEBSD
2159 #define OS_TYPE "FREEBSD"
2160 #define FREEBSD_STACKBOTTOM
2161 #ifdef __ELF__
2162 #define DYNAMIC_LOADING
2163 #endif
2164 #if !defined(ALPHA) && !defined(SPARC)
2165       extern char etext[];
2166 #define DATASTART GC_FreeBSDGetDataStart(0x1000, (ptr_t)etext)
2167 #define DATASTART_USES_BSDGETDATASTART
2168 #ifndef GC_FREEBSD_THREADS
2169 #define MPROTECT_VDB
2170 #endif
2171 #endif
2172 #endif
2173 #ifdef HAIKU
2174 #define OS_TYPE "HAIKU"
2175 #define DYNAMIC_LOADING
2176 #define MPROTECT_VDB
2177     EXTERN_C_END
2178 #include <OS.h>
2179     EXTERN_C_BEGIN
2180 #define GETPAGESIZE() (unsigned)B_PAGE_SIZE
2181 #endif
2182 #ifdef HPUX
2183 #define OS_TYPE "HPUX"
2184     extern int __data_start[];
2185 #define DATASTART ((ptr_t)(__data_start))
2186 #ifdef USE_MMAP
2187 #define USE_MMAP_ANON
2188 #endif
2189 #define DYNAMIC_LOADING
2190     EXTERN_C_END
2191 #include <unistd.h>
2192     EXTERN_C_BEGIN
2193 #define GETPAGESIZE() (unsigned)sysconf(_SC_PAGE_SIZE)
2194 #endif
2195 #ifdef LINUX
2196 #define OS_TYPE "LINUX"
2197     EXTERN_C_END
2198 #include <features.h>
2199     EXTERN_C_BEGIN
2200 #define COUNT_UNMAPPED_REGIONS
2201 #if !defined(MIPS) && !defined(POWERPC)
2202 #define LINUX_STACKBOTTOM
2203 #endif
2204 #if defined(__ELF__) && !defined(IA64)
2205 #define DYNAMIC_LOADING
2206 #endif
2207 #if defined(__ELF__) && !defined(ARC) && !defined(RISCV) \
2208        && !defined(S390) && !defined(TILEGX) && !defined(TILEPRO)
2209       extern int _end[];
2210 #define DATAEND ((ptr_t)(_end))
2211 #endif
2212 #endif
2213 #ifdef MACOS
2214 #define OS_TYPE "MACOS"
2215 #ifndef __LOWMEM__
2216       EXTERN_C_END
2217 #include <LowMem.h>
2218       EXTERN_C_BEGIN
2219 #endif
2220 #define STACKBOTTOM ((ptr_t)LMGetCurStackBase())
2221 #define DATAEND
2222 #endif
2223 #ifdef MSWIN32
2224 #define OS_TYPE "MSWIN32"
2225 #define DATAEND
2226 #define GWW_VDB
2227 #endif
2228 #ifdef MSWINCE
2229 #define OS_TYPE "MSWINCE"
2230 #define DATAEND
2231 #endif
2232 #ifdef NETBSD
2233 #define OS_TYPE "NETBSD"
2234 #define HEURISTIC2
2235 #ifdef __ELF__
2236       extern ptr_t GC_data_start;
2237 #define DATASTART GC_data_start
2238 #define DYNAMIC_LOADING
2239 #elif !defined(MIPS)
2240       extern char etext[];
2241 #define DATASTART ((ptr_t)(etext))
2242 #endif
2243 #endif
2244 #ifdef NEXT
2245 #define OS_TYPE "NEXT"
2246 #define DATASTART ((ptr_t)get_etext())
2247 #define DATASTART_IS_FUNC
2248 #define DATAEND
2249 #endif
2250 #ifdef OPENBSD
2251 #define OS_TYPE "OPENBSD"
2252 #if !defined(M68K)
2253 #ifndef GC_OPENBSD_THREADS
2254 #define HEURISTIC2
2255 #endif
2256       extern int __data_start[];
2257 #define DATASTART ((ptr_t)__data_start)
2258       extern int _end[];
2259 #define DATAEND ((ptr_t)(&_end))
2260 #define DYNAMIC_LOADING
2261 #endif
2262 #endif
2263 #ifdef SOLARIS
2264 #define OS_TYPE "SOLARIS"
2265     extern int _etext[], _end[];
2266     ptr_t GC_SysVGetDataStart(size_t, ptr_t);
2267 #define DATASTART_IS_FUNC
2268 #define DATAEND ((ptr_t)(_end))
2269 #if !defined(USE_MMAP) && defined(REDIRECT_MALLOC)
2270 #define USE_MMAP 1
2271 #endif
2272 #ifdef USE_MMAP
2273 #define HEAP_START (ptr_t)0x40000000
2274 #else
2275 #define HEAP_START DATAEND
2276 #endif
2277 #ifndef GC_THREADS
2278 #define MPROTECT_VDB
2279 #endif
2280 #define DYNAMIC_LOADING
2281     EXTERN_C_END
2282 #include <sys/vmparam.h>
2283 #include <unistd.h>
2284     EXTERN_C_BEGIN
2285 #ifdef USERLIMIT
2286 #define STACKBOTTOM ((ptr_t)USRSTACK)
2287 #else
2288 #define HEURISTIC2
2289 #endif
2290 #endif
2291 #define STACK_GRAN 0x1000000
2292 #ifdef SYMBIAN
2293 #define MACH_TYPE "SYMBIAN"
2294 #define OS_TYPE "SYMBIAN"
2295 #define CPP_WORDSZ 32
2296 #define ALIGNMENT 4
2297 #define DATASTART (ptr_t)ALIGNMENT
2298 #define DATAEND (ptr_t)ALIGNMENT
2299 #endif
2300 #ifdef M68K
2301 #define MACH_TYPE "M68K"
2302 #define ALIGNMENT 2
2303 #ifdef OPENBSD
2304 #define HEURISTIC2
2305 #ifdef __ELF__
2306           extern ptr_t GC_data_start;
2307 #define DATASTART GC_data_start
2308 #define DYNAMIC_LOADING
2309 #else
2310           extern char etext[];
2311 #define DATASTART ((ptr_t)(etext))
2312 #endif
2313 #endif
2314 #ifdef NETBSD
2315 #endif
2316 #ifdef LINUX
2317 #if !defined(REDIRECT_MALLOC)
2318 #define MPROTECT_VDB
2319 #endif
2320 #ifdef __ELF__
2321 #if defined(__GLIBC__) && __GLIBC__ >= 2
2322 #define SEARCH_FOR_DATA_START
2323 #else
2324             extern char **__environ;
2325 #define DATASTART ((ptr_t)(&__environ))
2326 #endif
2327 #else
2328           extern int etext[];
2329 #define DATASTART ((ptr_t)((((word)(etext)) + 0xfff) & ~0xfff))
2330 #endif
2331 #endif
2332 #ifdef AMIGA
2333 #define OS_TYPE "AMIGA"
2334 #define DATAEND
2335 #define GETPAGESIZE() 4096
2336 #endif
2337 #ifdef MACOS
2338 #define GETPAGESIZE() 4096
2339 #endif
2340 #ifdef NEXT
2341 #define STACKBOTTOM ((ptr_t)0x4000000)
2342 #endif
2343 #endif
2344 #ifdef POWERPC
2345 #define MACH_TYPE "POWERPC"
2346 #ifdef MACOS
2347 #define ALIGNMENT 2
2348 #endif
2349 #ifdef LINUX
2350 #if defined(__powerpc64__)
2351 #define ALIGNMENT 8
2352 #define CPP_WORDSZ 64
2353 #ifndef HBLKSIZE
2354 #define HBLKSIZE 4096
2355 #endif
2356 #else
2357 #define ALIGNMENT 4
2358 #endif
2359 #if defined(__bg__)
2360 #define HEURISTIC2
2361 #define NO_PTHREAD_GETATTR_NP
2362 #else
2363 #define LINUX_STACKBOTTOM
2364 #endif
2365 #define SEARCH_FOR_DATA_START
2366 #if !defined(REDIRECT_MALLOC)
2367 #define MPROTECT_VDB
2368 #endif
2369 #ifndef SOFT_VDB
2370 #define SOFT_VDB
2371 #endif
2372 #endif
2373 #ifdef DARWIN
2374 #if defined(__ppc64__)
2375 #define ALIGNMENT 8
2376 #define CPP_WORDSZ 64
2377 #define STACKBOTTOM ((ptr_t)0x7fff5fc00000)
2378 #define CACHE_LINE_SIZE 64
2379 #ifndef HBLKSIZE
2380 #define HBLKSIZE 4096
2381 #endif
2382 #else
2383 #define ALIGNMENT 4
2384 #define STACKBOTTOM ((ptr_t)0xc0000000)
2385 #endif
2386 #define MPROTECT_VDB
2387 #if defined(USE_PPC_PREFETCH) && defined(__GNUC__)
2388 #define PREFETCH(x) \
2389           __asm__ __volatile__ ("dcbt 0,%0" : : "r" ((const void *) (x)))
2390 #define GC_PREFETCH_FOR_WRITE(x) \
2391           __asm__ __volatile__ ("dcbtst 0,%0" : : "r" ((const void *) (x)))
2392 #endif
2393 #endif
2394 #ifdef OPENBSD
2395 #if defined(__powerpc64__)
2396 #define ALIGNMENT 8
2397 #define CPP_WORDSZ 64
2398 #else
2399 #define ALIGNMENT 4
2400 #endif
2401 #endif
2402 #ifdef FREEBSD
2403 #if defined(__powerpc64__)
2404 #define ALIGNMENT 8
2405 #define CPP_WORDSZ 64
2406 #ifndef HBLKSIZE
2407 #define HBLKSIZE 4096
2408 #endif
2409 #else
2410 #define ALIGNMENT 4
2411 #endif
2412 #define SIG_SUSPEND SIGUSR1
2413 #define SIG_THR_RESTART SIGUSR2
2414 #endif
2415 #ifdef NETBSD
2416 #define ALIGNMENT 4
2417 #endif
2418 #ifdef SN_TARGET_PS3
2419 #define OS_TYPE "SN_TARGET_PS3"
2420 #define NO_GETENV
2421 #define CPP_WORDSZ 32
2422 #define ALIGNMENT 4
2423       extern int _end[];
2424       extern int __bss_start;
2425 #define DATASTART ((ptr_t)(__bss_start))
2426 #define DATAEND ((ptr_t)(_end))
2427 #define STACKBOTTOM ((ptr_t)ps3_get_stack_bottom())
2428 #define NO_PTHREAD_TRYLOCK
2429 #endif
2430 #ifdef AIX
2431 #define OS_TYPE "AIX"
2432 #undef ALIGNMENT
2433 #undef IA64
2434 #ifdef __64BIT__
2435 #define ALIGNMENT 8
2436 #define CPP_WORDSZ 64
2437 #define STACKBOTTOM ((ptr_t)0x1000000000000000)
2438 #else
2439 #define ALIGNMENT 4
2440 #define CPP_WORDSZ 32
2441 #define STACKBOTTOM ((ptr_t)((ulong)&errno))
2442 #endif
2443 #define USE_MMAP_ANON
2444       extern int _data[], _end[];
2445 #define DATASTART ((ptr_t)((ulong)_data))
2446 #define DATAEND ((ptr_t)((ulong)_end))
2447       extern int errno;
2448 #define DYNAMIC_LOADING
2449 #endif
2450 #ifdef NOSYS
2451 #define OS_TYPE "NOSYS"
2452 #define ALIGNMENT 4
2453       extern void __end[], __dso_handle[];
2454 #define DATASTART ((ptr_t)__dso_handle)
2455 #define DATAEND ((ptr_t)(__end))
2456 #undef STACK_GRAN
2457 #define STACK_GRAN 0x10000000
2458 #define HEURISTIC1
2459 #endif
2460 #endif
2461 #ifdef NACL
2462 #define OS_TYPE "NACL"
2463 #if defined(__GLIBC__)
2464 #define DYNAMIC_LOADING
2465 #endif
2466 #define DATASTART ((ptr_t)0x10020000)
2467     extern int _end[];
2468 #define DATAEND ((ptr_t)_end)
2469 #undef STACK_GRAN
2470 #define STACK_GRAN 0x10000
2471 #define HEURISTIC1
2472 #define NO_PTHREAD_GETATTR_NP
2473 #define USE_MMAP_ANON
2474 #define GETPAGESIZE() 65536
2475 #define MAX_NACL_GC_THREADS 1024
2476 #endif
2477 #ifdef VAX
2478 #define MACH_TYPE "VAX"
2479 #define ALIGNMENT 4
2480     extern char etext[];
2481 #define DATASTART ((ptr_t)(etext))
2482 #ifdef BSD
2483 #define OS_TYPE "BSD"
2484 #define HEURISTIC1
2485 #endif
2486 #ifdef ULTRIX
2487 #define OS_TYPE "ULTRIX"
2488 #define STACKBOTTOM ((ptr_t)0x7fffc800)
2489 #endif
2490 #endif
2491 #ifdef SPARC
2492 #define MACH_TYPE "SPARC"
2493 #if defined(__arch64__) || defined(__sparcv9)
2494 #define ALIGNMENT 8
2495 #define CPP_WORDSZ 64
2496 #define ELF_CLASS ELFCLASS64
2497 #else
2498 #define ALIGNMENT 4
2499 #define CPP_WORDSZ 32
2500 #endif
2501 #ifdef SOLARIS
2502 #define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
2503 #define PROC_VDB
2504 #define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE)
2505 #endif
2506 #ifdef DRSNX
2507 #define OS_TYPE "DRSNX"
2508         extern int etext[];
2509         ptr_t GC_SysVGetDataStart(size_t, ptr_t);
2510 #define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)etext)
2511 #define DATASTART_IS_FUNC
2512 #define MPROTECT_VDB
2513 #define STACKBOTTOM ((ptr_t)0xdfff0000)
2514 #define DYNAMIC_LOADING
2515 #endif
2516 #ifdef LINUX
2517 #if !defined(__ELF__) && !defined(CPPCHECK)
2518 #error Linux SPARC a.out not supported
2519 #endif
2520 #define SVR4
2521       extern int _etext[];
2522       ptr_t GC_SysVGetDataStart(size_t, ptr_t);
2523 #ifdef __arch64__
2524 #define DATASTART GC_SysVGetDataStart(0x100000, (ptr_t)_etext)
2525 #else
2526 #define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
2527 #endif
2528 #define DATASTART_IS_FUNC
2529 #endif
2530 #ifdef OPENBSD
2531 #endif
2532 #ifdef NETBSD
2533 #endif
2534 #ifdef FREEBSD
2535 #define SIG_SUSPEND SIGUSR1
2536 #define SIG_THR_RESTART SIGUSR2
2537         extern char etext[];
2538         extern char edata[];
2539 #if !defined(CPPCHECK)
2540           extern char end[];
2541 #endif
2542 #define NEED_FIND_LIMIT
2543 #define DATASTART ((ptr_t)(&etext))
2544         void * GC_find_limit(void *, int);
2545 #define DATAEND (ptr_t)GC_find_limit(DATASTART, TRUE)
2546 #define DATAEND_IS_FUNC
2547 #define GC_HAVE_DATAREGION2
2548 #define DATASTART2 ((ptr_t)(&edata))
2549 #define DATAEND2 ((ptr_t)(&end))
2550 #endif
2551 #endif
2552 #ifdef I386
2553 #define MACH_TYPE "I386"
2554 #if (defined(__LP64__) || defined(_WIN64)) && !defined(CPPCHECK)
2555 #error This should be handled as X86_64
2556 #else
2557 #define CPP_WORDSZ 32
2558 #define ALIGNMENT 4
2559 #endif
2560 #ifdef SEQUENT
2561 #define OS_TYPE "SEQUENT"
2562         extern int etext[];
2563 #define DATASTART ((ptr_t)((((word)(etext)) + 0xfff) & ~0xfff))
2564 #define STACKBOTTOM ((ptr_t)0x3ffff000)
2565 #endif
2566 #ifdef EMSCRIPTEN
2567 #define OS_TYPE "EMSCRIPTEN"
2568 #define DATASTART (ptr_t)ALIGNMENT
2569 #define DATAEND (ptr_t)ALIGNMENT
2570 #define USE_MMAP_ANON
2571 #define STACK_GROWS_DOWN
2572 #endif
2573 #if defined(__QNX__)
2574 #define OS_TYPE "QNX"
2575 #define SA_RESTART 0
2576 #define HEURISTIC1
2577       extern char etext[];
2578       extern int _end[];
2579 #define DATASTART ((ptr_t)etext)
2580 #define DATAEND ((ptr_t)_end)
2581 #endif
2582 #ifdef HAIKU
2583       extern int etext[];
2584 #define DATASTART ((ptr_t)((((word)(etext)) + 0xfff) & ~0xfff))
2585 #endif
2586 #ifdef SOLARIS
2587 #define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)_etext)
2588 #ifdef SOLARIS25_PROC_VDB_BUG_FIXED
2589 #define PROC_VDB
2590 #endif
2591 #endif
2592 #ifdef SCO
2593 #define OS_TYPE "SCO"
2594         extern int etext[];
2595 #define DATASTART ((ptr_t)((((word)(etext)) + 0x3fffff) & ~0x3fffff) \
2596                                  + ((word)(etext) & 0xfff))
2597 #define STACKBOTTOM ((ptr_t)0x7ffffffc)
2598 #endif
2599 #ifdef SCO_ELF
2600 #define OS_TYPE "SCO_ELF"
2601         extern int etext[];
2602 #define DATASTART ((ptr_t)(etext))
2603 #define STACKBOTTOM ((ptr_t)0x08048000)
2604 #define DYNAMIC_LOADING
2605 #define ELF_CLASS ELFCLASS32
2606 #endif
2607 #ifdef DGUX
2608 #define OS_TYPE "DGUX"
2609         extern int _etext, _end;
2610         ptr_t GC_SysVGetDataStart(size_t, ptr_t);
2611 #define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)(&_etext))
2612 #define DATASTART_IS_FUNC
2613 #define DATAEND ((ptr_t)(&_end))
2614 #define STACK_GROWS_DOWN
2615 #define HEURISTIC2
2616         EXTERN_C_END
2617 #include <unistd.h>
2618         EXTERN_C_BEGIN
2619 #define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE)
2620 #define DYNAMIC_LOADING
2621 #ifndef USE_MMAP
2622 #define USE_MMAP 1
2623 #endif
2624 #define MAP_FAILED (void *) ((word)-1)
2625 #define HEAP_START (ptr_t)0x40000000
2626 #endif
2627 #ifdef LINUX
2628 #if !defined(REDIRECT_MALLOC)
2629 #define MPROTECT_VDB
2630 #else
2631 #endif
2632 #define HEAP_START (ptr_t)0x1000
2633 #ifdef __ELF__
2634 #if defined(__GLIBC__) && __GLIBC__ >= 2 \
2635                 || defined(HOST_ANDROID) || defined(HOST_TIZEN)
2636 #define SEARCH_FOR_DATA_START
2637 #else
2638                  extern char **__environ;
2639 #define DATASTART ((ptr_t)(&__environ))
2640 #endif
2641 #if !defined(GC_NO_SIGSETJMP) && (defined(HOST_TIZEN) \
2642                     || (defined(HOST_ANDROID) \
2643                         && !(GC_GNUC_PREREQ(4, 8) || GC_CLANG_PREREQ(3, 2) \
2644                              || __ANDROID_API__ >= 18)))
2645 #define GC_NO_SIGSETJMP 1
2646 #endif
2647 #else
2648              extern int etext[];
2649 #define DATASTART ((ptr_t)((((word)(etext)) + 0xfff) & ~0xfff))
2650 #endif
2651 #ifdef USE_I686_PREFETCH
2652 #define PREFETCH(x) \
2653             __asm__ __volatile__ ("prefetchnta %0" : : "m"(*(char *)(x)))
2654 #ifdef FORCE_WRITE_PREFETCH
2655 #define GC_PREFETCH_FOR_WRITE(x) \
2656               __asm__ __volatile__ ("prefetcht0 %0" : : "m"(*(char *)(x)))
2657 #else
2658 #define GC_NO_PREFETCH_FOR_WRITE
2659 #endif
2660 #elif defined(USE_3DNOW_PREFETCH)
2661 #define PREFETCH(x) \
2662             __asm__ __volatile__ ("prefetch %0" : : "m"(*(char *)(x)))
2663 #define GC_PREFETCH_FOR_WRITE(x) \
2664             __asm__ __volatile__ ("prefetchw %0" : : "m"(*(char *)(x)))
2665 #endif
2666 #if defined(__GLIBC__) && !defined(__UCLIBC__) \
2667            && !defined(GLIBC_TSX_BUG_FIXED)
2668 #define GLIBC_2_19_TSX_BUG
2669           EXTERN_C_END
2670 #include <gnu/libc-version.h>
2671           EXTERN_C_BEGIN
2672 #endif
2673 #ifndef SOFT_VDB
2674 #define SOFT_VDB
2675 #endif
2676 #endif
2677 #ifdef CYGWIN32
2678 #define WOW64_THREAD_CONTEXT_WORKAROUND
2679 #define DATASTART ((ptr_t)GC_DATASTART)
2680 #define DATAEND   ((ptr_t)GC_DATAEND)
2681 #ifndef USE_WINALLOC
2682 #
2683 #ifdef USE_MMAP
2684 #define NEED_FIND_LIMIT
2685 #endif
2686 #endif
2687 #endif
2688 #ifdef INTERIX
2689 #define OS_TYPE "INTERIX"
2690       extern int _data_start__[];
2691       extern int _bss_end__[];
2692 #define DATASTART ((ptr_t)_data_start__)
2693 #define DATAEND   ((ptr_t)_bss_end__)
2694 #define STACKBOTTOM ({ ptr_t rv; \
2695                             __asm__ __volatile__ ("movl %%fs:4, %%eax" \
2696                                                   : "=a" (rv)); \
2697                             rv; })
2698 #define USE_MMAP_ANON
2699 #endif
2700 #ifdef OS2
2701 #define OS_TYPE "OS2"
2702 #define DATAEND
2703 #endif
2704 #ifdef MSWIN32
2705 #define WOW64_THREAD_CONTEXT_WORKAROUND
2706 #define RETRY_GET_THREAD_CONTEXT
2707 #define MPROTECT_VDB
2708 #endif
2709 #ifdef MSWINCE
2710 #endif
2711 #ifdef DJGPP
2712 #define OS_TYPE "DJGPP"
2713         EXTERN_C_END
2714 #include "stubinfo.h"
2715         EXTERN_C_BEGIN
2716         extern int etext[];
2717         extern int _stklen;
2718         extern int __djgpp_stack_limit;
2719 #define DATASTART ((ptr_t)((((word)(etext)) + 0x1ff) & ~0x1ff))
2720 #define STACKBOTTOM ((ptr_t)((word)__djgpp_stack_limit + _stklen))
2721 #endif
2722 #ifdef OPENBSD
2723 #endif
2724 #ifdef FREEBSD
2725 #ifdef __GLIBC__
2726 #define SIG_SUSPEND          (32+6)
2727 #define SIG_THR_RESTART      (32+5)
2728             extern int _end[];
2729 #define DATAEND ((ptr_t)(_end))
2730 #else
2731 #define SIG_SUSPEND SIGUSR1
2732 #define SIG_THR_RESTART SIGUSR2
2733 #endif
2734 #endif
2735 #ifdef NETBSD
2736 #endif
2737 #ifdef THREE86BSD
2738 #define OS_TYPE "THREE86BSD"
2739 #define HEURISTIC2
2740         extern char etext[];
2741 #define DATASTART ((ptr_t)(etext))
2742 #endif
2743 #ifdef BSDI
2744 #define OS_TYPE "BSDI"
2745 #define HEURISTIC2
2746         extern char etext[];
2747 #define DATASTART ((ptr_t)(etext))
2748 #endif
2749 #ifdef NEXT
2750 #define STACKBOTTOM ((ptr_t)0xc0000000)
2751 #endif
2752 #ifdef RTEMS
2753 #define OS_TYPE "RTEMS"
2754         EXTERN_C_END
2755 #include <sys/unistd.h>
2756         EXTERN_C_BEGIN
2757         extern int etext[];
2758         void *rtems_get_stack_bottom(void);
2759 #define InitStackBottom rtems_get_stack_bottom()
2760 #define DATASTART ((ptr_t)etext)
2761 #define STACKBOTTOM ((ptr_t)InitStackBottom)
2762 #define SIG_SUSPEND SIGUSR1
2763 #define SIG_THR_RESTART SIGUSR2
2764 #endif
2765 #ifdef DOS4GW
2766 #define OS_TYPE "DOS4GW"
2767       extern long __nullarea;
2768       extern char _end;
2769       extern char *_STACKTOP;
2770       #pragma aux __nullarea "*";
2771       #pragma aux _end "*";
2772 #define STACKBOTTOM ((ptr_t)_STACKTOP)
2773 #define DATASTART ((ptr_t)(&__nullarea))
2774 #define DATAEND ((ptr_t)(&_end))
2775 #endif
2776 #ifdef HURD
2777 #define OS_TYPE "HURD"
2778 #define STACK_GROWS_DOWN
2779 #define HEURISTIC2
2780 #define SIG_SUSPEND SIGUSR1
2781 #define SIG_THR_RESTART SIGUSR2
2782 #define SEARCH_FOR_DATA_START
2783       extern int _end[];
2784 #define DATAEND ((ptr_t)(_end))
2785 #define DYNAMIC_LOADING
2786 #define USE_MMAP_ANON
2787 #endif
2788 #ifdef DARWIN
2789 #define DARWIN_DONT_PARSE_STACK 1
2790 #define STACKBOTTOM ((ptr_t)0xc0000000)
2791 #define MPROTECT_VDB
2792 #if TARGET_OS_IPHONE && !defined(NO_DYLD_BIND_FULLY_IMAGE)
2793 #define NO_DYLD_BIND_FULLY_IMAGE
2794 #endif
2795 #endif
2796 #endif
2797 #ifdef NS32K
2798 #define MACH_TYPE "NS32K"
2799 #define ALIGNMENT 4
2800     extern char **environ;
2801 #define DATASTART ((ptr_t)(&environ))
2802 #define STACKBOTTOM ((ptr_t)0xfffff000)
2803 #endif
2804 #ifdef LOONGARCH
2805 #define MACH_TYPE "LoongArch"
2806 #ifdef LINUX
2807 #pragma weak __data_start
2808       extern int __data_start[];
2809 #define DATASTART ((ptr_t)(__data_start))
2810 #define CPP_WORDSZ _LOONGARCH_SZPTR
2811 #define ALIGNMENT (_LOONGARCH_SZPTR/8)
2812 #endif
2813 #endif
2814 #ifdef MIPS
2815 #define MACH_TYPE "MIPS"
2816 #ifdef LINUX
2817 #pragma weak __data_start
2818       extern int __data_start[];
2819 #define DATASTART ((ptr_t)(__data_start))
2820 #ifdef _MIPS_SZPTR
2821 #define CPP_WORDSZ _MIPS_SZPTR
2822 #define ALIGNMENT (_MIPS_SZPTR/8)
2823 #else
2824 #define ALIGNMENT 4
2825 #endif
2826 #ifndef HBLKSIZE
2827 #define HBLKSIZE 4096
2828 #endif
2829 #if __GLIBC__ == 2 && __GLIBC_MINOR__ >= 2 || __GLIBC__ > 2
2830 #define LINUX_STACKBOTTOM
2831 #else
2832 #define STACKBOTTOM ((ptr_t)0x7fff8000)
2833 #endif
2834 #endif
2835 #ifdef EWS4800
2836 #define OS_TYPE "EWS4800"
2837 #define HEURISTIC2
2838 #if defined(_MIPS_SZPTR) && (_MIPS_SZPTR == 64)
2839         extern int _fdata[], _end[];
2840 #define DATASTART ((ptr_t)_fdata)
2841 #define DATAEND ((ptr_t)_end)
2842 #define CPP_WORDSZ _MIPS_SZPTR
2843 #define ALIGNMENT (_MIPS_SZPTR/8)
2844 #else
2845         extern int etext[], edata[];
2846 #if !defined(CPPCHECK)
2847           extern int end[];
2848 #endif
2849         extern int _DYNAMIC_LINKING[], _gp[];
2850 #define DATASTART ((ptr_t)((((word)(etext) + 0x3ffff) & ~0x3ffff) \
2851                                   + ((word)(etext) & 0xffff)))
2852 #define DATAEND ((ptr_t)(edata))
2853 #define GC_HAVE_DATAREGION2
2854 #define DATASTART2 (_DYNAMIC_LINKING \
2855                 ? (ptr_t)(((word)_gp + 0x8000 + 0x3ffff) & ~0x3ffff) \
2856                 : (ptr_t)edata)
2857 #define DATAEND2 ((ptr_t)(end))
2858 #define ALIGNMENT 4
2859 #endif
2860 #endif
2861 #ifdef ULTRIX
2862 #define OS_TYPE "ULTRIX"
2863 #define HEURISTIC2
2864 #define DATASTART ((ptr_t)0x10000000)
2865 #define ALIGNMENT 4
2866 #endif
2867 #ifdef IRIX5
2868 #define OS_TYPE "IRIX5"
2869 #define HEURISTIC2
2870         extern int _fdata[];
2871 #define DATASTART ((ptr_t)(_fdata))
2872 #ifdef USE_MMAP
2873 #define HEAP_START (ptr_t)0x30000000
2874 #else
2875 #define HEAP_START DATASTART
2876 #endif
2877 #ifdef _MIPS_SZPTR
2878 #define CPP_WORDSZ _MIPS_SZPTR
2879 #define ALIGNMENT (_MIPS_SZPTR/8)
2880 #else
2881 #define ALIGNMENT 4
2882 #endif
2883 #define DYNAMIC_LOADING
2884 #endif
2885 #ifdef MSWINCE
2886 #define ALIGNMENT 4
2887 #endif
2888 #ifdef NETBSD
2889 #define ALIGNMENT 4
2890 #ifndef __ELF__
2891 #define DATASTART ((ptr_t)0x10000000)
2892 #define STACKBOTTOM ((ptr_t)0x7ffff000)
2893 #endif
2894 #endif
2895 #ifdef OPENBSD
2896 #define CPP_WORDSZ 64
2897 #define ALIGNMENT 8
2898 #endif
2899 #ifdef FREEBSD
2900 #define ALIGNMENT 4
2901 #define SIG_SUSPEND SIGUSR1
2902 #define SIG_THR_RESTART SIGUSR2
2903 #endif
2904 #ifdef NONSTOP
2905 #define OS_TYPE "NONSTOP"
2906 #define CPP_WORDSZ 32
2907 #define ALIGNMENT 4
2908 #define DATASTART ((ptr_t)0x08000000)
2909      extern char **environ;
2910 #define DATAEND ((ptr_t)(environ - 0x10))
2911 #define STACKBOTTOM ((ptr_t)0x4fffffff)
2912 #endif
2913 #endif
2914 #ifdef NIOS2
2915 #define CPP_WORDSZ 32
2916 #define MACH_TYPE "NIOS2"
2917 #ifdef LINUX
2918       extern int __data_start[];
2919 #define DATASTART ((ptr_t)(__data_start))
2920 #define ALIGNMENT 4
2921 #ifndef HBLKSIZE
2922 #define HBLKSIZE 4096
2923 #endif
2924 #endif
2925 #endif
2926 #ifdef OR1K
2927 #define CPP_WORDSZ 32
2928 #define MACH_TYPE "OR1K"
2929 #ifdef LINUX
2930       extern int __data_start[];
2931 #define DATASTART ((ptr_t)(__data_start))
2932 #define ALIGNMENT 4
2933 #ifndef HBLKSIZE
2934 #define HBLKSIZE 4096
2935 #endif
2936 #endif
2937 #endif
2938 #ifdef HP_PA
2939 #define MACH_TYPE "HP_PA"
2940 #ifdef __LP64__
2941 #define CPP_WORDSZ 64
2942 #define ALIGNMENT 8
2943 #else
2944 #define CPP_WORDSZ 32
2945 #define ALIGNMENT 4
2946 #endif
2947 #define STACK_GROWS_UP
2948 #ifdef HPUX
2949 #ifndef GC_THREADS
2950 #define MPROTECT_VDB
2951 #endif
2952 #ifdef USE_HPUX_FIXED_STACKBOTTOM
2953 #define STACKBOTTOM ((ptr_t)0x7b033000)
2954 #elif defined(USE_ENVIRON_POINTER)
2955         extern char ** environ;
2956 #define STACKBOTTOM ((ptr_t)environ)
2957 #elif !defined(HEURISTIC2)
2958 #define HPUX_MAIN_STACKBOTTOM
2959 #define NEED_FIND_LIMIT
2960 #endif
2961 #ifndef __GNUC__
2962 #define PREFETCH(x)  do { \
2963                               register long addr = (long)(x); \
2964                               (void) _asm ("LDW", 0, 0, addr, 0); \
2965                             } while (0)
2966 #endif
2967 #endif
2968 #ifdef LINUX
2969 #define SEARCH_FOR_DATA_START
2970 #endif
2971 #ifdef OPENBSD
2972 #endif
2973 #endif
2974 #ifdef ALPHA
2975 #define MACH_TYPE "ALPHA"
2976 #define ALIGNMENT 8
2977 #define CPP_WORDSZ 64
2978 #ifdef NETBSD
2979 #define ELFCLASS32 32
2980 #define ELFCLASS64 64
2981 #define ELF_CLASS ELFCLASS64
2982 #endif
2983 #ifdef OPENBSD
2984 #endif
2985 #ifdef FREEBSD
2986 #define SIG_SUSPEND SIGUSR1
2987 #define SIG_THR_RESTART SIGUSR2
2988         extern char etext[];
2989         extern char edata[];
2990 #if !defined(CPPCHECK)
2991           extern char end[];
2992 #endif
2993 #define NEED_FIND_LIMIT
2994 #define DATASTART ((ptr_t)(&etext))
2995         void * GC_find_limit(void *, int);
2996 #define DATAEND (ptr_t)GC_find_limit(DATASTART, TRUE)
2997 #define DATAEND_IS_FUNC
2998 #define GC_HAVE_DATAREGION2
2999 #define DATASTART2 ((ptr_t)(&edata))
3000 #define DATAEND2 ((ptr_t)(&end))
3001 #endif
3002 #ifdef OSF1
3003 #define OS_TYPE "OSF1"
3004 #define DATASTART ((ptr_t)0x140000000)
3005         extern int _end[];
3006 #define DATAEND ((ptr_t)(&_end))
3007         extern char ** environ;
3008 #define STACKBOTTOM ((ptr_t)(((word)(environ) | (getpagesize()-1))+1))
3009         extern int __start[];
3010 #define HEURISTIC2_LIMIT ((ptr_t)((word)(__start) & ~(getpagesize()-1)))
3011 #ifndef GC_OSF1_THREADS
3012 #define MPROTECT_VDB
3013 #endif
3014 #define DYNAMIC_LOADING
3015 #endif
3016 #ifdef LINUX
3017 #ifdef __ELF__
3018 #define SEARCH_FOR_DATA_START
3019 #else
3020 #define DATASTART ((ptr_t)0x140000000)
3021           extern int _end[];
3022 #define DATAEND ((ptr_t)(_end))
3023 #endif
3024 #if !defined(REDIRECT_MALLOC)
3025 #define MPROTECT_VDB
3026 #endif
3027 #endif
3028 #endif
3029 #ifdef IA64
3030 #define MACH_TYPE "IA64"
3031 #ifdef HPUX
3032 #ifdef _ILP32
3033 #define CPP_WORDSZ 32
3034 #define ALIGNMENT 4
3035 #else
3036 #if !defined(_LP64) && !defined(CPPCHECK)
3037 #error Unknown ABI
3038 #endif
3039 #define CPP_WORDSZ 64
3040 #define ALIGNMENT 8
3041 #endif
3042         extern char ** environ;
3043 #define STACKBOTTOM ((ptr_t)environ)
3044 #define HPUX_STACKBOTTOM
3045 #define BACKING_STORE_DISPLACEMENT 0x1000000
3046 #define BACKING_STORE_ALIGNMENT 0x1000
3047         extern ptr_t GC_register_stackbottom;
3048 #define BACKING_STORE_BASE GC_register_stackbottom
3049 #endif
3050 #ifdef LINUX
3051 #define CPP_WORDSZ 64
3052 #define ALIGNMENT 8
3053         extern ptr_t GC_register_stackbottom;
3054 #define BACKING_STORE_BASE GC_register_stackbottom
3055 #define SEARCH_FOR_DATA_START
3056 #ifdef __GNUC__
3057 #define DYNAMIC_LOADING
3058 #else
3059 #endif
3060 #if !defined(REDIRECT_MALLOC)
3061 #define MPROTECT_VDB
3062 #endif
3063 #ifdef __GNUC__
3064 #ifndef __INTEL_COMPILER
3065 #define PREFETCH(x) \
3066               __asm__ ("        lfetch  [%0]": : "r"(x))
3067 #define GC_PREFETCH_FOR_WRITE(x) \
3068               __asm__ ("        lfetch.excl     [%0]": : "r"(x))
3069 #define CLEAR_DOUBLE(x) \
3070               __asm__ ("        stf.spill       [%0]=f0": : "r"((void *)(x)))
3071 #else
3072             EXTERN_C_END
3073 #include <ia64intrin.h>
3074             EXTERN_C_BEGIN
3075 #define PREFETCH(x) __lfetch(__lfhint_none, (x))
3076 #define GC_PREFETCH_FOR_WRITE(x) __lfetch(__lfhint_nta, (x))
3077 #define CLEAR_DOUBLE(x) __stf_spill((void *)(x), 0)
3078 #endif
3079 #endif
3080 #endif
3081 #ifdef MSWIN32
3082 #if defined(_WIN64)
3083 #define CPP_WORDSZ 64
3084 #else
3085 #define CPP_WORDSZ 32
3086 #endif
3087 #define ALIGNMENT 8
3088 #endif
3089 #endif
3090 #ifdef M88K
3091 #define MACH_TYPE "M88K"
3092 #define ALIGNMENT 4
3093 #define STACKBOTTOM ((char*)0xf0000000)
3094     extern int etext[];
3095 #ifdef CX_UX
3096 #define OS_TYPE "CX_UX"
3097 #define DATASTART ((ptr_t)((((word)(etext) + 0x3fffff) & ~0x3fffff) \
3098                                   + 0x10000))
3099 #endif
3100 #ifdef DGUX
3101 #define OS_TYPE "DGUX"
3102         ptr_t GC_SysVGetDataStart(size_t, ptr_t);
3103 #define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)etext)
3104 #define DATASTART_IS_FUNC
3105 #endif
3106 #endif
3107 #ifdef S370
3108 #define MACH_TYPE "S370"
3109 #define ALIGNMENT 4
3110 #ifdef UTS4
3111 #define OS_TYPE "UTS4"
3112         extern int etext[];
3113         extern int _etext[];
3114         extern int _end[];
3115         ptr_t GC_SysVGetDataStart(size_t, ptr_t);
3116 #define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
3117 #define DATASTART_IS_FUNC
3118 #define DATAEND ((ptr_t)(_end))
3119 #define HEURISTIC2
3120 #endif
3121 #endif
3122 #ifdef S390
3123 #define MACH_TYPE "S390"
3124 #ifndef __s390x__
3125 #define ALIGNMENT 4
3126 #define CPP_WORDSZ 32
3127 #else
3128 #define ALIGNMENT 8
3129 #define CPP_WORDSZ 64
3130 #ifndef HBLKSIZE
3131 #define HBLKSIZE 4096
3132 #endif
3133 #endif
3134 #ifdef LINUX
3135         extern int __data_start[] __attribute__((__weak__));
3136 #define DATASTART ((ptr_t)(__data_start))
3137         extern int _end[] __attribute__((__weak__));
3138 #define DATAEND ((ptr_t)(_end))
3139 #define CACHE_LINE_SIZE 256
3140 #define GETPAGESIZE() 4096
3141 #if !defined(REDIRECT_MALLOC)
3142 #define MPROTECT_VDB
3143 #endif
3144 #ifndef SOFT_VDB
3145 #define SOFT_VDB
3146 #endif
3147 #endif
3148 #endif
3149 #ifdef AARCH64
3150 #define MACH_TYPE "AARCH64"
3151 #ifdef __ILP32__
3152 #define CPP_WORDSZ 32
3153 #define ALIGNMENT 4
3154 #else
3155 #define CPP_WORDSZ 64
3156 #define ALIGNMENT 8
3157 #endif
3158 #ifndef HBLKSIZE
3159 #define HBLKSIZE 4096
3160 #endif
3161 #ifdef LINUX
3162 #if !defined(REDIRECT_MALLOC)
3163 #define MPROTECT_VDB
3164 #endif
3165 #if defined(HOST_ANDROID)
3166 #define SEARCH_FOR_DATA_START
3167 #else
3168         extern int __data_start[];
3169 #define DATASTART ((ptr_t)__data_start)
3170 #endif
3171 #endif
3172 #ifdef DARWIN
3173 #define DARWIN_DONT_PARSE_STACK 1
3174 #define STACKBOTTOM ((ptr_t)0x16fdfffff)
3175 #if TARGET_OS_IPHONE && !defined(NO_DYLD_BIND_FULLY_IMAGE)
3176 #define NO_DYLD_BIND_FULLY_IMAGE
3177 #endif
3178 #endif
3179 #ifdef FREEBSD
3180 #endif
3181 #ifdef NETBSD
3182 #define ELF_CLASS ELFCLASS64
3183 #endif
3184 #ifdef OPENBSD
3185 #endif
3186 #ifdef NINTENDO_SWITCH
3187 #define OS_TYPE "NINTENDO_SWITCH"
3188       extern int __bss_end[];
3189 #define NO_HANDLE_FORK 1
3190 #define DATASTART (ptr_t)ALIGNMENT
3191 #define DATAEND (ptr_t)(&__bss_end)
3192       void *switch_get_stack_bottom(void);
3193 #define STACKBOTTOM ((ptr_t)switch_get_stack_bottom())
3194 #endif
3195 #ifdef MSWIN32
3196 #endif
3197 #ifdef NOSYS
3198 #define OS_TYPE "NOSYS"
3199       extern int __data_start[];
3200 #define DATASTART ((ptr_t)__data_start)
3201       extern void *__stack_base__;
3202 #define STACKBOTTOM ((ptr_t)__stack_base__)
3203 #endif
3204 #endif
3205 #ifdef ARM32
3206 #if defined(NACL)
3207 #define MACH_TYPE "NACL"
3208 #else
3209 #define MACH_TYPE "ARM32"
3210 #endif
3211 #define CPP_WORDSZ 32
3212 #define ALIGNMENT 4
3213 #ifdef NETBSD
3214 #endif
3215 #ifdef LINUX
3216 #if !defined(REDIRECT_MALLOC)
3217 #define MPROTECT_VDB
3218 #endif
3219 #if defined(__GLIBC__) && __GLIBC__ >= 2 \
3220                 || defined(HOST_ANDROID) || defined(HOST_TIZEN)
3221 #define SEARCH_FOR_DATA_START
3222 #else
3223             extern char **__environ;
3224 #define DATASTART ((ptr_t)(&__environ))
3225 #endif
3226 #endif
3227 #ifdef MSWINCE
3228 #endif
3229 #ifdef FREEBSD
3230 #define SIG_SUSPEND SIGUSR1
3231 #define SIG_THR_RESTART SIGUSR2
3232 #endif
3233 #ifdef DARWIN
3234 #define DARWIN_DONT_PARSE_STACK 1
3235 #define STACKBOTTOM ((ptr_t)0x30000000)
3236 #if TARGET_OS_IPHONE && !defined(NO_DYLD_BIND_FULLY_IMAGE)
3237 #define NO_DYLD_BIND_FULLY_IMAGE
3238 #endif
3239 #endif
3240 #ifdef OPENBSD
3241 #endif
3242 #ifdef SN_TARGET_PSP2
3243 #define OS_TYPE "SN_TARGET_PSP2"
3244 #define NO_HANDLE_FORK 1
3245 #define DATASTART (ptr_t)ALIGNMENT
3246 #define DATAEND (ptr_t)ALIGNMENT
3247       void *psp2_get_stack_bottom(void);
3248 #define STACKBOTTOM ((ptr_t)psp2_get_stack_bottom())
3249 #endif
3250 #ifdef NN_PLATFORM_CTR
3251 #define OS_TYPE "NN_PLATFORM_CTR"
3252       extern unsigned char Image$$ZI$$ZI$$Base[];
3253 #define DATASTART (ptr_t)(Image$$ZI$$ZI$$Base)
3254       extern unsigned char Image$$ZI$$ZI$$Limit[];
3255 #define DATAEND (ptr_t)(Image$$ZI$$ZI$$Limit)
3256       void *n3ds_get_stack_bottom(void);
3257 #define STACKBOTTOM ((ptr_t)n3ds_get_stack_bottom())
3258 #endif
3259 #ifdef MSWIN32
3260 #endif
3261 #ifdef NOSYS
3262 #define OS_TYPE "NOSYS"
3263       extern int __data_start[];
3264 #define DATASTART ((ptr_t)(__data_start))
3265       extern void *__stack_base__;
3266 #define STACKBOTTOM ((ptr_t)(__stack_base__))
3267 #endif
3268 #endif
3269 #ifdef CRIS
3270 #define MACH_TYPE "CRIS"
3271 #define CPP_WORDSZ 32
3272 #define ALIGNMENT 1
3273 #ifdef LINUX
3274 #define SEARCH_FOR_DATA_START
3275 #endif
3276 #endif
3277 #if defined(SH) && !defined(SH4)
3278 #define MACH_TYPE "SH"
3279 #define ALIGNMENT 4
3280 #ifdef LINUX
3281 #define SEARCH_FOR_DATA_START
3282 #endif
3283 #ifdef NETBSD
3284 #endif
3285 #ifdef OPENBSD
3286 #endif
3287 #ifdef MSWINCE
3288 #endif
3289 #endif
3290 #ifdef SH4
3291 #define MACH_TYPE "SH4"
3292 #define ALIGNMENT 4
3293 #ifdef MSWINCE
3294 #endif
3295 #endif
3296 #ifdef AVR32
3297 #define MACH_TYPE "AVR32"
3298 #define CPP_WORDSZ 32
3299 #define ALIGNMENT 4
3300 #ifdef LINUX
3301 #define SEARCH_FOR_DATA_START
3302 #endif
3303 #endif
3304 #ifdef M32R
3305 #define CPP_WORDSZ 32
3306 #define MACH_TYPE "M32R"
3307 #define ALIGNMENT 4
3308 #ifdef LINUX
3309 #define SEARCH_FOR_DATA_START
3310 #endif
3311 #endif
3312 #ifdef X86_64
3313 #define MACH_TYPE "X86_64"
3314 #ifdef __ILP32__
3315 #define ALIGNMENT 4
3316 #define CPP_WORDSZ 32
3317 #else
3318 #define ALIGNMENT 8
3319 #define CPP_WORDSZ 64
3320 #endif
3321 #ifndef HBLKSIZE
3322 #define HBLKSIZE 4096
3323 #endif
3324 #ifndef CACHE_LINE_SIZE
3325 #define CACHE_LINE_SIZE 64
3326 #endif
3327 #ifdef PLATFORM_GETMEM
3328 #define OS_TYPE "PLATFORM_GETMEM"
3329 #define DATASTART (ptr_t)ALIGNMENT
3330 #define DATAEND (ptr_t)ALIGNMENT
3331       EXTERN_C_END
3332 #include <pthread.h>
3333       EXTERN_C_BEGIN
3334       void *platform_get_stack_bottom(void);
3335 #define STACKBOTTOM ((ptr_t)platform_get_stack_bottom())
3336 #endif
3337 #ifdef LINUX
3338 #if !defined(REDIRECT_MALLOC)
3339 #define MPROTECT_VDB
3340 #else
3341 #endif
3342 #define SEARCH_FOR_DATA_START
3343 #if defined(__GLIBC__) && !defined(__UCLIBC__)
3344 #define USE_MMAP_ANON
3345 #endif
3346 #if defined(__GLIBC__) && !defined(__UCLIBC__) \
3347            && !defined(GETCONTEXT_FPU_BUG_FIXED)
3348 #define GETCONTEXT_FPU_EXCMASK_BUG
3349 #endif
3350 #if defined(__GLIBC__) && !defined(__UCLIBC__) \
3351            && !defined(GLIBC_TSX_BUG_FIXED)
3352 #define GLIBC_2_19_TSX_BUG
3353           EXTERN_C_END
3354 #include <gnu/libc-version.h>
3355           EXTERN_C_BEGIN
3356 #endif
3357 #ifndef SOFT_VDB
3358 #define SOFT_VDB
3359 #endif
3360 #endif
3361 #ifdef DARWIN
3362 #define DARWIN_DONT_PARSE_STACK 1
3363 #define STACKBOTTOM ((ptr_t)0x7fff5fc00000)
3364 #define MPROTECT_VDB
3365 #if TARGET_OS_IPHONE && !defined(NO_DYLD_BIND_FULLY_IMAGE)
3366 #define NO_DYLD_BIND_FULLY_IMAGE
3367 #endif
3368 #endif
3369 #ifdef FREEBSD
3370 #ifdef __GLIBC__
3371 #define SIG_SUSPEND          (32+6)
3372 #define SIG_THR_RESTART      (32+5)
3373             extern int _end[];
3374 #define DATAEND ((ptr_t)(_end))
3375 #else
3376 #define SIG_SUSPEND SIGUSR1
3377 #define SIG_THR_RESTART SIGUSR2
3378 #endif
3379 #if defined(__DragonFly__)
3380 #define COUNT_UNMAPPED_REGIONS
3381 #endif
3382 #endif
3383 #ifdef NETBSD
3384 #endif
3385 #ifdef OPENBSD
3386 #endif
3387 #ifdef HAIKU
3388 #define HEURISTIC2
3389 #define SEARCH_FOR_DATA_START
3390 #endif
3391 #ifdef SOLARIS
3392 #define ELF_CLASS ELFCLASS64
3393 #define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)_etext)
3394 #ifdef SOLARIS25_PROC_VDB_BUG_FIXED
3395 #define PROC_VDB
3396 #endif
3397 #endif
3398 #ifdef CYGWIN32
3399 #ifndef USE_WINALLOC
3400 #if defined(THREAD_LOCAL_ALLOC)
3401 #else
3402 #define MPROTECT_VDB
3403 #endif
3404 #endif
3405 #endif
3406 #ifdef MSWIN_XBOX1
3407 #define OS_TYPE "MSWIN_XBOX1"
3408 #define NO_GETENV
3409 #define DATASTART (ptr_t)ALIGNMENT
3410 #define DATAEND (ptr_t)ALIGNMENT
3411       LONG64 durango_get_stack_bottom(void);
3412 #define STACKBOTTOM ((ptr_t)durango_get_stack_bottom())
3413 #define GETPAGESIZE() 4096
3414 #ifndef USE_MMAP
3415 #define USE_MMAP 1
3416 #endif
3417 #define PROT_NONE  0
3418 #define PROT_READ  1
3419 #define PROT_WRITE 2
3420 #define PROT_EXEC  4
3421 #define MAP_PRIVATE 2
3422 #define MAP_FIXED  0x10
3423 #define MAP_FAILED ((void *)-1)
3424 #endif
3425 #ifdef MSWIN32
3426 #define RETRY_GET_THREAD_CONTEXT
3427 #if !defined(__GNUC__) || defined(__INTEL_COMPILER) \
3428            || GC_GNUC_PREREQ(4, 7)
3429 #define MPROTECT_VDB
3430 #endif
3431 #endif
3432 #endif
3433 #ifdef ARC
3434 #define CPP_WORDSZ 32
3435 #define MACH_TYPE "ARC"
3436 #define ALIGNMENT 4
3437 #define CACHE_LINE_SIZE 64
3438 #ifdef LINUX
3439       extern int __data_start[] __attribute__((__weak__));
3440 #define DATASTART ((ptr_t)__data_start)
3441 #endif
3442 #endif
3443 #ifdef HEXAGON
3444 #define CPP_WORDSZ 32
3445 #define MACH_TYPE "HEXAGON"
3446 #define ALIGNMENT 4
3447 #ifdef LINUX
3448 #if !defined(REDIRECT_MALLOC)
3449 #define MPROTECT_VDB
3450 #endif
3451 #if defined(__GLIBC__)
3452 #define SEARCH_FOR_DATA_START
3453 #elif !defined(CPPCHECK)
3454 #error Unknown Hexagon libc configuration
3455 #endif
3456 #endif
3457 #endif
3458 #ifdef TILEPRO
3459 #define CPP_WORDSZ 32
3460 #define MACH_TYPE "TILEPro"
3461 #define ALIGNMENT 4
3462 #define PREFETCH(x) __insn_prefetch(x)
3463 #define CACHE_LINE_SIZE 64
3464 #ifdef LINUX
3465       extern int __data_start[];
3466 #define DATASTART ((ptr_t)__data_start)
3467 #endif
3468 #endif
3469 #ifdef TILEGX
3470 #define CPP_WORDSZ (__SIZEOF_POINTER__ * 8)
3471 #define MACH_TYPE "TILE-Gx"
3472 #define ALIGNMENT __SIZEOF_POINTER__
3473 #if CPP_WORDSZ < 64
3474 #define CLEAR_DOUBLE(x) (*(long long *)(x) = 0)
3475 #endif
3476 #define PREFETCH(x) __insn_prefetch_l1(x)
3477 #define CACHE_LINE_SIZE 64
3478 #ifdef LINUX
3479       extern int __data_start[];
3480 #define DATASTART ((ptr_t)__data_start)
3481 #endif
3482 #endif
3483 #ifdef RISCV
3484 #define MACH_TYPE "RISC-V"
3485 #define CPP_WORDSZ __riscv_xlen
3486 #define ALIGNMENT (CPP_WORDSZ/8)
3487 #ifdef FREEBSD
3488 #define SIG_SUSPEND SIGUSR1
3489 #define SIG_THR_RESTART SIGUSR2
3490 #endif
3491 #ifdef LINUX
3492       extern int __data_start[] __attribute__((__weak__));
3493 #define DATASTART ((ptr_t)__data_start)
3494 #endif
3495 #endif
3496 #if defined(__GLIBC__) && !defined(DONT_USE_LIBC_PRIVATES)
3497 #define USE_LIBC_PRIVATES
3498 #endif
3499 #ifdef NO_RETRY_GET_THREAD_CONTEXT
3500 #undef RETRY_GET_THREAD_CONTEXT
3501 #endif
3502 #if defined(LINUX_STACKBOTTOM) && defined(NO_PROC_STAT) \
3503     && !defined(USE_LIBC_PRIVATES)
3504 #undef LINUX_STACKBOTTOM
3505 #define HEURISTIC2
3506 #endif
3507 #if defined(USE_MMAP_ANON) && !defined(USE_MMAP)
3508 #define USE_MMAP 1
3509 #elif (defined(LINUX) || defined(OPENBSD)) && defined(USE_MMAP)
3510 #define USE_MMAP_ANON
3511 #endif
3512 #if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC) \
3513     && !defined(USE_PROC_FOR_LIBRARIES)
3514 #define USE_PROC_FOR_LIBRARIES
3515 #endif
3516 #ifndef STACK_GROWS_UP
3517 #define STACK_GROWS_DOWN
3518 #endif
3519 #ifndef CPP_WORDSZ
3520 #define CPP_WORDSZ 32
3521 #endif
3522 #ifndef OS_TYPE
3523 #define OS_TYPE ""
3524 #endif
3525 #ifndef DATAEND
3526 #if !defined(CPPCHECK)
3527     extern int end[];
3528 #endif
3529 #define DATAEND ((ptr_t)(end))
3530 #endif
3531 #if defined(HOST_ANDROID) && defined(__clang__) \
3532     && !defined(BROKEN_UUENDUU_SYM)
3533 #undef DATAEND
3534 #pragma weak __end__
3535   extern int __end__[];
3536 #define DATAEND (__end__ != 0 ? (ptr_t)__end__ : (ptr_t)_end)
3537 #endif
3538 #if (defined(SVR4) || defined(HOST_ANDROID) || defined(HOST_TIZEN)) \
3539     && !defined(GETPAGESIZE)
3540   EXTERN_C_END
3541 #include <unistd.h>
3542   EXTERN_C_BEGIN
3543 #define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE)
3544 #endif
3545 #ifndef GETPAGESIZE
3546 #if defined(SOLARIS) || defined(IRIX5) || defined(LINUX) \
3547      || defined(NETBSD) || defined(FREEBSD) || defined(HPUX)
3548     EXTERN_C_END
3549 #include <unistd.h>
3550     EXTERN_C_BEGIN
3551 #endif
3552 #define GETPAGESIZE() (unsigned)getpagesize()
3553 #endif
3554 #if defined(HOST_ANDROID) && !(__ANDROID_API__ >= 23) \
3555     && ((defined(MIPS) && (CPP_WORDSZ == 32)) \
3556         || defined(ARM32) || defined(I386) )
3557 #define USE_TKILL_ON_ANDROID
3558 #endif
3559 #if defined(SOLARIS) || defined(DRSNX) || defined(UTS4)
3560 #define SVR4
3561 #endif
3562 #if defined(SOLARIS) || defined(DRSNX)
3563 #define SOLARISDL
3564 #define SUNOS5SIGS
3565 #endif
3566 #if defined(HPUX)
3567 #define SUNOS5SIGS
3568 #endif
3569 #if defined(FREEBSD) && (defined(__DragonFly__) || __FreeBSD__ >= 4 \
3570                          || (__FreeBSD_kernel__ >= 4))
3571 #define SUNOS5SIGS
3572 #endif
3573 #if !defined(GC_EXPLICIT_SIGNALS_UNBLOCK) && defined(SUNOS5SIGS) \
3574     && !defined(GC_NO_PTHREAD_SIGMASK)
3575 #define GC_EXPLICIT_SIGNALS_UNBLOCK
3576 #endif
3577 #if !defined(NO_SIGNALS_UNBLOCK_IN_MAIN) && defined(GC_NO_PTHREAD_SIGMASK)
3578 #define NO_SIGNALS_UNBLOCK_IN_MAIN
3579 #endif
3580 #if !defined(NO_MARKER_SPECIAL_SIGMASK) \
3581     && (defined(NACL) || defined(GC_WIN32_PTHREADS))
3582 #define NO_MARKER_SPECIAL_SIGMASK
3583 #endif
3584 #ifdef GC_NETBSD_THREADS
3585 #define SIGRTMIN 33
3586 #define SIGRTMAX 63
3587 #define GC_NETBSD_THREADS_WORKAROUND
3588 #endif
3589 #ifdef GC_OPENBSD_THREADS
3590   EXTERN_C_END
3591 #include <sys/param.h>
3592   EXTERN_C_BEGIN
3593 #if OpenBSD < 201211
3594 #define GC_OPENBSD_UTHREADS 1
3595 #endif
3596 #endif
3597 #if defined(SVR4) || defined(LINUX) || defined(IRIX5) || defined(HPUX) \
3598     || defined(OPENBSD) || defined(NETBSD) || defined(FREEBSD) \
3599     || defined(DGUX) || defined(BSD) || defined(HAIKU) || defined(HURD) \
3600     || defined(AIX) || defined(DARWIN) || defined(OSF1)
3601 #define UNIX_LIKE
3602 #endif
3603 #if defined(CPPCHECK)
3604 #undef CPP_WORDSZ
3605 #define CPP_WORDSZ (__SIZEOF_POINTER__ * 8)
3606 #elif CPP_WORDSZ != 32 && CPP_WORDSZ != 64
3607 #error Bad word size
3608 #endif
3609 #if !defined(ALIGNMENT) && !defined(CPPCHECK)
3610 #error Undefined ALIGNMENT
3611 #endif
3612 #ifdef PCR
3613 #undef DYNAMIC_LOADING
3614 #undef STACKBOTTOM
3615 #undef HEURISTIC1
3616 #undef HEURISTIC2
3617 #undef PROC_VDB
3618 #undef MPROTECT_VDB
3619 #define PCR_VDB
3620 #endif
3621 #if !defined(STACKBOTTOM) && (defined(ECOS) || defined(NOSYS)) \
3622     && !defined(CPPCHECK)
3623 #error Undefined STACKBOTTOM
3624 #endif
3625 #ifdef IGNORE_DYNAMIC_LOADING
3626 #undef DYNAMIC_LOADING
3627 #endif
3628 #if defined(SMALL_CONFIG) && !defined(GC_DISABLE_INCREMENTAL)
3629 #define GC_DISABLE_INCREMENTAL
3630 #endif
3631 #if (defined(MSWIN32) || defined(MSWINCE)) && !defined(USE_WINALLOC)
3632 #define USE_WINALLOC 1
3633 #endif
3634 #ifdef USE_WINALLOC
3635 #undef USE_MMAP
3636 #endif
3637 #if defined(DARWIN) || defined(FREEBSD) || defined(HAIKU) \
3638     || defined(IRIX5) || defined(LINUX) || defined(NETBSD) \
3639     || defined(OPENBSD) || defined(SOLARIS) \
3640     || ((defined(CYGWIN32) || defined(USE_MMAP) || defined(USE_MUNMAP)) \
3641         && !defined(USE_WINALLOC))
3642 #define MMAP_SUPPORTED
3643 #endif
3644 #if defined(USE_MUNMAP) && !defined(MUNMAP_THRESHOLD) \
3645     && (defined(SN_TARGET_PS3) \
3646         || defined(SN_TARGET_PSP2) || defined(MSWIN_XBOX1))
3647 #define MUNMAP_THRESHOLD 2
3648 #endif
3649 #if defined(USE_MUNMAP) && defined(COUNT_UNMAPPED_REGIONS) \
3650     && !defined(GC_UNMAPPED_REGIONS_SOFT_LIMIT)
3651 #if defined(__DragonFly__)
3652 #define GC_UNMAPPED_REGIONS_SOFT_LIMIT (1000000 / 4)
3653 #else
3654 #define GC_UNMAPPED_REGIONS_SOFT_LIMIT 16384
3655 #endif
3656 #endif
3657 #if defined(GC_DISABLE_INCREMENTAL) || defined(DEFAULT_VDB)
3658 #undef GWW_VDB
3659 #undef MPROTECT_VDB
3660 #undef PCR_VDB
3661 #undef PROC_VDB
3662 #undef SOFT_VDB
3663 #endif
3664 #ifdef NO_GWW_VDB
3665 #undef GWW_VDB
3666 #endif
3667 #ifdef NO_MPROTECT_VDB
3668 #undef MPROTECT_VDB
3669 #endif
3670 #ifdef NO_SOFT_VDB
3671 #undef SOFT_VDB
3672 #endif
3673 #if defined(SOFT_VDB) && defined(SOFT_VDB_LINUX_VER_STATIC_CHECK)
3674   EXTERN_C_END
3675 #include <linux/version.h>
3676   EXTERN_C_BEGIN
3677 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0)
3678 #undef SOFT_VDB
3679 #endif
3680 #endif
3681 #ifdef GC_DISABLE_INCREMENTAL
3682 #undef CHECKSUMS
3683 #endif
3684 #ifdef USE_GLOBAL_ALLOC
3685 #undef GWW_VDB
3686 #endif
3687 #if defined(BASE_ATOMIC_OPS_EMULATED)
3688 #undef MPROTECT_VDB
3689 #endif
3690 #if defined(USE_PROC_FOR_LIBRARIES) && defined(GC_LINUX_THREADS)
3691 #undef MPROTECT_VDB
3692 #endif
3693 #if defined(MPROTECT_VDB) && defined(GC_PREFER_MPROTECT_VDB)
3694 #undef PCR_VDB
3695 #undef PROC_VDB
3696 #endif
3697 #ifdef PROC_VDB
3698 #undef MPROTECT_VDB
3699 #undef SOFT_VDB
3700 #endif
3701 #if defined(MPROTECT_VDB) && !defined(MSWIN32) && !defined(MSWINCE)
3702 #include <signal.h>
3703 #endif
3704 #if defined(SIGBUS) && !defined(HAVE_SIGBUS) && !defined(CPPCHECK)
3705 #define HAVE_SIGBUS
3706 #endif
3707 #ifndef SA_SIGINFO
3708 #define NO_SA_SIGACTION
3709 #endif
3710 #if (defined(NO_SA_SIGACTION) || defined(GC_NO_SIGSETJMP)) \
3711     && defined(MPROTECT_VDB) && !defined(DARWIN) \
3712     && !defined(MSWIN32) && !defined(MSWINCE)
3713 #undef MPROTECT_VDB
3714 #endif
3715 #if !defined(PCR_VDB) && !defined(PROC_VDB) && !defined(MPROTECT_VDB) \
3716     && !defined(GWW_VDB) && !defined(SOFT_VDB) && !defined(DEFAULT_VDB) \
3717     && !defined(GC_DISABLE_INCREMENTAL)
3718 #define DEFAULT_VDB
3719 #endif
3720 #if !defined(PROC_VDB) && !defined(SOFT_VDB) \
3721     && !defined(NO_VDB_FOR_STATIC_ROOTS)
3722 #define NO_VDB_FOR_STATIC_ROOTS
3723 #endif
3724 #if ((defined(UNIX_LIKE) && (defined(DARWIN) || defined(HAIKU) \
3725                              || defined(HURD) || defined(OPENBSD) \
3726                              || defined(ARM32) \
3727                              || defined(AVR32) || defined(MIPS) \
3728                              || defined(NIOS2) || defined(OR1K))) \
3729      || (defined(LINUX) && !defined(__gnu_linux__)) \
3730      || (defined(RTEMS) && defined(I386)) || defined(HOST_ANDROID)) \
3731     && !defined(NO_GETCONTEXT)
3732 #define NO_GETCONTEXT 1
3733 #endif
3734 #ifndef PREFETCH
3735 #if GC_GNUC_PREREQ(3, 0) && !defined(NO_PREFETCH)
3736 #define PREFETCH(x) __builtin_prefetch((x), 0, 0)
3737 #else
3738 #define PREFETCH(x) (void)0
3739 #endif
3740 #endif
3741 #ifndef GC_PREFETCH_FOR_WRITE
3742 #if GC_GNUC_PREREQ(3, 0) && !defined(GC_NO_PREFETCH_FOR_WRITE)
3743 #define GC_PREFETCH_FOR_WRITE(x) __builtin_prefetch((x), 1)
3744 #else
3745 #define GC_PREFETCH_FOR_WRITE(x) (void)0
3746 #endif
3747 #endif
3748 #ifndef CACHE_LINE_SIZE
3749 #define CACHE_LINE_SIZE 32
3750 #endif
3751 #ifndef STATIC
3752 #ifdef GC_ASSERTIONS
3753 #define STATIC
3754 #else
3755 #define STATIC static
3756 #endif
3757 #endif
3758 #if defined(LINUX) && (defined(USE_PROC_FOR_LIBRARIES) || defined(IA64) \
3759                        || !defined(SMALL_CONFIG))
3760 #define NEED_PROC_MAPS
3761 #endif
3762 #if defined(LINUX) || defined(HURD) || defined(__GLIBC__)
3763 #define REGISTER_LIBRARIES_EARLY
3764 #endif
3765 #if defined(SEARCH_FOR_DATA_START)
3766   extern ptr_t GC_data_start;
3767 #define DATASTART GC_data_start
3768 #endif
3769 #ifndef HEAP_START
3770 #define HEAP_START ((ptr_t)0)
3771 #endif
3772 #ifndef CLEAR_DOUBLE
3773 #define CLEAR_DOUBLE(x) (((word*)(x))[0] = 0, ((word*)(x))[1] = 0)
3774 #endif
3775 #if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC) \
3776     && !defined(INCLUDE_LINUX_THREAD_DESCR)
3777 #define INCLUDE_LINUX_THREAD_DESCR
3778 #endif
3779 #if !defined(CPPCHECK)
3780 #if defined(GC_IRIX_THREADS) && !defined(IRIX5)
3781 #error Inconsistent configuration
3782 #endif
3783 #if defined(GC_LINUX_THREADS) && !defined(LINUX) && !defined(NACL)
3784 #error Inconsistent configuration
3785 #endif
3786 #if defined(GC_NETBSD_THREADS) && !defined(NETBSD)
3787 #error Inconsistent configuration
3788 #endif
3789 #if defined(GC_FREEBSD_THREADS) && !defined(FREEBSD)
3790 #error Inconsistent configuration
3791 #endif
3792 #if defined(GC_SOLARIS_THREADS) && !defined(SOLARIS)
3793 #error Inconsistent configuration
3794 #endif
3795 #if defined(GC_HPUX_THREADS) && !defined(HPUX)
3796 #error Inconsistent configuration
3797 #endif
3798 #if defined(GC_AIX_THREADS) && !defined(_AIX)
3799 #error Inconsistent configuration
3800 #endif
3801 #if defined(GC_WIN32_THREADS) && !defined(CYGWIN32) && !defined(MSWIN32) \
3802      && !defined(MSWINCE) && !defined(MSWIN_XBOX1)
3803 #error Inconsistent configuration
3804 #endif
3805 #if defined(GC_WIN32_PTHREADS) && defined(CYGWIN32)
3806 #error Inconsistent configuration
3807 #endif
3808 #endif
3809 #if defined(PCR) || defined(GC_WIN32_THREADS) || defined(GC_PTHREADS) \
3810     || ((defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) \
3811          || defined(SN_TARGET_PS3) \
3812          || defined(SN_TARGET_PSP2)) && defined(GC_THREADS))
3813 #define THREADS
3814 #endif
3815 #if defined(PARALLEL_MARK) && !defined(THREADS) && !defined(CPPCHECK)
3816 #error Invalid config: PARALLEL_MARK requires GC_THREADS
3817 #endif
3818 #if defined(GWW_VDB) && !defined(USE_WINALLOC) && !defined(CPPCHECK)
3819 #error Invalid config: GWW_VDB requires USE_WINALLOC
3820 #endif
3821 #if (((defined(MSWIN32) || defined(MSWINCE)) && !defined(__GNUC__)) \
3822         || (defined(MSWIN32) && defined(I386))  \
3823         || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))) \
3824     && !defined(NO_CRT) && !defined(NO_WRAP_MARK_SOME)
3825 #define WRAP_MARK_SOME
3826 #endif
3827 #if defined(PARALLEL_MARK) && !defined(DEFAULT_STACK_MAYBE_SMALL) \
3828     && (defined(HPUX) || defined(GC_DGUX386_THREADS) \
3829         || defined(NO_GETCONTEXT) )
3830 #define DEFAULT_STACK_MAYBE_SMALL
3831 #endif
3832 #ifdef PARALLEL_MARK
3833 #define MIN_STACK_SIZE (8 * HBLKSIZE * sizeof(word))
3834 #endif
3835 #if defined(HOST_ANDROID) && !defined(THREADS) \
3836     && !defined(USE_GET_STACKBASE_FOR_MAIN)
3837 #define USE_GET_STACKBASE_FOR_MAIN
3838 #endif
3839 #if ((defined(FREEBSD) && defined(__GLIBC__))  \
3840      || defined(LINUX) || defined(NETBSD) || defined(HOST_ANDROID)) \
3841     && !defined(NO_PTHREAD_GETATTR_NP)
3842 #define HAVE_PTHREAD_GETATTR_NP 1
3843 #elif defined(FREEBSD) && !defined(__GLIBC__) \
3844       && !defined(NO_PTHREAD_ATTR_GET_NP)
3845 #define HAVE_PTHREAD_NP_H 1
3846 #define HAVE_PTHREAD_ATTR_GET_NP 1
3847 #endif
3848 #if defined(UNIX_LIKE) && defined(THREADS) && !defined(NO_CANCEL_SAFE) \
3849     && !defined(HOST_ANDROID)
3850 #define CANCEL_SAFE
3851 #endif
3852 #ifdef CANCEL_SAFE
3853 #define IF_CANCEL(x) x
3854 #else
3855 #define IF_CANCEL(x)
3856 #endif
3857 #if !defined(CAN_HANDLE_FORK) && !defined(NO_HANDLE_FORK) \
3858     && !defined(HAVE_NO_FORK) \
3859     && ((defined(GC_PTHREADS) && !defined(NACL) \
3860          && !defined(GC_WIN32_PTHREADS) && !defined(USE_WINALLOC)) \
3861         || (defined(DARWIN) && defined(MPROTECT_VDB)) || defined(HANDLE_FORK))
3862 #define CAN_HANDLE_FORK
3863 #endif
3864 #if defined(CAN_HANDLE_FORK) && !defined(CAN_CALL_ATFORK) \
3865     && !defined(GC_NO_CAN_CALL_ATFORK) && !defined(HOST_TIZEN) \
3866     && !defined(HURD) && (!defined(HOST_ANDROID) || __ANDROID_API__ >= 21)
3867 #define CAN_CALL_ATFORK
3868 #endif
3869 #if !defined(CAN_HANDLE_FORK) && !defined(HAVE_NO_FORK) \
3870     && !(defined(CYGWIN32) || defined(SOLARIS) || defined(UNIX_LIKE))
3871 #define HAVE_NO_FORK
3872 #endif
3873 #if !defined(USE_MARK_BITS) && !defined(USE_MARK_BYTES) \
3874     && defined(PARALLEL_MARK)
3875 #define USE_MARK_BYTES
3876 #endif
3877 #if (defined(MSWINCE) && !defined(__CEGCC__) || defined(MSWINRT_FLAVOR)) \
3878     && !defined(NO_GETENV)
3879 #define NO_GETENV
3880 #endif
3881 #if (defined(NO_GETENV) || defined(MSWINCE)) && !defined(NO_GETENV_WIN32)
3882 #define NO_GETENV_WIN32
3883 #endif
3884 #if !defined(MSGBOX_ON_ERROR) && !defined(NO_MSGBOX_ON_ERROR) \
3885     && !defined(SMALL_CONFIG) && defined(MSWIN32) \
3886     && !defined(MSWINRT_FLAVOR) && !defined(MSWIN_XBOX1)
3887 #define MSGBOX_ON_ERROR
3888 #endif
3889 #ifndef STRTOULL
3890 #if defined(_WIN64) && !defined(__GNUC__)
3891 #define STRTOULL _strtoui64
3892 #elif defined(_LLP64) || defined(__LLP64__) || defined(_WIN64)
3893 #define STRTOULL strtoull
3894 #else
3895 #define STRTOULL strtoul
3896 #endif
3897 #endif
3898 #ifndef GC_WORD_C
3899 #if defined(_WIN64) && !defined(__GNUC__)
3900 #define GC_WORD_C(val) val##ui64
3901 #elif defined(_LLP64) || defined(__LLP64__) || defined(_WIN64)
3902 #define GC_WORD_C(val) val##ULL
3903 #else
3904 #define GC_WORD_C(val) ((word)val##UL)
3905 #endif
3906 #endif
3907 #if defined(__has_feature)
3908 #if __has_feature(address_sanitizer) && !defined(ADDRESS_SANITIZER)
3909 #define ADDRESS_SANITIZER
3910 #endif
3911 #if __has_feature(memory_sanitizer) && !defined(MEMORY_SANITIZER)
3912 #define MEMORY_SANITIZER
3913 #endif
3914 #if __has_feature(thread_sanitizer) && !defined(THREAD_SANITIZER)
3915 #define THREAD_SANITIZER
3916 #endif
3917 #else
3918 #ifdef __SANITIZE_ADDRESS__
3919 #define ADDRESS_SANITIZER
3920 #endif
3921 #endif
3922 #if defined(SPARC)
3923 #define ASM_CLEAR_CODE
3924 #endif
3925 #if defined(SPARC)
3926 #define CAN_SAVE_CALL_ARGS
3927 #endif
3928 #if (defined(I386) || defined(X86_64)) \
3929     && (defined(LINUX) || defined(__GLIBC__))
3930 #define CAN_SAVE_CALL_ARGS
3931 #endif
3932 #if defined(SAVE_CALL_COUNT) && !defined(GC_ADD_CALLER) \
3933     && defined(GC_CAN_SAVE_CALL_STACKS)
3934 #define SAVE_CALL_CHAIN
3935 #endif
3936 #ifdef SAVE_CALL_CHAIN
3937 #if defined(SAVE_CALL_NARGS) && defined(CAN_SAVE_CALL_ARGS)
3938 #define NARGS SAVE_CALL_NARGS
3939 #else
3940 #define NARGS 0
3941 #endif
3942 #endif
3943 #ifdef SAVE_CALL_CHAIN
3944 #if !defined(SAVE_CALL_COUNT) || defined(CPPCHECK)
3945 #define NFRAMES 6
3946 #else
3947 #define NFRAMES ((SAVE_CALL_COUNT + 1) & ~1)
3948 #endif
3949 #define NEED_CALLINFO
3950 #endif
3951 #ifdef GC_ADD_CALLER
3952 #define NFRAMES 1
3953 #define NARGS 0
3954 #define NEED_CALLINFO
3955 #endif
3956 #if (defined(FREEBSD) || (defined(DARWIN) && !defined(_POSIX_C_SOURCE)) \
3957         || (defined(SOLARIS) && (!defined(_XOPEN_SOURCE) \
3958                                  || defined(__EXTENSIONS__))) \
3959         || defined(LINUX)) && !defined(HAVE_DLADDR)
3960 #define HAVE_DLADDR 1
3961 #endif
3962 #if defined(MAKE_BACK_GRAPH) && !defined(DBG_HDRS_ALL)
3963 #define DBG_HDRS_ALL 1
3964 #endif
3965 #if defined(POINTER_MASK) && !defined(POINTER_SHIFT)
3966 #define POINTER_SHIFT 0
3967 #endif
3968 #if defined(POINTER_SHIFT) && !defined(POINTER_MASK)
3969 #define POINTER_MASK ((word)(-1))
3970 #endif
3971 #if !defined(FIXUP_POINTER) && defined(POINTER_MASK)
3972 #define FIXUP_POINTER(p) (p = ((p) & POINTER_MASK) << POINTER_SHIFT)
3973 #endif
3974 #if defined(FIXUP_POINTER)
3975 #define NEED_FIXUP_POINTER
3976 #else
3977 #define FIXUP_POINTER(p)
3978 #endif
3979 #if !defined(MARK_BIT_PER_GRANULE) && !defined(MARK_BIT_PER_OBJ)
3980 #define MARK_BIT_PER_GRANULE
3981 #endif
3982 #if !defined(CPPCHECK)
3983 #if defined(MARK_BIT_PER_GRANULE) && defined(MARK_BIT_PER_OBJ)
3984 #error Define only one of MARK_BIT_PER_GRANULE and MARK_BIT_PER_OBJ
3985 #endif
3986 #if defined(STACK_GROWS_UP) && defined(STACK_GROWS_DOWN)
3987 #error Only one of STACK_GROWS_UP and STACK_GROWS_DOWN should be defined
3988 #endif
3989 #if !defined(STACK_GROWS_UP) && !defined(STACK_GROWS_DOWN)
3990 #error One of STACK_GROWS_UP and STACK_GROWS_DOWN should be defined
3991 #endif
3992 #if defined(REDIRECT_MALLOC) && defined(THREADS) && !defined(LINUX) \
3993      && !defined(REDIRECT_MALLOC_IN_HEADER)
3994 #error REDIRECT_MALLOC with THREADS works at most on Linux
3995 #endif
3996 #endif
3997 #ifdef GC_PRIVATE_H
3998         struct hblk;
3999 #if defined(PCR)
4000     char * real_malloc(size_t bytes);
4001 #define GET_MEM(bytes) HBLKPTR(real_malloc(SIZET_SAT_ADD(bytes, \
4002                                                             GC_page_size)) \
4003                                           + GC_page_size-1)
4004 #elif defined(OS2)
4005     void * os2_alloc(size_t bytes);
4006 #define GET_MEM(bytes) HBLKPTR((ptr_t)os2_alloc( \
4007                                             SIZET_SAT_ADD(bytes, \
4008                                                           GC_page_size)) \
4009                                   + GC_page_size-1)
4010 #elif defined(NEXT) || defined(DOS4GW) || defined(NONSTOP) \
4011         || (defined(AMIGA) && !defined(GC_AMIGA_FASTALLOC)) \
4012         || (defined(SOLARIS) && !defined(USE_MMAP)) || defined(RTEMS) \
4013         || defined(__CC_ARM)
4014 #define GET_MEM(bytes) HBLKPTR((size_t)calloc(1, \
4015                                             SIZET_SAT_ADD(bytes, \
4016                                                           GC_page_size)) \
4017                                   + GC_page_size - 1)
4018 #elif defined(MSWIN_XBOX1)
4019     ptr_t GC_durango_get_mem(size_t bytes);
4020 #define GET_MEM(bytes) (struct hblk *)GC_durango_get_mem(bytes)
4021 #elif defined(MSWIN32) || defined(CYGWIN32)
4022     ptr_t GC_win32_get_mem(size_t bytes);
4023 #define GET_MEM(bytes) (struct hblk *)GC_win32_get_mem(bytes)
4024 #elif defined(MACOS)
4025 #if defined(USE_TEMPORARY_MEMORY)
4026       Ptr GC_MacTemporaryNewPtr(size_t size, Boolean clearMemory);
4027 #define GET_MEM(bytes) HBLKPTR(GC_MacTemporaryNewPtr( \
4028                                         SIZET_SAT_ADD(bytes, \
4029                                                       GC_page_size), true) \
4030                         + GC_page_size-1)
4031 #else
4032 #define GET_MEM(bytes) HBLKPTR(NewPtrClear(SIZET_SAT_ADD(bytes, \
4033                                                               GC_page_size)) \
4034                                     + GC_page_size-1)
4035 #endif
4036 #elif defined(MSWINCE)
4037     ptr_t GC_wince_get_mem(size_t bytes);
4038 #define GET_MEM(bytes) (struct hblk *)GC_wince_get_mem(bytes)
4039 #elif defined(AMIGA) && defined(GC_AMIGA_FASTALLOC)
4040     void *GC_amiga_get_mem(size_t bytes);
4041 #define GET_MEM(bytes) HBLKPTR((size_t)GC_amiga_get_mem( \
4042                                             SIZET_SAT_ADD(bytes, \
4043                                                           GC_page_size)) \
4044                           + GC_page_size-1)
4045 #elif defined(PLATFORM_GETMEM)
4046     void *platform_get_mem(size_t bytes);
4047 #define GET_MEM(bytes) (struct hblk*)platform_get_mem(bytes)
4048 #elif defined(SN_TARGET_PS3)
4049     void *ps3_get_mem(size_t bytes);
4050 #define GET_MEM(bytes) (struct hblk*)ps3_get_mem(bytes)
4051 #elif defined(SN_TARGET_PSP2)
4052     void *psp2_get_mem(size_t bytes);
4053 #define GET_MEM(bytes) (struct hblk*)psp2_get_mem(bytes)
4054 #elif defined(NINTENDO_SWITCH)
4055     void *switch_get_mem(size_t bytes);
4056 #define GET_MEM(bytes) (struct hblk*)switch_get_mem(bytes)
4057 #elif defined(HAIKU)
4058     ptr_t GC_haiku_get_mem(size_t bytes);
4059 #define GET_MEM(bytes) (struct hblk*)GC_haiku_get_mem(bytes)
4060 #elif defined(EMSCRIPTEN_TINY)
4061     void *emmalloc_memalign(size_t alignment, size_t size);
4062 #define GET_MEM(bytes) (struct hblk*)emmalloc_memalign(GC_page_size, bytes)
4063 #else
4064     ptr_t GC_unix_get_mem(size_t bytes);
4065 #define GET_MEM(bytes) (struct hblk *)GC_unix_get_mem(bytes)
4066 #endif
4067 #endif
4068 EXTERN_C_END
4069 #endif
4070 #if !defined(GC_ATOMIC_UNCOLLECTABLE) && defined(ATOMIC_UNCOLLECTABLE)
4071 #define GC_ATOMIC_UNCOLLECTABLE
4072 #endif
4073 #ifndef GC_INNER
4074 #if defined(GC_DLL) && defined(__GNUC__) && !defined(MSWIN32) \
4075         && !defined(MSWINCE) && !defined(CYGWIN32)
4076 #if GC_GNUC_PREREQ(4, 0) && !defined(GC_NO_VISIBILITY)
4077 #define GC_INNER __attribute__((__visibility__("hidden")))
4078 #else
4079 #define GC_INNER
4080 #endif
4081 #else
4082 #define GC_INNER
4083 #endif
4084 #define GC_EXTERN extern GC_INNER
4085 #endif
4086 #ifdef __cplusplus
4087 #define REGISTER
4088 #else
4089 #define REGISTER register
4090 #endif
4091 #if defined(CPPCHECK)
4092 #define MACRO_BLKSTMT_BEGIN {
4093 #define MACRO_BLKSTMT_END   }
4094 #else
4095 #define MACRO_BLKSTMT_BEGIN do {
4096 #define MACRO_BLKSTMT_END   } while (0)
4097 #endif
4098 #if defined(M68K) && defined(__GNUC__)
4099 #define GC_ATTR_WORD_ALIGNED __attribute__((__aligned__(sizeof(word))))
4100 #else
4101 #define GC_ATTR_WORD_ALIGNED
4102 #endif
4103 #ifndef HEADERS_H
4104 #ifndef GC_HEADERS_H
4105 #define GC_HEADERS_H
4106 #if CPP_WORDSZ != 32 && CPP_WORDSZ < 36 && !defined(CPPCHECK)
4107 #error Get a real machine
4108 #endif
4109 EXTERN_C_BEGIN
4110 typedef struct hblkhdr hdr;
4111 #if CPP_WORDSZ > 32
4112 #define HASH_TL
4113 #endif
4114 #if defined(LARGE_CONFIG) || !defined(SMALL_CONFIG)
4115 #define LOG_BOTTOM_SZ 10
4116 #else
4117 #define LOG_BOTTOM_SZ 11
4118 #endif
4119 #define BOTTOM_SZ (1 << LOG_BOTTOM_SZ)
4120 #ifndef HASH_TL
4121 #define LOG_TOP_SZ (WORDSZ - LOG_BOTTOM_SZ - LOG_HBLKSIZE)
4122 #else
4123 #define LOG_TOP_SZ 11
4124 #endif
4125 #define TOP_SZ (1 << LOG_TOP_SZ)
4126 #ifdef COUNT_HDR_CACHE_HITS
4127   extern word GC_hdr_cache_hits;
4128   extern word GC_hdr_cache_misses;
4129 #define HC_HIT() (void)(++GC_hdr_cache_hits)
4130 #define HC_MISS() (void)(++GC_hdr_cache_misses)
4131 #else
4132 #define HC_HIT()
4133 #define HC_MISS()
4134 #endif
4135 typedef struct hce {
4136   word block_addr;
4137   hdr * hce_hdr;
4138 } hdr_cache_entry;
4139 #define HDR_CACHE_SIZE 8
4140 #define DECLARE_HDR_CACHE \
4141         hdr_cache_entry hdr_cache[HDR_CACHE_SIZE]
4142 #define INIT_HDR_CACHE BZERO(hdr_cache, sizeof(hdr_cache))
4143 #define HCE(h) \
4144         (hdr_cache + (((word)(h) >> LOG_HBLKSIZE) & (HDR_CACHE_SIZE-1)))
4145 #define HCE_VALID_FOR(hce, h) ((hce) -> block_addr == \
4146                                 ((word)(h) >> LOG_HBLKSIZE))
4147 #define HCE_HDR(h) ((hce) -> hce_hdr)
4148 #ifdef PRINT_BLACK_LIST
4149   GC_INNER hdr * GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce,
4150                                       ptr_t source);
4151 #define HEADER_CACHE_MISS(p, hce, source) \
4152           GC_header_cache_miss(p, hce, source)
4153 #else
4154   GC_INNER hdr * GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce);
4155 #define HEADER_CACHE_MISS(p, hce, source) GC_header_cache_miss(p, hce)
4156 #endif
4157 #define HC_GET_HDR(p, hhdr, source) \
4158         {  \
4159           hdr_cache_entry * hce = HCE(p); \
4160           if (EXPECT(HCE_VALID_FOR(hce, p), TRUE)) { \
4161             HC_HIT(); \
4162             hhdr = hce -> hce_hdr; \
4163           } else { \
4164             hhdr = HEADER_CACHE_MISS(p, hce, source); \
4165             if (NULL == hhdr) break;  \
4166           } \
4167         }
4168 typedef struct bi {
4169     hdr * index[BOTTOM_SZ];
4170     struct bi * asc_link;
4171     struct bi * desc_link;
4172     word key;
4173 #ifdef HASH_TL
4174     struct bi * hash_link;
4175 #endif
4176 } bottom_index;
4177 #define MAX_JUMP (HBLKSIZE - 1)
4178 #define HDR_FROM_BI(bi, p) \
4179                 ((bi)->index[((word)(p) >> LOG_HBLKSIZE) & (BOTTOM_SZ - 1)])
4180 #ifndef HASH_TL
4181 #define BI(p) (GC_top_index \
4182               [(word)(p) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE)])
4183 #define HDR_INNER(p) HDR_FROM_BI(BI(p),p)
4184 #ifdef SMALL_CONFIG
4185 #define HDR(p) GC_find_header((ptr_t)(p))
4186 #else
4187 #define HDR(p) HDR_INNER(p)
4188 #endif
4189 #define GET_BI(p, bottom_indx) (void)((bottom_indx) = BI(p))
4190 #define GET_HDR(p, hhdr) (void)((hhdr) = HDR(p))
4191 #define SET_HDR(p, hhdr) (void)(HDR_INNER(p) = (hhdr))
4192 #define GET_HDR_ADDR(p, ha) (void)((ha) = &HDR_INNER(p))
4193 #else
4194 #define TL_HASH(hi) ((hi) & (TOP_SZ - 1))
4195 #define GET_BI(p, bottom_indx) \
4196         do { \
4197           REGISTER word hi = (word)(p) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE); \
4198           REGISTER bottom_index * _bi = GC_top_index[TL_HASH(hi)]; \
4199           while (_bi -> key != hi && _bi != GC_all_nils) \
4200               _bi = _bi -> hash_link; \
4201           (bottom_indx) = _bi; \
4202         } while (0)
4203 #define GET_HDR_ADDR(p, ha) \
4204         do { \
4205           REGISTER bottom_index * bi; \
4206           GET_BI(p, bi); \
4207           (ha) = &HDR_FROM_BI(bi, p); \
4208         } while (0)
4209 #define GET_HDR(p, hhdr) \
4210         do { \
4211           REGISTER hdr ** _ha; \
4212           GET_HDR_ADDR(p, _ha); \
4213           (hhdr) = *_ha; \
4214         } while (0)
4215 #define SET_HDR(p, hhdr) \
4216         do { \
4217           REGISTER hdr ** _ha; \
4218           GET_HDR_ADDR(p, _ha); \
4219           *_ha = (hhdr); \
4220         } while (0)
4221 #define HDR(p) GC_find_header((ptr_t)(p))
4222 #endif
4223 #define IS_FORWARDING_ADDR_OR_NIL(hhdr) ((size_t) (hhdr) <= MAX_JUMP)
4224 #define FORWARDED_ADDR(h, hhdr) ((struct hblk *)(h) - (size_t)(hhdr))
4225 EXTERN_C_END
4226 #endif
4227 #endif
4228 #ifndef GC_ATTR_NO_SANITIZE_ADDR
4229 #ifndef ADDRESS_SANITIZER
4230 #define GC_ATTR_NO_SANITIZE_ADDR
4231 #elif GC_CLANG_PREREQ(3, 8)
4232 #define GC_ATTR_NO_SANITIZE_ADDR __attribute__((no_sanitize("address")))
4233 #else
4234 #define GC_ATTR_NO_SANITIZE_ADDR __attribute__((no_sanitize_address))
4235 #endif
4236 #endif
4237 #ifndef GC_ATTR_NO_SANITIZE_MEMORY
4238 #ifndef MEMORY_SANITIZER
4239 #define GC_ATTR_NO_SANITIZE_MEMORY
4240 #elif GC_CLANG_PREREQ(3, 8)
4241 #define GC_ATTR_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory")))
4242 #else
4243 #define GC_ATTR_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))
4244 #endif
4245 #endif
4246 #ifndef GC_ATTR_NO_SANITIZE_THREAD
4247 #ifndef THREAD_SANITIZER
4248 #define GC_ATTR_NO_SANITIZE_THREAD
4249 #elif GC_CLANG_PREREQ(3, 8)
4250 #define GC_ATTR_NO_SANITIZE_THREAD __attribute__((no_sanitize("thread")))
4251 #else
4252 #define GC_ATTR_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))
4253 #endif
4254 #endif
4255 #ifndef GC_ATTR_UNUSED
4256 #if GC_GNUC_PREREQ(3, 4)
4257 #define GC_ATTR_UNUSED __attribute__((__unused__))
4258 #else
4259 #define GC_ATTR_UNUSED
4260 #endif
4261 #endif
4262 #ifdef HAVE_CONFIG_H
4263 #define GC_INLINE static inline
4264 #elif defined(_MSC_VER) || defined(__INTEL_COMPILER) || defined(__DMC__) \
4265         || (GC_GNUC_PREREQ(3, 0) && defined(__STRICT_ANSI__)) \
4266         || defined(__WATCOMC__)
4267 #define GC_INLINE static __inline
4268 #elif GC_GNUC_PREREQ(3, 0) || defined(__sun)
4269 #define GC_INLINE static inline
4270 #else
4271 #define GC_INLINE static
4272 #endif
4273 #ifndef GC_ATTR_NOINLINE
4274 #if GC_GNUC_PREREQ(4, 0)
4275 #define GC_ATTR_NOINLINE __attribute__((__noinline__))
4276 #elif _MSC_VER >= 1400
4277 #define GC_ATTR_NOINLINE __declspec(noinline)
4278 #else
4279 #define GC_ATTR_NOINLINE
4280 #endif
4281 #endif
4282 #ifndef GC_API_OSCALL
4283 #if defined(__GNUC__)
4284 #if GC_GNUC_PREREQ(4, 0) && !defined(GC_NO_VISIBILITY)
4285 #define GC_API_OSCALL extern __attribute__((__visibility__("default")))
4286 #else
4287 #define GC_API_OSCALL extern
4288 #endif
4289 #else
4290 #define GC_API_OSCALL GC_API
4291 #endif
4292 #endif
4293 #ifndef GC_API_PRIV
4294 #define GC_API_PRIV GC_API
4295 #endif
4296 #if defined(THREADS) && !defined(NN_PLATFORM_CTR)
4297 #ifndef GC_ATOMIC_OPS_H
4298 #define GC_ATOMIC_OPS_H
4299 #ifdef GC_BUILTIN_ATOMIC
4300 #ifdef __cplusplus
4301     extern "C" {
4302 #endif
4303   typedef GC_word AO_t;
4304 #ifdef GC_PRIVATE_H
4305 #define AO_INLINE GC_INLINE
4306 #else
4307 #define AO_INLINE static __inline
4308 #endif
4309   typedef unsigned char AO_TS_t;
4310 #define AO_TS_CLEAR 0
4311 #define AO_TS_INITIALIZER (AO_TS_t)AO_TS_CLEAR
4312 #if defined(__GCC_ATOMIC_TEST_AND_SET_TRUEVAL) && !defined(CPPCHECK)
4313 #define AO_TS_SET __GCC_ATOMIC_TEST_AND_SET_TRUEVAL
4314 #else
4315 #define AO_TS_SET (AO_TS_t)1
4316 #endif
4317 #define AO_CLEAR(p) __atomic_clear(p, __ATOMIC_RELEASE)
4318 #define AO_test_and_set_acquire(p) __atomic_test_and_set(p, __ATOMIC_ACQUIRE)
4319 #define AO_HAVE_test_and_set_acquire
4320 #define AO_compiler_barrier() __atomic_signal_fence(__ATOMIC_SEQ_CST)
4321 #define AO_nop_full() __atomic_thread_fence(__ATOMIC_SEQ_CST)
4322 #define AO_HAVE_nop_full
4323 #define AO_fetch_and_add(p, v) __atomic_fetch_add(p, v, __ATOMIC_RELAXED)
4324 #define AO_HAVE_fetch_and_add
4325 #define AO_fetch_and_add1(p) AO_fetch_and_add(p, 1)
4326 #define AO_HAVE_fetch_and_add1
4327 #define AO_or(p, v) (void)__atomic_or_fetch(p, v, __ATOMIC_RELAXED)
4328 #define AO_HAVE_or
4329 #define AO_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
4330 #define AO_HAVE_load
4331 #define AO_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
4332 #define AO_HAVE_load_acquire
4333 #define AO_load_acquire_read(p) AO_load_acquire(p)
4334 #define AO_HAVE_load_acquire_read
4335 #define AO_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
4336 #define AO_HAVE_store
4337 #define AO_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
4338 #define AO_HAVE_store_release
4339 #define AO_store_release_write(p, v) AO_store_release(p, v)
4340 #define AO_HAVE_store_release_write
4341 #define AO_char_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
4342 #define AO_HAVE_char_load
4343 #define AO_char_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
4344 #define AO_HAVE_char_store
4345 #ifdef AO_REQUIRE_CAS
4346     AO_INLINE int
4347     AO_compare_and_swap(volatile AO_t *p, AO_t ov, AO_t nv)
4348     {
4349       return (int)__atomic_compare_exchange_n(p, &ov, nv, 0,
4350                                         __ATOMIC_RELAXED, __ATOMIC_RELAXED);
4351     }
4352     AO_INLINE int
4353     AO_compare_and_swap_release(volatile AO_t *p, AO_t ov, AO_t nv)
4354     {
4355       return (int)__atomic_compare_exchange_n(p, &ov, nv, 0,
4356                                         __ATOMIC_RELEASE, __ATOMIC_RELAXED);
4357     }
4358 #define AO_HAVE_compare_and_swap_release
4359 #endif
4360 #ifdef __cplusplus
4361     }
4362 #endif
4363 #ifndef NO_LOCKFREE_AO_OR
4364 #define HAVE_LOCKFREE_AO_OR 1
4365 #endif
4366 #else
4367 #include "atomic_ops.h"
4368 #if (!defined(AO_HAVE_load) || !defined(AO_HAVE_store)) && !defined(CPPCHECK)
4369 #error AO_load or AO_store is missing; probably old version of atomic_ops
4370 #endif
4371 #endif
4372 #endif
4373 #ifndef AO_HAVE_compiler_barrier
4374 #define AO_HAVE_compiler_barrier 1
4375 #endif
4376 #endif
4377 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
4378 #ifndef WIN32_LEAN_AND_MEAN
4379 #define WIN32_LEAN_AND_MEAN 1
4380 #endif
4381 #define NOSERVICE
4382 #include <windows.h>
4383 #include <winbase.h>
4384 #endif
4385 #ifndef GC_LOCKS_H
4386 #define GC_LOCKS_H
4387 #ifdef THREADS
4388 #ifdef PCR
4389 #include <base/PCR_Base.h>
4390 #include <th/PCR_Th.h>
4391 #endif
4392    EXTERN_C_BEGIN
4393 #ifdef PCR
4394      GC_EXTERN PCR_Th_ML GC_allocate_ml;
4395 #if defined(CPPCHECK)
4396 #define DCL_LOCK_STATE
4397 #else
4398 #define DCL_LOCK_STATE \
4399          PCR_ERes GC_fastLockRes; PCR_sigset_t GC_old_sig_mask
4400 #endif
4401 #define UNCOND_LOCK() PCR_Th_ML_Acquire(&GC_allocate_ml)
4402 #define UNCOND_UNLOCK() PCR_Th_ML_Release(&GC_allocate_ml)
4403 #elif defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH)
4404       extern void GC_lock(void);
4405       extern void GC_unlock(void);
4406 #define UNCOND_LOCK() GC_lock()
4407 #define UNCOND_UNLOCK() GC_unlock()
4408 #endif
4409 #if (!defined(AO_HAVE_test_and_set_acquire) || defined(GC_RTEMS_PTHREADS) \
4410        || defined(SN_TARGET_PS3) \
4411        || defined(GC_WIN32_THREADS) || defined(BASE_ATOMIC_OPS_EMULATED) \
4412        || defined(LINT2)) && defined(GC_PTHREADS)
4413 #define USE_PTHREAD_LOCKS
4414 #undef USE_SPIN_LOCK
4415 #if defined(LINT2) && !defined(NO_PTHREAD_TRYLOCK)
4416 #define NO_PTHREAD_TRYLOCK
4417 #endif
4418 #endif
4419 #if defined(GC_WIN32_THREADS) && !defined(USE_PTHREAD_LOCKS)
4420 #define NO_THREAD (DWORD)(-1)
4421      GC_EXTERN CRITICAL_SECTION GC_allocate_ml;
4422 #ifdef GC_ASSERTIONS
4423        GC_EXTERN DWORD GC_lock_holder;
4424 #define SET_LOCK_HOLDER() GC_lock_holder = GetCurrentThreadId()
4425 #define UNSET_LOCK_HOLDER() GC_lock_holder = NO_THREAD
4426 #define I_HOLD_LOCK() (!GC_need_to_lock \
4427                            || GC_lock_holder == GetCurrentThreadId())
4428 #ifdef THREAD_SANITIZER
4429 #define I_DONT_HOLD_LOCK() TRUE
4430 #else
4431 #define I_DONT_HOLD_LOCK() (!GC_need_to_lock \
4432                            || GC_lock_holder != GetCurrentThreadId())
4433 #endif
4434 #define UNCOND_LOCK() \
4435                 { GC_ASSERT(I_DONT_HOLD_LOCK()); \
4436                   EnterCriticalSection(&GC_allocate_ml); \
4437                   SET_LOCK_HOLDER(); }
4438 #define UNCOND_UNLOCK() \
4439                 { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \
4440                   LeaveCriticalSection(&GC_allocate_ml); }
4441 #else
4442 #define UNCOND_LOCK() EnterCriticalSection(&GC_allocate_ml)
4443 #define UNCOND_UNLOCK() LeaveCriticalSection(&GC_allocate_ml)
4444 #endif
4445 #elif defined(GC_PTHREADS)
4446      EXTERN_C_END
4447 #include <pthread.h>
4448      EXTERN_C_BEGIN
4449 #if !defined(GC_WIN32_PTHREADS)
4450 #define NUMERIC_THREAD_ID(id) ((unsigned long)(id))
4451 #define THREAD_EQUAL(id1, id2) ((id1) == (id2))
4452 #define NUMERIC_THREAD_ID_UNIQUE
4453 #elif defined(__WINPTHREADS_VERSION_MAJOR)
4454 #define NUMERIC_THREAD_ID(id) ((unsigned long)(id))
4455 #define THREAD_EQUAL(id1, id2) ((id1) == (id2))
4456 #ifndef _WIN64
4457 #define NUMERIC_THREAD_ID_UNIQUE
4458 #endif
4459 #else
4460 #define NUMERIC_THREAD_ID(id) ((unsigned long)(word)(id.p))
4461 #define THREAD_EQUAL(id1, id2) ((id1.p == id2.p) && (id1.x == id2.x))
4462 #undef NUMERIC_THREAD_ID_UNIQUE
4463 #endif
4464 #define NO_THREAD ((unsigned long)(-1l))
4465 #ifdef SN_TARGET_PSP2
4466        EXTERN_C_END
4467 #include "psp2-support.h"
4468        EXTERN_C_BEGIN
4469        GC_EXTERN WapiMutex GC_allocate_ml_PSP2;
4470 #define UNCOND_LOCK() { int res; GC_ASSERT(I_DONT_HOLD_LOCK()); \
4471                               res = PSP2_MutexLock(&GC_allocate_ml_PSP2); \
4472                               GC_ASSERT(0 == res); (void)res; \
4473                               SET_LOCK_HOLDER(); }
4474 #define UNCOND_UNLOCK() { int res; GC_ASSERT(I_HOLD_LOCK()); \
4475                               UNSET_LOCK_HOLDER(); \
4476                               res = PSP2_MutexUnlock(&GC_allocate_ml_PSP2); \
4477                               GC_ASSERT(0 == res); (void)res; }
4478 #elif (!defined(THREAD_LOCAL_ALLOC) || defined(USE_SPIN_LOCK)) \
4479           && !defined(USE_PTHREAD_LOCKS)
4480 #undef USE_SPIN_LOCK
4481 #define USE_SPIN_LOCK
4482       GC_EXTERN volatile AO_TS_t GC_allocate_lock;
4483       GC_INNER void GC_lock(void);
4484 #ifdef GC_ASSERTIONS
4485 #define UNCOND_LOCK() \
4486               { GC_ASSERT(I_DONT_HOLD_LOCK()); \
4487                 if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_SET) \
4488                   GC_lock(); \
4489                 SET_LOCK_HOLDER(); }
4490 #define UNCOND_UNLOCK() \
4491               { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \
4492                 AO_CLEAR(&GC_allocate_lock); }
4493 #else
4494 #define UNCOND_LOCK() \
4495               { if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_SET) \
4496                   GC_lock(); }
4497 #define UNCOND_UNLOCK() AO_CLEAR(&GC_allocate_lock)
4498 #endif
4499 #else
4500 #ifndef USE_PTHREAD_LOCKS
4501 #define USE_PTHREAD_LOCKS
4502 #endif
4503 #endif
4504 #ifdef USE_PTHREAD_LOCKS
4505        EXTERN_C_END
4506 #include <pthread.h>
4507        EXTERN_C_BEGIN
4508        GC_EXTERN pthread_mutex_t GC_allocate_ml;
4509 #ifdef GC_ASSERTIONS
4510          GC_INNER void GC_lock(void);
4511 #define UNCOND_LOCK() { GC_ASSERT(I_DONT_HOLD_LOCK()); \
4512                                 GC_lock(); SET_LOCK_HOLDER(); }
4513 #define UNCOND_UNLOCK() \
4514                 { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \
4515                   pthread_mutex_unlock(&GC_allocate_ml); }
4516 #else
4517 #if defined(NO_PTHREAD_TRYLOCK)
4518 #define UNCOND_LOCK() pthread_mutex_lock(&GC_allocate_ml)
4519 #else
4520            GC_INNER void GC_lock(void);
4521 #define UNCOND_LOCK() \
4522               { if (0 != pthread_mutex_trylock(&GC_allocate_ml)) \
4523                   GC_lock(); }
4524 #endif
4525 #define UNCOND_UNLOCK() pthread_mutex_unlock(&GC_allocate_ml)
4526 #endif
4527 #endif
4528 #ifdef GC_ASSERTIONS
4529        GC_EXTERN unsigned long GC_lock_holder;
4530 #define SET_LOCK_HOLDER() \
4531                 GC_lock_holder = NUMERIC_THREAD_ID(pthread_self())
4532 #define UNSET_LOCK_HOLDER() GC_lock_holder = NO_THREAD
4533 #define I_HOLD_LOCK() \
4534                 (!GC_need_to_lock \
4535                  || GC_lock_holder == NUMERIC_THREAD_ID(pthread_self()))
4536 #if !defined(NUMERIC_THREAD_ID_UNIQUE) || defined(THREAD_SANITIZER)
4537 #define I_DONT_HOLD_LOCK() TRUE
4538 #else
4539 #define I_DONT_HOLD_LOCK() \
4540                 (!GC_need_to_lock \
4541                  || GC_lock_holder != NUMERIC_THREAD_ID(pthread_self()))
4542 #endif
4543 #endif
4544 #ifndef GC_WIN32_THREADS
4545        GC_EXTERN volatile GC_bool GC_collecting;
4546 #ifdef AO_HAVE_char_store
4547 #define ENTER_GC() AO_char_store((unsigned char*)&GC_collecting, TRUE)
4548 #define EXIT_GC() AO_char_store((unsigned char*)&GC_collecting, FALSE)
4549 #else
4550 #define ENTER_GC() (void)(GC_collecting = TRUE)
4551 #define EXIT_GC() (void)(GC_collecting = FALSE)
4552 #endif
4553 #endif
4554 #endif
4555 #if defined(GC_ALWAYS_MULTITHREADED) \
4556       && (defined(USE_PTHREAD_LOCKS) || defined(USE_SPIN_LOCK))
4557 #define GC_need_to_lock TRUE
4558 #define set_need_to_lock() (void)0
4559 #else
4560 #if defined(GC_ALWAYS_MULTITHREADED) && !defined(CPPCHECK)
4561 #error Runtime initialization of GC lock is needed!
4562 #endif
4563 #undef GC_ALWAYS_MULTITHREADED
4564      GC_EXTERN GC_bool GC_need_to_lock;
4565 #ifdef THREAD_SANITIZER
4566 #define set_need_to_lock() \
4567                 (void)(*(GC_bool volatile *)&GC_need_to_lock \
4568                         ? FALSE \
4569                         : (GC_need_to_lock = TRUE))
4570 #else
4571 #define set_need_to_lock() (void)(GC_need_to_lock = TRUE)
4572 #endif
4573 #endif
4574    EXTERN_C_END
4575 #else
4576 #define LOCK() (void)0
4577 #define UNLOCK() (void)0
4578 #ifdef GC_ASSERTIONS
4579 #define I_HOLD_LOCK() TRUE
4580 #define I_DONT_HOLD_LOCK() TRUE
4581 #endif
4582 #endif
4583 #if defined(UNCOND_LOCK) && !defined(LOCK)
4584 #if (defined(LINT2) && defined(USE_PTHREAD_LOCKS)) \
4585      || defined(GC_ALWAYS_MULTITHREADED)
4586 #define LOCK() UNCOND_LOCK()
4587 #define UNLOCK() UNCOND_UNLOCK()
4588 #else
4589 #define LOCK() do { if (GC_need_to_lock) UNCOND_LOCK(); } while (0)
4590 #define UNLOCK() do { if (GC_need_to_lock) UNCOND_UNLOCK(); } while (0)
4591 #endif
4592 #endif
4593 #ifndef ENTER_GC
4594 #define ENTER_GC()
4595 #define EXIT_GC()
4596 #endif
4597 #ifndef DCL_LOCK_STATE
4598 #define DCL_LOCK_STATE
4599 #endif
4600 #endif
4601 #define GC_WORD_MAX (~(word)0)
4602 #ifdef STACK_GROWS_DOWN
4603 #define COOLER_THAN >
4604 #define HOTTER_THAN <
4605 #define MAKE_COOLER(x,y) if ((word)((x) + (y)) > (word)(x)) {(x) += (y);} \
4606                             else (x) = (ptr_t)GC_WORD_MAX
4607 #define MAKE_HOTTER(x,y) (x) -= (y)
4608 #else
4609 #define COOLER_THAN <
4610 #define HOTTER_THAN >
4611 #define MAKE_COOLER(x,y) if ((word)((x) - (y)) < (word)(x)) {(x) -= (y);} \
4612                             else (x) = 0
4613 #define MAKE_HOTTER(x,y) (x) += (y)
4614 #endif
4615 #if defined(AMIGA) && defined(__SASC)
4616 #define GC_FAR __far
4617 #else
4618 #define GC_FAR
4619 #endif
4620 EXTERN_C_BEGIN
4621 #ifndef GC_NO_FINALIZATION
4622 #define GC_INVOKE_FINALIZERS() GC_notify_or_invoke_finalizers()
4623   GC_INNER void GC_notify_or_invoke_finalizers(void);
4624   GC_INNER void GC_finalize(void);
4625 #ifndef GC_TOGGLE_REFS_NOT_NEEDED
4626     GC_INNER void GC_process_togglerefs(void);
4627 #endif
4628 #ifndef SMALL_CONFIG
4629     GC_INNER void GC_print_finalization_stats(void);
4630 #endif
4631 #else
4632 #define GC_INVOKE_FINALIZERS() (void)0
4633 #endif
4634 #if !defined(DONT_ADD_BYTE_AT_END)
4635 #ifdef LINT2
4636 #define EXTRA_BYTES ((size_t)(GC_all_interior_pointers? 1 : 0))
4637 #else
4638 #define EXTRA_BYTES (size_t)GC_all_interior_pointers
4639 #endif
4640 #define MAX_EXTRA_BYTES 1
4641 #else
4642 #define EXTRA_BYTES 0
4643 #define MAX_EXTRA_BYTES 0
4644 #endif
4645 #ifndef LARGE_CONFIG
4646 #define MINHINCR 16
4647 #define MAXHINCR 2048
4648 #else
4649 #define MINHINCR 64
4650 #define MAXHINCR 4096
4651 #endif
4652 #define BL_LIMIT GC_black_list_spacing
4653 #ifdef NEED_CALLINFO
4654     struct callinfo {
4655         word ci_pc;
4656 #if NARGS > 0
4657             word ci_arg[NARGS];
4658 #endif
4659 #if (NFRAMES * (NARGS + 1)) % 2 == 1
4660             word ci_dummy;
4661 #endif
4662     };
4663 #endif
4664 #ifdef SAVE_CALL_CHAIN
4665   GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]);
4666   GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]);
4667 #endif
4668 EXTERN_C_END
4669 #ifndef NO_CLOCK
4670 #ifdef BSD_TIME
4671 #undef CLOCK_TYPE
4672 #undef GET_TIME
4673 #undef MS_TIME_DIFF
4674 #define CLOCK_TYPE struct timeval
4675 #define CLOCK_TYPE_INITIALIZER { 0, 0 }
4676 #define GET_TIME(x) \
4677                 do { \
4678                   struct rusage rusage; \
4679                   getrusage(RUSAGE_SELF, &rusage); \
4680                   x = rusage.ru_utime; \
4681                 } while (0)
4682 #define MS_TIME_DIFF(a,b) ((unsigned long)((long)(a.tv_sec-b.tv_sec) * 1000 \
4683                 + (long)(a.tv_usec - b.tv_usec) / 1000 \
4684                 - (a.tv_usec < b.tv_usec \
4685                    && (long)(a.tv_usec - b.tv_usec) % 1000 != 0 ? 1 : 0)))
4686 #define NS_FRAC_TIME_DIFF(a, b) ((unsigned long) \
4687                 ((a.tv_usec < b.tv_usec \
4688                   && (long)(a.tv_usec - b.tv_usec) % 1000 != 0 ? 1000L : 0) \
4689                  + (long)(a.tv_usec - b.tv_usec) % 1000) * 1000)
4690 #elif defined(MSWIN32) || defined(MSWINCE) || defined(WINXP_USE_PERF_COUNTER)
4691 #if defined(MSWINRT_FLAVOR) || defined(WINXP_USE_PERF_COUNTER)
4692 #define CLOCK_TYPE ULONGLONG
4693 #define GET_TIME(x) \
4694                 do { \
4695                   LARGE_INTEGER freq, tc; \
4696                   if (!QueryPerformanceFrequency(&freq) \
4697                       || !QueryPerformanceCounter(&tc)) \
4698                     ABORT("QueryPerformanceCounter requires WinXP+"); \
4699                   x = (CLOCK_TYPE)((double)tc.QuadPart/freq.QuadPart * 1e9); \
4700                 } while (0)
4701 #define MS_TIME_DIFF(a, b) ((unsigned long)(((a) - (b)) / 1000000UL))
4702 #define NS_FRAC_TIME_DIFF(a, b) ((unsigned long)(((a) - (b)) % 1000000UL))
4703 #else
4704 #define CLOCK_TYPE DWORD
4705 #define GET_TIME(x) (void)(x = GetTickCount())
4706 #define MS_TIME_DIFF(a, b) ((unsigned long)((a) - (b)))
4707 #define NS_FRAC_TIME_DIFF(a, b) 0UL
4708 #endif
4709 #elif defined(NN_PLATFORM_CTR)
4710 #define CLOCK_TYPE long long
4711   EXTERN_C_BEGIN
4712   CLOCK_TYPE n3ds_get_system_tick(void);
4713   CLOCK_TYPE n3ds_convert_tick_to_ms(CLOCK_TYPE tick);
4714   EXTERN_C_END
4715 #define GET_TIME(x) (void)(x = n3ds_get_system_tick())
4716 #define MS_TIME_DIFF(a,b) ((unsigned long)n3ds_convert_tick_to_ms((a)-(b)))
4717 #define NS_FRAC_TIME_DIFF(a, b) 0UL
4718 #elif defined(NINTENDO_SWITCH) \
4719       || (((defined(LINUX) && defined(__USE_POSIX199309)) \
4720            || defined(CYGWIN32)) && defined(_POSIX_TIMERS))
4721 #include <time.h>
4722 #define HAVE_CLOCK_GETTIME 1
4723 #define CLOCK_TYPE struct timespec
4724 #define CLOCK_TYPE_INITIALIZER { 0, 0 }
4725 #if defined(_POSIX_MONOTONIC_CLOCK) && !defined(NINTENDO_SWITCH)
4726 #define GET_TIME(x) \
4727                 do { \
4728                   if (clock_gettime(CLOCK_MONOTONIC, &x) == -1) \
4729                     ABORT("clock_gettime failed"); \
4730                 } while (0)
4731 #else
4732 #define GET_TIME(x) \
4733                 do { \
4734                   if (clock_gettime(CLOCK_REALTIME, &x) == -1) \
4735                     ABORT("clock_gettime failed"); \
4736                 } while (0)
4737 #endif
4738 #define MS_TIME_DIFF(a, b) \
4739      \
4740     ((unsigned long)((a).tv_nsec + (1000000L*1000 - (b).tv_nsec)) / 1000000UL \
4741      + ((unsigned long)((a).tv_sec - (b).tv_sec) * 1000UL) - 1000UL)
4742 #define NS_FRAC_TIME_DIFF(a, b) \
4743     ((unsigned long)((a).tv_nsec + (1000000L*1000 - (b).tv_nsec)) % 1000000UL)
4744 #else
4745 #include <time.h>
4746 #if defined(FREEBSD) && !defined(CLOCKS_PER_SEC)
4747 #include <machine/limits.h>
4748 #define CLOCKS_PER_SEC CLK_TCK
4749 #endif
4750 #if !defined(CLOCKS_PER_SEC)
4751 #define CLOCKS_PER_SEC 1000000
4752 #endif
4753 #define CLOCK_TYPE clock_t
4754 #define GET_TIME(x) (void)(x = clock())
4755 #define MS_TIME_DIFF(a,b) (CLOCKS_PER_SEC % 1000 == 0 ? \
4756         (unsigned long)((a) - (b)) / (unsigned long)(CLOCKS_PER_SEC / 1000) \
4757         : ((unsigned long)((a) - (b)) * 1000) / (unsigned long)CLOCKS_PER_SEC)
4758 #define NS_FRAC_TIME_DIFF(a, b) (CLOCKS_PER_SEC <= 1000 ? 0UL \
4759     : (unsigned long)(CLOCKS_PER_SEC <= (clock_t)1000000UL \
4760         ? (((a) - (b)) * ((clock_t)1000000UL / CLOCKS_PER_SEC) % 1000) * 1000 \
4761         : (CLOCKS_PER_SEC <= (clock_t)1000000UL * 1000 \
4762             ? ((a) - (b)) * ((clock_t)1000000UL * 1000 / CLOCKS_PER_SEC) \
4763             : (((a) - (b)) * (clock_t)1000000UL * 1000) / CLOCKS_PER_SEC) \
4764           % (clock_t)1000000UL))
4765 #endif
4766 #ifndef CLOCK_TYPE_INITIALIZER
4767 #define CLOCK_TYPE_INITIALIZER 0
4768 #endif
4769 #endif
4770 #if defined(SPARC) && defined(SUNOS4) \
4771      || (defined(M68K) && defined(NEXT)) || defined(VAX)
4772 #define BCOPY_EXISTS
4773 #elif defined(AMIGA) || defined(DARWIN)
4774 #include <string.h>
4775 #define BCOPY_EXISTS
4776 #elif defined(MACOS) && defined(POWERPC)
4777 #include <MacMemory.h>
4778 #define bcopy(x,y,n) BlockMoveData(x, y, n)
4779 #define bzero(x,n) BlockZero(x, n)
4780 #define BCOPY_EXISTS
4781 #endif
4782 #if !defined(BCOPY_EXISTS) || defined(CPPCHECK)
4783 #include <string.h>
4784 #define BCOPY(x,y,n) memcpy(y, x, (size_t)(n))
4785 #define BZERO(x,n)  memset(x, 0, (size_t)(n))
4786 #else
4787 #define BCOPY(x,y,n) bcopy((void *)(x),(void *)(y),(size_t)(n))
4788 #define BZERO(x,n) bzero((void *)(x),(size_t)(n))
4789 #endif
4790 #ifdef PCR
4791 #include "th/PCR_ThCtl.h"
4792 #endif
4793 EXTERN_C_BEGIN
4794 #ifdef PCR
4795 #define STOP_WORLD() \
4796         PCR_ThCtl_SetExclusiveMode(PCR_ThCtl_ExclusiveMode_stopNormal, \
4797                                    PCR_allSigsBlocked, \
4798                                    PCR_waitForever)
4799 #define START_WORLD() \
4800         PCR_ThCtl_SetExclusiveMode(PCR_ThCtl_ExclusiveMode_null, \
4801                                    PCR_allSigsBlocked, \
4802                                    PCR_waitForever)
4803 #else
4804 #if defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) \
4805        || defined(GC_WIN32_THREADS) || defined(GC_PTHREADS)
4806       GC_INNER void GC_stop_world(void);
4807       GC_INNER void GC_start_world(void);
4808 #define STOP_WORLD() GC_stop_world()
4809 #define START_WORLD() GC_start_world()
4810 #else
4811 #define STOP_WORLD() GC_ASSERT(GC_blocked_sp == NULL)
4812 #define START_WORLD()
4813 #endif
4814 #endif
4815 #ifdef THREADS
4816   GC_EXTERN GC_on_thread_event_proc GC_on_thread_event;
4817 #endif
4818 #if defined(SMALL_CONFIG) || defined(PCR)
4819 #define GC_on_abort(msg) (void)0
4820 #else
4821     GC_API_PRIV GC_abort_func GC_on_abort;
4822 #endif
4823 #if defined(CPPCHECK)
4824 #define ABORT(msg) { GC_on_abort(msg); abort(); }
4825 #elif defined(PCR)
4826 #define ABORT(s) PCR_Base_Panic(s)
4827 #else
4828 #if defined(MSWIN_XBOX1) && !defined(DebugBreak)
4829 #define DebugBreak() __debugbreak()
4830 #elif defined(MSWINCE) && !defined(DebugBreak) \
4831        && (!defined(UNDER_CE) || (defined(__MINGW32CE__) && !defined(ARM32)))
4832 #define DebugBreak() _exit(-1)
4833 #endif
4834 #if defined(MSWIN32) && (defined(NO_DEBUGGING) || defined(LINT2))
4835 #define ABORT(msg) (GC_on_abort(msg), _exit(-1))
4836 #elif defined(MSWINCE) && defined(NO_DEBUGGING)
4837 #define ABORT(msg) (GC_on_abort(msg), ExitProcess(-1))
4838 #elif defined(MSWIN32) || defined(MSWINCE)
4839 #if defined(_CrtDbgBreak) && defined(_DEBUG) && defined(_MSC_VER)
4840 #define ABORT(msg) { GC_on_abort(msg); \
4841                             _CrtDbgBreak() ; }
4842 #else
4843 #define ABORT(msg) { GC_on_abort(msg); DebugBreak(); }
4844 #endif
4845 #else
4846 #define ABORT(msg) (GC_on_abort(msg), abort())
4847 #endif
4848 #endif
4849 #define ABORT_ARG1(C_msg, C_fmt, arg1) \
4850                 MACRO_BLKSTMT_BEGIN \
4851                   GC_ERRINFO_PRINTF(C_msg  C_fmt "\n", arg1); \
4852                   ABORT(C_msg); \
4853                 MACRO_BLKSTMT_END
4854 #define ABORT_ARG2(C_msg, C_fmt, arg1, arg2) \
4855                 MACRO_BLKSTMT_BEGIN \
4856                   GC_ERRINFO_PRINTF(C_msg  C_fmt "\n", arg1, arg2); \
4857                   ABORT(C_msg); \
4858                 MACRO_BLKSTMT_END
4859 #define ABORT_ARG3(C_msg, C_fmt, arg1, arg2, arg3) \
4860                 MACRO_BLKSTMT_BEGIN \
4861                   GC_ERRINFO_PRINTF(C_msg  C_fmt "\n", \
4862                                     arg1, arg2, arg3); \
4863                   ABORT(C_msg); \
4864                 MACRO_BLKSTMT_END
4865 #define ABORT_RET(msg) \
4866               if ((signed_word)GC_current_warn_proc == -1) {} else ABORT(msg)
4867 #ifdef PCR
4868 #define EXIT() PCR_Base_Exit(1,PCR_waitForever)
4869 #else
4870 #define EXIT() (GC_on_abort(NULL), exit(1 ))
4871 #endif
4872 #define WARN(msg, arg) \
4873     (*GC_current_warn_proc)(( char *)("GC Warning: " msg), \
4874                             (word)(arg))
4875 GC_EXTERN GC_warn_proc GC_current_warn_proc;
4876 #ifndef WARN_PRIdPTR
4877 #define WARN_PRIdPTR "ld"
4878 #endif
4879 #define TRUSTED_STRING(s) (char*)COVERT_DATAFLOW(s)
4880 #ifdef GC_READ_ENV_FILE
4881   GC_INNER char * GC_envfile_getenv(const char *name);
4882 #define GETENV(name) GC_envfile_getenv(name)
4883 #elif defined(NO_GETENV) && !defined(CPPCHECK)
4884 #define GETENV(name) NULL
4885 #elif defined(EMPTY_GETENV_RESULTS)
4886   GC_INLINE char * fixed_getenv(const char *name)
4887   {
4888     char *value = getenv(name);
4889     return value != NULL && *value != '\0' ? value : NULL;
4890   }
4891 #define GETENV(name) fixed_getenv(name)
4892 #else
4893 #define GETENV(name) getenv(name)
4894 #endif
4895 EXTERN_C_END
4896 #if defined(DARWIN)
4897 #include <mach/thread_status.h>
4898 #ifndef MAC_OS_X_VERSION_MAX_ALLOWED
4899 #include <AvailabilityMacros.h>
4900 #endif
4901 #if defined(POWERPC)
4902 #if CPP_WORDSZ == 32
4903 #define GC_THREAD_STATE_T          ppc_thread_state_t
4904 #else
4905 #define GC_THREAD_STATE_T          ppc_thread_state64_t
4906 #define GC_MACH_THREAD_STATE       PPC_THREAD_STATE64
4907 #define GC_MACH_THREAD_STATE_COUNT PPC_THREAD_STATE64_COUNT
4908 #endif
4909 #elif defined(I386) || defined(X86_64)
4910 #if CPP_WORDSZ == 32
4911 #if defined(i386_THREAD_STATE_COUNT) && !defined(x86_THREAD_STATE32_COUNT)
4912 #define GC_THREAD_STATE_T                i386_thread_state_t
4913 #define GC_MACH_THREAD_STATE             i386_THREAD_STATE
4914 #define GC_MACH_THREAD_STATE_COUNT       i386_THREAD_STATE_COUNT
4915 #else
4916 #define GC_THREAD_STATE_T                x86_thread_state32_t
4917 #define GC_MACH_THREAD_STATE             x86_THREAD_STATE32
4918 #define GC_MACH_THREAD_STATE_COUNT       x86_THREAD_STATE32_COUNT
4919 #endif
4920 #else
4921 #define GC_THREAD_STATE_T          x86_thread_state64_t
4922 #define GC_MACH_THREAD_STATE       x86_THREAD_STATE64
4923 #define GC_MACH_THREAD_STATE_COUNT x86_THREAD_STATE64_COUNT
4924 #endif
4925 #elif defined(ARM32) && defined(ARM_UNIFIED_THREAD_STATE) \
4926        && !defined(CPPCHECK)
4927 #define GC_THREAD_STATE_T            arm_unified_thread_state_t
4928 #define GC_MACH_THREAD_STATE         ARM_UNIFIED_THREAD_STATE
4929 #define GC_MACH_THREAD_STATE_COUNT   ARM_UNIFIED_THREAD_STATE_COUNT
4930 #elif defined(ARM32)
4931 #define GC_THREAD_STATE_T            arm_thread_state_t
4932 #ifdef ARM_MACHINE_THREAD_STATE_COUNT
4933 #define GC_MACH_THREAD_STATE       ARM_MACHINE_THREAD_STATE
4934 #define GC_MACH_THREAD_STATE_COUNT ARM_MACHINE_THREAD_STATE_COUNT
4935 #endif
4936 #elif defined(AARCH64)
4937 #define GC_THREAD_STATE_T            arm_thread_state64_t
4938 #define GC_MACH_THREAD_STATE         ARM_THREAD_STATE64
4939 #define GC_MACH_THREAD_STATE_COUNT   ARM_THREAD_STATE64_COUNT
4940 #elif !defined(CPPCHECK)
4941 #error define GC_THREAD_STATE_T
4942 #endif
4943 #ifndef GC_MACH_THREAD_STATE
4944 #define GC_MACH_THREAD_STATE         MACHINE_THREAD_STATE
4945 #define GC_MACH_THREAD_STATE_COUNT   MACHINE_THREAD_STATE_COUNT
4946 #endif
4947 #if CPP_WORDSZ == 32
4948 #define GC_MACH_HEADER   mach_header
4949 #define GC_MACH_SECTION  section
4950 #define GC_GETSECTBYNAME getsectbynamefromheader
4951 #else
4952 #define GC_MACH_HEADER   mach_header_64
4953 #define GC_MACH_SECTION  section_64
4954 #define GC_GETSECTBYNAME getsectbynamefromheader_64
4955 #endif
4956 #if __DARWIN_UNIX03
4957 #define THREAD_FLD_NAME(x) __ ## x
4958 #else
4959 #define THREAD_FLD_NAME(x) x
4960 #endif
4961 #if defined(ARM32) && defined(ARM_UNIFIED_THREAD_STATE)
4962 #define THREAD_FLD(x) ts_32.THREAD_FLD_NAME(x)
4963 #else
4964 #define THREAD_FLD(x) THREAD_FLD_NAME(x)
4965 #endif
4966 #endif
4967 #include <setjmp.h>
4968 #if __STDC_VERSION__ >= 201112L
4969 #include <assert.h>
4970 #endif
4971 EXTERN_C_BEGIN
4972 #if CPP_WORDSZ == 32
4973 #define WORDS_TO_BYTES(x) ((x)<<2)
4974 #define BYTES_TO_WORDS(x) ((x)>>2)
4975 #define LOGWL             ((word)5)
4976 #define modWORDSZ(n) ((n) & 0x1f)
4977 #if ALIGNMENT != 4
4978 #define UNALIGNED_PTRS
4979 #endif
4980 #endif
4981 #if CPP_WORDSZ == 64
4982 #define WORDS_TO_BYTES(x)   ((x)<<3)
4983 #define BYTES_TO_WORDS(x)   ((x)>>3)
4984 #define LOGWL               ((word)6)
4985 #define modWORDSZ(n) ((n) & 0x3f)
4986 #if ALIGNMENT != 8
4987 #define UNALIGNED_PTRS
4988 #endif
4989 #endif
4990 #define GRANULE_BYTES GC_GRANULE_BYTES
4991 #define TINY_FREELISTS GC_TINY_FREELISTS
4992 #define WORDSZ ((word)CPP_WORDSZ)
4993 #define SIGNB  ((word)1 << (WORDSZ-1))
4994 #define BYTES_PER_WORD      ((word)(sizeof (word)))
4995 #define divWORDSZ(n) ((n) >> LOGWL)
4996 #if GRANULE_BYTES == 8
4997 #define BYTES_TO_GRANULES(n) ((n)>>3)
4998 #define GRANULES_TO_BYTES(n) ((n)<<3)
4999 #if CPP_WORDSZ == 64
5000 #define GRANULES_TO_WORDS(n) (n)
5001 #elif CPP_WORDSZ == 32
5002 #define GRANULES_TO_WORDS(n) ((n)<<1)
5003 #else
5004 #define GRANULES_TO_WORDS(n) BYTES_TO_WORDS(GRANULES_TO_BYTES(n))
5005 #endif
5006 #elif GRANULE_BYTES == 16
5007 #define BYTES_TO_GRANULES(n) ((n)>>4)
5008 #define GRANULES_TO_BYTES(n) ((n)<<4)
5009 #if CPP_WORDSZ == 64
5010 #define GRANULES_TO_WORDS(n) ((n)<<1)
5011 #elif CPP_WORDSZ == 32
5012 #define GRANULES_TO_WORDS(n) ((n)<<2)
5013 #else
5014 #define GRANULES_TO_WORDS(n) BYTES_TO_WORDS(GRANULES_TO_BYTES(n))
5015 #endif
5016 #else
5017 #error Bad GRANULE_BYTES value
5018 #endif
5019 #ifndef HBLKSIZE
5020 #if defined(LARGE_CONFIG) || !defined(SMALL_CONFIG)
5021 #ifdef ALPHA
5022 #define CPP_LOG_HBLKSIZE 13
5023 #elif defined(SN_TARGET_PSP2)
5024 #define CPP_LOG_HBLKSIZE 16
5025 #else
5026 #define CPP_LOG_HBLKSIZE 12
5027 #endif
5028 #else
5029 #define CPP_LOG_HBLKSIZE 10
5030 #endif
5031 #else
5032 #if HBLKSIZE == 512
5033 #define CPP_LOG_HBLKSIZE 9
5034 #elif HBLKSIZE == 1024
5035 #define CPP_LOG_HBLKSIZE 10
5036 #elif HBLKSIZE == 2048
5037 #define CPP_LOG_HBLKSIZE 11
5038 #elif HBLKSIZE == 4096
5039 #define CPP_LOG_HBLKSIZE 12
5040 #elif HBLKSIZE == 8192
5041 #define CPP_LOG_HBLKSIZE 13
5042 #elif HBLKSIZE == 16384
5043 #define CPP_LOG_HBLKSIZE 14
5044 #elif !defined(CPPCHECK)
5045 #error Bad HBLKSIZE value
5046 #endif
5047 #undef HBLKSIZE
5048 #endif
5049 #define CPP_HBLKSIZE (1 << CPP_LOG_HBLKSIZE)
5050 #define LOG_HBLKSIZE   ((size_t)CPP_LOG_HBLKSIZE)
5051 #define HBLKSIZE ((size_t)CPP_HBLKSIZE)
5052 #define GC_SQRT_SIZE_MAX ((((size_t)1) << (WORDSZ / 2)) - 1)
5053 #define CPP_MAXOBJBYTES (CPP_HBLKSIZE/2)
5054 #define MAXOBJBYTES ((size_t)CPP_MAXOBJBYTES)
5055 #define CPP_MAXOBJWORDS BYTES_TO_WORDS(CPP_MAXOBJBYTES)
5056 #define MAXOBJWORDS ((size_t)CPP_MAXOBJWORDS)
5057 #define CPP_MAXOBJGRANULES BYTES_TO_GRANULES(CPP_MAXOBJBYTES)
5058 #define MAXOBJGRANULES ((size_t)CPP_MAXOBJGRANULES)
5059 #define divHBLKSZ(n) ((n) >> LOG_HBLKSIZE)
5060 #define HBLK_PTR_DIFF(p,q) divHBLKSZ((ptr_t)p - (ptr_t)q)
5061 #define modHBLKSZ(n) ((n) & (HBLKSIZE-1))
5062 #define HBLKPTR(objptr) ((struct hblk *)(((word)(objptr)) \
5063                                           & ~(word)(HBLKSIZE-1)))
5064 #define HBLKDISPL(objptr) (((size_t) (objptr)) & (HBLKSIZE-1))
5065 #define ROUNDUP_GRANULE_SIZE(lb)  \
5066             (SIZET_SAT_ADD(lb, GRANULE_BYTES - 1) & ~(GRANULE_BYTES - 1))
5067 #define ROUNDED_UP_GRANULES(lb)  \
5068         BYTES_TO_GRANULES(SIZET_SAT_ADD(lb, GRANULE_BYTES - 1 + EXTRA_BYTES))
5069 #if MAX_EXTRA_BYTES == 0
5070 #define SMALL_OBJ(bytes) EXPECT((bytes) <= (MAXOBJBYTES), TRUE)
5071 #else
5072 #define SMALL_OBJ(bytes) \
5073             (EXPECT((bytes) <= (MAXOBJBYTES - MAX_EXTRA_BYTES), TRUE) \
5074              || (bytes) <= MAXOBJBYTES - EXTRA_BYTES)
5075 #endif
5076 #define ADD_SLOP(lb)  \
5077                 SIZET_SAT_ADD(lb, EXTRA_BYTES)
5078 #ifndef LOG_PHT_ENTRIES
5079 #ifdef LARGE_CONFIG
5080 #if CPP_WORDSZ == 32
5081 #define LOG_PHT_ENTRIES 20
5082 #else
5083 #define LOG_PHT_ENTRIES 21
5084 #endif
5085 #elif !defined(SMALL_CONFIG)
5086 #define LOG_PHT_ENTRIES  18
5087 #else
5088 #define LOG_PHT_ENTRIES  15
5089 #endif
5090 #endif
5091 #define PHT_ENTRIES ((word)1 << LOG_PHT_ENTRIES)
5092 #define PHT_SIZE (PHT_ENTRIES >> LOGWL)
5093 typedef word page_hash_table[PHT_SIZE];
5094 #define PHT_HASH(addr) ((((word)(addr)) >> LOG_HBLKSIZE) & (PHT_ENTRIES - 1))
5095 #define get_pht_entry_from_index(bl, index) \
5096                 (((bl)[divWORDSZ(index)] >> modWORDSZ(index)) & 1)
5097 #define set_pht_entry_from_index(bl, index) \
5098                 (void)((bl)[divWORDSZ(index)] |= (word)1 << modWORDSZ(index))
5099 #if defined(THREADS) && defined(AO_HAVE_or)
5100 #define set_pht_entry_from_index_concurrent(bl, index) \
5101                 AO_or((volatile AO_t *)&(bl)[divWORDSZ(index)], \
5102                       (AO_t)((word)1 << modWORDSZ(index)))
5103 #else
5104 #define set_pht_entry_from_index_concurrent(bl, index) \
5105                 set_pht_entry_from_index(bl, index)
5106 #endif
5107 #define HBLKMASK   (HBLKSIZE-1)
5108 #define MARK_BITS_PER_HBLK (HBLKSIZE/GRANULE_BYTES)
5109 union word_ptr_ao_u {
5110   word w;
5111   signed_word sw;
5112   void *vp;
5113 #ifdef PARALLEL_MARK
5114     volatile AO_t ao;
5115 #endif
5116 };
5117 struct hblkhdr {
5118     struct hblk * hb_next;
5119     struct hblk * hb_prev;
5120     struct hblk * hb_block;
5121     unsigned char hb_obj_kind;
5122     unsigned char hb_flags;
5123 #define IGNORE_OFF_PAGE  1
5124 #define WAS_UNMAPPED 2
5125 #define FREE_BLK 4
5126 #ifdef ENABLE_DISCLAIM
5127 #define HAS_DISCLAIM 8
5128 #define MARK_UNCONDITIONALLY 0x10
5129 #endif
5130 #ifdef MARK_BIT_PER_GRANULE
5131 #define LARGE_BLOCK 0x20
5132 #endif
5133     unsigned short hb_last_reclaimed;
5134 #ifdef MARK_BIT_PER_OBJ
5135       unsigned32 hb_inv_sz;
5136 #define LARGE_INV_SZ (1 << 16)
5137 #endif
5138     word hb_sz;
5139     word hb_descr;
5140 #ifdef MARK_BIT_PER_GRANULE
5141       unsigned short * hb_map;
5142 #endif
5143 #ifdef PARALLEL_MARK
5144       volatile AO_t hb_n_marks;
5145 #else
5146       size_t hb_n_marks;
5147 #endif
5148 #ifdef USE_MARK_BYTES
5149 #define MARK_BITS_SZ (MARK_BITS_PER_HBLK + 1)
5150       union {
5151         char _hb_marks[MARK_BITS_SZ];
5152         word dummy;
5153       } _mark_byte_union;
5154 #define hb_marks _mark_byte_union._hb_marks
5155 #else
5156 #define MARK_BITS_SZ (MARK_BITS_PER_HBLK/CPP_WORDSZ + 1)
5157       word hb_marks[MARK_BITS_SZ];
5158 #endif
5159 };
5160 #define ANY_INDEX 23
5161 #define HBLK_WORDS (HBLKSIZE/sizeof(word))
5162 #define HBLK_GRANULES (HBLKSIZE/GRANULE_BYTES)
5163 #define HBLK_OBJS(sz_in_bytes) (HBLKSIZE/(sz_in_bytes))
5164 struct hblk {
5165     char hb_body[HBLKSIZE];
5166 };
5167 #define HBLK_IS_FREE(hdr) (((hdr) -> hb_flags & FREE_BLK) != 0)
5168 #define OBJ_SZ_TO_BLOCKS(lb) divHBLKSZ((lb) + HBLKSIZE-1)
5169 #define OBJ_SZ_TO_BLOCKS_CHECKED(lb)  \
5170                                 divHBLKSZ(SIZET_SAT_ADD(lb, HBLKSIZE - 1))
5171 #define obj_link(p) (*(void  **)(p))
5172 #define LOG_MAX_MARK_PROCS 6
5173 #define MAX_MARK_PROCS (1 << LOG_MAX_MARK_PROCS)
5174 #ifdef LARGE_CONFIG
5175 #define MAX_ROOT_SETS 8192
5176 #elif !defined(SMALL_CONFIG)
5177 #define MAX_ROOT_SETS 2048
5178 #else
5179 #define MAX_ROOT_SETS 512
5180 #endif
5181 #define MAX_EXCLUSIONS (MAX_ROOT_SETS/4)
5182 struct exclusion {
5183     ptr_t e_start;
5184     ptr_t e_end;
5185 };
5186 struct roots {
5187         ptr_t r_start;
5188         ptr_t r_end;
5189 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
5190           struct roots * r_next;
5191 #endif
5192         GC_bool r_tmp;
5193 };
5194 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
5195 #define LOG_RT_SIZE 6
5196 #define RT_SIZE (1 << LOG_RT_SIZE)
5197 #endif
5198 #if !defined(MAX_HEAP_SECTS) && (defined(CYGWIN32) || defined(MSWIN32) \
5199                     || defined(MSWINCE) || defined(USE_PROC_FOR_LIBRARIES))
5200 #ifdef LARGE_CONFIG
5201 #if CPP_WORDSZ > 32
5202 #define MAX_HEAP_SECTS 81920
5203 #else
5204 #define MAX_HEAP_SECTS 7680
5205 #endif
5206 #elif defined(SMALL_CONFIG) && !defined(USE_PROC_FOR_LIBRARIES)
5207 #if defined(PARALLEL_MARK) && (defined(MSWIN32) || defined(CYGWIN32))
5208 #define MAX_HEAP_SECTS 384
5209 #else
5210 #define MAX_HEAP_SECTS 128
5211 #endif
5212 #elif CPP_WORDSZ > 32
5213 #define MAX_HEAP_SECTS 1024
5214 #else
5215 #define MAX_HEAP_SECTS 512
5216 #endif
5217 #endif
5218 typedef struct GC_ms_entry {
5219     ptr_t mse_start;
5220     union word_ptr_ao_u mse_descr;
5221 } mse;
5222 typedef int mark_state_t;
5223 struct disappearing_link;
5224 struct finalizable_object;
5225 struct dl_hashtbl_s {
5226     struct disappearing_link **head;
5227     word entries;
5228     unsigned log_size;
5229 };
5230 struct fnlz_roots_s {
5231   struct finalizable_object **fo_head;
5232   struct finalizable_object *finalize_now;
5233 };
5234 union toggle_ref_u {
5235   void *strong_ref;
5236   GC_hidden_pointer weak_ref;
5237 };
5238 typedef struct {
5239     word ed_bitmap;
5240     GC_bool ed_continued;
5241 } typed_ext_descr_t;
5242 struct HeapSect {
5243     ptr_t hs_start;
5244     size_t hs_bytes;
5245 };
5246 struct _GC_arrays {
5247   word _heapsize;
5248   word _requested_heapsize;
5249   ptr_t _last_heap_addr;
5250   word _large_free_bytes;
5251   word _large_allocd_bytes;
5252   word _max_large_allocd_bytes;
5253   word _bytes_allocd_before_gc;
5254 #define GC_our_mem_bytes GC_arrays._our_mem_bytes
5255   word _our_mem_bytes;
5256 #ifndef SEPARATE_GLOBALS
5257 #define GC_bytes_allocd GC_arrays._bytes_allocd
5258     word _bytes_allocd;
5259 #endif
5260   word _bytes_dropped;
5261   word _bytes_finalized;
5262   word _bytes_freed;
5263   word _finalizer_bytes_freed;
5264   bottom_index *_all_bottom_indices;
5265   bottom_index *_all_bottom_indices_end;
5266   ptr_t _scratch_free_ptr;
5267   hdr *_hdr_free_list;
5268   ptr_t _scratch_end_ptr;
5269 #if defined(IRIX5) || (defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX))
5270 #define USE_SCRATCH_LAST_END_PTR
5271 #define GC_scratch_last_end_ptr GC_arrays._scratch_last_end_ptr
5272     ptr_t _scratch_last_end_ptr;
5273 #endif
5274   mse *_mark_stack;
5275   mse *_mark_stack_limit;
5276 #ifdef PARALLEL_MARK
5277     mse *volatile _mark_stack_top;
5278 #else
5279     mse *_mark_stack_top;
5280 #endif
5281   word _composite_in_use;
5282   word _atomic_in_use;
5283 #ifdef USE_MUNMAP
5284 #define GC_unmapped_bytes GC_arrays._unmapped_bytes
5285     word _unmapped_bytes;
5286 #ifdef COUNT_UNMAPPED_REGIONS
5287 #define GC_num_unmapped_regions GC_arrays._num_unmapped_regions
5288       signed_word _num_unmapped_regions;
5289 #endif
5290 #else
5291 #define GC_unmapped_bytes 0
5292 #endif
5293   bottom_index * _all_nils;
5294 #define GC_scan_ptr GC_arrays._scan_ptr
5295   struct hblk * _scan_ptr;
5296 #ifdef PARALLEL_MARK
5297 #define GC_main_local_mark_stack GC_arrays._main_local_mark_stack
5298     mse *_main_local_mark_stack;
5299 #define GC_first_nonempty GC_arrays._first_nonempty
5300     volatile AO_t _first_nonempty;
5301 #endif
5302 #define GC_mark_stack_size GC_arrays._mark_stack_size
5303   size_t _mark_stack_size;
5304 #define GC_mark_state GC_arrays._mark_state
5305   mark_state_t _mark_state;
5306 #define GC_mark_stack_too_small GC_arrays._mark_stack_too_small
5307   GC_bool _mark_stack_too_small;
5308 #define GC_objects_are_marked GC_arrays._objects_are_marked
5309   GC_bool _objects_are_marked;
5310 #ifdef ENABLE_TRACE
5311 #define GC_trace_addr GC_arrays._trace_addr
5312     ptr_t _trace_addr;
5313 #endif
5314 #define GC_capacity_heap_sects GC_arrays._capacity_heap_sects
5315   size_t _capacity_heap_sects;
5316 #define GC_n_heap_sects GC_arrays._n_heap_sects
5317   word _n_heap_sects;
5318 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
5319 #define GC_n_heap_bases GC_arrays._n_heap_bases
5320     word _n_heap_bases;
5321 #endif
5322 #ifdef USE_PROC_FOR_LIBRARIES
5323 #define GC_n_memory GC_arrays._n_memory
5324     word _n_memory;
5325 #endif
5326 #ifdef GC_GCJ_SUPPORT
5327 #define GC_gcjobjfreelist GC_arrays._gcjobjfreelist
5328     ptr_t *_gcjobjfreelist;
5329 #endif
5330 #define GC_fo_entries GC_arrays._fo_entries
5331   word _fo_entries;
5332 #ifndef GC_NO_FINALIZATION
5333 #define GC_dl_hashtbl GC_arrays._dl_hashtbl
5334 #define GC_fnlz_roots GC_arrays._fnlz_roots
5335 #define GC_log_fo_table_size GC_arrays._log_fo_table_size
5336 #ifndef GC_LONG_REFS_NOT_NEEDED
5337 #define GC_ll_hashtbl GC_arrays._ll_hashtbl
5338       struct dl_hashtbl_s _ll_hashtbl;
5339 #endif
5340     struct dl_hashtbl_s _dl_hashtbl;
5341     struct fnlz_roots_s _fnlz_roots;
5342     unsigned _log_fo_table_size;
5343 #ifndef GC_TOGGLE_REFS_NOT_NEEDED
5344 #define GC_toggleref_arr GC_arrays._toggleref_arr
5345 #define GC_toggleref_array_size GC_arrays._toggleref_array_size
5346 #define GC_toggleref_array_capacity GC_arrays._toggleref_array_capacity
5347       union toggle_ref_u *_toggleref_arr;
5348       size_t _toggleref_array_size;
5349       size_t _toggleref_array_capacity;
5350 #endif
5351 #endif
5352 #ifdef TRACE_BUF
5353 #define GC_trace_buf_ptr GC_arrays._trace_buf_ptr
5354     int _trace_buf_ptr;
5355 #endif
5356 #ifdef ENABLE_DISCLAIM
5357 #define GC_finalized_kind GC_arrays._finalized_kind
5358     int _finalized_kind;
5359 #endif
5360 #define n_root_sets GC_arrays._n_root_sets
5361 #define GC_excl_table_entries GC_arrays._excl_table_entries
5362   int _n_root_sets;
5363   size_t _excl_table_entries;
5364 #ifdef THREADS
5365 #define GC_roots_were_cleared GC_arrays._roots_were_cleared
5366     GC_bool _roots_were_cleared;
5367 #endif
5368 #define GC_explicit_typing_initialized GC_arrays._explicit_typing_initialized
5369 #define GC_ed_size GC_arrays._ed_size
5370 #define GC_avail_descr GC_arrays._avail_descr
5371 #define GC_ext_descriptors GC_arrays._ext_descriptors
5372 #ifdef AO_HAVE_load_acquire
5373     volatile AO_t _explicit_typing_initialized;
5374 #else
5375     GC_bool _explicit_typing_initialized;
5376 #endif
5377   size_t _ed_size;
5378   size_t _avail_descr;
5379   typed_ext_descr_t *_ext_descriptors;
5380   GC_mark_proc _mark_procs[MAX_MARK_PROCS];
5381   char _modws_valid_offsets[sizeof(word)];
5382 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
5383 #define GC_root_index GC_arrays._root_index
5384     struct roots * _root_index[RT_SIZE];
5385 #endif
5386 #ifdef SAVE_CALL_CHAIN
5387 #define GC_last_stack GC_arrays._last_stack
5388     struct callinfo _last_stack[NFRAMES];
5389 #endif
5390 #ifndef SEPARATE_GLOBALS
5391 #define GC_objfreelist GC_arrays._objfreelist
5392     void *_objfreelist[MAXOBJGRANULES+1];
5393 #define GC_aobjfreelist GC_arrays._aobjfreelist
5394     void *_aobjfreelist[MAXOBJGRANULES+1];
5395 #endif
5396   void *_uobjfreelist[MAXOBJGRANULES+1];
5397 #ifdef GC_ATOMIC_UNCOLLECTABLE
5398 #define GC_auobjfreelist GC_arrays._auobjfreelist
5399     void *_auobjfreelist[MAXOBJGRANULES+1];
5400 #endif
5401   size_t _size_map[MAXOBJBYTES+1];
5402 #ifdef MARK_BIT_PER_GRANULE
5403 #define GC_obj_map GC_arrays._obj_map
5404     unsigned short * _obj_map[MAXOBJGRANULES + 1];
5405 #define MAP_LEN BYTES_TO_GRANULES(HBLKSIZE)
5406 #endif
5407 #define VALID_OFFSET_SZ HBLKSIZE
5408   char _valid_offsets[VALID_OFFSET_SZ];
5409 #ifndef GC_DISABLE_INCREMENTAL
5410 #define GC_grungy_pages GC_arrays._grungy_pages
5411     page_hash_table _grungy_pages;
5412 #define GC_dirty_pages GC_arrays._dirty_pages
5413     volatile page_hash_table _dirty_pages;
5414 #endif
5415 #if (defined(CHECKSUMS) && (defined(GWW_VDB) || defined(SOFT_VDB))) \
5416      || defined(PROC_VDB)
5417 #define GC_written_pages GC_arrays._written_pages
5418     page_hash_table _written_pages;
5419 #endif
5420 #define GC_heap_sects GC_arrays._heap_sects
5421   struct HeapSect *_heap_sects;
5422 #if defined(USE_PROC_FOR_LIBRARIES)
5423 #define GC_our_memory GC_arrays._our_memory
5424     struct HeapSect _our_memory[MAX_HEAP_SECTS];
5425 #endif
5426 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
5427 #define GC_heap_bases GC_arrays._heap_bases
5428     ptr_t _heap_bases[MAX_HEAP_SECTS];
5429 #endif
5430 #ifdef MSWINCE
5431 #define GC_heap_lengths GC_arrays._heap_lengths
5432     word _heap_lengths[MAX_HEAP_SECTS];
5433 #endif
5434   struct roots _static_roots[MAX_ROOT_SETS];
5435   struct exclusion _excl_table[MAX_EXCLUSIONS];
5436   bottom_index * _top_index[TOP_SZ];
5437 };
5438 GC_API_PRIV GC_FAR struct _GC_arrays GC_arrays;
5439 #define GC_all_nils GC_arrays._all_nils
5440 #define GC_atomic_in_use GC_arrays._atomic_in_use
5441 #define GC_bytes_allocd_before_gc GC_arrays._bytes_allocd_before_gc
5442 #define GC_bytes_dropped GC_arrays._bytes_dropped
5443 #define GC_bytes_finalized GC_arrays._bytes_finalized
5444 #define GC_bytes_freed GC_arrays._bytes_freed
5445 #define GC_composite_in_use GC_arrays._composite_in_use
5446 #define GC_excl_table GC_arrays._excl_table
5447 #define GC_finalizer_bytes_freed GC_arrays._finalizer_bytes_freed
5448 #define GC_heapsize GC_arrays._heapsize
5449 #define GC_large_allocd_bytes GC_arrays._large_allocd_bytes
5450 #define GC_large_free_bytes GC_arrays._large_free_bytes
5451 #define GC_last_heap_addr GC_arrays._last_heap_addr
5452 #define GC_mark_stack GC_arrays._mark_stack
5453 #define GC_mark_stack_limit GC_arrays._mark_stack_limit
5454 #define GC_mark_stack_top GC_arrays._mark_stack_top
5455 #define GC_mark_procs GC_arrays._mark_procs
5456 #define GC_max_large_allocd_bytes GC_arrays._max_large_allocd_bytes
5457 #define GC_modws_valid_offsets GC_arrays._modws_valid_offsets
5458 #define GC_requested_heapsize GC_arrays._requested_heapsize
5459 #define GC_all_bottom_indices GC_arrays._all_bottom_indices
5460 #define GC_all_bottom_indices_end GC_arrays._all_bottom_indices_end
5461 #define GC_scratch_free_ptr GC_arrays._scratch_free_ptr
5462 #define GC_hdr_free_list GC_arrays._hdr_free_list
5463 #define GC_scratch_end_ptr GC_arrays._scratch_end_ptr
5464 #define GC_size_map GC_arrays._size_map
5465 #define GC_static_roots GC_arrays._static_roots
5466 #define GC_top_index GC_arrays._top_index
5467 #define GC_uobjfreelist GC_arrays._uobjfreelist
5468 #define GC_valid_offsets GC_arrays._valid_offsets
5469 #define beginGC_arrays ((ptr_t)(&GC_arrays))
5470 #define endGC_arrays (((ptr_t)(&GC_arrays)) + (sizeof GC_arrays))
5471 #define USED_HEAP_SIZE (GC_heapsize - GC_large_free_bytes)
5472 #ifndef MAXOBJKINDS
5473 #define MAXOBJKINDS 16
5474 #endif
5475 GC_EXTERN struct obj_kind {
5476    void **ok_freelist;
5477    struct hblk **ok_reclaim_list;
5478    word ok_descriptor;
5479    GC_bool ok_relocate_descr;
5480    GC_bool ok_init;
5481 #ifdef ENABLE_DISCLAIM
5482      GC_bool ok_mark_unconditionally;
5483      int (GC_CALLBACK *ok_disclaim_proc)(void * );
5484 #define OK_DISCLAIM_INITZ , FALSE, 0
5485 #else
5486 #define OK_DISCLAIM_INITZ
5487 #endif
5488 } GC_obj_kinds[MAXOBJKINDS];
5489 #define beginGC_obj_kinds ((ptr_t)(&GC_obj_kinds))
5490 #define endGC_obj_kinds (beginGC_obj_kinds + (sizeof GC_obj_kinds))
5491 #ifdef SEPARATE_GLOBALS
5492   extern word GC_bytes_allocd;
5493   extern ptr_t GC_objfreelist[MAXOBJGRANULES+1];
5494 #define beginGC_objfreelist ((ptr_t)(&GC_objfreelist))
5495 #define endGC_objfreelist (beginGC_objfreelist + sizeof(GC_objfreelist))
5496   extern ptr_t GC_aobjfreelist[MAXOBJGRANULES+1];
5497 #define beginGC_aobjfreelist ((ptr_t)(&GC_aobjfreelist))
5498 #define endGC_aobjfreelist (beginGC_aobjfreelist + sizeof(GC_aobjfreelist))
5499 #endif
5500 #define PTRFREE 0
5501 #define NORMAL  1
5502 #define UNCOLLECTABLE 2
5503 #ifdef GC_ATOMIC_UNCOLLECTABLE
5504 #define AUNCOLLECTABLE 3
5505 #define IS_UNCOLLECTABLE(k) (((k) & ~1) == UNCOLLECTABLE)
5506 #define GC_N_KINDS_INITIAL_VALUE 4
5507 #else
5508 #define IS_UNCOLLECTABLE(k) ((k) == UNCOLLECTABLE)
5509 #define GC_N_KINDS_INITIAL_VALUE 3
5510 #endif
5511 GC_EXTERN unsigned GC_n_kinds;
5512 GC_EXTERN size_t GC_page_size;
5513 #define ROUNDUP_PAGESIZE(lb)  \
5514             (SIZET_SAT_ADD(lb, GC_page_size - 1) & ~(GC_page_size - 1))
5515 #ifdef MMAP_SUPPORTED
5516 #define ROUNDUP_PAGESIZE_IF_MMAP(lb) ROUNDUP_PAGESIZE(lb)
5517 #else
5518 #define ROUNDUP_PAGESIZE_IF_MMAP(lb) (lb)
5519 #endif
5520 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
5521   GC_EXTERN SYSTEM_INFO GC_sysinfo;
5522   GC_INNER GC_bool GC_is_heap_base(void *p);
5523 #endif
5524 GC_EXTERN word GC_black_list_spacing;
5525 #ifdef GC_GCJ_SUPPORT
5526   extern struct hblk * GC_hblkfreelist[];
5527   extern word GC_free_bytes[];
5528 #endif
5529 GC_EXTERN word GC_root_size;
5530 GC_EXTERN GC_bool GC_debugging_started;
5531 struct blocking_data {
5532     GC_fn_type fn;
5533     void * client_data;
5534 };
5535 struct GC_traced_stack_sect_s {
5536   ptr_t saved_stack_ptr;
5537 #ifdef IA64
5538     ptr_t saved_backing_store_ptr;
5539     ptr_t backing_store_end;
5540 #endif
5541   struct GC_traced_stack_sect_s *prev;
5542 };
5543 #ifdef THREADS
5544   GC_INNER void GC_push_all_stack_sections(ptr_t lo, ptr_t hi,
5545                         struct GC_traced_stack_sect_s *traced_stack_sect);
5546   GC_EXTERN word GC_total_stacksize;
5547 #else
5548   GC_EXTERN ptr_t GC_blocked_sp;
5549   GC_EXTERN struct GC_traced_stack_sect_s *GC_traced_stack_sect;
5550 #endif
5551 #ifdef IA64
5552   GC_INNER void GC_push_all_register_sections(ptr_t bs_lo, ptr_t bs_hi,
5553                   int eager, struct GC_traced_stack_sect_s *traced_stack_sect);
5554 #endif
5555 #ifdef USE_MARK_BYTES
5556 #define mark_bit_from_hdr(hhdr,n) ((hhdr)->hb_marks[n])
5557 #define set_mark_bit_from_hdr(hhdr,n) ((hhdr)->hb_marks[n] = 1)
5558 #define clear_mark_bit_from_hdr(hhdr,n) ((hhdr)->hb_marks[n] = 0)
5559 #else
5560 #if defined(PARALLEL_MARK) || (defined(THREAD_SANITIZER) && defined(THREADS))
5561 #define OR_WORD(addr, bits) AO_or((volatile AO_t *)(addr), (AO_t)(bits))
5562 #else
5563 #define OR_WORD(addr, bits) (void)(*(addr) |= (bits))
5564 #endif
5565 #define mark_bit_from_hdr(hhdr,n) \
5566               (((hhdr)->hb_marks[divWORDSZ(n)] >> modWORDSZ(n)) & (word)1)
5567 #define set_mark_bit_from_hdr(hhdr,n) \
5568               OR_WORD((hhdr)->hb_marks+divWORDSZ(n), (word)1 << modWORDSZ(n))
5569 #define clear_mark_bit_from_hdr(hhdr,n) \
5570               ((hhdr)->hb_marks[divWORDSZ(n)] &= ~((word)1 << modWORDSZ(n)))
5571 #endif
5572 #ifdef MARK_BIT_PER_OBJ
5573 #define MARK_BIT_NO(offset, sz) (((word)(offset))/(sz))
5574 #define MARK_BIT_OFFSET(sz) 1
5575 #define IF_PER_OBJ(x) x
5576 #define FINAL_MARK_BIT(sz) ((sz) > MAXOBJBYTES? 1 : HBLK_OBJS(sz))
5577 #else
5578 #define MARK_BIT_NO(offset, sz) BYTES_TO_GRANULES((word)(offset))
5579 #define MARK_BIT_OFFSET(sz) BYTES_TO_GRANULES(sz)
5580 #define IF_PER_OBJ(x)
5581 #define FINAL_MARK_BIT(sz) \
5582                 ((sz) > MAXOBJBYTES ? MARK_BITS_PER_HBLK \
5583                                 : BYTES_TO_GRANULES((sz) * HBLK_OBJS(sz)))
5584 #endif
5585 GC_INNER ptr_t GC_approx_sp(void);
5586 GC_INNER GC_bool GC_should_collect(void);
5587 void GC_apply_to_all_blocks(void (*fn)(struct hblk *h, word client_data),
5588                             word client_data);
5589 GC_INNER struct hblk * GC_next_block(struct hblk *h, GC_bool allow_free);
5590 GC_INNER struct hblk * GC_prev_block(struct hblk * h);
5591 GC_INNER void GC_mark_init(void);
5592 GC_INNER void GC_clear_marks(void);
5593 GC_INNER void GC_invalidate_mark_state(void);
5594 GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame);
5595 GC_INNER void GC_initiate_gc(void);
5596 GC_INNER GC_bool GC_collection_in_progress(void);
5597 #define GC_PUSH_ALL_SYM(sym) \
5598                 GC_push_all(( void *)&(sym), \
5599                             ( void *)(&(sym) + 1))
5600 GC_INNER void GC_push_all_stack(ptr_t b, ptr_t t);
5601 #ifdef NO_VDB_FOR_STATIC_ROOTS
5602 #define GC_push_conditional_static(b, t, all) \
5603                 ((void)(all), GC_push_all(b, t))
5604 #else
5605   GC_INNER void GC_push_conditional_static(void *b, void *t, GC_bool all);
5606 #endif
5607 #if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK)
5608   GC_INNER void GC_push_conditional_eager(void *bottom, void *top,
5609                                           GC_bool all);
5610 #endif
5611 GC_INNER void GC_push_roots(GC_bool all, ptr_t cold_gc_frame);
5612 GC_API_PRIV GC_push_other_roots_proc GC_push_other_roots;
5613 #ifdef THREADS
5614   void GC_push_thread_structures(void);
5615 #endif
5616 GC_EXTERN void (*GC_push_typed_structures)(void);
5617 GC_INNER void GC_with_callee_saves_pushed(void (*fn)(ptr_t, void *),
5618                                           volatile ptr_t arg);
5619 #if defined(SPARC) || defined(IA64)
5620   ptr_t GC_save_regs_in_stack(void);
5621 #endif
5622 #if defined(AMIGA) || defined(MACOS) || defined(GC_DARWIN_THREADS)
5623   void GC_push_one(word p);
5624 #endif
5625 #ifdef GC_WIN32_THREADS
5626   GC_INNER void GC_push_many_regs(const word *regs, unsigned count);
5627 #endif
5628 #if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
5629   GC_INNER void GC_mark_and_push_stack(ptr_t p, ptr_t source);
5630 #else
5631   GC_INNER void GC_mark_and_push_stack(ptr_t p);
5632 #endif
5633 GC_INNER void GC_clear_hdr_marks(hdr * hhdr);
5634 GC_INNER void GC_set_hdr_marks(hdr * hhdr);
5635 GC_INNER void GC_set_fl_marks(ptr_t p);
5636 #if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
5637   void GC_check_fl_marks(void **);
5638 #endif
5639 void GC_add_roots_inner(ptr_t b, ptr_t e, GC_bool tmp);
5640 #ifdef USE_PROC_FOR_LIBRARIES
5641   GC_INNER void GC_remove_roots_subregion(ptr_t b, ptr_t e);
5642 #endif
5643 GC_INNER void GC_exclude_static_roots_inner(void *start, void *finish);
5644 #if defined(DYNAMIC_LOADING) || defined(MSWIN32) || defined(MSWINCE) \
5645     || defined(CYGWIN32) || defined(PCR)
5646   GC_INNER void GC_register_dynamic_libraries(void);
5647 #endif
5648 GC_INNER void GC_cond_register_dynamic_libraries(void);
5649 ptr_t GC_get_main_stack_base(void);
5650 #ifdef IA64
5651   GC_INNER ptr_t GC_get_register_stack_base(void);
5652 #endif
5653 void GC_register_data_segments(void);
5654 #ifdef THREADS
5655   GC_INNER void GC_thr_init(void);
5656   GC_INNER void GC_init_parallel(void);
5657 #else
5658   GC_INNER GC_bool GC_is_static_root(void *p);
5659 #ifdef TRACE_BUF
5660     void GC_add_trace_entry(char *kind, word arg1, word arg2);
5661 #endif
5662 #endif
5663 #ifdef PRINT_BLACK_LIST
5664   GC_INNER void GC_add_to_black_list_normal(word p, ptr_t source);
5665 #define GC_ADD_TO_BLACK_LIST_NORMAL(bits, source) \
5666                 if (GC_all_interior_pointers) { \
5667                   GC_add_to_black_list_stack((word)(bits), (source)); \
5668                 } else \
5669                   GC_add_to_black_list_normal((word)(bits), (source))
5670   GC_INNER void GC_add_to_black_list_stack(word p, ptr_t source);
5671 #define GC_ADD_TO_BLACK_LIST_STACK(bits, source) \
5672             GC_add_to_black_list_stack((word)(bits), (source))
5673 #else
5674   GC_INNER void GC_add_to_black_list_normal(word p);
5675 #define GC_ADD_TO_BLACK_LIST_NORMAL(bits, source) \
5676                 if (GC_all_interior_pointers) { \
5677                   GC_add_to_black_list_stack((word)(bits)); \
5678                 } else \
5679                   GC_add_to_black_list_normal((word)(bits))
5680   GC_INNER void GC_add_to_black_list_stack(word p);
5681 #define GC_ADD_TO_BLACK_LIST_STACK(bits, source) \
5682             GC_add_to_black_list_stack((word)(bits))
5683 #endif
5684 struct hblk * GC_is_black_listed(struct hblk * h, word len);
5685 GC_INNER void GC_promote_black_lists(void);
5686 GC_INNER void GC_unpromote_black_lists(void);
5687 GC_INNER ptr_t GC_scratch_alloc(size_t bytes);
5688 #ifdef GWW_VDB
5689 #else
5690 #define GC_scratch_recycle_no_gww GC_scratch_recycle_inner
5691 #endif
5692 GC_INNER void GC_scratch_recycle_inner(void *ptr, size_t bytes);
5693 #ifdef MARK_BIT_PER_GRANULE
5694   GC_INNER GC_bool GC_add_map_entry(size_t sz);
5695 #endif
5696 GC_INNER void GC_register_displacement_inner(size_t offset);
5697 GC_INNER void GC_new_hblk(size_t size_in_granules, int kind);
5698 GC_INNER ptr_t GC_build_fl(struct hblk *h, size_t words, GC_bool clear,
5699                            ptr_t list);
5700 GC_INNER struct hblk * GC_allochblk(size_t size_in_bytes, int kind,
5701                                     unsigned flags);
5702 GC_INNER ptr_t GC_alloc_large(size_t lb, int k, unsigned flags);
5703 GC_INNER void GC_freehblk(struct hblk * p);
5704 GC_INNER GC_bool GC_expand_hp_inner(word n);
5705 GC_INNER void GC_start_reclaim(GC_bool abort_if_found);
5706 GC_INNER void GC_continue_reclaim(word sz, int kind);
5707 GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old);
5708 GC_INNER ptr_t GC_reclaim_generic(struct hblk * hbp, hdr *hhdr, size_t sz,
5709                                   GC_bool init, ptr_t list,
5710                                   signed_word *count);
5711 GC_INNER GC_bool GC_block_empty(hdr * hhdr);
5712 GC_INNER int GC_CALLBACK GC_never_stop_func(void);
5713 GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func f);
5714 #define GC_gcollect_inner() \
5715                 (void)GC_try_to_collect_inner(GC_never_stop_func)
5716 #ifdef THREADS
5717   GC_EXTERN GC_bool GC_in_thread_creation;
5718 #endif
5719 GC_EXTERN GC_bool GC_is_initialized;
5720 GC_INNER void GC_collect_a_little_inner(int n);
5721 GC_INNER void * GC_generic_malloc_inner(size_t lb, int k);
5722 #if defined(DBG_HDRS_ALL) || defined(GC_GCJ_SUPPORT) \
5723     || !defined(GC_NO_FINALIZATION)
5724   GC_INNER void * GC_generic_malloc_inner_ignore_off_page(size_t lb, int k);
5725 #endif
5726 GC_INNER GC_bool GC_collect_or_expand(word needed_blocks,
5727                                       GC_bool ignore_off_page, GC_bool retry);
5728 GC_INNER ptr_t GC_allocobj(size_t sz, int kind);
5729 #ifdef GC_ADD_CALLER
5730 #ifdef GC_HAVE_RETURN_ADDR_PARENT
5731 #define GC_DBG_EXTRAS GC_RETURN_ADDR_PARENT, NULL, 0
5732 #else
5733 #define GC_DBG_EXTRAS GC_RETURN_ADDR, NULL, 0
5734 #endif
5735 #else
5736 #define GC_DBG_EXTRAS "unknown", 0
5737 #endif
5738 #ifdef GC_COLLECT_AT_MALLOC
5739   extern size_t GC_dbg_collect_at_malloc_min_lb;
5740 #define GC_DBG_COLLECT_AT_MALLOC(lb) \
5741                 (void)((lb) >= GC_dbg_collect_at_malloc_min_lb ? \
5742                             (GC_gcollect(), 0) : 0)
5743 #else
5744 #define GC_DBG_COLLECT_AT_MALLOC(lb) (void)0
5745 #endif
5746 #if defined(THREAD_LOCAL_ALLOC) && defined(GC_GCJ_SUPPORT)
5747     GC_INNER void * GC_core_gcj_malloc(size_t, void *);
5748 #endif
5749 GC_INNER void GC_init_headers(void);
5750 GC_INNER struct hblkhdr * GC_install_header(struct hblk *h);
5751 GC_INNER GC_bool GC_install_counts(struct hblk * h, size_t sz);
5752 GC_INNER void GC_remove_header(struct hblk * h);
5753 GC_INNER void GC_remove_counts(struct hblk * h, size_t sz);
5754 GC_INNER hdr * GC_find_header(ptr_t h);
5755 #ifdef USE_PROC_FOR_LIBRARIES
5756   GC_INNER void GC_add_to_our_memory(ptr_t p, size_t bytes);
5757 #else
5758 #define GC_add_to_our_memory(p, bytes) \
5759                 (GC_our_mem_bytes += (bytes), (void)(p))
5760 #endif
5761 GC_INNER void GC_print_all_errors(void);
5762 GC_EXTERN void (*GC_check_heap)(void);
5763 GC_EXTERN void (*GC_print_all_smashed)(void);
5764 GC_EXTERN void (*GC_print_heap_obj)(ptr_t p);
5765 #if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
5766   void GC_print_address_map(void);
5767 #endif
5768 #ifndef SHORT_DBG_HDRS
5769   GC_EXTERN GC_bool GC_findleak_delay_free;
5770   GC_INNER GC_bool GC_check_leaked(ptr_t base);
5771 #endif
5772 GC_EXTERN GC_bool GC_have_errors;
5773 #define VERBOSE 2
5774 #if !defined(NO_CLOCK) || !defined(SMALL_CONFIG)
5775   extern int GC_print_stats;
5776 #else
5777 #define GC_print_stats 0
5778 #endif
5779 #ifdef KEEP_BACK_PTRS
5780   GC_EXTERN long GC_backtraces;
5781   GC_INNER void GC_generate_random_backtrace_no_gc(void);
5782 #endif
5783 #ifdef LINT2
5784 #define GC_RAND_MAX (~0U >> 1)
5785   GC_API_PRIV long GC_random(void);
5786 #endif
5787 GC_EXTERN GC_bool GC_print_back_height;
5788 #ifdef MAKE_BACK_GRAPH
5789   void GC_print_back_graph_stats(void);
5790 #endif
5791 #ifdef THREADS
5792   GC_INNER void GC_free_inner(void * p);
5793 #endif
5794 #ifdef DBG_HDRS_ALL
5795   GC_INNER void * GC_debug_generic_malloc_inner(size_t lb, int k);
5796   GC_INNER void * GC_debug_generic_malloc_inner_ignore_off_page(size_t lb,
5797                                                                 int k);
5798 #define GC_INTERNAL_MALLOC GC_debug_generic_malloc_inner
5799 #define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE \
5800                GC_debug_generic_malloc_inner_ignore_off_page
5801 #ifdef THREADS
5802     GC_INNER void GC_debug_free_inner(void * p);
5803 #define GC_INTERNAL_FREE GC_debug_free_inner
5804 #else
5805 #define GC_INTERNAL_FREE GC_debug_free
5806 #endif
5807 #else
5808 #define GC_INTERNAL_MALLOC GC_generic_malloc_inner
5809 #define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE \
5810                GC_generic_malloc_inner_ignore_off_page
5811 #ifdef THREADS
5812 #define GC_INTERNAL_FREE GC_free_inner
5813 #else
5814 #define GC_INTERNAL_FREE GC_free
5815 #endif
5816 #endif
5817 #ifdef USE_MUNMAP
5818   GC_INNER void GC_unmap_old(void);
5819   GC_INNER void GC_merge_unmapped(void);
5820   GC_INNER void GC_unmap(ptr_t start, size_t bytes);
5821   GC_INNER void GC_remap(ptr_t start, size_t bytes);
5822   GC_INNER void GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2,
5823                              size_t bytes2);
5824   GC_INLINE ptr_t GC_unmap_end(ptr_t start, size_t bytes)
5825   {
5826      return (ptr_t)((word)(start + bytes) & ~(GC_page_size - 1));
5827   }
5828 #endif
5829 #ifdef CAN_HANDLE_FORK
5830   GC_EXTERN int GC_handle_fork;
5831 #endif
5832 #ifdef GC_DISABLE_INCREMENTAL
5833 #define GC_incremental FALSE
5834 #define GC_auto_incremental FALSE
5835 #define GC_manual_vdb FALSE
5836 #define GC_dirty(p) (void)(p)
5837 #define REACHABLE_AFTER_DIRTY(p) (void)(p)
5838 #else
5839   GC_EXTERN GC_bool GC_incremental;
5840   GC_INNER void GC_read_dirty(GC_bool output_unneeded);
5841   GC_INNER GC_bool GC_page_was_dirty(struct hblk *h);
5842   GC_INNER void GC_remove_protection(struct hblk *h, word nblocks,
5843                                    GC_bool pointerfree);
5844 #if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB)
5845     GC_INNER GC_bool GC_is_vdb_for_static_roots(void);
5846 #endif
5847 #ifdef CAN_HANDLE_FORK
5848 #if defined(PROC_VDB) || defined(SOFT_VDB)
5849       GC_INNER void GC_dirty_update_child(void);
5850 #else
5851 #define GC_dirty_update_child() (void)0
5852 #endif
5853 #endif
5854   GC_INNER GC_bool GC_dirty_init(void);
5855   GC_EXTERN GC_bool GC_manual_vdb;
5856 #define GC_auto_incremental (GC_incremental && !GC_manual_vdb)
5857   GC_INNER void GC_dirty_inner(const void *p);
5858 #define GC_dirty(p) (GC_manual_vdb ? GC_dirty_inner(p) : (void)0)
5859 #define REACHABLE_AFTER_DIRTY(p) GC_reachable_here(p)
5860 #endif
5861 #define GC_base_C(p) ((const void *)GC_base(( void *)(p)))
5862 void GC_print_block_list(void);
5863 void GC_print_hblkfreelist(void);
5864 void GC_print_heap_sects(void);
5865 void GC_print_static_roots(void);
5866 #ifdef KEEP_BACK_PTRS
5867    GC_INNER void GC_store_back_pointer(ptr_t source, ptr_t dest);
5868    GC_INNER void GC_marked_for_finalization(ptr_t dest);
5869 #define GC_STORE_BACK_PTR(source, dest) GC_store_back_pointer(source, dest)
5870 #define GC_MARKED_FOR_FINALIZATION(dest) GC_marked_for_finalization(dest)
5871 #else
5872 #define GC_STORE_BACK_PTR(source, dest) (void)(source)
5873 #define GC_MARKED_FOR_FINALIZATION(dest)
5874 #endif
5875 void GC_noop6(word, word, word, word, word, word);
5876 GC_API void GC_CALL GC_noop1(word);
5877 #ifndef GC_ATTR_FORMAT_PRINTF
5878 #if GC_GNUC_PREREQ(3, 0)
5879 #define GC_ATTR_FORMAT_PRINTF(spec_argnum, first_checked) \
5880         __attribute__((__format__(__printf__, spec_argnum, first_checked)))
5881 #else
5882 #define GC_ATTR_FORMAT_PRINTF(spec_argnum, first_checked)
5883 #endif
5884 #endif
5885 GC_API_PRIV void GC_printf(const char * format, ...)
5886                         GC_ATTR_FORMAT_PRINTF(1, 2);
5887 GC_API_PRIV void GC_err_printf(const char * format, ...)
5888                         GC_ATTR_FORMAT_PRINTF(1, 2);
5889 GC_API_PRIV void GC_log_printf(const char * format, ...)
5890                         GC_ATTR_FORMAT_PRINTF(1, 2);
5891 #ifndef GC_ANDROID_LOG
5892 #define GC_PRINT_STATS_FLAG (GC_print_stats != 0)
5893 #define GC_INFOLOG_PRINTF GC_COND_LOG_PRINTF
5894 #define GC_verbose_log_printf GC_log_printf
5895 #else
5896   extern GC_bool GC_quiet;
5897 #define GC_PRINT_STATS_FLAG (!GC_quiet)
5898 #ifndef GC_INFOLOG_PRINTF
5899 #define GC_INFOLOG_PRINTF if (GC_quiet) {} else GC_info_log_printf
5900 #endif
5901   GC_INNER void GC_info_log_printf(const char *format, ...)
5902                         GC_ATTR_FORMAT_PRINTF(1, 2);
5903   GC_INNER void GC_verbose_log_printf(const char *format, ...)
5904                         GC_ATTR_FORMAT_PRINTF(1, 2);
5905 #endif
5906 #if defined(SMALL_CONFIG) || defined(GC_ANDROID_LOG)
5907 #define GC_ERRINFO_PRINTF GC_INFOLOG_PRINTF
5908 #else
5909 #define GC_ERRINFO_PRINTF GC_log_printf
5910 #endif
5911 #define GC_COND_LOG_PRINTF \
5912                 if (EXPECT(!GC_print_stats, TRUE)) {} else GC_log_printf
5913 #define GC_VERBOSE_LOG_PRINTF \
5914     if (EXPECT(GC_print_stats != VERBOSE, TRUE)) {} else GC_verbose_log_printf
5915 #ifndef GC_DBGLOG_PRINTF
5916 #define GC_DBGLOG_PRINTF if (!GC_PRINT_STATS_FLAG) {} else GC_log_printf
5917 #endif
5918 void GC_err_puts(const char *s);
5919 #define TO_KiB_UL(v) ((unsigned long)(((v) + ((1 << 9) - 1)) >> 10))
5920 GC_EXTERN unsigned GC_fail_count;
5921 GC_EXTERN long GC_large_alloc_warn_interval;
5922 GC_EXTERN signed_word GC_bytes_found;
5923 #ifndef GC_GET_HEAP_USAGE_NOT_NEEDED
5924   GC_EXTERN word GC_reclaimed_bytes_before_gc;
5925 #endif
5926 #ifdef USE_MUNMAP
5927   GC_EXTERN int GC_unmap_threshold;
5928   GC_EXTERN GC_bool GC_force_unmap_on_gcollect;
5929 #endif
5930 #ifdef MSWIN32
5931   GC_EXTERN GC_bool GC_no_win32_dlls;
5932   GC_EXTERN GC_bool GC_wnt;
5933 #endif
5934 #ifdef THREADS
5935 #if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
5936     GC_EXTERN CRITICAL_SECTION GC_write_cs;
5937 #ifdef GC_ASSERTIONS
5938       GC_EXTERN GC_bool GC_write_disabled;
5939 #endif
5940 #endif
5941 #if defined(GC_DISABLE_INCREMENTAL) || defined(HAVE_LOCKFREE_AO_OR)
5942 #define GC_acquire_dirty_lock() (void)0
5943 #define GC_release_dirty_lock() (void)0
5944 #else
5945 #define GC_acquire_dirty_lock() \
5946         do {  \
5947         } while (AO_test_and_set_acquire(&GC_fault_handler_lock) == AO_TS_SET)
5948 #define GC_release_dirty_lock() AO_CLEAR(&GC_fault_handler_lock)
5949     GC_EXTERN volatile AO_TS_t GC_fault_handler_lock;
5950 #endif
5951 #ifdef MSWINCE
5952     GC_EXTERN GC_bool GC_dont_query_stack_min;
5953 #endif
5954 #elif defined(IA64)
5955   GC_EXTERN ptr_t GC_save_regs_ret_val;
5956 #endif
5957 #ifdef THREAD_LOCAL_ALLOC
5958   GC_EXTERN GC_bool GC_world_stopped;
5959   GC_INNER void GC_mark_thread_local_free_lists(void);
5960 #endif
5961 #if defined(GLIBC_2_19_TSX_BUG) && defined(THREADS)
5962   GC_INNER int GC_parse_version(int *pminor, const char *pverstr);
5963 #endif
5964 #if defined(MPROTECT_VDB) && defined(GWW_VDB)
5965     GC_INNER GC_bool GC_gww_dirty_init(void);
5966 #endif
5967 #if defined(CHECKSUMS) || defined(PROC_VDB)
5968   GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk * h);
5969 #endif
5970 #ifdef CHECKSUMS
5971 #if defined(MPROTECT_VDB) && !defined(DARWIN)
5972     void GC_record_fault(struct hblk * h);
5973 #endif
5974   void GC_check_dirty(void);
5975 #endif
5976 GC_INNER void GC_default_print_heap_obj_proc(ptr_t p);
5977 GC_INNER void GC_setpagesize(void);
5978 GC_INNER void GC_initialize_offsets(void);
5979 GC_INNER void GC_bl_init(void);
5980 GC_INNER void GC_bl_init_no_interiors(void);
5981 GC_INNER void GC_start_debugging_inner(void);
5982 GC_INNER void *GC_store_debug_info_inner(void *p, word sz, const char *str,
5983                                          int linenum);
5984 #ifdef REDIRECT_MALLOC
5985 #ifdef GC_LINUX_THREADS
5986     GC_INNER GC_bool GC_text_mapping(char *nm, ptr_t *startp, ptr_t *endp);
5987 #endif
5988 #elif defined(USE_WINALLOC)
5989   GC_INNER void GC_add_current_malloc_heap(void);
5990 #endif
5991 #ifdef MAKE_BACK_GRAPH
5992   GC_INNER void GC_build_back_graph(void);
5993   GC_INNER void GC_traverse_back_graph(void);
5994 #endif
5995 #ifdef MSWIN32
5996   GC_INNER void GC_init_win32(void);
5997 #endif
5998 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
5999   GC_INNER void * GC_roots_present(ptr_t);
6000 #endif
6001 #ifdef GC_WIN32_THREADS
6002   GC_INNER void GC_get_next_stack(char *start, char * limit, char **lo,
6003                                   char **hi);
6004 #if defined(MPROTECT_VDB) && !defined(CYGWIN32)
6005     GC_INNER void GC_set_write_fault_handler(void);
6006 #endif
6007 #if defined(WRAP_MARK_SOME) && !defined(GC_PTHREADS)
6008     GC_INNER GC_bool GC_started_thread_while_stopped(void);
6009 #endif
6010 #endif
6011 #ifdef THREADS
6012   GC_INNER void GC_reset_finalizer_nested(void);
6013   GC_INNER unsigned char *GC_check_finalizer_nested(void);
6014   GC_INNER void GC_do_blocking_inner(ptr_t data, void * context);
6015   GC_INNER void GC_push_all_stacks(void);
6016 #ifdef USE_PROC_FOR_LIBRARIES
6017     GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo, ptr_t hi);
6018 #endif
6019 #ifdef IA64
6020     GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound);
6021 #endif
6022 #endif
6023 #ifdef DYNAMIC_LOADING
6024   GC_INNER GC_bool GC_register_main_static_data(void);
6025 #ifdef DARWIN
6026     GC_INNER void GC_init_dyld(void);
6027 #endif
6028 #endif
6029 #ifdef SEARCH_FOR_DATA_START
6030   GC_INNER void GC_init_linux_data_start(void);
6031   void * GC_find_limit(void *, int);
6032 #endif
6033 #if defined(NETBSD) && defined(__ELF__)
6034   GC_INNER void GC_init_netbsd_elf(void);
6035   void * GC_find_limit(void *, int);
6036 #endif
6037 #ifdef UNIX_LIKE
6038   GC_INNER void GC_set_and_save_fault_handler(void (*handler)(int));
6039 #endif
6040 #ifdef NEED_PROC_MAPS
6041 #if defined(DYNAMIC_LOADING) && defined(USE_PROC_FOR_LIBRARIES)
6042     GC_INNER const char *GC_parse_map_entry(const char *maps_ptr,
6043                                             ptr_t *start, ptr_t *end,
6044                                             const char **prot,
6045                                             unsigned *maj_dev,
6046                                             const char **mapping_name);
6047 #endif
6048 #if defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR)
6049     GC_INNER GC_bool GC_enclosing_mapping(ptr_t addr,
6050                                           ptr_t *startp, ptr_t *endp);
6051 #endif
6052   GC_INNER const char *GC_get_maps(void);
6053 #endif
6054 #ifdef GC_ASSERTIONS
6055 #define GC_ASSERT(expr) \
6056               do { \
6057                 if (!(expr)) { \
6058                   GC_err_printf("Assertion failure: %s:%d\n", \
6059                                 __FILE__, __LINE__); \
6060                   ABORT("assertion failure"); \
6061                 } \
6062               } while (0)
6063   GC_INNER word GC_compute_large_free_bytes(void);
6064   GC_INNER word GC_compute_root_size(void);
6065 #else
6066 #define GC_ASSERT(expr)
6067 #endif
6068 #if _MSC_VER >= 1700
6069 #define GC_STATIC_ASSERT(expr) \
6070                 static_assert(expr, "static assertion failed: " #expr)
6071 #elif defined(static_assert) && __STDC_VERSION__ >= 201112L
6072 #define GC_STATIC_ASSERT(expr) static_assert(expr, #expr)
6073 #elif defined(mips) && !defined(__GNUC__)
6074 #define GC_STATIC_ASSERT(expr) \
6075     do { if (0) { char j[(expr)? 1 : -1]; j[0]='\0'; j[0]=j[0]; } } while(0)
6076 #else
6077 #define GC_STATIC_ASSERT(expr) (void)sizeof(char[(expr)? 1 : -1])
6078 #endif
6079 #if GC_GNUC_PREREQ(4, 0)
6080 #define NONNULL_ARG_NOT_NULL(arg) (*(volatile void **)&(arg) != NULL)
6081 #else
6082 #define NONNULL_ARG_NOT_NULL(arg) (NULL != (arg))
6083 #endif
6084 #define COND_DUMP_CHECKS \
6085           do { \
6086             GC_ASSERT(GC_compute_large_free_bytes() == GC_large_free_bytes); \
6087             GC_ASSERT(GC_compute_root_size() == GC_root_size); \
6088           } while (0)
6089 #ifndef NO_DEBUGGING
6090   GC_EXTERN GC_bool GC_dump_regularly;
6091 #define COND_DUMP if (EXPECT(GC_dump_regularly, FALSE)) { \
6092                         GC_dump_named(NULL); \
6093                    } else COND_DUMP_CHECKS
6094 #else
6095 #define COND_DUMP COND_DUMP_CHECKS
6096 #endif
6097 #if defined(PARALLEL_MARK)
6098 #define GC_markers_m1 GC_parallel
6099   GC_EXTERN GC_bool GC_parallel_mark_disabled;
6100   GC_INNER void GC_wait_for_markers_init(void);
6101   GC_INNER void GC_acquire_mark_lock(void);
6102   GC_INNER void GC_release_mark_lock(void);
6103   GC_INNER void GC_notify_all_builder(void);
6104   GC_INNER void GC_wait_for_reclaim(void);
6105   GC_EXTERN signed_word GC_fl_builder_count;
6106   GC_INNER void GC_notify_all_marker(void);
6107   GC_INNER void GC_wait_marker(void);
6108   GC_EXTERN word GC_mark_no;
6109   GC_INNER void GC_help_marker(word my_mark_no);
6110   GC_INNER void GC_start_mark_threads_inner(void);
6111 #endif
6112 #if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS) && !defined(NACL) \
6113     && !defined(GC_DARWIN_THREADS) && !defined(SIG_SUSPEND)
6114 #if (defined(GC_LINUX_THREADS) || defined(GC_DGUX386_THREADS)) \
6115      && !defined(GC_USESIGRT_SIGNALS)
6116 #if defined(SPARC) && !defined(SIGPWR)
6117 #define SIG_SUSPEND SIGLOST
6118 #else
6119 #define SIG_SUSPEND SIGPWR
6120 #endif
6121 #elif defined(GC_OPENBSD_THREADS)
6122 #ifndef GC_OPENBSD_UTHREADS
6123 #define SIG_SUSPEND SIGXFSZ
6124 #endif
6125 #elif defined(_SIGRTMIN) && !defined(CPPCHECK)
6126 #define SIG_SUSPEND _SIGRTMIN + 6
6127 #else
6128 #define SIG_SUSPEND SIGRTMIN + 6
6129 #endif
6130 #endif
6131 #if defined(GC_PTHREADS) && !defined(GC_SEM_INIT_PSHARED)
6132 #define GC_SEM_INIT_PSHARED 0
6133 #endif
6134 #if (defined(UNIX_LIKE) || (defined(NEED_FIND_LIMIT) && defined(CYGWIN32))) \
6135     && !defined(GC_NO_SIGSETJMP)
6136 #if defined(SUNOS5SIGS) && !defined(FREEBSD) && !defined(LINUX)
6137     EXTERN_C_END
6138 #include <sys/siginfo.h>
6139     EXTERN_C_BEGIN
6140 #endif
6141 #define SETJMP(env) sigsetjmp(env, 1)
6142 #define LONGJMP(env, val) siglongjmp(env, val)
6143 #define JMP_BUF sigjmp_buf
6144 #else
6145 #ifdef ECOS
6146 #define SETJMP(env) hal_setjmp(env)
6147 #else
6148 #define SETJMP(env) setjmp(env)
6149 #endif
6150 #define LONGJMP(env, val) longjmp(env, val)
6151 #define JMP_BUF jmp_buf
6152 #endif
6153 #if defined(HEURISTIC2) || defined(SEARCH_FOR_DATA_START) \
6154     || ((defined(SVR4) || defined(AIX) || defined(DGUX) \
6155          || (defined(LINUX) && defined(SPARC))) && !defined(PCR))
6156 #define NEED_FIND_LIMIT
6157 #endif
6158 #if defined(DATASTART_USES_BSDGETDATASTART)
6159   EXTERN_C_END
6160 #include <machine/trap.h>
6161   EXTERN_C_BEGIN
6162 #if !defined(PCR)
6163 #define NEED_FIND_LIMIT
6164 #endif
6165   GC_INNER ptr_t GC_FreeBSDGetDataStart(size_t, ptr_t);
6166 #define DATASTART_IS_FUNC
6167 #endif
6168 #if (defined(NETBSD) || defined(OPENBSD)) && defined(__ELF__) \
6169     && !defined(NEED_FIND_LIMIT)
6170 #define NEED_FIND_LIMIT
6171 #endif
6172 #if defined(IA64) && !defined(NEED_FIND_LIMIT)
6173 #define NEED_FIND_LIMIT
6174 #endif
6175 #if defined(NEED_FIND_LIMIT) \
6176      || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))
6177   GC_EXTERN JMP_BUF GC_jmp_buf;
6178   GC_INNER void GC_setup_temporary_fault_handler(void);
6179   GC_INNER void GC_reset_fault_handler(void);
6180 #endif
6181 #if defined(CANCEL_SAFE)
6182 #if defined(GC_ASSERTIONS) \
6183      && (defined(USE_COMPILER_TLS) \
6184          || (defined(LINUX) && !defined(ARM32) && GC_GNUC_PREREQ(3, 3) \
6185              || defined(HPUX) ))
6186     extern __thread unsigned char GC_cancel_disable_count;
6187 #define NEED_CANCEL_DISABLE_COUNT
6188 #define INCR_CANCEL_DISABLE() ++GC_cancel_disable_count
6189 #define DECR_CANCEL_DISABLE() --GC_cancel_disable_count
6190 #define ASSERT_CANCEL_DISABLED() GC_ASSERT(GC_cancel_disable_count > 0)
6191 #else
6192 #define INCR_CANCEL_DISABLE()
6193 #define DECR_CANCEL_DISABLE()
6194 #define ASSERT_CANCEL_DISABLED() (void)0
6195 #endif
6196 #define DISABLE_CANCEL(state) \
6197         do { pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &state); \
6198           INCR_CANCEL_DISABLE(); } while (0)
6199 #define RESTORE_CANCEL(state) \
6200         do { ASSERT_CANCEL_DISABLED(); \
6201           pthread_setcancelstate(state, NULL); \
6202           DECR_CANCEL_DISABLE(); } while (0)
6203 #else
6204 #define DISABLE_CANCEL(state) (void)0
6205 #define RESTORE_CANCEL(state) (void)0
6206 #define ASSERT_CANCEL_DISABLED() (void)0
6207 #endif
6208 EXTERN_C_END
6209 #endif
6210 #ifdef KEEP_BACK_PTRS
6211 #ifndef GC_BACKPTR_H
6212 #define GC_BACKPTR_H
6213 #ifndef GC_H
6214 #endif
6215 #ifdef __cplusplus
6216   extern "C" {
6217 #endif
6218 typedef enum {
6219     GC_UNREFERENCED,
6220     GC_NO_SPACE,
6221     GC_REFD_FROM_ROOT,
6222     GC_REFD_FROM_REG,
6223     GC_REFD_FROM_HEAP,
6224     GC_FINALIZER_REFD
6225 } GC_ref_kind;
6226 GC_API GC_ref_kind GC_CALL GC_get_back_ptr_info(void * ,
6227                                 void ** , size_t * )
6228                                 GC_ATTR_NONNULL(1);
6229 GC_API void * GC_CALL GC_generate_random_heap_address(void);
6230 GC_API void * GC_CALL GC_generate_random_valid_address(void);
6231 GC_API void GC_CALL GC_generate_random_backtrace(void);
6232 GC_API void GC_CALL GC_print_backtrace(void *) GC_ATTR_NONNULL(1);
6233 #ifdef __cplusplus
6234   }
6235 #endif
6236 #endif
6237 #endif
6238 EXTERN_C_BEGIN
6239 #if CPP_WORDSZ == 32
6240 #define START_FLAG (word)0xfedcedcb
6241 #define END_FLAG (word)0xbcdecdef
6242 #else
6243 #define START_FLAG GC_WORD_C(0xFEDCEDCBfedcedcb)
6244 #define END_FLAG GC_WORD_C(0xBCDECDEFbcdecdef)
6245 #endif
6246 #if defined(KEEP_BACK_PTRS) || defined(PRINT_BLACK_LIST) \
6247     || defined(MAKE_BACK_GRAPH)
6248 #define NOT_MARKED (ptr_t)0
6249 #define MARKED_FOR_FINALIZATION ((ptr_t)(word)2)
6250 #define MARKED_FROM_REGISTER ((ptr_t)(word)4)
6251 #endif
6252 typedef struct {
6253 #if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
6254 #if ALIGNMENT == 1
6255 #define HIDE_BACK_PTR(p) GC_HIDE_POINTER(~1 & (word)(p))
6256 #else
6257 #define HIDE_BACK_PTR(p) GC_HIDE_POINTER(p)
6258 #endif
6259 #ifdef KEEP_BACK_PTRS
6260       GC_hidden_pointer oh_back_ptr;
6261 #endif
6262 #ifdef MAKE_BACK_GRAPH
6263       GC_hidden_pointer oh_bg_ptr;
6264 #endif
6265 #if defined(KEEP_BACK_PTRS) != defined(MAKE_BACK_GRAPH)
6266       word oh_dummy;
6267 #endif
6268 #endif
6269   const char * oh_string;
6270   signed_word oh_int;
6271 #ifdef NEED_CALLINFO
6272     struct callinfo oh_ci[NFRAMES];
6273 #endif
6274 #ifndef SHORT_DBG_HDRS
6275     word oh_sz;
6276     word oh_sf;
6277 #endif
6278 } oh;
6279 #ifdef SHORT_DBG_HDRS
6280 #define DEBUG_BYTES (sizeof (oh))
6281 #define UNCOLLECTABLE_DEBUG_BYTES DEBUG_BYTES
6282 #else
6283 #define UNCOLLECTABLE_DEBUG_BYTES (sizeof (oh) + sizeof (word))
6284 #define DEBUG_BYTES (UNCOLLECTABLE_DEBUG_BYTES - EXTRA_BYTES)
6285 #endif
6286 #define SIMPLE_ROUNDED_UP_WORDS(n) BYTES_TO_WORDS((n) + WORDS_TO_BYTES(1) - 1)
6287 #if defined(SAVE_CALL_CHAIN)
6288   struct callinfo;
6289   GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]);
6290   GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]);
6291 #define ADD_CALL_CHAIN(base, ra) GC_save_callers(((oh *)(base)) -> oh_ci)
6292 #define PRINT_CALL_CHAIN(base) GC_print_callers(((oh *)(base)) -> oh_ci)
6293 #elif defined(GC_ADD_CALLER)
6294   struct callinfo;
6295   GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]);
6296 #define ADD_CALL_CHAIN(base, ra) ((oh *)(base)) -> oh_ci[0].ci_pc = (ra)
6297 #define PRINT_CALL_CHAIN(base) GC_print_callers(((oh *)(base)) -> oh_ci)
6298 #else
6299 #define ADD_CALL_CHAIN(base, ra)
6300 #define PRINT_CALL_CHAIN(base)
6301 #endif
6302 #ifdef GC_ADD_CALLER
6303 #define OPT_RA ra,
6304 #else
6305 #define OPT_RA
6306 #endif
6307 #ifdef SHORT_DBG_HDRS
6308 #define GC_has_other_debug_info(p) 1
6309 #else
6310   GC_INNER int GC_has_other_debug_info(ptr_t p);
6311 #endif
6312 #if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
6313 #if defined(SHORT_DBG_HDRS) && !defined(CPPCHECK)
6314 #error Non-ptr stored in object results in GC_HAS_DEBUG_INFO malfunction
6315 #endif
6316 #if defined(PARALLEL_MARK) && defined(KEEP_BACK_PTRS)
6317 #define GC_HAS_DEBUG_INFO(p) \
6318                 ((AO_load((volatile AO_t *)(p)) & 1) != 0 \
6319                  && GC_has_other_debug_info(p) > 0)
6320 #else
6321 #define GC_HAS_DEBUG_INFO(p) \
6322                 ((*(word *)(p) & 1) && GC_has_other_debug_info(p) > 0)
6323 #endif
6324 #else
6325 #define GC_HAS_DEBUG_INFO(p) (GC_has_other_debug_info(p) > 0)
6326 #endif
6327 EXTERN_C_END
6328 #endif
6329 #ifdef MAKE_BACK_GRAPH
6330 #define MAX_IN  10
6331 #if (!defined(DBG_HDRS_ALL) || (ALIGNMENT != CPP_WORDSZ/8) \
6332      ) && !defined(CPPCHECK)
6333 #error The configuration does not support MAKE_BACK_GRAPH
6334 #endif
6335 #define FLAG_MANY 2
6336 typedef struct back_edges_struct {
6337   word n_edges;
6338   unsigned short flags;
6339 #define RETAIN 1
6340   unsigned short height_gc_no;
6341   signed_word height;
6342 #define HEIGHT_UNKNOWN      (-2)
6343 #define HEIGHT_IN_PROGRESS  (-1)
6344   ptr_t edges[MAX_IN];
6345   struct back_edges_struct *cont;
6346 } back_edges;
6347 #define MAX_BACK_EDGE_STRUCTS 100000
6348 static back_edges *back_edge_space = 0;
6349 STATIC int GC_n_back_edge_structs = 0;
6350 static back_edges *avail_back_edges = 0;
6351 static back_edges * new_back_edges(void)
6352 {
6353   if (0 == back_edge_space) {
6354     size_t bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(MAX_BACK_EDGE_STRUCTS
6355                                                    * sizeof(back_edges));
6356     GC_ASSERT(GC_page_size != 0);
6357     back_edge_space = (back_edges *)GET_MEM(bytes_to_get);
6358     if (NULL == back_edge_space)
6359       ABORT("Insufficient memory for back edges");
6360     GC_add_to_our_memory((ptr_t)back_edge_space, bytes_to_get);
6361   }
6362   if (0 != avail_back_edges) {
6363     back_edges * result = avail_back_edges;
6364     avail_back_edges = result -> cont;
6365     result -> cont = 0;
6366     return result;
6367   }
6368   if (GC_n_back_edge_structs >= MAX_BACK_EDGE_STRUCTS - 1) {
6369     ABORT("Needed too much space for back edges: adjust "
6370           "MAX_BACK_EDGE_STRUCTS");
6371   }
6372   return back_edge_space + (GC_n_back_edge_structs++);
6373 }
6374 static void deallocate_back_edges(back_edges *p)
6375 {
6376    back_edges *last = p;
6377    while (0 != last -> cont) last = last -> cont;
6378    last -> cont = avail_back_edges;
6379    avail_back_edges = p;
6380 }
6381 #define INITIAL_IN_PROGRESS 10000
6382 static ptr_t * in_progress_space = 0;
6383 static size_t in_progress_size = 0;
6384 static size_t n_in_progress = 0;
6385 static void push_in_progress(ptr_t p)
6386 {
6387   if (n_in_progress >= in_progress_size) {
6388     ptr_t * new_in_progress_space;
6389     GC_ASSERT(GC_page_size != 0);
6390     if (NULL == in_progress_space) {
6391       in_progress_size = ROUNDUP_PAGESIZE_IF_MMAP(INITIAL_IN_PROGRESS
6392                                                         * sizeof(ptr_t))
6393                                 / sizeof(ptr_t);
6394       new_in_progress_space =
6395                         (ptr_t *)GET_MEM(in_progress_size * sizeof(ptr_t));
6396     } else {
6397       in_progress_size *= 2;
6398       new_in_progress_space = (ptr_t *)
6399                                 GET_MEM(in_progress_size * sizeof(ptr_t));
6400       if (new_in_progress_space != NULL)
6401         BCOPY(in_progress_space, new_in_progress_space,
6402               n_in_progress * sizeof(ptr_t));
6403     }
6404     if (EXPECT(new_in_progress_space != NULL, TRUE))
6405       GC_add_to_our_memory((ptr_t)new_in_progress_space,
6406                            in_progress_size * sizeof(ptr_t));
6407 #ifndef GWW_VDB
6408       GC_scratch_recycle_no_gww(in_progress_space,
6409                                 n_in_progress * sizeof(ptr_t));
6410 #elif defined(LINT2)
6411       GC_noop1((word)in_progress_space);
6412 #endif
6413     in_progress_space = new_in_progress_space;
6414   }
6415   if (in_progress_space == 0)
6416       ABORT("MAKE_BACK_GRAPH: Out of in-progress space: "
6417             "Huge linear data structure?");
6418   in_progress_space[n_in_progress++] = p;
6419 }
6420 static GC_bool is_in_progress(ptr_t p)
6421 {
6422   size_t i;
6423   for (i = 0; i < n_in_progress; ++i) {
6424     if (in_progress_space[i] == p) return TRUE;
6425   }
6426   return FALSE;
6427 }
6428 GC_INLINE void pop_in_progress(ptr_t p GC_ATTR_UNUSED)
6429 {
6430   --n_in_progress;
6431   GC_ASSERT(in_progress_space[n_in_progress] == p);
6432 }
6433 #define GET_OH_BG_PTR(p) \
6434                 (ptr_t)GC_REVEAL_POINTER(((oh *)(p)) -> oh_bg_ptr)
6435 #define SET_OH_BG_PTR(p,q) (((oh *)(p)) -> oh_bg_ptr = GC_HIDE_POINTER(q))
6436 static void ensure_struct(ptr_t p)
6437 {
6438   ptr_t old_back_ptr = GET_OH_BG_PTR(p);
6439   if (!((word)old_back_ptr & FLAG_MANY)) {
6440     back_edges *be = new_back_edges();
6441     be -> flags = 0;
6442     if (0 == old_back_ptr) {
6443       be -> n_edges = 0;
6444     } else {
6445       be -> n_edges = 1;
6446       be -> edges[0] = old_back_ptr;
6447     }
6448     be -> height = HEIGHT_UNKNOWN;
6449     be -> height_gc_no = (unsigned short)(GC_gc_no - 1);
6450     GC_ASSERT((word)be >= (word)back_edge_space);
6451     SET_OH_BG_PTR(p, (word)be | FLAG_MANY);
6452   }
6453 }
6454 static void add_edge(ptr_t p, ptr_t q)
6455 {
6456     ptr_t pred = GET_OH_BG_PTR(q);
6457     back_edges * be, *be_cont;
6458     word i;
6459     GC_ASSERT(p == GC_base(p) && q == GC_base(q));
6460     if (!GC_HAS_DEBUG_INFO(q) || !GC_HAS_DEBUG_INFO(p)) {
6461       return;
6462     }
6463     if (NULL == pred) {
6464       static unsigned random_number = 13;
6465 #define GOT_LUCKY_NUMBER (((++random_number) & 0x7f) == 0)
6466         SET_OH_BG_PTR(q, p);
6467         if (GOT_LUCKY_NUMBER) ensure_struct(q);
6468         return;
6469     }
6470     {
6471       back_edges *e = (back_edges *)((word)pred & ~FLAG_MANY);
6472       word n_edges;
6473       word total;
6474       int local = 0;
6475       if (((word)pred & FLAG_MANY) != 0) {
6476         n_edges = e -> n_edges;
6477       } else if (((word)COVERT_DATAFLOW(pred) & 1) == 0) {
6478         n_edges = 1;
6479         local = -1;
6480       } else {
6481         n_edges = 0;
6482       }
6483       for (total = 0; total < n_edges; ++total) {
6484         if (local == MAX_IN) {
6485           e = e -> cont;
6486           local = 0;
6487         }
6488         if (local >= 0)
6489           pred = e -> edges[local++];
6490         if (pred == p)
6491           return;
6492       }
6493     }
6494     ensure_struct(q);
6495     be = (back_edges *)((word)GET_OH_BG_PTR(q) & ~FLAG_MANY);
6496     for (i = be -> n_edges, be_cont = be; i > MAX_IN; i -= MAX_IN)
6497         be_cont = be_cont -> cont;
6498     if (i == MAX_IN) {
6499         be_cont -> cont = new_back_edges();
6500         be_cont = be_cont -> cont;
6501         i = 0;
6502     }
6503     be_cont -> edges[i] = p;
6504     be -> n_edges++;
6505 #ifdef DEBUG_PRINT_BIG_N_EDGES
6506       if (GC_print_stats == VERBOSE && be -> n_edges == 100) {
6507         GC_err_printf("The following object has big in-degree:\n");
6508         GC_print_heap_obj(q);
6509       }
6510 #endif
6511 }
6512 typedef void (*per_object_func)(ptr_t p, size_t n_bytes, word gc_descr);
6513 static void per_object_helper(struct hblk *h, word fn)
6514 {
6515   hdr * hhdr = HDR(h);
6516   size_t sz = (size_t)hhdr->hb_sz;
6517   word descr = hhdr -> hb_descr;
6518   per_object_func f = (per_object_func)fn;
6519   size_t i = 0;
6520   do {
6521     f((ptr_t)(h -> hb_body + i), sz, descr);
6522     i += sz;
6523   } while (i + sz <= BYTES_TO_WORDS(HBLKSIZE));
6524 }
6525 GC_INLINE void GC_apply_to_each_object(per_object_func f)
6526 {
6527   GC_apply_to_all_blocks(per_object_helper, (word)f);
6528 }
6529 static void reset_back_edge(ptr_t p, size_t n_bytes GC_ATTR_UNUSED,
6530                             word gc_descr GC_ATTR_UNUSED)
6531 {
6532   if (GC_HAS_DEBUG_INFO(p)) {
6533     ptr_t old_back_ptr = GET_OH_BG_PTR(p);
6534     if ((word)old_back_ptr & FLAG_MANY) {
6535       back_edges *be = (back_edges *)((word)old_back_ptr & ~FLAG_MANY);
6536       if (!(be -> flags & RETAIN)) {
6537         deallocate_back_edges(be);
6538         SET_OH_BG_PTR(p, 0);
6539       } else {
6540         GC_ASSERT(GC_is_marked(p));
6541           be -> n_edges = 0;
6542           if (0 != be -> cont) {
6543             deallocate_back_edges(be -> cont);
6544             be -> cont = 0;
6545           }
6546         GC_ASSERT(GC_is_marked(p));
6547           be -> flags &= ~RETAIN;
6548       }
6549     } else  {
6550       SET_OH_BG_PTR(p, 0);
6551     }
6552   }
6553 }
6554 static void add_back_edges(ptr_t p, size_t n_bytes, word gc_descr)
6555 {
6556   word *currentp = (word *)(p + sizeof(oh));
6557     if((gc_descr & GC_DS_TAGS) != GC_DS_LENGTH) {
6558       gc_descr = n_bytes;
6559     }
6560   while ((word)currentp < (word)(p + gc_descr)) {
6561     word current = *currentp++;
6562     FIXUP_POINTER(current);
6563     if (current >= (word)GC_least_plausible_heap_addr &&
6564         current <= (word)GC_greatest_plausible_heap_addr) {
6565        ptr_t target = (ptr_t)GC_base((void *)current);
6566        if (0 != target) {
6567          add_edge(p, target);
6568        }
6569     }
6570   }
6571 }
6572 GC_INNER void GC_build_back_graph(void)
6573 {
6574   GC_ASSERT(I_HOLD_LOCK());
6575   GC_apply_to_each_object(add_back_edges);
6576 }
6577 static word backwards_height(ptr_t p)
6578 {
6579   word result;
6580   ptr_t pred = GET_OH_BG_PTR(p);
6581   back_edges *be;
6582   if (NULL == pred)
6583     return 1;
6584   if (((word)pred & FLAG_MANY) == 0) {
6585     if (is_in_progress(p)) return 0;
6586     push_in_progress(p);
6587     result = backwards_height(pred) + 1;
6588     pop_in_progress(p);
6589     return result;
6590   }
6591   be = (back_edges *)((word)pred & ~FLAG_MANY);
6592   if (be -> height >= 0 && be -> height_gc_no == (unsigned short)GC_gc_no)
6593       return be -> height;
6594     if (be -> height == HEIGHT_IN_PROGRESS) return 0;
6595   result = (be -> height > 0? be -> height : 1);
6596   be -> height = HEIGHT_IN_PROGRESS;
6597   {
6598       back_edges *e = be;
6599       word n_edges;
6600       word total;
6601       int local = 0;
6602       if (((word)pred & FLAG_MANY) != 0) {
6603         n_edges = e -> n_edges;
6604       } else if (((word)pred & 1) == 0) {
6605         n_edges = 1;
6606         local = -1;
6607       } else {
6608         n_edges = 0;
6609       }
6610       for (total = 0; total < n_edges; ++total) {
6611         word this_height;
6612         if (local == MAX_IN) {
6613           e = e -> cont;
6614           local = 0;
6615         }
6616         if (local >= 0)
6617           pred = e -> edges[local++];
6618         if (GC_is_marked(pred) && ((word)GET_OH_BG_PTR(p) & FLAG_MANY) == 0) {
6619           GC_COND_LOG_PRINTF("Found bogus pointer from %p to %p\n",
6620                              (void *)pred, (void *)p);
6621           this_height = 1;
6622         } else {
6623           this_height = backwards_height(pred);
6624         }
6625         if (this_height >= result)
6626           result = this_height + 1;
6627       }
6628   }
6629   be -> height = result;
6630   be -> height_gc_no = (unsigned short)GC_gc_no;
6631   return result;
6632 }
6633 STATIC word GC_max_height = 0;
6634 STATIC ptr_t GC_deepest_obj = NULL;
6635 static void update_max_height(ptr_t p, size_t n_bytes GC_ATTR_UNUSED,
6636                               word gc_descr GC_ATTR_UNUSED)
6637 {
6638   if (GC_is_marked(p) && GC_HAS_DEBUG_INFO(p)) {
6639     word p_height = 0;
6640     ptr_t p_deepest_obj = 0;
6641     ptr_t back_ptr;
6642     back_edges *be = 0;
6643     back_ptr = GET_OH_BG_PTR(p);
6644     if (0 != back_ptr && ((word)back_ptr & FLAG_MANY)) {
6645       be = (back_edges *)((word)back_ptr & ~FLAG_MANY);
6646       if (be -> height != HEIGHT_UNKNOWN) p_height = be -> height;
6647     }
6648     {
6649       ptr_t pred = GET_OH_BG_PTR(p);
6650       back_edges *e = (back_edges *)((word)pred & ~FLAG_MANY);
6651       word n_edges;
6652       word total;
6653       int local = 0;
6654       if (((word)pred & FLAG_MANY) != 0) {
6655         n_edges = e -> n_edges;
6656       } else if (pred != NULL && ((word)pred & 1) == 0) {
6657         n_edges = 1;
6658         local = -1;
6659       } else {
6660         n_edges = 0;
6661       }
6662       for (total = 0; total < n_edges; ++total) {
6663         if (local == MAX_IN) {
6664           e = e -> cont;
6665           local = 0;
6666         }
6667         if (local >= 0)
6668           pred = e -> edges[local++];
6669         if (!GC_is_marked(pred) && GC_HAS_DEBUG_INFO(pred)) {
6670           word this_height = backwards_height(pred);
6671           if (this_height > p_height) {
6672             p_height = this_height;
6673             p_deepest_obj = pred;
6674           }
6675         }
6676       }
6677     }
6678     if (p_height > 0) {
6679         if (be == 0) {
6680           ensure_struct(p);
6681           back_ptr = GET_OH_BG_PTR(p);
6682           be = (back_edges *)((word)back_ptr & ~FLAG_MANY);
6683         }
6684         be -> flags |= RETAIN;
6685         be -> height = p_height;
6686         be -> height_gc_no = (unsigned short)GC_gc_no;
6687     }
6688     if (p_height > GC_max_height) {
6689         GC_max_height = p_height;
6690         GC_deepest_obj = p_deepest_obj;
6691     }
6692   }
6693 }
6694 STATIC word GC_max_max_height = 0;
6695 GC_INNER void GC_traverse_back_graph(void)
6696 {
6697   GC_ASSERT(I_HOLD_LOCK());
6698   GC_max_height = 0;
6699   GC_apply_to_each_object(update_max_height);
6700   if (0 != GC_deepest_obj)
6701     GC_set_mark_bit(GC_deepest_obj);
6702 }
6703 void GC_print_back_graph_stats(void)
6704 {
6705   GC_ASSERT(I_HOLD_LOCK());
6706   GC_printf("Maximum backwards height of reachable objects"
6707             " at GC #%lu is %lu\n",
6708             (unsigned long)GC_gc_no, (unsigned long)GC_max_height);
6709   if (GC_max_height > GC_max_max_height) {
6710     ptr_t obj = GC_deepest_obj;
6711     GC_max_max_height = GC_max_height;
6712     UNLOCK();
6713     GC_err_printf(
6714             "The following unreachable object is last in a longest chain "
6715             "of unreachable objects:\n");
6716     GC_print_heap_obj(obj);
6717     LOCK();
6718   }
6719   GC_COND_LOG_PRINTF("Needed max total of %d back-edge structs\n",
6720                      GC_n_back_edge_structs);
6721   GC_apply_to_each_object(reset_back_edge);
6722   GC_deepest_obj = 0;
6723 }
6724 #endif
6725 STATIC word * GC_old_normal_bl = NULL;
6726 STATIC word * GC_incomplete_normal_bl = NULL;
6727 STATIC word * GC_old_stack_bl = NULL;
6728 STATIC word * GC_incomplete_stack_bl = NULL;
6729 STATIC word GC_total_stack_black_listed = 0;
6730 GC_INNER word GC_black_list_spacing = MINHINCR * HBLKSIZE;
6731 STATIC void GC_clear_bl(word *);
6732 GC_INNER void GC_default_print_heap_obj_proc(ptr_t p)
6733 {
6734     ptr_t base = (ptr_t)GC_base(p);
6735     int kind = HDR(base)->hb_obj_kind;
6736     GC_err_printf("object at %p of appr. %lu bytes (%s)\n",
6737                   (void *)base, (unsigned long)GC_size(base),
6738                   kind == PTRFREE ? "atomic" :
6739                     IS_UNCOLLECTABLE(kind) ? "uncollectable" : "composite");
6740 }
6741 GC_INNER void (*GC_print_heap_obj)(ptr_t p) = GC_default_print_heap_obj_proc;
6742 #ifdef PRINT_BLACK_LIST
6743   STATIC void GC_print_blacklisted_ptr(word p, ptr_t source,
6744                                        const char *kind_str)
6745   {
6746     ptr_t base = (ptr_t)GC_base(source);
6747     if (0 == base) {
6748         GC_err_printf("Black listing (%s) %p referenced from %p in %s\n",
6749                       kind_str, (void *)p, (void *)source,
6750                       NULL != source ? "root set" : "register");
6751     } else {
6752         GC_err_printf("Black listing (%s) %p referenced from %p in"
6753                       " object at %p of appr. %lu bytes\n",
6754                       kind_str, (void *)p, (void *)source,
6755                       (void *)base, (unsigned long)GC_size(base));
6756     }
6757   }
6758 #endif
6759 GC_INNER void GC_bl_init_no_interiors(void)
6760 {
6761   if (GC_incomplete_normal_bl == 0) {
6762     GC_old_normal_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table));
6763     GC_incomplete_normal_bl = (word *)GC_scratch_alloc(
6764                                                   sizeof(page_hash_table));
6765     if (GC_old_normal_bl == 0 || GC_incomplete_normal_bl == 0) {
6766       GC_err_printf("Insufficient memory for black list\n");
6767       EXIT();
6768     }
6769     GC_clear_bl(GC_old_normal_bl);
6770     GC_clear_bl(GC_incomplete_normal_bl);
6771   }
6772 }
6773 GC_INNER void GC_bl_init(void)
6774 {
6775     if (!GC_all_interior_pointers) {
6776       GC_bl_init_no_interiors();
6777     }
6778     GC_ASSERT(NULL == GC_old_stack_bl && NULL == GC_incomplete_stack_bl);
6779     GC_old_stack_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table));
6780     GC_incomplete_stack_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table));
6781     if (GC_old_stack_bl == 0 || GC_incomplete_stack_bl == 0) {
6782         GC_err_printf("Insufficient memory for black list\n");
6783         EXIT();
6784     }
6785     GC_clear_bl(GC_old_stack_bl);
6786     GC_clear_bl(GC_incomplete_stack_bl);
6787 }
6788 STATIC void GC_clear_bl(word *doomed)
6789 {
6790     BZERO(doomed, sizeof(page_hash_table));
6791 }
6792 STATIC void GC_copy_bl(word *old, word *dest)
6793 {
6794     BCOPY(old, dest, sizeof(page_hash_table));
6795 }
6796 static word total_stack_black_listed(void);
6797 GC_INNER void GC_promote_black_lists(void)
6798 {
6799     word * very_old_normal_bl = GC_old_normal_bl;
6800     word * very_old_stack_bl = GC_old_stack_bl;
6801     GC_old_normal_bl = GC_incomplete_normal_bl;
6802     GC_old_stack_bl = GC_incomplete_stack_bl;
6803     if (!GC_all_interior_pointers) {
6804       GC_clear_bl(very_old_normal_bl);
6805     }
6806     GC_clear_bl(very_old_stack_bl);
6807     GC_incomplete_normal_bl = very_old_normal_bl;
6808     GC_incomplete_stack_bl = very_old_stack_bl;
6809     GC_total_stack_black_listed = total_stack_black_listed();
6810     GC_VERBOSE_LOG_PRINTF(
6811                 "%lu bytes in heap blacklisted for interior pointers\n",
6812                 (unsigned long)GC_total_stack_black_listed);
6813     if (GC_total_stack_black_listed != 0) {
6814         GC_black_list_spacing =
6815                 HBLKSIZE*(GC_heapsize/GC_total_stack_black_listed);
6816     }
6817     if (GC_black_list_spacing < 3 * HBLKSIZE) {
6818         GC_black_list_spacing = 3 * HBLKSIZE;
6819     }
6820     if (GC_black_list_spacing > MAXHINCR * HBLKSIZE) {
6821         GC_black_list_spacing = MAXHINCR * HBLKSIZE;
6822     }
6823 }
6824 GC_INNER void GC_unpromote_black_lists(void)
6825 {
6826     if (!GC_all_interior_pointers) {
6827       GC_copy_bl(GC_old_normal_bl, GC_incomplete_normal_bl);
6828     }
6829     GC_copy_bl(GC_old_stack_bl, GC_incomplete_stack_bl);
6830 }
6831 #if defined(PARALLEL_MARK) && defined(THREAD_SANITIZER)
6832 #define backlist_set_pht_entry_from_index(db, index) \
6833                         set_pht_entry_from_index_concurrent(db, index)
6834 #else
6835 #define backlist_set_pht_entry_from_index(bl, index) \
6836                         set_pht_entry_from_index(bl, index)
6837 #endif
6838 #ifdef PRINT_BLACK_LIST
6839   GC_INNER void GC_add_to_black_list_normal(word p, ptr_t source)
6840 #else
6841   GC_INNER void GC_add_to_black_list_normal(word p)
6842 #endif
6843 {
6844   if (GC_modws_valid_offsets[p & (sizeof(word)-1)]) {
6845     word index = PHT_HASH((word)p);
6846     if (HDR(p) == 0 || get_pht_entry_from_index(GC_old_normal_bl, index)) {
6847 #ifdef PRINT_BLACK_LIST
6848         if (!get_pht_entry_from_index(GC_incomplete_normal_bl, index)) {
6849           GC_print_blacklisted_ptr(p, source, "normal");
6850         }
6851 #endif
6852       backlist_set_pht_entry_from_index(GC_incomplete_normal_bl, index);
6853     }
6854   }
6855 }
6856 #ifdef PRINT_BLACK_LIST
6857   GC_INNER void GC_add_to_black_list_stack(word p, ptr_t source)
6858 #else
6859   GC_INNER void GC_add_to_black_list_stack(word p)
6860 #endif
6861 {
6862   word index = PHT_HASH((word)p);
6863   if (HDR(p) == 0 || get_pht_entry_from_index(GC_old_stack_bl, index)) {
6864 #ifdef PRINT_BLACK_LIST
6865       if (!get_pht_entry_from_index(GC_incomplete_stack_bl, index)) {
6866         GC_print_blacklisted_ptr(p, source, "stack");
6867       }
6868 #endif
6869     backlist_set_pht_entry_from_index(GC_incomplete_stack_bl, index);
6870   }
6871 }
6872 struct hblk * GC_is_black_listed(struct hblk *h, word len)
6873 {
6874     word index = PHT_HASH((word)h);
6875     word i;
6876     word nblocks;
6877     if (!GC_all_interior_pointers
6878         && (get_pht_entry_from_index(GC_old_normal_bl, index)
6879             || get_pht_entry_from_index(GC_incomplete_normal_bl, index))) {
6880       return (h+1);
6881     }
6882     nblocks = divHBLKSZ(len);
6883     for (i = 0;;) {
6884         if (GC_old_stack_bl[divWORDSZ(index)] == 0
6885             && GC_incomplete_stack_bl[divWORDSZ(index)] == 0) {
6886           i += WORDSZ - modWORDSZ(index);
6887         } else {
6888           if (get_pht_entry_from_index(GC_old_stack_bl, index)
6889               || get_pht_entry_from_index(GC_incomplete_stack_bl, index)) {
6890             return(h+i+1);
6891           }
6892           i++;
6893         }
6894         if (i >= nblocks) break;
6895         index = PHT_HASH((word)(h+i));
6896     }
6897     return(0);
6898 }
6899 STATIC word GC_number_stack_black_listed(struct hblk *start,
6900                                          struct hblk *endp1)
6901 {
6902     struct hblk * h;
6903     word result = 0;
6904     for (h = start; (word)h < (word)endp1; h++) {
6905         word index = PHT_HASH((word)h);
6906         if (get_pht_entry_from_index(GC_old_stack_bl, index)) result++;
6907     }
6908     return(result);
6909 }
6910 static word total_stack_black_listed(void)
6911 {
6912     unsigned i;
6913     word total = 0;
6914     for (i = 0; i < GC_n_heap_sects; i++) {
6915         struct hblk * start = (struct hblk *) GC_heap_sects[i].hs_start;
6916         struct hblk * endp1 = start + divHBLKSZ(GC_heap_sects[i].hs_bytes);
6917         total += GC_number_stack_black_listed(start, endp1);
6918     }
6919     return(total * HBLKSIZE);
6920 }
6921 #ifdef CHECKSUMS
6922 #define NSUMS 10000
6923 #define OFFSET 0x10000
6924 typedef struct {
6925         GC_bool new_valid;
6926         word old_sum;
6927         word new_sum;
6928         struct hblk * block;
6929 } page_entry;
6930 page_entry GC_sums[NSUMS];
6931 STATIC word GC_faulted[NSUMS] = { 0 };
6932 STATIC size_t GC_n_faulted = 0;
6933 #if defined(MPROTECT_VDB) && !defined(DARWIN)
6934   void GC_record_fault(struct hblk * h)
6935   {
6936     word page = (word)h & ~(GC_page_size - 1);
6937     GC_ASSERT(GC_page_size != 0);
6938     if (GC_n_faulted >= NSUMS) ABORT("write fault log overflowed");
6939     GC_faulted[GC_n_faulted++] = page;
6940   }
6941 #endif
6942 STATIC GC_bool GC_was_faulted(struct hblk *h)
6943 {
6944     size_t i;
6945     word page = (word)h & ~(GC_page_size - 1);
6946     for (i = 0; i < GC_n_faulted; ++i) {
6947         if (GC_faulted[i] == page) return TRUE;
6948     }
6949     return FALSE;
6950 }
6951 STATIC word GC_checksum(struct hblk *h)
6952 {
6953     word *p = (word *)h;
6954     word *lim = (word *)(h+1);
6955     word result = 0;
6956     while ((word)p < (word)lim) {
6957         result += *p++;
6958     }
6959     return(result | 0x80000000 );
6960 }
6961 int GC_n_dirty_errors = 0;
6962 int GC_n_faulted_dirty_errors = 0;
6963 unsigned long GC_n_clean = 0;
6964 unsigned long GC_n_dirty = 0;
6965 STATIC void GC_update_check_page(struct hblk *h, int index)
6966 {
6967     page_entry *pe = GC_sums + index;
6968     hdr * hhdr = HDR(h);
6969     struct hblk *b;
6970     if (pe -> block != 0 && pe -> block != h + OFFSET) ABORT("goofed");
6971     pe -> old_sum = pe -> new_sum;
6972     pe -> new_sum = GC_checksum(h);
6973 #if !defined(MSWIN32) && !defined(MSWINCE)
6974         if (pe -> new_sum != 0x80000000 && !GC_page_was_ever_dirty(h)) {
6975             GC_err_printf("GC_page_was_ever_dirty(%p) is wrong\n", (void *)h);
6976         }
6977 #endif
6978     if (GC_page_was_dirty(h)) {
6979         GC_n_dirty++;
6980     } else {
6981         GC_n_clean++;
6982     }
6983     b = h;
6984     while (IS_FORWARDING_ADDR_OR_NIL(hhdr) && hhdr != 0) {
6985         b -= (word)hhdr;
6986         hhdr = HDR(b);
6987     }
6988     if (pe -> new_valid
6989         && hhdr != 0 && hhdr -> hb_descr != 0
6990         && pe -> old_sum != pe -> new_sum) {
6991         if (!GC_page_was_dirty(h) || !GC_page_was_ever_dirty(h)) {
6992             GC_bool was_faulted = GC_was_faulted(h);
6993             GC_n_dirty_errors++;
6994             if (was_faulted) GC_n_faulted_dirty_errors++;
6995         }
6996     }
6997     pe -> new_valid = TRUE;
6998     pe -> block = h + OFFSET;
6999 }
7000 word GC_bytes_in_used_blocks = 0;
7001 STATIC void GC_add_block(struct hblk *h, word dummy GC_ATTR_UNUSED)
7002 {
7003    hdr * hhdr = HDR(h);
7004    GC_bytes_in_used_blocks += (hhdr->hb_sz + HBLKSIZE-1) & ~(HBLKSIZE-1);
7005 }
7006 STATIC void GC_check_blocks(void)
7007 {
7008     word bytes_in_free_blocks = GC_large_free_bytes;
7009     GC_bytes_in_used_blocks = 0;
7010     GC_apply_to_all_blocks(GC_add_block, (word)0);
7011     GC_COND_LOG_PRINTF("GC_bytes_in_used_blocks= %lu,"
7012                        " bytes_in_free_blocks= %lu, heapsize= %lu\n",
7013                        (unsigned long)GC_bytes_in_used_blocks,
7014                        (unsigned long)bytes_in_free_blocks,
7015                        (unsigned long)GC_heapsize);
7016     if (GC_bytes_in_used_blocks + bytes_in_free_blocks != GC_heapsize) {
7017         GC_err_printf("LOST SOME BLOCKS!!\n");
7018     }
7019 }
7020 void GC_check_dirty(void)
7021 {
7022     int index;
7023     unsigned i;
7024     struct hblk *h;
7025     ptr_t start;
7026     GC_check_blocks();
7027     GC_n_dirty_errors = 0;
7028     GC_n_faulted_dirty_errors = 0;
7029     GC_n_clean = 0;
7030     GC_n_dirty = 0;
7031     index = 0;
7032     for (i = 0; i < GC_n_heap_sects; i++) {
7033         start = GC_heap_sects[i].hs_start;
7034         for (h = (struct hblk *)start;
7035              (word)h < (word)(start + GC_heap_sects[i].hs_bytes); h++) {
7036              GC_update_check_page(h, index);
7037              index++;
7038              if (index >= NSUMS) goto out;
7039         }
7040     }
7041 out:
7042     GC_COND_LOG_PRINTF("Checked %lu clean and %lu dirty pages\n",
7043                        GC_n_clean, GC_n_dirty);
7044     if (GC_n_dirty_errors > 0) {
7045         GC_err_printf("Found %d dirty bit errors (%d were faulted)\n",
7046                       GC_n_dirty_errors, GC_n_faulted_dirty_errors);
7047     }
7048     for (i = 0; i < GC_n_faulted; ++i) {
7049         GC_faulted[i] = 0;
7050     }
7051     GC_n_faulted = 0;
7052 }
7053 #endif
7054 #ifndef GC_PMARK_H
7055 #define GC_PMARK_H
7056 #if defined(HAVE_CONFIG_H) && !defined(GC_PRIVATE_H)
7057 #endif
7058 #ifndef GC_BUILD
7059 #define GC_BUILD
7060 #endif
7061 #if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__)) \
7062     && !defined(_GNU_SOURCE) && defined(GC_PTHREADS) \
7063     && !defined(GC_NO_PTHREAD_SIGMASK)
7064 #define _GNU_SOURCE 1
7065 #endif
7066 #if defined(KEEP_BACK_PTRS) || defined(PRINT_BLACK_LIST)
7067 #endif
7068 EXTERN_C_BEGIN
7069 #ifndef MARK_DESCR_OFFSET
7070 #define MARK_DESCR_OFFSET sizeof(word)
7071 #endif
7072 #define BITMAP_BITS (WORDSZ - GC_DS_TAG_BITS)
7073 #define PROC(descr) \
7074       (GC_mark_procs[((descr) >> GC_DS_TAG_BITS) & (GC_MAX_MARK_PROCS-1)])
7075 #define ENV(descr) \
7076       ((descr) >> (GC_DS_TAG_BITS + GC_LOG_MAX_MARK_PROCS))
7077 #define MAX_ENV \
7078       (((word)1 << (WORDSZ - GC_DS_TAG_BITS - GC_LOG_MAX_MARK_PROCS)) - 1)
7079 GC_EXTERN unsigned GC_n_mark_procs;
7080 #define GC_MARK_STACK_DISCARDS (INITIAL_MARK_STACK_SIZE/8)
7081 #ifdef PARALLEL_MARK
7082 #endif
7083 GC_INNER mse * GC_signal_mark_stack_overflow(mse *msp);
7084 GC_INLINE mse * GC_push_obj(ptr_t obj, hdr * hhdr,  mse * mark_stack_top,
7085                             mse * mark_stack_limit)
7086 {
7087   word descr = hhdr -> hb_descr;
7088   GC_ASSERT(!HBLK_IS_FREE(hhdr));
7089   if (descr != 0) {
7090     mark_stack_top++;
7091     if ((word)mark_stack_top >= (word)mark_stack_limit) {
7092       mark_stack_top = GC_signal_mark_stack_overflow(mark_stack_top);
7093     }
7094     mark_stack_top -> mse_start = obj;
7095     mark_stack_top -> mse_descr.w = descr;
7096   }
7097   return mark_stack_top;
7098 }
7099 #define PUSH_CONTENTS(current, mark_stack_top, mark_stack_limit, source) \
7100   do { \
7101     hdr * my_hhdr; \
7102     HC_GET_HDR(current, my_hhdr, source);  \
7103     mark_stack_top = GC_push_contents_hdr(current, mark_stack_top, \
7104                                           mark_stack_limit, \
7105                                           source, my_hhdr, TRUE); \
7106   } while (0)
7107 #ifdef USE_MARK_BYTES
7108 #if defined(PARALLEL_MARK) && defined(AO_HAVE_char_store) \
7109      && !defined(BASE_ATOMIC_OPS_EMULATED)
7110 #define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
7111       {  \
7112         volatile unsigned char * mark_byte_addr = \
7113                         (unsigned char *)(hhdr)->hb_marks + (bit_no); \
7114          \
7115         if (AO_char_load(mark_byte_addr) != 0) \
7116           break;  \
7117         AO_char_store(mark_byte_addr, 1); \
7118       }
7119 #else
7120 #define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
7121       {  \
7122         char * mark_byte_addr = (char *)(hhdr)->hb_marks + (bit_no); \
7123         if (*mark_byte_addr != 0) break;  \
7124         *mark_byte_addr = 1; \
7125       }
7126 #endif
7127 #else
7128 #ifdef PARALLEL_MARK
7129 #ifdef THREAD_SANITIZER
7130 #define OR_WORD_EXIT_IF_SET(addr, bits) \
7131         {  \
7132           if (!((word)AO_load((volatile AO_t *)(addr)) & (bits))) { \
7133                  \
7134             AO_or((volatile AO_t *)(addr), (AO_t)(bits)); \
7135           } else { \
7136             break;  \
7137           } \
7138         }
7139 #else
7140 #define OR_WORD_EXIT_IF_SET(addr, bits) \
7141         {  \
7142           if (!(*(addr) & (bits))) { \
7143             AO_or((volatile AO_t *)(addr), (AO_t)(bits)); \
7144           } else { \
7145             break;  \
7146           } \
7147         }
7148 #endif
7149 #else
7150 #define OR_WORD_EXIT_IF_SET(addr, bits) \
7151         {  \
7152            word old = *(addr); \
7153            word my_bits = (bits); \
7154            if ((old & my_bits) != 0) \
7155              break;  \
7156            *(addr) = old | my_bits; \
7157         }
7158 #endif
7159 #define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
7160     {  \
7161         word * mark_word_addr = (hhdr)->hb_marks + divWORDSZ(bit_no); \
7162         OR_WORD_EXIT_IF_SET(mark_word_addr, \
7163                 (word)1 << modWORDSZ(bit_no));  \
7164     }
7165 #endif
7166 #ifdef PARALLEL_MARK
7167 #define INCR_MARKS(hhdr) \
7168                 AO_store(&hhdr->hb_n_marks, AO_load(&hhdr->hb_n_marks) + 1)
7169 #else
7170 #define INCR_MARKS(hhdr) (void)(++hhdr->hb_n_marks)
7171 #endif
7172 #ifdef ENABLE_TRACE
7173 #define TRACE(source, cmd) \
7174         if (GC_trace_addr != 0 && (ptr_t)(source) == GC_trace_addr) cmd
7175 #define TRACE_TARGET(target, cmd) \
7176         if (GC_trace_addr != 0 && (target) == *(ptr_t *)GC_trace_addr) cmd
7177 #else
7178 #define TRACE(source, cmd)
7179 #define TRACE_TARGET(source, cmd)
7180 #endif
7181 #if defined(I386) && defined(__GNUC__) && !defined(NACL)
7182 #define LONG_MULT(hprod, lprod, x, y) \
7183     do { \
7184         __asm__ __volatile__("mull %2" : "=a"(lprod), "=d"(hprod) \
7185                              : "g"(y), "0"(x)); \
7186     } while (0)
7187 #else
7188 #if defined(__int64) && !defined(__GNUC__) && !defined(CPPCHECK)
7189 #define ULONG_MULT_T unsigned __int64
7190 #else
7191 #define ULONG_MULT_T unsigned long long
7192 #endif
7193 #define LONG_MULT(hprod, lprod, x, y) \
7194     do { \
7195         ULONG_MULT_T prod = (ULONG_MULT_T)(x) * (ULONG_MULT_T)(y); \
7196         GC_STATIC_ASSERT(sizeof(x) + sizeof(y) <= sizeof(prod)); \
7197         hprod = prod >> 32; \
7198         lprod = (unsigned32)prod; \
7199     } while (0)
7200 #endif
7201 GC_INLINE mse * GC_push_contents_hdr(ptr_t current, mse * mark_stack_top,
7202                                      mse * mark_stack_limit, ptr_t source,
7203                                      hdr * hhdr, GC_bool do_offset_check)
7204 {
7205   do {
7206     size_t displ = HBLKDISPL(current);
7207     ptr_t base = current;
7208 #ifdef MARK_BIT_PER_GRANULE
7209       size_t gran_displ = BYTES_TO_GRANULES(displ);
7210       size_t gran_offset = hhdr -> hb_map[gran_displ];
7211       size_t byte_offset = displ & (GRANULE_BYTES - 1);
7212       if (EXPECT((gran_offset | byte_offset) != 0, FALSE))
7213 #else
7214       unsigned32 gran_displ;
7215       unsigned32 inv_sz = hhdr -> hb_inv_sz;
7216 #endif
7217     {
7218 #ifdef MARK_BIT_PER_GRANULE
7219         if ((hhdr -> hb_flags & LARGE_BLOCK) != 0)
7220 #else
7221         if (EXPECT(inv_sz == LARGE_INV_SZ, FALSE))
7222 #endif
7223       {
7224         size_t obj_displ;
7225         base = (ptr_t)hhdr->hb_block;
7226         obj_displ = current - base;
7227         if (obj_displ != displ) {
7228           GC_ASSERT(obj_displ < hhdr -> hb_sz);
7229         } else if (do_offset_check && !GC_valid_offsets[obj_displ]) {
7230           GC_ADD_TO_BLACK_LIST_NORMAL(current, source);
7231           break;
7232         }
7233         GC_ASSERT(hhdr -> hb_sz > HBLKSIZE
7234                   || hhdr -> hb_block == HBLKPTR(current));
7235         GC_ASSERT((word)hhdr->hb_block <= (word)current);
7236         gran_displ = 0;
7237       } else {
7238 #ifdef MARK_BIT_PER_GRANULE
7239           size_t obj_displ = GRANULES_TO_BYTES(gran_offset) + byte_offset;
7240 #else
7241           unsigned32 low_prod;
7242           LONG_MULT(gran_displ, low_prod, (unsigned32)displ, inv_sz);
7243           if ((low_prod >> 16) != 0)
7244 #endif
7245         {
7246 #if defined(MARK_BIT_PER_OBJ) \
7247              && !defined(MARK_BIT_PER_GRANULE)
7248             size_t obj_displ;
7249             GC_STATIC_ASSERT(HBLKSIZE <= (1 << 15));
7250             obj_displ = (((low_prod >> 16) + 1) * (size_t)hhdr->hb_sz) >> 16;
7251 #endif
7252           if (do_offset_check && !GC_valid_offsets[obj_displ]) {
7253             GC_ADD_TO_BLACK_LIST_NORMAL(current, source);
7254             break;
7255           }
7256 #ifdef MARK_BIT_PER_GRANULE
7257             gran_displ -= gran_offset;
7258 #endif
7259           base -= obj_displ;
7260         }
7261       }
7262     }
7263 #ifdef MARK_BIT_PER_GRANULE
7264       GC_ASSERT(hhdr == GC_find_header(base));
7265       GC_ASSERT(gran_displ % BYTES_TO_GRANULES(hhdr -> hb_sz) == 0);
7266 #else
7267       GC_ASSERT(gran_displ <= HBLK_OBJS(hhdr -> hb_sz));
7268 #endif
7269     TRACE(source, GC_log_printf("GC #%lu: passed validity tests\n",
7270                                 (unsigned long)GC_gc_no));
7271     SET_MARK_BIT_EXIT_IF_SET(hhdr, gran_displ);
7272     TRACE(source, GC_log_printf("GC #%lu: previously unmarked\n",
7273                                 (unsigned long)GC_gc_no));
7274     TRACE_TARGET(base, GC_log_printf("GC #%lu: marking %p from %p instead\n",
7275                                      (unsigned long)GC_gc_no, (void *)base,
7276                                      (void *)source));
7277     INCR_MARKS(hhdr);
7278     GC_STORE_BACK_PTR(source, base);
7279     mark_stack_top = GC_push_obj(base, hhdr, mark_stack_top,
7280                                  mark_stack_limit);
7281   } while (0);
7282   return mark_stack_top;
7283 }
7284 #if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
7285 #define PUSH_ONE_CHECKED_STACK(p, source) \
7286         GC_mark_and_push_stack((ptr_t)(p), (ptr_t)(source))
7287 #else
7288 #define PUSH_ONE_CHECKED_STACK(p, source) \
7289         GC_mark_and_push_stack((ptr_t)(p))
7290 #endif
7291 #ifdef NEED_FIXUP_POINTER
7292 #define GC_PUSH_ONE_STACK(p, source) \
7293     do { \
7294       if ((word)(p) >= (word)GC_least_plausible_heap_addr \
7295           && (word)(p) < (word)GC_greatest_plausible_heap_addr) { \
7296          PUSH_ONE_CHECKED_STACK(p, source); \
7297       } \
7298       FIXUP_POINTER(p); \
7299       if ((word)(p) >= (word)GC_least_plausible_heap_addr \
7300           && (word)(p) < (word)GC_greatest_plausible_heap_addr) { \
7301          PUSH_ONE_CHECKED_STACK(p, source); \
7302       } \
7303     } while (0)
7304 #else
7305 #define GC_PUSH_ONE_STACK(p, source) \
7306     do { \
7307       if ((word)(p) >= (word)GC_least_plausible_heap_addr \
7308           && (word)(p) < (word)GC_greatest_plausible_heap_addr) { \
7309          PUSH_ONE_CHECKED_STACK(p, source); \
7310       } \
7311     } while (0)
7312 #endif
7313 #define GC_PUSH_ONE_HEAP(p,source,mark_stack_top) \
7314     do { \
7315       FIXUP_POINTER(p); \
7316       if ((word)(p) >= (word)GC_least_plausible_heap_addr \
7317           && (word)(p) < (word)GC_greatest_plausible_heap_addr) \
7318         mark_stack_top = GC_mark_and_push((void *)(p), mark_stack_top, \
7319                                 GC_mark_stack_limit, (void * *)(source)); \
7320     } while (0)
7321 GC_INNER mse * GC_mark_from(mse * top, mse * bottom, mse *limit);
7322 #define MARK_FROM_MARK_STACK() \
7323         GC_mark_stack_top = GC_mark_from(GC_mark_stack_top, \
7324                                          GC_mark_stack, \
7325                                          GC_mark_stack + GC_mark_stack_size);
7326 #define GC_mark_stack_empty() ((word)GC_mark_stack_top < (word)GC_mark_stack)
7327 #define GC_MARK_FO(real_ptr, mark_proc) \
7328   do { \
7329     (*(mark_proc))(real_ptr); \
7330     while (!GC_mark_stack_empty()) MARK_FROM_MARK_STACK(); \
7331     if (GC_mark_state != MS_NONE) { \
7332         GC_set_mark_bit(real_ptr); \
7333         while (!GC_mark_some((ptr_t)0)) {  } \
7334     } \
7335   } while (0)
7336 #define MS_NONE 0
7337 #define MS_PUSH_RESCUERS 1
7338 #define MS_PUSH_UNCOLLECTABLE 2
7339 #define MS_ROOTS_PUSHED 3
7340 #define MS_PARTIALLY_INVALID 4
7341 #define MS_INVALID 5
7342 EXTERN_C_END
7343 #endif
7344 #ifdef GC_GCJ_SUPPORT
7345 #ifndef GC_GCJ_H
7346 #define GC_GCJ_H
7347 #ifndef GC_H
7348 #endif
7349 #ifdef __cplusplus
7350   extern "C" {
7351 #endif
7352 GC_API void GC_CALL GC_init_gcj_malloc(int ,
7353                                 void *  );
7354 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
7355         GC_gcj_malloc(size_t ,
7356                       void * );
7357 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
7358         GC_debug_gcj_malloc(size_t ,
7359                             void * ,
7360                             GC_EXTRA_PARAMS);
7361 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
7362         GC_gcj_malloc_ignore_off_page(size_t ,
7363                                 void * );
7364 GC_API int GC_gcj_kind;
7365 GC_API int GC_gcj_debug_kind;
7366 #ifdef GC_DEBUG
7367 #define GC_GCJ_MALLOC(s,d) GC_debug_gcj_malloc(s,d,GC_EXTRAS)
7368 #define GC_GCJ_MALLOC_IGNORE_OFF_PAGE(s,d) GC_debug_gcj_malloc(s,d,GC_EXTRAS)
7369 #else
7370 #define GC_GCJ_MALLOC(s,d) GC_gcj_malloc(s,d)
7371 #define GC_GCJ_MALLOC_IGNORE_OFF_PAGE(s,d) GC_gcj_malloc_ignore_off_page(s,d)
7372 #endif
7373 #ifdef __cplusplus
7374   }
7375 #endif
7376 #endif
7377 int GC_gcj_kind = 0;
7378 int GC_gcj_debug_kind = 0;
7379 STATIC struct GC_ms_entry * GC_gcj_fake_mark_proc(word * addr GC_ATTR_UNUSED,
7380                         struct GC_ms_entry *mark_stack_ptr,
7381                         struct GC_ms_entry * mark_stack_limit GC_ATTR_UNUSED,
7382                         word env GC_ATTR_UNUSED)
7383 {
7384     ABORT_RET("No client gcj mark proc is specified");
7385     return mark_stack_ptr;
7386 }
7387 GC_API void GC_CALL GC_init_gcj_malloc(int mp_index,
7388                                        void * mp)
7389 {
7390 #ifndef GC_IGNORE_GCJ_INFO
7391       GC_bool ignore_gcj_info;
7392 #endif
7393     DCL_LOCK_STATE;
7394     if (mp == 0)
7395       mp = (void *)(word)GC_gcj_fake_mark_proc;
7396     GC_init();
7397     LOCK();
7398     if (GC_gcjobjfreelist != NULL) {
7399       UNLOCK();
7400       return;
7401     }
7402 #ifdef GC_IGNORE_GCJ_INFO
7403 #define ignore_gcj_info TRUE
7404 #else
7405       ignore_gcj_info = (0 != GETENV("GC_IGNORE_GCJ_INFO"));
7406 #endif
7407     if (ignore_gcj_info) {
7408       GC_COND_LOG_PRINTF("Gcj-style type information is disabled!\n");
7409     }
7410     GC_ASSERT(GC_mark_procs[mp_index] == (GC_mark_proc)0);
7411     GC_mark_procs[mp_index] = (GC_mark_proc)(word)mp;
7412     if ((unsigned)mp_index >= GC_n_mark_procs)
7413         ABORT("GC_init_gcj_malloc: bad index");
7414     GC_gcjobjfreelist = (ptr_t *)GC_new_free_list_inner();
7415     if (ignore_gcj_info) {
7416         GC_gcj_kind = GC_new_kind_inner((void **)GC_gcjobjfreelist,
7417                                          GC_DS_LENGTH,
7418                                         TRUE, TRUE);
7419         GC_gcj_debug_kind = GC_gcj_kind;
7420     } else {
7421         GC_gcj_kind = GC_new_kind_inner(
7422                         (void **)GC_gcjobjfreelist,
7423                         (((word)(-(signed_word)MARK_DESCR_OFFSET
7424                                  - GC_INDIR_PER_OBJ_BIAS))
7425                          | GC_DS_PER_OBJECT),
7426                         FALSE, TRUE);
7427         GC_gcj_debug_kind = GC_new_kind_inner(GC_new_free_list_inner(),
7428                                 GC_MAKE_PROC(mp_index,
7429                                              1 ),
7430                                 FALSE, TRUE);
7431     }
7432     UNLOCK();
7433 #undef ignore_gcj_info
7434 }
7435 #define GENERAL_MALLOC_INNER(lb,k) \
7436     GC_clear_stack(GC_generic_malloc_inner(lb, k))
7437 #define GENERAL_MALLOC_INNER_IOP(lb,k) \
7438     GC_clear_stack(GC_generic_malloc_inner_ignore_off_page(lb, k))
7439 static void maybe_finalize(void)
7440 {
7441    static word last_finalized_no = 0;
7442    DCL_LOCK_STATE;
7443    if (GC_gc_no == last_finalized_no ||
7444        !EXPECT(GC_is_initialized, TRUE)) return;
7445    UNLOCK();
7446    GC_INVOKE_FINALIZERS();
7447    LOCK();
7448    last_finalized_no = GC_gc_no;
7449 }
7450 #ifdef THREAD_LOCAL_ALLOC
7451   GC_INNER void * GC_core_gcj_malloc(size_t lb,
7452                                      void * ptr_to_struct_containing_descr)
7453 #else
7454   GC_API GC_ATTR_MALLOC void * GC_CALL GC_gcj_malloc(size_t lb,
7455                                       void * ptr_to_struct_containing_descr)
7456 #endif
7457 {
7458     ptr_t op;
7459     DCL_LOCK_STATE;
7460     GC_DBG_COLLECT_AT_MALLOC(lb);
7461     if(SMALL_OBJ(lb)) {
7462         word lg;
7463         LOCK();
7464         lg = GC_size_map[lb];
7465         op = GC_gcjobjfreelist[lg];
7466         if(EXPECT(0 == op, FALSE)) {
7467             maybe_finalize();
7468             op = (ptr_t)GENERAL_MALLOC_INNER((word)lb, GC_gcj_kind);
7469             if (0 == op) {
7470                 GC_oom_func oom_fn = GC_oom_fn;
7471                 UNLOCK();
7472                 return((*oom_fn)(lb));
7473             }
7474         } else {
7475             GC_gcjobjfreelist[lg] = (ptr_t)obj_link(op);
7476             GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
7477         }
7478         GC_ASSERT(((void **)op)[1] == 0);
7479     } else {
7480         LOCK();
7481         maybe_finalize();
7482         op = (ptr_t)GENERAL_MALLOC_INNER((word)lb, GC_gcj_kind);
7483         if (0 == op) {
7484             GC_oom_func oom_fn = GC_oom_fn;
7485             UNLOCK();
7486             return((*oom_fn)(lb));
7487         }
7488     }
7489     *(void **)op = ptr_to_struct_containing_descr;
7490     UNLOCK();
7491     GC_dirty(op);
7492     REACHABLE_AFTER_DIRTY(ptr_to_struct_containing_descr);
7493     return (void *)op;
7494 }
7495 GC_API GC_ATTR_MALLOC void * GC_CALL GC_debug_gcj_malloc(size_t lb,
7496                 void * ptr_to_struct_containing_descr, GC_EXTRA_PARAMS)
7497 {
7498     void * result;
7499     DCL_LOCK_STATE;
7500     LOCK();
7501     maybe_finalize();
7502     result = GC_generic_malloc_inner(SIZET_SAT_ADD(lb, DEBUG_BYTES),
7503                                      GC_gcj_debug_kind);
7504     if (result == 0) {
7505         GC_oom_func oom_fn = GC_oom_fn;
7506         UNLOCK();
7507         GC_err_printf("GC_debug_gcj_malloc(%lu, %p) returning NULL (%s:%d)\n",
7508                 (unsigned long)lb, ptr_to_struct_containing_descr, s, i);
7509         return((*oom_fn)(lb));
7510     }
7511     *((void **)((ptr_t)result + sizeof(oh))) = ptr_to_struct_containing_descr;
7512     if (!GC_debugging_started) {
7513         GC_start_debugging_inner();
7514     }
7515     ADD_CALL_CHAIN(result, ra);
7516     result = GC_store_debug_info_inner(result, (word)lb, s, i);
7517     UNLOCK();
7518     GC_dirty(result);
7519     REACHABLE_AFTER_DIRTY(ptr_to_struct_containing_descr);
7520     return result;
7521 }
7522 GC_API GC_ATTR_MALLOC void * GC_CALL GC_gcj_malloc_ignore_off_page(size_t lb,
7523                                      void * ptr_to_struct_containing_descr)
7524 {
7525     ptr_t op;
7526     DCL_LOCK_STATE;
7527     GC_DBG_COLLECT_AT_MALLOC(lb);
7528     if(SMALL_OBJ(lb)) {
7529         word lg;
7530         LOCK();
7531         lg = GC_size_map[lb];
7532         op = GC_gcjobjfreelist[lg];
7533         if (EXPECT(0 == op, FALSE)) {
7534             maybe_finalize();
7535             op = (ptr_t)GENERAL_MALLOC_INNER_IOP(lb, GC_gcj_kind);
7536             if (0 == op) {
7537                 GC_oom_func oom_fn = GC_oom_fn;
7538                 UNLOCK();
7539                 return((*oom_fn)(lb));
7540             }
7541         } else {
7542             GC_gcjobjfreelist[lg] = (ptr_t)obj_link(op);
7543             GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
7544         }
7545     } else {
7546         LOCK();
7547         maybe_finalize();
7548         op = (ptr_t)GENERAL_MALLOC_INNER_IOP(lb, GC_gcj_kind);
7549         if (0 == op) {
7550             GC_oom_func oom_fn = GC_oom_fn;
7551             UNLOCK();
7552             return((*oom_fn)(lb));
7553         }
7554     }
7555     *(void **)op = ptr_to_struct_containing_descr;
7556     UNLOCK();
7557     GC_dirty(op);
7558     REACHABLE_AFTER_DIRTY(ptr_to_struct_containing_descr);
7559     return (void *)op;
7560 }
7561 #endif
7562 GC_INNER hdr * GC_find_header(ptr_t h)
7563 {
7564 #ifdef HASH_TL
7565         hdr * result;
7566         GET_HDR(h, result);
7567         return(result);
7568 #else
7569         return(HDR_INNER(h));
7570 #endif
7571 }
7572 GC_INNER hdr *
7573 #ifdef PRINT_BLACK_LIST
7574   GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce, ptr_t source)
7575 #else
7576   GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce)
7577 #endif
7578 {
7579   hdr *hhdr;
7580   HC_MISS();
7581   GET_HDR(p, hhdr);
7582   if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
7583     if (GC_all_interior_pointers) {
7584       if (hhdr != 0) {
7585         ptr_t current = p;
7586         current = (ptr_t)HBLKPTR(current);
7587         do {
7588             current = current - HBLKSIZE*(word)hhdr;
7589             hhdr = HDR(current);
7590         } while(IS_FORWARDING_ADDR_OR_NIL(hhdr));
7591         if (hhdr -> hb_flags & IGNORE_OFF_PAGE)
7592             return 0;
7593         if (HBLK_IS_FREE(hhdr)
7594             || p - current >= (ptrdiff_t)(hhdr->hb_sz)) {
7595             GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
7596             return 0;
7597         }
7598       } else {
7599         GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
7600       }
7601       GC_ASSERT(hhdr == 0 || !HBLK_IS_FREE(hhdr));
7602       return hhdr;
7603     } else {
7604       if (hhdr == 0) {
7605         GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
7606       }
7607       return 0;
7608     }
7609   } else {
7610     if (HBLK_IS_FREE(hhdr)) {
7611       GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
7612       return 0;
7613     } else {
7614       hce -> block_addr = (word)(p) >> LOG_HBLKSIZE;
7615       hce -> hce_hdr = hhdr;
7616       return hhdr;
7617     }
7618   }
7619 }
7620 GC_INNER ptr_t GC_scratch_alloc(size_t bytes)
7621 {
7622     ptr_t result = GC_scratch_free_ptr;
7623     size_t bytes_to_get;
7624     bytes = ROUNDUP_GRANULE_SIZE(bytes);
7625     for (;;) {
7626         GC_ASSERT((word)GC_scratch_end_ptr >= (word)result);
7627         if (bytes <= (word)GC_scratch_end_ptr - (word)result) {
7628             GC_scratch_free_ptr = result + bytes;
7629             return result;
7630         }
7631         GC_ASSERT(GC_page_size != 0);
7632         if (bytes >= MINHINCR * HBLKSIZE) {
7633             bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(bytes);
7634             result = (ptr_t)GET_MEM(bytes_to_get);
7635             if (result != NULL) {
7636               GC_add_to_our_memory(result, bytes_to_get);
7637 #ifdef USE_SCRATCH_LAST_END_PTR
7638                 GC_scratch_last_end_ptr = result + bytes;
7639 #endif
7640             }
7641             return result;
7642         }
7643         bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(MINHINCR * HBLKSIZE);
7644         result = (ptr_t)GET_MEM(bytes_to_get);
7645         if (EXPECT(NULL == result, FALSE)) {
7646             WARN("Out of memory - trying to allocate requested amount"
7647                  " (%" WARN_PRIdPTR " bytes)...\n", (word)bytes);
7648             bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(bytes);
7649             result = (ptr_t)GET_MEM(bytes_to_get);
7650             if (result != NULL) {
7651               GC_add_to_our_memory(result, bytes_to_get);
7652 #ifdef USE_SCRATCH_LAST_END_PTR
7653                 GC_scratch_last_end_ptr = result + bytes;
7654 #endif
7655             }
7656             return result;
7657         }
7658         GC_add_to_our_memory(result, bytes_to_get);
7659         GC_scratch_free_ptr = result;
7660         GC_scratch_end_ptr = GC_scratch_free_ptr + bytes_to_get;
7661 #ifdef USE_SCRATCH_LAST_END_PTR
7662           GC_scratch_last_end_ptr = GC_scratch_end_ptr;
7663 #endif
7664     }
7665 }
7666 static hdr * alloc_hdr(void)
7667 {
7668     hdr * result;
7669     if (NULL == GC_hdr_free_list) {
7670         result = (hdr *)GC_scratch_alloc(sizeof(hdr));
7671     } else {
7672         result = GC_hdr_free_list;
7673         GC_hdr_free_list = (hdr *) result -> hb_next;
7674     }
7675     return(result);
7676 }
7677 GC_INLINE void free_hdr(hdr * hhdr)
7678 {
7679     hhdr -> hb_next = (struct hblk *) GC_hdr_free_list;
7680     GC_hdr_free_list = hhdr;
7681 }
7682 #ifdef COUNT_HDR_CACHE_HITS
7683   word GC_hdr_cache_hits = 0;
7684   word GC_hdr_cache_misses = 0;
7685 #endif
7686 GC_INNER void GC_init_headers(void)
7687 {
7688     unsigned i;
7689     GC_ASSERT(NULL == GC_all_nils);
7690     GC_all_nils = (bottom_index *)GC_scratch_alloc(sizeof(bottom_index));
7691     if (GC_all_nils == NULL) {
7692       GC_err_printf("Insufficient memory for GC_all_nils\n");
7693       EXIT();
7694     }
7695     BZERO(GC_all_nils, sizeof(bottom_index));
7696     for (i = 0; i < TOP_SZ; i++) {
7697         GC_top_index[i] = GC_all_nils;
7698     }
7699 }
7700 static GC_bool get_index(word addr)
7701 {
7702     word hi = (word)(addr) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE);
7703     bottom_index * r;
7704     bottom_index * p;
7705     bottom_index ** prev;
7706     bottom_index *pi;
7707     word i;
7708     GC_ASSERT(I_HOLD_LOCK());
7709 #ifdef HASH_TL
7710       i = TL_HASH(hi);
7711       pi = p = GC_top_index[i];
7712       while(p != GC_all_nils) {
7713           if (p -> key == hi) return(TRUE);
7714           p = p -> hash_link;
7715       }
7716 #else
7717       if (GC_top_index[hi] != GC_all_nils)
7718         return TRUE;
7719       i = hi;
7720 #endif
7721     r = (bottom_index *)GC_scratch_alloc(sizeof(bottom_index));
7722     if (EXPECT(NULL == r, FALSE))
7723       return FALSE;
7724     BZERO(r, sizeof(bottom_index));
7725     r -> key = hi;
7726 #ifdef HASH_TL
7727       r -> hash_link = pi;
7728 #endif
7729       prev = &GC_all_bottom_indices;
7730       pi = 0;
7731       while ((p = *prev) != 0 && p -> key < hi) {
7732         pi = p;
7733         prev = &(p -> asc_link);
7734       }
7735       r -> desc_link = pi;
7736       if (0 == p) {
7737         GC_all_bottom_indices_end = r;
7738       } else {
7739         p -> desc_link = r;
7740       }
7741       r -> asc_link = p;
7742       *prev = r;
7743       GC_top_index[i] = r;
7744     return(TRUE);
7745 }
7746 GC_INNER struct hblkhdr * GC_install_header(struct hblk *h)
7747 {
7748     hdr * result;
7749     if (!get_index((word) h)) return(0);
7750     result = alloc_hdr();
7751     if (result) {
7752       SET_HDR(h, result);
7753 #ifdef USE_MUNMAP
7754         result -> hb_last_reclaimed = (unsigned short)GC_gc_no;
7755 #endif
7756     }
7757     return(result);
7758 }
7759 GC_INNER GC_bool GC_install_counts(struct hblk *h, size_t sz)
7760 {
7761     struct hblk * hbp;
7762     for (hbp = h; (word)hbp < (word)h + sz; hbp += BOTTOM_SZ) {
7763         if (!get_index((word)hbp))
7764             return FALSE;
7765         if ((word)hbp > GC_WORD_MAX - (word)BOTTOM_SZ * HBLKSIZE)
7766             break;
7767     }
7768     if (!get_index((word)h + sz - 1))
7769         return FALSE;
7770     for (hbp = h + 1; (word)hbp < (word)h + sz; hbp += 1) {
7771         word i = HBLK_PTR_DIFF(hbp, h);
7772         SET_HDR(hbp, (hdr *)(i > MAX_JUMP? MAX_JUMP : i));
7773     }
7774     return TRUE;
7775 }
7776 GC_INNER void GC_remove_header(struct hblk *h)
7777 {
7778     hdr **ha;
7779     GET_HDR_ADDR(h, ha);
7780     free_hdr(*ha);
7781     *ha = 0;
7782 }
7783 GC_INNER void GC_remove_counts(struct hblk *h, size_t sz)
7784 {
7785     struct hblk * hbp;
7786     for (hbp = h+1; (word)hbp < (word)h + sz; hbp += 1) {
7787         SET_HDR(hbp, 0);
7788     }
7789 }
7790 void GC_apply_to_all_blocks(void (*fn)(struct hblk *h, word client_data),
7791                             word client_data)
7792 {
7793     signed_word j;
7794     bottom_index * index_p;
7795     for (index_p = GC_all_bottom_indices; index_p != 0;
7796          index_p = index_p -> asc_link) {
7797         for (j = BOTTOM_SZ-1; j >= 0;) {
7798             if (!IS_FORWARDING_ADDR_OR_NIL(index_p->index[j])) {
7799                 if (!HBLK_IS_FREE(index_p->index[j])) {
7800                     (*fn)(((struct hblk *)
7801                               (((index_p->key << LOG_BOTTOM_SZ) + (word)j)
7802                                << LOG_HBLKSIZE)),
7803                           client_data);
7804                 }
7805                 j--;
7806              } else if (index_p->index[j] == 0) {
7807                 j--;
7808              } else {
7809                 j -= (signed_word)(index_p->index[j]);
7810              }
7811          }
7812      }
7813 }
7814 GC_INNER struct hblk * GC_next_block(struct hblk *h, GC_bool allow_free)
7815 {
7816     REGISTER bottom_index * bi;
7817     REGISTER word j = ((word)h >> LOG_HBLKSIZE) & (BOTTOM_SZ-1);
7818     GC_ASSERT(I_HOLD_LOCK());
7819     GET_BI(h, bi);
7820     if (bi == GC_all_nils) {
7821         REGISTER word hi = (word)h >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE);
7822         bi = GC_all_bottom_indices;
7823         while (bi != 0 && bi -> key < hi) bi = bi -> asc_link;
7824         j = 0;
7825     }
7826     while (bi != 0) {
7827         while (j < BOTTOM_SZ) {
7828             hdr * hhdr = bi -> index[j];
7829             if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
7830                 j++;
7831             } else {
7832                 if (allow_free || !HBLK_IS_FREE(hhdr)) {
7833                     return ((struct hblk *)
7834                               (((bi -> key << LOG_BOTTOM_SZ) + j)
7835                                << LOG_HBLKSIZE));
7836                 } else {
7837                     j += divHBLKSZ(hhdr -> hb_sz);
7838                 }
7839             }
7840         }
7841         j = 0;
7842         bi = bi -> asc_link;
7843     }
7844     return(0);
7845 }
7846 GC_INNER struct hblk * GC_prev_block(struct hblk *h)
7847 {
7848     bottom_index * bi;
7849     signed_word j = ((word)h >> LOG_HBLKSIZE) & (BOTTOM_SZ-1);
7850     GC_ASSERT(I_HOLD_LOCK());
7851     GET_BI(h, bi);
7852     if (bi == GC_all_nils) {
7853         word hi = (word)h >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE);
7854         bi = GC_all_bottom_indices_end;
7855         while (bi != 0 && bi -> key > hi) bi = bi -> desc_link;
7856         j = BOTTOM_SZ - 1;
7857     }
7858     while(bi != 0) {
7859         while (j >= 0) {
7860             hdr * hhdr = bi -> index[j];
7861             if (0 == hhdr) {
7862                 --j;
7863             } else if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
7864                 j -= (signed_word)hhdr;
7865             } else {
7866                 return((struct hblk *)
7867                           (((bi -> key << LOG_BOTTOM_SZ) + j)
7868                                << LOG_HBLKSIZE));
7869             }
7870         }
7871         j = BOTTOM_SZ - 1;
7872         bi = bi -> desc_link;
7873     }
7874     return(0);
7875 }
7876 #include <stdio.h>
7877 #ifndef SMALL_CONFIG
7878   STATIC ptr_t GC_build_fl_clear2(struct hblk *h, ptr_t ofl)
7879   {
7880     word * p = (word *)(h -> hb_body);
7881     word * lim = (word *)(h + 1);
7882     p[0] = (word)ofl;
7883     p[1] = 0;
7884     p[2] = (word)p;
7885     p[3] = 0;
7886     p += 4;
7887     for (; (word)p < (word)lim; p += 4) {
7888         p[0] = (word)(p-2);
7889         p[1] = 0;
7890         p[2] = (word)p;
7891         p[3] = 0;
7892     }
7893     return((ptr_t)(p-2));
7894   }
7895   STATIC ptr_t GC_build_fl_clear4(struct hblk *h, ptr_t ofl)
7896   {
7897     word * p = (word *)(h -> hb_body);
7898     word * lim = (word *)(h + 1);
7899     p[0] = (word)ofl;
7900     p[1] = 0;
7901     p[2] = 0;
7902     p[3] = 0;
7903     p += 4;
7904     for (; (word)p < (word)lim; p += 4) {
7905         GC_PREFETCH_FOR_WRITE((ptr_t)(p + 64));
7906         p[0] = (word)(p-4);
7907         p[1] = 0;
7908         CLEAR_DOUBLE(p+2);
7909     }
7910     return((ptr_t)(p-4));
7911   }
7912   STATIC ptr_t GC_build_fl2(struct hblk *h, ptr_t ofl)
7913   {
7914     word * p = (word *)(h -> hb_body);
7915     word * lim = (word *)(h + 1);
7916     p[0] = (word)ofl;
7917     p[2] = (word)p;
7918     p += 4;
7919     for (; (word)p < (word)lim; p += 4) {
7920         p[0] = (word)(p-2);
7921         p[2] = (word)p;
7922     }
7923     return((ptr_t)(p-2));
7924   }
7925   STATIC ptr_t GC_build_fl4(struct hblk *h, ptr_t ofl)
7926   {
7927     word * p = (word *)(h -> hb_body);
7928     word * lim = (word *)(h + 1);
7929     p[0] = (word)ofl;
7930     p[4] = (word)p;
7931     p += 8;
7932     for (; (word)p < (word)lim; p += 8) {
7933         GC_PREFETCH_FOR_WRITE((ptr_t)(p + 64));
7934         p[0] = (word)(p-4);
7935         p[4] = (word)p;
7936     }
7937     return((ptr_t)(p-4));
7938   }
7939 #endif
7940 GC_INNER ptr_t GC_build_fl(struct hblk *h, size_t sz, GC_bool clear,
7941                            ptr_t list)
7942 {
7943   word *p, *prev;
7944   word *last_object;
7945     GC_PREFETCH_FOR_WRITE((ptr_t)h);
7946     GC_PREFETCH_FOR_WRITE((ptr_t)h + 128);
7947     GC_PREFETCH_FOR_WRITE((ptr_t)h + 256);
7948     GC_PREFETCH_FOR_WRITE((ptr_t)h + 378);
7949 #ifndef SMALL_CONFIG
7950     switch (sz) {
7951         case 2: if (clear) {
7952                     return GC_build_fl_clear2(h, list);
7953                 } else {
7954                     return GC_build_fl2(h, list);
7955                 }
7956         case 4: if (clear) {
7957                     return GC_build_fl_clear4(h, list);
7958                 } else {
7959                     return GC_build_fl4(h, list);
7960                 }
7961         default:
7962                 break;
7963     }
7964 #endif
7965     if (clear) BZERO(h, HBLKSIZE);
7966     p = (word *)(h -> hb_body) + sz;
7967     prev = (word *)(h -> hb_body);
7968     last_object = (word *)((char *)h + HBLKSIZE);
7969     last_object -= sz;
7970     while ((word)p <= (word)last_object) {
7971         obj_link(p) = (ptr_t)prev;
7972         prev = p;
7973         p += sz;
7974     }
7975     p -= sz;
7976     *(ptr_t *)h = list;
7977     return ((ptr_t)p);
7978 }
7979 GC_INNER void GC_new_hblk(size_t gran, int kind)
7980 {
7981   struct hblk *h;
7982   GC_bool clear = GC_obj_kinds[kind].ok_init;
7983   GC_STATIC_ASSERT((sizeof (struct hblk)) == HBLKSIZE);
7984   GC_ASSERT(I_HOLD_LOCK());
7985   if (GC_debugging_started) clear = TRUE;
7986     h = GC_allochblk(GRANULES_TO_BYTES(gran), kind, 0);
7987     if (h == 0) return;
7988       if (IS_UNCOLLECTABLE(kind)) GC_set_hdr_marks(HDR(h));
7989       GC_obj_kinds[kind].ok_freelist[gran] =
7990         GC_build_fl(h, GRANULES_TO_WORDS(gran), clear,
7991                     (ptr_t)GC_obj_kinds[kind].ok_freelist[gran]);
7992 }
7993 GC_API void GC_CALL GC_register_displacement(size_t offset)
7994 {
7995     DCL_LOCK_STATE;
7996     LOCK();
7997     GC_register_displacement_inner(offset);
7998     UNLOCK();
7999 }
8000 GC_INNER void GC_register_displacement_inner(size_t offset)
8001 {
8002     GC_ASSERT(I_HOLD_LOCK());
8003     if (offset >= VALID_OFFSET_SZ) {
8004         ABORT("Bad argument to GC_register_displacement");
8005     }
8006     if (!GC_valid_offsets[offset]) {
8007       GC_valid_offsets[offset] = TRUE;
8008       GC_modws_valid_offsets[offset % sizeof(word)] = TRUE;
8009     }
8010 }
8011 #ifdef MARK_BIT_PER_GRANULE
8012   GC_INNER GC_bool GC_add_map_entry(size_t granules)
8013   {
8014     unsigned displ;
8015     unsigned short * new_map;
8016     if (granules > BYTES_TO_GRANULES(MAXOBJBYTES)) granules = 0;
8017     if (GC_obj_map[granules] != 0) {
8018         return(TRUE);
8019     }
8020     new_map = (unsigned short *)GC_scratch_alloc(MAP_LEN * sizeof(short));
8021     if (new_map == 0) return(FALSE);
8022     GC_COND_LOG_PRINTF(
8023                 "Adding block map for size of %u granules (%u bytes)\n",
8024                 (unsigned)granules, (unsigned)GRANULES_TO_BYTES(granules));
8025     if (granules == 0) {
8026       for (displ = 0; displ < BYTES_TO_GRANULES(HBLKSIZE); displ++) {
8027         new_map[displ] = 1;
8028       }
8029     } else {
8030       for (displ = 0; displ < BYTES_TO_GRANULES(HBLKSIZE); displ++) {
8031         new_map[displ] = (unsigned short)(displ % granules);
8032       }
8033     }
8034     GC_obj_map[granules] = new_map;
8035     return(TRUE);
8036   }
8037 #endif
8038 GC_INNER void GC_initialize_offsets(void)
8039 {
8040   unsigned i;
8041   if (GC_all_interior_pointers) {
8042     for (i = 0; i < VALID_OFFSET_SZ; ++i)
8043       GC_valid_offsets[i] = TRUE;
8044   } else {
8045     BZERO(GC_valid_offsets, sizeof(GC_valid_offsets));
8046     for (i = 0; i < sizeof(word); ++i)
8047       GC_modws_valid_offsets[i] = FALSE;
8048   }
8049 }
8050 STATIC void GC_CALLBACK GC_default_same_obj_print_proc(void * p, void * q)
8051 {
8052     ABORT_ARG2("GC_same_obj test failed",
8053                ": %p and %p are not in the same object", p, q);
8054 }
8055 void (GC_CALLBACK *GC_same_obj_print_proc) (void *, void *)
8056                 = GC_default_same_obj_print_proc;
8057 GC_API void * GC_CALL GC_same_obj(void *p, void *q)
8058 {
8059     struct hblk *h;
8060     hdr *hhdr;
8061     ptr_t base, limit;
8062     word sz;
8063     if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
8064     hhdr = HDR((word)p);
8065     if (hhdr == 0) {
8066         if (divHBLKSZ((word)p) != divHBLKSZ((word)q)
8067             && HDR((word)q) != 0) {
8068             goto fail;
8069         }
8070         return(p);
8071     }
8072     if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
8073         h = HBLKPTR(p) - (word)hhdr;
8074         hhdr = HDR(h);
8075         while (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
8076            h = FORWARDED_ADDR(h, hhdr);
8077            hhdr = HDR(h);
8078         }
8079         limit = (ptr_t)h + hhdr -> hb_sz;
8080         if ((word)p >= (word)limit || (word)q >= (word)limit
8081             || (word)q < (word)h) {
8082             goto fail;
8083         }
8084         return(p);
8085     }
8086     sz = hhdr -> hb_sz;
8087     if (sz > MAXOBJBYTES) {
8088       base = (ptr_t)HBLKPTR(p);
8089       limit = base + sz;
8090       if ((word)p >= (word)limit) {
8091         goto fail;
8092       }
8093     } else {
8094       size_t offset;
8095       size_t pdispl = HBLKDISPL(p);
8096       offset = pdispl % sz;
8097       if (HBLKPTR(p) != HBLKPTR(q)) goto fail;
8098       base = (ptr_t)p - offset;
8099       limit = base + sz;
8100     }
8101     if ((word)q >= (word)limit || (word)q < (word)base) {
8102         goto fail;
8103     }
8104     return(p);
8105 fail:
8106     (*GC_same_obj_print_proc)((ptr_t)p, (ptr_t)q);
8107     return(p);
8108 }
8109 STATIC void GC_CALLBACK GC_default_is_valid_displacement_print_proc (void *p)
8110 {
8111     ABORT_ARG1("GC_is_valid_displacement test failed", ": %p not valid", p);
8112 }
8113 void (GC_CALLBACK *GC_is_valid_displacement_print_proc)(void *) =
8114         GC_default_is_valid_displacement_print_proc;
8115 GC_API void * GC_CALL GC_is_valid_displacement(void *p)
8116 {
8117     hdr *hhdr;
8118     word pdispl;
8119     word offset;
8120     struct hblk *h;
8121     word sz;
8122     if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
8123     hhdr = HDR((word)p);
8124     if (hhdr == 0) return(p);
8125     h = HBLKPTR(p);
8126     if (GC_all_interior_pointers) {
8127         while (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
8128            h = FORWARDED_ADDR(h, hhdr);
8129            hhdr = HDR(h);
8130         }
8131     } else if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
8132         goto fail;
8133     }
8134     sz = hhdr -> hb_sz;
8135     pdispl = HBLKDISPL(p);
8136     offset = pdispl % sz;
8137     if ((sz > MAXOBJBYTES && (word)p >= (word)h + sz)
8138         || !GC_valid_offsets[offset]
8139         || ((word)p + (sz - offset) > (word)(h + 1)
8140             && !IS_FORWARDING_ADDR_OR_NIL(HDR(h + 1)))) {
8141         goto fail;
8142     }
8143     return(p);
8144 fail:
8145     (*GC_is_valid_displacement_print_proc)((ptr_t)p);
8146     return(p);
8147 }
8148 STATIC void GC_CALLBACK GC_default_is_visible_print_proc(void * p)
8149 {
8150     ABORT_ARG1("GC_is_visible test failed", ": %p not GC-visible", p);
8151 }
8152 void (GC_CALLBACK *GC_is_visible_print_proc)(void * p) =
8153                 GC_default_is_visible_print_proc;
8154 #ifndef THREADS
8155    STATIC GC_bool GC_on_stack(void *p)
8156    {
8157 #ifdef STACK_GROWS_DOWN
8158        if ((word)p >= (word)GC_approx_sp()
8159            && (word)p < (word)GC_stackbottom) {
8160          return(TRUE);
8161        }
8162 #else
8163        if ((word)p <= (word)GC_approx_sp()
8164            && (word)p > (word)GC_stackbottom) {
8165          return(TRUE);
8166        }
8167 #endif
8168      return(FALSE);
8169    }
8170 #endif
8171 GC_API void * GC_CALL GC_is_visible(void *p)
8172 {
8173     hdr *hhdr;
8174     if ((word)p & (ALIGNMENT - 1)) goto fail;
8175     if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
8176 #ifdef THREADS
8177         hhdr = HDR((word)p);
8178         if (hhdr != 0 && GC_base(p) == 0) {
8179             goto fail;
8180         } else {
8181             return(p);
8182         }
8183 #else
8184           if (GC_on_stack(p)) return(p);
8185         hhdr = HDR((word)p);
8186         if (hhdr == 0) {
8187             if (GC_is_static_root(p)) return(p);
8188 #if defined(DYNAMIC_LOADING) || defined(MSWIN32) \
8189                 || defined(MSWINCE) || defined(CYGWIN32) || defined(PCR)
8190               GC_register_dynamic_libraries();
8191               if (GC_is_static_root(p))
8192                 return(p);
8193 #endif
8194             goto fail;
8195         } else {
8196             word descr;
8197             ptr_t base = (ptr_t)GC_base(p);
8198             if (NULL == base) goto fail;
8199             if (HBLKPTR(base) != HBLKPTR(p))
8200                 hhdr = HDR(base);
8201             descr = hhdr -> hb_descr;
8202     retry:
8203             switch(descr & GC_DS_TAGS) {
8204                 case GC_DS_LENGTH:
8205                     if ((word)p - (word)base > descr) goto fail;
8206                     break;
8207                 case GC_DS_BITMAP:
8208                     if ((word)p - (word)base >= WORDS_TO_BYTES(BITMAP_BITS)
8209                         || ((word)p & (sizeof(word) - 1))) goto fail;
8210                     if (!(((word)1 << (WORDSZ - ((ptr_t)p - (ptr_t)base) - 1))
8211                           & descr)) goto fail;
8212                     break;
8213                 case GC_DS_PROC:
8214                     break;
8215                 case GC_DS_PER_OBJECT:
8216                     if ((signed_word)descr >= 0) {
8217                       descr = *(word *)((ptr_t)base + (descr & ~GC_DS_TAGS));
8218                     } else {
8219                       ptr_t type_descr = *(ptr_t *)base;
8220                       descr = *(word *)(type_descr
8221                                         - (descr - (word)(GC_DS_PER_OBJECT
8222                                            - GC_INDIR_PER_OBJ_BIAS)));
8223                     }
8224                     goto retry;
8225             }
8226             return(p);
8227         }
8228 #endif
8229 fail:
8230     (*GC_is_visible_print_proc)((ptr_t)p);
8231     return(p);
8232 }
8233 GC_API void * GC_CALL GC_pre_incr (void **p, ptrdiff_t how_much)
8234 {
8235     void * initial = *p;
8236     void * result = GC_same_obj((void *)((ptr_t)initial + how_much), initial);
8237     if (!GC_all_interior_pointers) {
8238         (void) GC_is_valid_displacement(result);
8239     }
8240     return (*p = result);
8241 }
8242 GC_API void * GC_CALL GC_post_incr (void **p, ptrdiff_t how_much)
8243 {
8244     void * initial = *p;
8245     void * result = GC_same_obj((void *)((ptr_t)initial + how_much), initial);
8246     if (!GC_all_interior_pointers) {
8247         (void) GC_is_valid_displacement(result);
8248     }
8249     *p = result;
8250     return(initial);
8251 }
8252 #ifndef GC_INLINE_H
8253 #define GC_INLINE_H
8254 #if GC_GNUC_PREREQ(3, 0)
8255 #define GC_EXPECT(expr, outcome) __builtin_expect(expr,outcome)
8256 #else
8257 #define GC_EXPECT(expr, outcome) (expr)
8258 #endif
8259 #ifndef GC_ASSERT
8260 #ifdef NDEBUG
8261 #define GC_ASSERT(expr)
8262 #else
8263 #include <assert.h>
8264 #define GC_ASSERT(expr) assert(expr)
8265 #endif
8266 #endif
8267 #ifdef __cplusplus
8268   extern "C" {
8269 #endif
8270 #ifndef GC_PREFETCH_FOR_WRITE
8271 #if GC_GNUC_PREREQ(3, 0) && !defined(GC_NO_PREFETCH_FOR_WRITE)
8272 #define GC_PREFETCH_FOR_WRITE(x) __builtin_prefetch((x), 1)
8273 #else
8274 #define GC_PREFETCH_FOR_WRITE(x) (void)0
8275 #endif
8276 #endif
8277 #define GC_I_PTRFREE 0
8278 #define GC_I_NORMAL 1
8279 GC_API void GC_CALL GC_generic_malloc_many(size_t , int ,
8280                                            void ** );
8281 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
8282         GC_malloc_kind(size_t , int );
8283 #ifdef GC_THREADS
8284   GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
8285         GC_malloc_kind_global(size_t , int );
8286 #else
8287 #define GC_malloc_kind_global GC_malloc_kind
8288 #endif
8289 #if defined(GC_THREADS) && defined(AO_HAVE_store)
8290 #define GC_FAST_M_AO_STORE(my_fl, next) \
8291                 AO_store((volatile AO_t *)(my_fl), (AO_t)(next))
8292 #else
8293 #define GC_FAST_M_AO_STORE(my_fl, next) (void)(*(my_fl) = (next))
8294 #endif
8295 #define GC_FAST_MALLOC_GRANS(result,granules,tiny_fl,num_direct, \
8296                               kind,default_expr,init) \
8297   do { \
8298     if (GC_EXPECT((granules) >= GC_TINY_FREELISTS,0)) { \
8299         result = (default_expr); \
8300     } else { \
8301         void **my_fl = (tiny_fl) + (granules); \
8302         void *my_entry=*my_fl; \
8303         void *next; \
8304     \
8305         for (;;) { \
8306             if (GC_EXPECT((GC_word)my_entry \
8307                           > (num_direct) + GC_TINY_FREELISTS + 1, 1)) { \
8308                 next = *(void **)(my_entry); \
8309                 result = (void *)my_entry; \
8310                 GC_FAST_M_AO_STORE(my_fl, next); \
8311                 init; \
8312                 GC_PREFETCH_FOR_WRITE(next); \
8313                 if ((kind) != GC_I_PTRFREE) { \
8314                     GC_end_stubborn_change(my_fl); \
8315                     GC_reachable_here(next); \
8316                 } \
8317                 GC_ASSERT(GC_size(result) >= (granules)*GC_GRANULE_BYTES); \
8318                 GC_ASSERT((kind) == GC_I_PTRFREE \
8319                           || ((GC_word *)result)[1] == 0); \
8320                 break; \
8321             } \
8322              \
8323             if ((GC_signed_word)my_entry - (GC_signed_word)(num_direct) <= 0 \
8324                      \
8325                     && my_entry != 0 ) { \
8326                  \
8327                 GC_FAST_M_AO_STORE(my_fl, (char *)my_entry \
8328                                           + (granules) + 1); \
8329                 result = (default_expr); \
8330                 break; \
8331             } else { \
8332                  \
8333                 GC_generic_malloc_many(((granules) == 0? GC_GRANULE_BYTES : \
8334                                         GC_RAW_BYTES_FROM_INDEX(granules)), \
8335                                        kind, my_fl); \
8336                 my_entry = *my_fl; \
8337                 if (my_entry == 0) { \
8338                     result = (*GC_get_oom_fn())((granules)*GC_GRANULE_BYTES); \
8339                     break; \
8340                 } \
8341             } \
8342         } \
8343     } \
8344   } while (0)
8345 #define GC_WORDS_TO_WHOLE_GRANULES(n) \
8346         GC_WORDS_TO_GRANULES((n) + GC_GRANULE_WORDS - 1)
8347 #define GC_MALLOC_WORDS_KIND(result,n,tiny_fl,kind,init) \
8348     do { \
8349       size_t granules = GC_WORDS_TO_WHOLE_GRANULES(n); \
8350       GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, 0, kind, \
8351                            GC_malloc_kind(granules*GC_GRANULE_BYTES, kind), \
8352                            init); \
8353     } while (0)
8354 #define GC_MALLOC_WORDS(result,n,tiny_fl) \
8355         GC_MALLOC_WORDS_KIND(result, n, tiny_fl, GC_I_NORMAL, \
8356                              *(void **)(result) = 0)
8357 #define GC_MALLOC_ATOMIC_WORDS(result,n,tiny_fl) \
8358         GC_MALLOC_WORDS_KIND(result, n, tiny_fl, GC_I_PTRFREE, (void)0)
8359 #define GC_CONS(result, first, second, tiny_fl) \
8360     do { \
8361       void *l = (void *)(first); \
8362       void *r = (void *)(second); \
8363       GC_MALLOC_WORDS_KIND(result, 2, tiny_fl, GC_I_NORMAL, (void)0); \
8364       if ((result) != 0 ) { \
8365         *(void **)(result) = l; \
8366         GC_PTR_STORE_AND_DIRTY((void **)(result) + 1, r); \
8367         GC_reachable_here(l); \
8368       } \
8369     } while (0)
8370 GC_API void GC_CALL GC_print_free_list(int ,
8371                                        size_t );
8372 #ifdef __cplusplus
8373   }
8374 #endif
8375 #endif
8376 #include <stdio.h>
8377 #ifdef GC_USE_ENTIRE_HEAP
8378   int GC_use_entire_heap = TRUE;
8379 #else
8380   int GC_use_entire_heap = FALSE;
8381 #endif
8382 #define MAX_BLACK_LIST_ALLOC (2*HBLKSIZE)
8383 #define UNIQUE_THRESHOLD 32
8384 #define HUGE_THRESHOLD 256
8385 #define FL_COMPRESSION 8
8386 #define N_HBLK_FLS ((HUGE_THRESHOLD - UNIQUE_THRESHOLD) / FL_COMPRESSION \
8387                      + UNIQUE_THRESHOLD)
8388 #ifndef GC_GCJ_SUPPORT
8389   STATIC
8390 #endif
8391   struct hblk * GC_hblkfreelist[N_HBLK_FLS+1] = { 0 };
8392 #ifndef GC_GCJ_SUPPORT
8393   STATIC
8394 #endif
8395   word GC_free_bytes[N_HBLK_FLS+1] = { 0 };
8396 GC_INLINE int GC_enough_large_bytes_left(void)
8397 {
8398     int n;
8399     word bytes = GC_large_allocd_bytes;
8400     GC_ASSERT(GC_max_large_allocd_bytes <= GC_heapsize);
8401     for (n = N_HBLK_FLS; n >= 0; --n) {
8402         bytes += GC_free_bytes[n];
8403         if (bytes >= GC_max_large_allocd_bytes) return n;
8404     }
8405     return 0;
8406 }
8407 STATIC int GC_hblk_fl_from_blocks(word blocks_needed)
8408 {
8409     if (blocks_needed <= UNIQUE_THRESHOLD) return (int)blocks_needed;
8410     if (blocks_needed >= HUGE_THRESHOLD) return N_HBLK_FLS;
8411     return (int)(blocks_needed - UNIQUE_THRESHOLD)/FL_COMPRESSION
8412                                         + UNIQUE_THRESHOLD;
8413 }
8414 #define PHDR(hhdr) HDR((hhdr) -> hb_prev)
8415 #define NHDR(hhdr) HDR((hhdr) -> hb_next)
8416 #ifdef USE_MUNMAP
8417 #define IS_MAPPED(hhdr) (((hhdr) -> hb_flags & WAS_UNMAPPED) == 0)
8418 #else
8419 #define IS_MAPPED(hhdr) TRUE
8420 #endif
8421 #if !defined(NO_DEBUGGING) || defined(GC_ASSERTIONS)
8422   GC_INNER word GC_compute_large_free_bytes(void)
8423   {
8424       word total_free = 0;
8425       unsigned i;
8426       for (i = 0; i <= N_HBLK_FLS; ++i) {
8427         struct hblk * h;
8428         hdr * hhdr;
8429         for (h = GC_hblkfreelist[i]; h != 0; h = hhdr->hb_next) {
8430           hhdr = HDR(h);
8431           total_free += hhdr->hb_sz;
8432         }
8433       }
8434       return total_free;
8435   }
8436 #endif
8437 #if !defined(NO_DEBUGGING)
8438 void GC_print_hblkfreelist(void)
8439 {
8440     unsigned i;
8441     word total;
8442     for (i = 0; i <= N_HBLK_FLS; ++i) {
8443       struct hblk * h = GC_hblkfreelist[i];
8444       if (0 != h) GC_printf("Free list %u (total size %lu):\n",
8445                             i, (unsigned long)GC_free_bytes[i]);
8446       while (h ) {
8447         hdr * hhdr = HDR(h);
8448         GC_printf("\t%p size %lu %s black listed\n",
8449                 (void *)h, (unsigned long) hhdr -> hb_sz,
8450                 GC_is_black_listed(h, HBLKSIZE) != 0 ? "start" :
8451                 GC_is_black_listed(h, hhdr -> hb_sz) != 0 ? "partially" :
8452                                                         "not");
8453         h = hhdr -> hb_next;
8454       }
8455     }
8456     GC_printf("GC_large_free_bytes: %lu\n",
8457               (unsigned long)GC_large_free_bytes);
8458     if ((total = GC_compute_large_free_bytes()) != GC_large_free_bytes)
8459           GC_err_printf("GC_large_free_bytes INCONSISTENT!! Should be: %lu\n",
8460                         (unsigned long)total);
8461 }
8462 static int free_list_index_of(hdr *wanted)
8463 {
8464     int i;
8465     for (i = 0; i <= N_HBLK_FLS; ++i) {
8466       struct hblk * h;
8467       hdr * hhdr;
8468       for (h = GC_hblkfreelist[i]; h != 0; h = hhdr -> hb_next) {
8469         hhdr = HDR(h);
8470         if (hhdr == wanted) return i;
8471       }
8472     }
8473     return -1;
8474 }
8475 GC_API void GC_CALL GC_dump_regions(void)
8476 {
8477     unsigned i;
8478     for (i = 0; i < GC_n_heap_sects; ++i) {
8479         ptr_t start = GC_heap_sects[i].hs_start;
8480         size_t bytes = GC_heap_sects[i].hs_bytes;
8481         ptr_t end = start + bytes;
8482         ptr_t p;
8483           while (i+1 < GC_n_heap_sects && GC_heap_sects[i+1].hs_start == end) {
8484             ++i;
8485             end = GC_heap_sects[i].hs_start + GC_heap_sects[i].hs_bytes;
8486           }
8487         GC_printf("***Section from %p to %p\n", (void *)start, (void *)end);
8488         for (p = start; (word)p < (word)end; ) {
8489             hdr *hhdr = HDR(p);
8490             if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
8491                 GC_printf("\t%p Missing header!!(%p)\n",
8492                           (void *)p, (void *)hhdr);
8493                 p += HBLKSIZE;
8494                 continue;
8495             }
8496             if (HBLK_IS_FREE(hhdr)) {
8497                 int correct_index = GC_hblk_fl_from_blocks(
8498                                         divHBLKSZ(hhdr -> hb_sz));
8499                 int actual_index;
8500                 GC_printf("\t%p\tfree block of size 0x%lx bytes%s\n",
8501                           (void *)p, (unsigned long)(hhdr -> hb_sz),
8502                           IS_MAPPED(hhdr) ? "" : " (unmapped)");
8503                 actual_index = free_list_index_of(hhdr);
8504                 if (-1 == actual_index) {
8505                     GC_printf("\t\tBlock not on free list %d!!\n",
8506                               correct_index);
8507                 } else if (correct_index != actual_index) {
8508                     GC_printf("\t\tBlock on list %d, should be on %d!!\n",
8509                               actual_index, correct_index);
8510                 }
8511                 p += hhdr -> hb_sz;
8512             } else {
8513                 GC_printf("\t%p\tused for blocks of size 0x%lx bytes\n",
8514                           (void *)p, (unsigned long)(hhdr -> hb_sz));
8515                 p += HBLKSIZE * OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
8516             }
8517         }
8518     }
8519 }
8520 #endif
8521 static GC_bool setup_header(hdr * hhdr, struct hblk *block, size_t byte_sz,
8522                             int kind, unsigned flags)
8523 {
8524     word descr;
8525 #ifdef MARK_BIT_PER_GRANULE
8526       if (byte_sz > MAXOBJBYTES)
8527         flags |= LARGE_BLOCK;
8528 #endif
8529 #ifdef ENABLE_DISCLAIM
8530       if (GC_obj_kinds[kind].ok_disclaim_proc)
8531         flags |= HAS_DISCLAIM;
8532       if (GC_obj_kinds[kind].ok_mark_unconditionally)
8533         flags |= MARK_UNCONDITIONALLY;
8534 #endif
8535       hhdr -> hb_sz = byte_sz;
8536       hhdr -> hb_obj_kind = (unsigned char)kind;
8537       hhdr -> hb_flags = (unsigned char)flags;
8538       hhdr -> hb_block = block;
8539       descr = GC_obj_kinds[kind].ok_descriptor;
8540       if (GC_obj_kinds[kind].ok_relocate_descr) descr += byte_sz;
8541       hhdr -> hb_descr = descr;
8542 #ifdef MARK_BIT_PER_OBJ
8543       if (byte_sz > MAXOBJBYTES) {
8544          hhdr -> hb_inv_sz = LARGE_INV_SZ;
8545       } else {
8546         word inv_sz;
8547 #if CPP_WORDSZ == 64
8548           inv_sz = ((word)1 << 32)/byte_sz;
8549           if (((inv_sz*byte_sz) >> 32) == 0) ++inv_sz;
8550 #else
8551           GC_ASSERT(byte_sz >= 4);
8552           inv_sz = ((unsigned)1 << 31)/byte_sz;
8553           inv_sz *= 2;
8554           while (inv_sz*byte_sz > byte_sz) ++inv_sz;
8555 #endif
8556 #ifdef INV_SZ_COMPUTATION_CHECK
8557           GC_ASSERT(((1ULL << 32) + byte_sz - 1) / byte_sz == inv_sz);
8558 #endif
8559         hhdr -> hb_inv_sz = inv_sz;
8560       }
8561 #endif
8562 #ifdef MARK_BIT_PER_GRANULE
8563     {
8564       size_t granules = BYTES_TO_GRANULES(byte_sz);
8565       if (EXPECT(!GC_add_map_entry(granules), FALSE)) {
8566         hhdr -> hb_sz = HBLKSIZE;
8567         hhdr -> hb_descr = 0;
8568         hhdr -> hb_flags |= LARGE_BLOCK;
8569         hhdr -> hb_map = 0;
8570         return FALSE;
8571       }
8572       hhdr -> hb_map = GC_obj_map[(hhdr -> hb_flags & LARGE_BLOCK) != 0 ?
8573                                     0 : granules];
8574     }
8575 #endif
8576     GC_clear_hdr_marks(hhdr);
8577     hhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
8578     return(TRUE);
8579 }
8580 STATIC void GC_remove_from_fl_at(hdr *hhdr, int index)
8581 {
8582     GC_ASSERT(((hhdr -> hb_sz) & (HBLKSIZE-1)) == 0);
8583     if (hhdr -> hb_prev == 0) {
8584         GC_ASSERT(HDR(GC_hblkfreelist[index]) == hhdr);
8585         GC_hblkfreelist[index] = hhdr -> hb_next;
8586     } else {
8587         hdr *phdr;
8588         GET_HDR(hhdr -> hb_prev, phdr);
8589         phdr -> hb_next = hhdr -> hb_next;
8590     }
8591     GC_ASSERT(GC_free_bytes[index] >= hhdr -> hb_sz);
8592     GC_free_bytes[index] -= hhdr -> hb_sz;
8593     if (0 != hhdr -> hb_next) {
8594         hdr * nhdr;
8595         GC_ASSERT(!IS_FORWARDING_ADDR_OR_NIL(NHDR(hhdr)));
8596         GET_HDR(hhdr -> hb_next, nhdr);
8597         nhdr -> hb_prev = hhdr -> hb_prev;
8598     }
8599 }
8600 GC_INLINE void GC_remove_from_fl(hdr *hhdr)
8601 {
8602   GC_remove_from_fl_at(hhdr, GC_hblk_fl_from_blocks(divHBLKSZ(hhdr->hb_sz)));
8603 }
8604 static struct hblk * get_block_ending_at(struct hblk *h)
8605 {
8606     struct hblk * p = h - 1;
8607     hdr * phdr;
8608     GET_HDR(p, phdr);
8609     while (0 != phdr && IS_FORWARDING_ADDR_OR_NIL(phdr)) {
8610         p = FORWARDED_ADDR(p,phdr);
8611         phdr = HDR(p);
8612     }
8613     if (0 != phdr) {
8614         return p;
8615     }
8616     p = GC_prev_block(h - 1);
8617     if (p) {
8618         phdr = HDR(p);
8619         if ((ptr_t)p + phdr -> hb_sz == (ptr_t)h) {
8620             return p;
8621         }
8622     }
8623     return NULL;
8624 }
8625 STATIC struct hblk * GC_free_block_ending_at(struct hblk *h)
8626 {
8627     struct hblk * p = get_block_ending_at(h);
8628     if (p ) {
8629       hdr * phdr = HDR(p);
8630       if (HBLK_IS_FREE(phdr)) {
8631         return p;
8632       }
8633     }
8634     return 0;
8635 }
8636 STATIC void GC_add_to_fl(struct hblk *h, hdr *hhdr)
8637 {
8638     int index = GC_hblk_fl_from_blocks(divHBLKSZ(hhdr -> hb_sz));
8639     struct hblk *second = GC_hblkfreelist[index];
8640 #if defined(GC_ASSERTIONS) && !defined(USE_MUNMAP)
8641       struct hblk *next = (struct hblk *)((word)h + hhdr -> hb_sz);
8642       hdr * nexthdr = HDR(next);
8643       struct hblk *prev = GC_free_block_ending_at(h);
8644       hdr * prevhdr = HDR(prev);
8645       GC_ASSERT(nexthdr == 0 || !HBLK_IS_FREE(nexthdr)
8646                 || (GC_heapsize & SIGNB) != 0);
8647       GC_ASSERT(prev == 0 || !HBLK_IS_FREE(prevhdr)
8648                 || (GC_heapsize & SIGNB) != 0);
8649 #endif
8650     GC_ASSERT(((hhdr -> hb_sz) & (HBLKSIZE-1)) == 0);
8651     GC_hblkfreelist[index] = h;
8652     GC_free_bytes[index] += hhdr -> hb_sz;
8653     GC_ASSERT(GC_free_bytes[index] <= GC_large_free_bytes);
8654     hhdr -> hb_next = second;
8655     hhdr -> hb_prev = 0;
8656     if (second ) {
8657       hdr * second_hdr;
8658       GET_HDR(second, second_hdr);
8659       second_hdr -> hb_prev = h;
8660     }
8661     hhdr -> hb_flags |= FREE_BLK;
8662 }
8663 #ifdef USE_MUNMAP
8664 #ifndef MUNMAP_THRESHOLD
8665 #define MUNMAP_THRESHOLD 6
8666 #endif
8667 GC_INNER int GC_unmap_threshold = MUNMAP_THRESHOLD;
8668 #ifdef COUNT_UNMAPPED_REGIONS
8669   static int calc_num_unmapped_regions_delta(struct hblk *h, hdr *hhdr)
8670   {
8671     struct hblk * prev = get_block_ending_at(h);
8672     struct hblk * next;
8673     GC_bool prev_unmapped = FALSE;
8674     GC_bool next_unmapped = FALSE;
8675     next = GC_next_block((struct hblk *)((ptr_t)h + hhdr->hb_sz), TRUE);
8676     if ((ptr_t)next != GC_unmap_end((ptr_t)h, (size_t)hhdr->hb_sz)) {
8677       next = NULL;
8678     }
8679     if (prev != NULL) {
8680       hdr * prevhdr = HDR(prev);
8681       prev_unmapped = !IS_MAPPED(prevhdr);
8682     }
8683     if (next != NULL) {
8684       hdr * nexthdr = HDR(next);
8685       next_unmapped = !IS_MAPPED(nexthdr);
8686     }
8687     if (prev_unmapped && next_unmapped) {
8688       return IS_MAPPED(hhdr) ? -1 : 1;
8689     }
8690     if (!prev_unmapped && !next_unmapped) {
8691       return IS_MAPPED(hhdr) ? 1 : -1;
8692     }
8693     return 0;
8694   }
8695 #endif
8696 GC_INLINE void GC_adjust_num_unmapped(struct hblk *h GC_ATTR_UNUSED,
8697                                       hdr *hhdr GC_ATTR_UNUSED)
8698 {
8699 #ifdef COUNT_UNMAPPED_REGIONS
8700     GC_num_unmapped_regions += calc_num_unmapped_regions_delta(h, hhdr);
8701 #endif
8702 }
8703 GC_INNER void GC_unmap_old(void)
8704 {
8705     int i;
8706     if (GC_unmap_threshold == 0)
8707       return;
8708 #ifdef COUNT_UNMAPPED_REGIONS
8709     if (GC_num_unmapped_regions >= GC_UNMAPPED_REGIONS_SOFT_LIMIT)
8710       return;
8711 #endif
8712     for (i = 0; i <= N_HBLK_FLS; ++i) {
8713       struct hblk * h;
8714       hdr * hhdr;
8715       for (h = GC_hblkfreelist[i]; 0 != h; h = hhdr -> hb_next) {
8716         hhdr = HDR(h);
8717         if (!IS_MAPPED(hhdr)) continue;
8718         if ((unsigned short)(GC_gc_no - hhdr->hb_last_reclaimed) >
8719                 (unsigned short)GC_unmap_threshold) {
8720 #ifdef COUNT_UNMAPPED_REGIONS
8721             int delta = calc_num_unmapped_regions_delta(h, hhdr);
8722             signed_word regions = GC_num_unmapped_regions + delta;
8723             if (delta >= 0 && regions >= GC_UNMAPPED_REGIONS_SOFT_LIMIT) {
8724               GC_COND_LOG_PRINTF("Unmapped regions limit reached!\n");
8725               return;
8726             }
8727             GC_num_unmapped_regions = regions;
8728 #endif
8729           GC_unmap((ptr_t)h, (size_t)hhdr->hb_sz);
8730           hhdr -> hb_flags |= WAS_UNMAPPED;
8731         }
8732       }
8733     }
8734 }
8735 GC_INNER void GC_merge_unmapped(void)
8736 {
8737     int i;
8738     for (i = 0; i <= N_HBLK_FLS; ++i) {
8739       struct hblk *h = GC_hblkfreelist[i];
8740       while (h != 0) {
8741         struct hblk *next;
8742         hdr *hhdr, *nexthdr;
8743         word size, nextsize;
8744         GET_HDR(h, hhdr);
8745         size = hhdr->hb_sz;
8746         next = (struct hblk *)((word)h + size);
8747         GET_HDR(next, nexthdr);
8748           if (0 != nexthdr && HBLK_IS_FREE(nexthdr)
8749               && (signed_word) (size + (nextsize = nexthdr->hb_sz)) > 0
8750                  ) {
8751             if (IS_MAPPED(hhdr) && !IS_MAPPED(nexthdr)) {
8752                 if (size > nextsize) {
8753                   GC_adjust_num_unmapped(next, nexthdr);
8754                   GC_remap((ptr_t)next, nextsize);
8755                 } else {
8756                   GC_adjust_num_unmapped(h, hhdr);
8757                   GC_unmap((ptr_t)h, size);
8758                   GC_unmap_gap((ptr_t)h, size, (ptr_t)next, nextsize);
8759                   hhdr -> hb_flags |= WAS_UNMAPPED;
8760                 }
8761             } else if (IS_MAPPED(nexthdr) && !IS_MAPPED(hhdr)) {
8762               if (size > nextsize) {
8763                 GC_adjust_num_unmapped(next, nexthdr);
8764                 GC_unmap((ptr_t)next, nextsize);
8765                 GC_unmap_gap((ptr_t)h, size, (ptr_t)next, nextsize);
8766               } else {
8767                 GC_adjust_num_unmapped(h, hhdr);
8768                 GC_remap((ptr_t)h, size);
8769                 hhdr -> hb_flags &= ~WAS_UNMAPPED;
8770                 hhdr -> hb_last_reclaimed = nexthdr -> hb_last_reclaimed;
8771               }
8772             } else if (!IS_MAPPED(hhdr) && !IS_MAPPED(nexthdr)) {
8773                 GC_unmap_gap((ptr_t)h, size, (ptr_t)next, nextsize);
8774             }
8775             GC_remove_from_fl_at(hhdr, i);
8776             GC_remove_from_fl(nexthdr);
8777             hhdr -> hb_sz += nexthdr -> hb_sz;
8778             GC_remove_header(next);
8779             GC_add_to_fl(h, hhdr);
8780             h = GC_hblkfreelist[i];
8781           } else  {
8782             h = hhdr -> hb_next;
8783           }
8784       }
8785     }
8786 }
8787 #endif
8788 STATIC struct hblk * GC_get_first_part(struct hblk *h, hdr *hhdr,
8789                                        size_t bytes, int index)
8790 {
8791     word total_size = hhdr -> hb_sz;
8792     struct hblk * rest;
8793     hdr * rest_hdr;
8794     GC_ASSERT((total_size & (HBLKSIZE-1)) == 0);
8795     GC_remove_from_fl_at(hhdr, index);
8796     if (total_size == bytes) return h;
8797     rest = (struct hblk *)((word)h + bytes);
8798     rest_hdr = GC_install_header(rest);
8799     if (0 == rest_hdr) {
8800         WARN("Header allocation failed: dropping block\n", 0);
8801         return(0);
8802     }
8803     rest_hdr -> hb_sz = total_size - bytes;
8804     rest_hdr -> hb_flags = 0;
8805 #ifdef GC_ASSERTIONS
8806         hhdr -> hb_flags &= ~FREE_BLK;
8807 #endif
8808     GC_add_to_fl(rest, rest_hdr);
8809     return h;
8810 }
8811 STATIC void GC_split_block(struct hblk *h, hdr *hhdr, struct hblk *n,
8812                            hdr *nhdr, int index )
8813 {
8814     word total_size = hhdr -> hb_sz;
8815     word h_size = (word)n - (word)h;
8816     struct hblk *prev = hhdr -> hb_prev;
8817     struct hblk *next = hhdr -> hb_next;
8818       nhdr -> hb_prev = prev;
8819       nhdr -> hb_next = next;
8820       nhdr -> hb_sz = total_size - h_size;
8821       nhdr -> hb_flags = 0;
8822       if (prev ) {
8823         HDR(prev) -> hb_next = n;
8824       } else {
8825         GC_hblkfreelist[index] = n;
8826       }
8827       if (next ) {
8828         HDR(next) -> hb_prev = n;
8829       }
8830       GC_ASSERT(GC_free_bytes[index] > h_size);
8831       GC_free_bytes[index] -= h_size;
8832 #ifdef USE_MUNMAP
8833       hhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
8834 #endif
8835     hhdr -> hb_sz = h_size;
8836     GC_add_to_fl(h, hhdr);
8837     nhdr -> hb_flags |= FREE_BLK;
8838 }
8839 STATIC struct hblk *
8840 GC_allochblk_nth(size_t sz , int kind, unsigned flags, int n,
8841                  int may_split);
8842 #define AVOID_SPLIT_REMAPPED 2
8843 GC_INNER struct hblk *
8844 GC_allochblk(size_t sz, int kind, unsigned flags)
8845 {
8846     word blocks;
8847     int start_list;
8848     struct hblk *result;
8849     int may_split;
8850     int split_limit;
8851     GC_ASSERT(I_HOLD_LOCK());
8852     GC_ASSERT((sz & (GRANULE_BYTES - 1)) == 0);
8853     blocks = OBJ_SZ_TO_BLOCKS_CHECKED(sz);
8854     if ((signed_word)(blocks * HBLKSIZE) < 0) {
8855       return 0;
8856     }
8857     start_list = GC_hblk_fl_from_blocks(blocks);
8858     result = GC_allochblk_nth(sz, kind, flags, start_list, FALSE);
8859     if (0 != result) return result;
8860     may_split = TRUE;
8861     if (GC_use_entire_heap || GC_dont_gc
8862         || USED_HEAP_SIZE < GC_requested_heapsize
8863         || GC_incremental || !GC_should_collect()) {
8864         split_limit = N_HBLK_FLS;
8865     } else if (GC_finalizer_bytes_freed > (GC_heapsize >> 4)) {
8866           split_limit = 0;
8867     } else {
8868           split_limit = GC_enough_large_bytes_left();
8869 #ifdef USE_MUNMAP
8870             if (split_limit > 0)
8871               may_split = AVOID_SPLIT_REMAPPED;
8872 #endif
8873     }
8874     if (start_list < UNIQUE_THRESHOLD) {
8875       ++start_list;
8876     }
8877     for (; start_list <= split_limit; ++start_list) {
8878         result = GC_allochblk_nth(sz, kind, flags, start_list, may_split);
8879         if (0 != result)
8880             break;
8881     }
8882     return result;
8883 }
8884 STATIC long GC_large_alloc_warn_suppressed = 0;
8885 STATIC struct hblk *
8886 GC_allochblk_nth(size_t sz, int kind, unsigned flags, int n, int may_split)
8887 {
8888     struct hblk *hbp;
8889     hdr * hhdr;
8890     struct hblk *thishbp;
8891     hdr * thishdr;
8892     signed_word size_needed = HBLKSIZE * OBJ_SZ_TO_BLOCKS_CHECKED(sz);
8893         for (hbp = GC_hblkfreelist[n];; hbp = hhdr -> hb_next) {
8894             signed_word size_avail;
8895             if (hbp ) {
8896             } else {
8897               return NULL;
8898             }
8899             GET_HDR(hbp, hhdr);
8900             size_avail = (signed_word)hhdr->hb_sz;
8901             if (size_avail < size_needed) continue;
8902             if (size_avail != size_needed) {
8903               if (!may_split) continue;
8904               thishbp = hhdr -> hb_next;
8905               if (thishbp ) {
8906                 signed_word next_size;
8907                 GET_HDR(thishbp, thishdr);
8908                 next_size = (signed_word)(thishdr -> hb_sz);
8909                 if (next_size < size_avail
8910                     && next_size >= size_needed
8911                     && !GC_is_black_listed(thishbp, (word)size_needed)) {
8912                     continue;
8913                 }
8914               }
8915             }
8916             if (!IS_UNCOLLECTABLE(kind) && (kind != PTRFREE
8917                         || size_needed > (signed_word)MAX_BLACK_LIST_ALLOC)) {
8918               struct hblk * lasthbp = hbp;
8919               ptr_t search_end = (ptr_t)hbp + size_avail - size_needed;
8920               signed_word orig_avail = size_avail;
8921               signed_word eff_size_needed = (flags & IGNORE_OFF_PAGE) != 0 ?
8922                                                 (signed_word)HBLKSIZE
8923                                                 : size_needed;
8924               while ((word)lasthbp <= (word)search_end
8925                      && (thishbp = GC_is_black_listed(lasthbp,
8926                                             (word)eff_size_needed)) != 0) {
8927                 lasthbp = thishbp;
8928               }
8929               size_avail -= (ptr_t)lasthbp - (ptr_t)hbp;
8930               thishbp = lasthbp;
8931               if (size_avail >= size_needed) {
8932                 if (thishbp != hbp) {
8933 #ifdef USE_MUNMAP
8934                     if (may_split == AVOID_SPLIT_REMAPPED && !IS_MAPPED(hhdr))
8935                       continue;
8936 #endif
8937                   thishdr = GC_install_header(thishbp);
8938                   if (0 != thishdr) {
8939 #ifdef USE_MUNMAP
8940                       if (!IS_MAPPED(hhdr)) {
8941                         GC_adjust_num_unmapped(hbp, hhdr);
8942                         GC_remap((ptr_t)hbp, (size_t)hhdr->hb_sz);
8943                         hhdr -> hb_flags &= ~WAS_UNMAPPED;
8944                       }
8945 #endif
8946                       GC_split_block(hbp, hhdr, thishbp, thishdr, n);
8947                       hbp = thishbp;
8948                       hhdr = thishdr;
8949                   }
8950                 }
8951               } else if (size_needed > (signed_word)BL_LIMIT
8952                          && orig_avail - size_needed
8953                             > (signed_word)BL_LIMIT) {
8954                 if (++GC_large_alloc_warn_suppressed
8955                     >= GC_large_alloc_warn_interval) {
8956                   WARN("Repeated allocation of very large block "
8957                        "(appr. size %" WARN_PRIdPTR "):\n"
8958                        "\tMay lead to memory leak and poor performance\n",
8959                        size_needed);
8960                   GC_large_alloc_warn_suppressed = 0;
8961                 }
8962                 size_avail = orig_avail;
8963               } else if (size_avail == 0
8964                          && size_needed == (signed_word)HBLKSIZE
8965                          && IS_MAPPED(hhdr)) {
8966                 if (!GC_find_leak) {
8967                   static unsigned count = 0;
8968                   if ((++count & 3) == 0) {
8969                       word total_size = hhdr -> hb_sz;
8970                       struct hblk * limit = hbp + divHBLKSZ(total_size);
8971                       struct hblk * h;
8972                       struct hblk * prev = hhdr -> hb_prev;
8973                       GC_large_free_bytes -= total_size;
8974                       GC_bytes_dropped += total_size;
8975                       GC_remove_from_fl_at(hhdr, n);
8976                       for (h = hbp; (word)h < (word)limit; h++) {
8977                         if (h != hbp) {
8978                           hhdr = GC_install_header(h);
8979                         }
8980                         if (NULL != hhdr) {
8981                           (void)setup_header(hhdr, h, HBLKSIZE, PTRFREE, 0);
8982                           if (GC_debugging_started) {
8983                             BZERO(h, HBLKSIZE);
8984                           }
8985                         }
8986                       }
8987                       hbp = prev;
8988                       if (0 == hbp) {
8989                         return GC_allochblk_nth(sz, kind, flags, n, may_split);
8990                       }
8991                       hhdr = HDR(hbp);
8992                   }
8993                 }
8994               }
8995             }
8996             if( size_avail >= size_needed ) {
8997 #ifdef USE_MUNMAP
8998                   if (!IS_MAPPED(hhdr)) {
8999                     GC_adjust_num_unmapped(hbp, hhdr);
9000                     GC_remap((ptr_t)hbp, (size_t)hhdr->hb_sz);
9001                     hhdr -> hb_flags &= ~WAS_UNMAPPED;
9002                   }
9003 #endif
9004                 hbp = GC_get_first_part(hbp, hhdr, size_needed, n);
9005                 break;
9006             }
9007         }
9008     if (0 == hbp) return 0;
9009         if (!GC_install_counts(hbp, (word)size_needed)) return(0);
9010         if (!setup_header(hhdr, hbp, sz, kind, flags)) {
9011             GC_remove_counts(hbp, (word)size_needed);
9012             return(0);
9013         }
9014 #ifndef GC_DISABLE_INCREMENTAL
9015         GC_ASSERT((size_needed & (HBLKSIZE-1)) == 0);
9016         GC_remove_protection(hbp, divHBLKSZ(size_needed),
9017                              (hhdr -> hb_descr == 0) );
9018 #endif
9019     GC_fail_count = 0;
9020     GC_large_free_bytes -= size_needed;
9021     GC_ASSERT(IS_MAPPED(hhdr));
9022     return( hbp );
9023 }
9024 GC_INNER void GC_freehblk(struct hblk *hbp)
9025 {
9026     struct hblk *next, *prev;
9027     hdr *hhdr, *prevhdr, *nexthdr;
9028     word size;
9029     GET_HDR(hbp, hhdr);
9030     size = HBLKSIZE * OBJ_SZ_TO_BLOCKS(hhdr->hb_sz);
9031     if ((size & SIGNB) != 0)
9032       ABORT("Deallocating excessively large block.  Too large an allocation?");
9033     GC_remove_counts(hbp, size);
9034     hhdr->hb_sz = size;
9035 #ifdef USE_MUNMAP
9036       hhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
9037 #endif
9038       if (HBLK_IS_FREE(hhdr)) {
9039         ABORT_ARG1("Duplicate large block deallocation",
9040                    " of %p", (void *)hbp);
9041       }
9042     GC_ASSERT(IS_MAPPED(hhdr));
9043     hhdr -> hb_flags |= FREE_BLK;
9044     next = (struct hblk *)((ptr_t)hbp + size);
9045     GET_HDR(next, nexthdr);
9046     prev = GC_free_block_ending_at(hbp);
9047       if(0 != nexthdr && HBLK_IS_FREE(nexthdr) && IS_MAPPED(nexthdr)
9048          && (signed_word)(hhdr -> hb_sz + nexthdr -> hb_sz) > 0
9049          ) {
9050         GC_remove_from_fl(nexthdr);
9051         hhdr -> hb_sz += nexthdr -> hb_sz;
9052         GC_remove_header(next);
9053       }
9054       if (prev ) {
9055         prevhdr = HDR(prev);
9056         if (IS_MAPPED(prevhdr)
9057             && (signed_word)(hhdr -> hb_sz + prevhdr -> hb_sz) > 0) {
9058           GC_remove_from_fl(prevhdr);
9059           prevhdr -> hb_sz += hhdr -> hb_sz;
9060 #ifdef USE_MUNMAP
9061             prevhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
9062 #endif
9063           GC_remove_header(hbp);
9064           hbp = prev;
9065           hhdr = prevhdr;
9066         }
9067       }
9068     GC_large_free_bytes += size;
9069     GC_add_to_fl(hbp, hhdr);
9070 }
9071 #include <stdio.h>
9072 #if !defined(MACOS) && !defined(MSWINCE)
9073 #include <signal.h>
9074 #if !defined(GC_NO_TYPES) && !defined(SN_TARGET_PSP2) \
9075      && !defined(__CC_ARM)
9076 #include <sys/types.h>
9077 #endif
9078 #endif
9079 word GC_non_gc_bytes = 0;
9080 word GC_gc_no = 0;
9081 #ifndef NO_CLOCK
9082   static unsigned long full_gc_total_time = 0;
9083   static unsigned full_gc_total_ns_frac = 0;
9084   static GC_bool measure_performance = FALSE;
9085   GC_API void GC_CALL GC_start_performance_measurement(void)
9086   {
9087     measure_performance = TRUE;
9088   }
9089   GC_API unsigned long GC_CALL GC_get_full_gc_total_time(void)
9090   {
9091     return full_gc_total_time;
9092   }
9093 #endif
9094 #ifndef GC_DISABLE_INCREMENTAL
9095   GC_INNER GC_bool GC_incremental = FALSE;
9096   STATIC GC_bool GC_should_start_incremental_collection = FALSE;
9097 #endif
9098 GC_API int GC_CALL GC_is_incremental_mode(void)
9099 {
9100   return (int)GC_incremental;
9101 }
9102 #ifdef THREADS
9103   int GC_parallel = FALSE;
9104 #endif
9105 #if defined(GC_FULL_FREQ) && !defined(CPPCHECK)
9106   int GC_full_freq = GC_FULL_FREQ;
9107 #else
9108   int GC_full_freq = 19;
9109 #endif
9110 STATIC GC_bool GC_need_full_gc = FALSE;
9111 #ifdef THREAD_LOCAL_ALLOC
9112   GC_INNER GC_bool GC_world_stopped = FALSE;
9113 #endif
9114 STATIC GC_bool GC_disable_automatic_collection = FALSE;
9115 GC_API void GC_CALL GC_set_disable_automatic_collection(int value)
9116 {
9117   DCL_LOCK_STATE;
9118   LOCK();
9119   GC_disable_automatic_collection = (GC_bool)value;
9120   UNLOCK();
9121 }
9122 GC_API int GC_CALL GC_get_disable_automatic_collection(void)
9123 {
9124   int value;
9125   DCL_LOCK_STATE;
9126   LOCK();
9127   value = (int)GC_disable_automatic_collection;
9128   UNLOCK();
9129   return value;
9130 }
9131 STATIC word GC_used_heap_size_after_full = 0;
9132 EXTERN_C_BEGIN
9133 extern const char * const GC_copyright[];
9134 EXTERN_C_END
9135 const char * const GC_copyright[] =
9136 {"Copyright 1988, 1989 Hans-J. Boehm and Alan J. Demers ",
9137 "Copyright (c) 1991-1995 by Xerox Corporation.  All rights reserved. ",
9138 "Copyright (c) 1996-1998 by Silicon Graphics.  All rights reserved. ",
9139 "Copyright (c) 1999-2009 by Hewlett-Packard Company.  All rights reserved. ",
9140 "Copyright (c) 2008-2021 Ivan Maidanski ",
9141 "THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY",
9142 " EXPRESSED OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.",
9143 "See source code for details." };
9144 #ifndef GC_NO_VERSION_VAR
9145   EXTERN_C_BEGIN
9146   extern const unsigned GC_version;
9147   EXTERN_C_END
9148   const unsigned GC_version = ((GC_VERSION_MAJOR << 16) |
9149                         (GC_VERSION_MINOR << 8) | GC_VERSION_MICRO);
9150 #endif
9151 GC_API unsigned GC_CALL GC_get_version(void)
9152 {
9153   return (GC_VERSION_MAJOR << 16) | (GC_VERSION_MINOR << 8) |
9154           GC_VERSION_MICRO;
9155 }
9156 #ifdef GC_DONT_EXPAND
9157   int GC_dont_expand = TRUE;
9158 #else
9159   int GC_dont_expand = FALSE;
9160 #endif
9161 #if defined(GC_FREE_SPACE_DIVISOR) && !defined(CPPCHECK)
9162   word GC_free_space_divisor = GC_FREE_SPACE_DIVISOR;
9163 #else
9164   word GC_free_space_divisor = 3;
9165 #endif
9166 GC_INNER int GC_CALLBACK GC_never_stop_func(void)
9167 {
9168   return(0);
9169 }
9170 #if defined(GC_TIME_LIMIT) && !defined(CPPCHECK)
9171   unsigned long GC_time_limit = GC_TIME_LIMIT;
9172 #elif defined(PARALLEL_MARK)
9173   unsigned long GC_time_limit = GC_TIME_UNLIMITED;
9174 #else
9175   unsigned long GC_time_limit = 50;
9176 #endif
9177 #ifndef NO_CLOCK
9178   STATIC unsigned long GC_time_lim_nsec = 0;
9179 #define TV_NSEC_LIMIT (1000UL * 1000)
9180   GC_API void GC_CALL GC_set_time_limit_tv(struct GC_timeval_s tv)
9181   {
9182     GC_ASSERT(tv.tv_ms <= GC_TIME_UNLIMITED);
9183     GC_ASSERT(tv.tv_nsec < TV_NSEC_LIMIT);
9184     GC_time_limit = tv.tv_ms;
9185     GC_time_lim_nsec = tv.tv_nsec;
9186   }
9187   GC_API struct GC_timeval_s GC_CALL GC_get_time_limit_tv(void)
9188   {
9189     struct GC_timeval_s tv;
9190     tv.tv_ms = GC_time_limit;
9191     tv.tv_nsec = GC_time_lim_nsec;
9192     return tv;
9193   }
9194   STATIC CLOCK_TYPE GC_start_time = CLOCK_TYPE_INITIALIZER;
9195 #endif
9196 STATIC int GC_n_attempts = 0;
9197 STATIC GC_stop_func GC_default_stop_func = GC_never_stop_func;
9198 GC_API void GC_CALL GC_set_stop_func(GC_stop_func stop_func)
9199 {
9200   DCL_LOCK_STATE;
9201   GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func));
9202   LOCK();
9203   GC_default_stop_func = stop_func;
9204   UNLOCK();
9205 }
9206 GC_API GC_stop_func GC_CALL GC_get_stop_func(void)
9207 {
9208   GC_stop_func stop_func;
9209   DCL_LOCK_STATE;
9210   LOCK();
9211   stop_func = GC_default_stop_func;
9212   UNLOCK();
9213   return stop_func;
9214 }
9215 #if defined(GC_DISABLE_INCREMENTAL) || defined(NO_CLOCK)
9216 #define GC_timeout_stop_func GC_default_stop_func
9217 #else
9218   STATIC int GC_CALLBACK GC_timeout_stop_func (void)
9219   {
9220     CLOCK_TYPE current_time;
9221     static unsigned count = 0;
9222     unsigned long time_diff, nsec_diff;
9223     if ((*GC_default_stop_func)())
9224       return(1);
9225     if ((count++ & 3) != 0) return(0);
9226     GET_TIME(current_time);
9227     time_diff = MS_TIME_DIFF(current_time,GC_start_time);
9228     nsec_diff = NS_FRAC_TIME_DIFF(current_time, GC_start_time);
9229 #if defined(CPPCHECK)
9230       GC_noop1((word)&nsec_diff);
9231 #endif
9232     if (time_diff >= GC_time_limit
9233         && (time_diff > GC_time_limit || nsec_diff >= GC_time_lim_nsec)) {
9234       GC_COND_LOG_PRINTF("Abandoning stopped marking after %lu ms %lu ns"
9235                          " (attempt %d)\n",
9236                          time_diff, nsec_diff, GC_n_attempts);
9237       return 1;
9238     }
9239     return(0);
9240   }
9241 #endif
9242 #ifdef THREADS
9243   GC_INNER word GC_total_stacksize = 0;
9244 #endif
9245 static size_t min_bytes_allocd_minimum = 1;
9246 GC_API void GC_CALL GC_set_min_bytes_allocd(size_t value)
9247 {
9248     GC_ASSERT(value > 0);
9249     min_bytes_allocd_minimum = value;
9250 }
9251 GC_API size_t GC_CALL GC_get_min_bytes_allocd(void)
9252 {
9253     return min_bytes_allocd_minimum;
9254 }
9255 static word min_bytes_allocd(void)
9256 {
9257     word result;
9258     word stack_size;
9259     word total_root_size;
9260     word scan_size;
9261 #ifdef THREADS
9262       if (GC_need_to_lock) {
9263         stack_size = GC_total_stacksize;
9264 #ifdef DEBUG_THREADS
9265           GC_log_printf("Total stacks size: %lu\n",
9266                         (unsigned long)stack_size);
9267 #endif
9268       } else
9269 #endif
9270      {
9271 #ifdef STACK_NOT_SCANNED
9272         stack_size = 0;
9273 #elif defined(STACK_GROWS_UP)
9274         stack_size = GC_approx_sp() - GC_stackbottom;
9275 #else
9276         stack_size = GC_stackbottom - GC_approx_sp();
9277 #endif
9278     }
9279     total_root_size = 2 * stack_size + GC_root_size;
9280     scan_size = 2 * GC_composite_in_use + GC_atomic_in_use / 4
9281                 + total_root_size;
9282     result = scan_size / GC_free_space_divisor;
9283     if (GC_incremental) {
9284       result /= 2;
9285     }
9286     return result > min_bytes_allocd_minimum
9287             ? result : min_bytes_allocd_minimum;
9288 }
9289 STATIC word GC_non_gc_bytes_at_gc = 0;
9290 STATIC word GC_adj_bytes_allocd(void)
9291 {
9292     signed_word result;
9293     signed_word expl_managed = (signed_word)GC_non_gc_bytes
9294                                 - (signed_word)GC_non_gc_bytes_at_gc;
9295     result = (signed_word)GC_bytes_allocd
9296              + (signed_word)GC_bytes_dropped
9297              - (signed_word)GC_bytes_freed
9298              + (signed_word)GC_finalizer_bytes_freed
9299              - expl_managed;
9300     if (result > (signed_word)GC_bytes_allocd) {
9301         result = GC_bytes_allocd;
9302     }
9303     result += GC_bytes_finalized;
9304     if (result < (signed_word)(GC_bytes_allocd >> 3)) {
9305         return(GC_bytes_allocd >> 3);
9306     } else {
9307         return(result);
9308     }
9309 }
9310 STATIC void GC_clear_a_few_frames(void)
9311 {
9312 #ifndef CLEAR_NWORDS
9313 #define CLEAR_NWORDS 64
9314 #endif
9315     volatile word frames[CLEAR_NWORDS];
9316     BZERO((word *)frames, CLEAR_NWORDS * sizeof(word));
9317 }
9318 STATIC word GC_collect_at_heapsize = GC_WORD_MAX;
9319 GC_API void GC_CALL GC_start_incremental_collection(void)
9320 {
9321 #ifndef GC_DISABLE_INCREMENTAL
9322     DCL_LOCK_STATE;
9323     if (!GC_incremental) return;
9324     LOCK();
9325     GC_should_start_incremental_collection = TRUE;
9326     ENTER_GC();
9327     GC_collect_a_little_inner(1);
9328     EXIT_GC();
9329     UNLOCK();
9330 #endif
9331 }
9332 GC_INNER GC_bool GC_should_collect(void)
9333 {
9334     static word last_min_bytes_allocd;
9335     static word last_gc_no;
9336     GC_ASSERT(I_HOLD_LOCK());
9337     if (last_gc_no != GC_gc_no) {
9338       last_min_bytes_allocd = min_bytes_allocd();
9339       last_gc_no = GC_gc_no;
9340     }
9341 #ifndef GC_DISABLE_INCREMENTAL
9342     if (GC_should_start_incremental_collection) {
9343       GC_should_start_incremental_collection = FALSE;
9344       return TRUE;
9345     }
9346 #endif
9347     if (GC_disable_automatic_collection) return FALSE;
9348     return(GC_adj_bytes_allocd() >= last_min_bytes_allocd
9349            || GC_heapsize >= GC_collect_at_heapsize);
9350 }
9351  GC_start_callback_proc GC_start_call_back = 0;
9352 GC_API void GC_CALL GC_set_start_callback(GC_start_callback_proc fn)
9353 {
9354     DCL_LOCK_STATE;
9355     LOCK();
9356     GC_start_call_back = fn;
9357     UNLOCK();
9358 }
9359 GC_API GC_start_callback_proc GC_CALL GC_get_start_callback(void)
9360 {
9361     GC_start_callback_proc fn;
9362     DCL_LOCK_STATE;
9363     LOCK();
9364     fn = GC_start_call_back;
9365     UNLOCK();
9366     return fn;
9367 }
9368 GC_INLINE void GC_notify_full_gc(void)
9369 {
9370     if (GC_start_call_back != 0) {
9371         (*GC_start_call_back)();
9372     }
9373 }
9374 STATIC GC_bool GC_is_full_gc = FALSE;
9375 STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func);
9376 STATIC void GC_finish_collection(void);
9377 STATIC void GC_maybe_gc(void)
9378 {
9379     GC_ASSERT(I_HOLD_LOCK());
9380     ASSERT_CANCEL_DISABLED();
9381     if (GC_should_collect()) {
9382         static int n_partial_gcs = 0;
9383         if (!GC_incremental) {
9384             GC_try_to_collect_inner(GC_never_stop_func);
9385             n_partial_gcs = 0;
9386             return;
9387         } else {
9388 #ifdef PARALLEL_MARK
9389             if (GC_parallel)
9390               GC_wait_for_reclaim();
9391 #endif
9392           if (GC_need_full_gc || n_partial_gcs >= GC_full_freq) {
9393             GC_COND_LOG_PRINTF(
9394                 "***>Full mark for collection #%lu after %lu allocd bytes\n",
9395                 (unsigned long)GC_gc_no + 1, (unsigned long)GC_bytes_allocd);
9396             GC_promote_black_lists();
9397             (void)GC_reclaim_all((GC_stop_func)0, TRUE);
9398             GC_notify_full_gc();
9399             GC_clear_marks();
9400             n_partial_gcs = 0;
9401             GC_is_full_gc = TRUE;
9402           } else {
9403             n_partial_gcs++;
9404           }
9405         }
9406 #ifndef NO_CLOCK
9407           if (GC_time_limit != GC_TIME_UNLIMITED) { GET_TIME(GC_start_time); }
9408 #endif
9409         if (GC_stopped_mark(GC_time_limit == GC_TIME_UNLIMITED?
9410                             GC_never_stop_func : GC_timeout_stop_func)) {
9411 #ifdef SAVE_CALL_CHAIN
9412                 GC_save_callers(GC_last_stack);
9413 #endif
9414             GC_finish_collection();
9415         } else {
9416             if (!GC_is_full_gc) {
9417                 GC_n_attempts++;
9418             }
9419         }
9420     }
9421 }
9422 STATIC GC_on_collection_event_proc GC_on_collection_event = 0;
9423 GC_API void GC_CALL GC_set_on_collection_event(GC_on_collection_event_proc fn)
9424 {
9425     DCL_LOCK_STATE;
9426     LOCK();
9427     GC_on_collection_event = fn;
9428     UNLOCK();
9429 }
9430 GC_API GC_on_collection_event_proc GC_CALL GC_get_on_collection_event(void)
9431 {
9432     GC_on_collection_event_proc fn;
9433     DCL_LOCK_STATE;
9434     LOCK();
9435     fn = GC_on_collection_event;
9436     UNLOCK();
9437     return fn;
9438 }
9439 GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func stop_func)
9440 {
9441 #ifndef NO_CLOCK
9442       CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
9443       GC_bool start_time_valid;
9444 #endif
9445     ASSERT_CANCEL_DISABLED();
9446     GC_ASSERT(I_HOLD_LOCK());
9447     if (GC_dont_gc || (*stop_func)()) return FALSE;
9448     if (GC_on_collection_event)
9449       GC_on_collection_event(GC_EVENT_START);
9450     if (GC_incremental && GC_collection_in_progress()) {
9451       GC_COND_LOG_PRINTF(
9452             "GC_try_to_collect_inner: finishing collection in progress\n");
9453         while(GC_collection_in_progress()) {
9454             if ((*stop_func)()) {
9455               return(FALSE);
9456             }
9457             ENTER_GC();
9458             GC_collect_a_little_inner(1);
9459             EXIT_GC();
9460         }
9461     }
9462     GC_notify_full_gc();
9463 #ifndef NO_CLOCK
9464       start_time_valid = FALSE;
9465       if ((GC_print_stats | (int)measure_performance) != 0) {
9466         if (GC_print_stats)
9467           GC_log_printf("Initiating full world-stop collection!\n");
9468         start_time_valid = TRUE;
9469         GET_TIME(start_time);
9470       }
9471 #endif
9472     GC_promote_black_lists();
9473 #ifdef PARALLEL_MARK
9474           if (GC_parallel)
9475             GC_wait_for_reclaim();
9476 #endif
9477         if ((GC_find_leak || stop_func != GC_never_stop_func)
9478             && !GC_reclaim_all(stop_func, FALSE)) {
9479             return(FALSE);
9480         }
9481     GC_invalidate_mark_state();
9482     GC_clear_marks();
9483 #ifdef SAVE_CALL_CHAIN
9484         GC_save_callers(GC_last_stack);
9485 #endif
9486     GC_is_full_gc = TRUE;
9487     if (!GC_stopped_mark(stop_func)) {
9488       if (!GC_incremental) {
9489         GC_invalidate_mark_state();
9490         GC_unpromote_black_lists();
9491       }
9492       return(FALSE);
9493     }
9494     GC_finish_collection();
9495 #ifndef NO_CLOCK
9496       if (start_time_valid) {
9497         CLOCK_TYPE current_time;
9498         unsigned long time_diff, ns_frac_diff;
9499         GET_TIME(current_time);
9500         time_diff = MS_TIME_DIFF(current_time, start_time);
9501         ns_frac_diff = NS_FRAC_TIME_DIFF(current_time, start_time);
9502         if (measure_performance) {
9503           full_gc_total_time += time_diff;
9504           full_gc_total_ns_frac += (unsigned)ns_frac_diff;
9505           if (full_gc_total_ns_frac >= 1000000U) {
9506             full_gc_total_ns_frac -= 1000000U;
9507             full_gc_total_time++;
9508           }
9509         }
9510         if (GC_print_stats)
9511           GC_log_printf("Complete collection took %lu ms %lu ns\n",
9512                         time_diff, ns_frac_diff);
9513       }
9514 #endif
9515     if (GC_on_collection_event)
9516       GC_on_collection_event(GC_EVENT_END);
9517     return(TRUE);
9518 }
9519 #ifndef GC_RATE
9520 #define GC_RATE 10
9521 #endif
9522 #ifndef MAX_PRIOR_ATTEMPTS
9523 #define MAX_PRIOR_ATTEMPTS 1
9524 #endif
9525 STATIC int GC_deficit = 0;
9526 STATIC int GC_rate = GC_RATE;
9527 GC_API void GC_CALL GC_set_rate(int value)
9528 {
9529     GC_ASSERT(value > 0);
9530     GC_rate = value;
9531 }
9532 GC_API int GC_CALL GC_get_rate(void)
9533 {
9534     return GC_rate;
9535 }
9536 static int max_prior_attempts = MAX_PRIOR_ATTEMPTS;
9537 GC_API void GC_CALL GC_set_max_prior_attempts(int value)
9538 {
9539     GC_ASSERT(value >= 0);
9540     max_prior_attempts = value;
9541 }
9542 GC_API int GC_CALL GC_get_max_prior_attempts(void)
9543 {
9544     return max_prior_attempts;
9545 }
9546 GC_INNER void GC_collect_a_little_inner(int n)
9547 {
9548     IF_CANCEL(int cancel_state;)
9549     GC_ASSERT(I_HOLD_LOCK());
9550     if (GC_dont_gc) return;
9551     DISABLE_CANCEL(cancel_state);
9552     if (GC_incremental && GC_collection_in_progress()) {
9553         int i;
9554         int max_deficit = GC_rate * n;
9555 #ifdef PARALLEL_MARK
9556             if (GC_time_limit != GC_TIME_UNLIMITED)
9557                 GC_parallel_mark_disabled = TRUE;
9558 #endif
9559         for (i = GC_deficit; i < max_deficit; i++) {
9560             if (GC_mark_some(NULL))
9561                 break;
9562         }
9563 #ifdef PARALLEL_MARK
9564             GC_parallel_mark_disabled = FALSE;
9565 #endif
9566         if (i < max_deficit) {
9567 #ifdef SAVE_CALL_CHAIN
9568                 GC_save_callers(GC_last_stack);
9569 #endif
9570 #ifdef PARALLEL_MARK
9571                 if (GC_parallel)
9572                     GC_wait_for_reclaim();
9573 #endif
9574             if (GC_n_attempts < max_prior_attempts
9575                 && GC_time_limit != GC_TIME_UNLIMITED) {
9576 #ifndef NO_CLOCK
9577                     GET_TIME(GC_start_time);
9578 #endif
9579                 if (GC_stopped_mark(GC_timeout_stop_func)) {
9580                     GC_finish_collection();
9581                 } else {
9582                     GC_n_attempts++;
9583                 }
9584             } else {
9585                 (void)GC_stopped_mark(GC_never_stop_func);
9586                 GC_finish_collection();
9587             }
9588         }
9589         if (GC_deficit > 0) {
9590             GC_deficit -= max_deficit;
9591             if (GC_deficit < 0)
9592                 GC_deficit = 0;
9593         }
9594     } else {
9595         GC_maybe_gc();
9596     }
9597     RESTORE_CANCEL(cancel_state);
9598 }
9599 GC_INNER void (*GC_check_heap)(void) = 0;
9600 GC_INNER void (*GC_print_all_smashed)(void) = 0;
9601 GC_API int GC_CALL GC_collect_a_little(void)
9602 {
9603     int result;
9604     DCL_LOCK_STATE;
9605     LOCK();
9606     ENTER_GC();
9607     GC_collect_a_little_inner(1);
9608     EXIT_GC();
9609     result = (int)GC_collection_in_progress();
9610     UNLOCK();
9611     if (!result && GC_debugging_started) GC_print_all_smashed();
9612     return(result);
9613 }
9614 #ifndef NO_CLOCK
9615   static unsigned world_stopped_total_time = 0;
9616   static unsigned world_stopped_total_divisor = 0;
9617 #ifndef MAX_TOTAL_TIME_DIVISOR
9618 #define MAX_TOTAL_TIME_DIVISOR 1000
9619 #endif
9620 #endif
9621 #ifdef USE_MUNMAP
9622 #define IF_USE_MUNMAP(x) x
9623 #define COMMA_IF_USE_MUNMAP(x) , x
9624 #else
9625 #define IF_USE_MUNMAP(x)
9626 #define COMMA_IF_USE_MUNMAP(x)
9627 #endif
9628 STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func)
9629 {
9630     int i;
9631 #ifndef NO_CLOCK
9632       CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
9633 #endif
9634     GC_ASSERT(I_HOLD_LOCK());
9635 #if !defined(REDIRECT_MALLOC) && defined(USE_WINALLOC)
9636         GC_add_current_malloc_heap();
9637 #endif
9638 #if defined(REGISTER_LIBRARIES_EARLY)
9639         GC_cond_register_dynamic_libraries();
9640 #endif
9641 #ifndef NO_CLOCK
9642       if (GC_PRINT_STATS_FLAG)
9643         GET_TIME(start_time);
9644 #endif
9645 #if !defined(GC_NO_FINALIZATION) && !defined(GC_TOGGLE_REFS_NOT_NEEDED)
9646       GC_process_togglerefs();
9647 #endif
9648 #ifdef THREADS
9649       if (GC_on_collection_event)
9650         GC_on_collection_event(GC_EVENT_PRE_STOP_WORLD);
9651 #endif
9652     STOP_WORLD();
9653 #ifdef THREADS
9654       if (GC_on_collection_event)
9655         GC_on_collection_event(GC_EVENT_POST_STOP_WORLD);
9656 #endif
9657 #ifdef THREAD_LOCAL_ALLOC
9658       GC_world_stopped = TRUE;
9659 #endif
9660     GC_COND_LOG_PRINTF(
9661               "\n--> Marking for collection #%lu after %lu allocated bytes\n",
9662               (unsigned long)GC_gc_no + 1, (unsigned long) GC_bytes_allocd);
9663 #ifdef MAKE_BACK_GRAPH
9664       if (GC_print_back_height) {
9665         GC_build_back_graph();
9666       }
9667 #endif
9668         if (GC_on_collection_event)
9669           GC_on_collection_event(GC_EVENT_MARK_START);
9670             GC_clear_a_few_frames();
9671             GC_noop6(0,0,0,0,0,0);
9672         GC_initiate_gc();
9673 #ifdef PARALLEL_MARK
9674           if (stop_func != GC_never_stop_func)
9675             GC_parallel_mark_disabled = TRUE;
9676 #endif
9677         for (i = 0; !(*stop_func)(); i++) {
9678           if (GC_mark_some(GC_approx_sp())) {
9679 #ifdef PARALLEL_MARK
9680               if (GC_parallel && GC_parallel_mark_disabled) {
9681                 GC_COND_LOG_PRINTF("Stopped marking done after %d iterations"
9682                                    " with disabled parallel marker\n", i);
9683               }
9684 #endif
9685             i = -1;
9686             break;
9687           }
9688         }
9689 #ifdef PARALLEL_MARK
9690           GC_parallel_mark_disabled = FALSE;
9691 #endif
9692         if (i >= 0) {
9693           GC_COND_LOG_PRINTF("Abandoned stopped marking after"
9694                              " %d iterations\n", i);
9695           GC_deficit = i;
9696 #ifdef THREAD_LOCAL_ALLOC
9697             GC_world_stopped = FALSE;
9698 #endif
9699 #ifdef THREADS
9700             if (GC_on_collection_event)
9701               GC_on_collection_event(GC_EVENT_PRE_START_WORLD);
9702 #endif
9703           START_WORLD();
9704 #ifdef THREADS
9705             if (GC_on_collection_event)
9706               GC_on_collection_event(GC_EVENT_POST_START_WORLD);
9707 #endif
9708           return FALSE;
9709         }
9710     GC_gc_no++;
9711 #ifdef USE_MUNMAP
9712       GC_ASSERT(GC_heapsize >= GC_unmapped_bytes);
9713 #endif
9714     GC_ASSERT(GC_our_mem_bytes >= GC_heapsize);
9715     GC_DBGLOG_PRINTF("GC #%lu freed %ld bytes, heap %lu KiB ("
9716                      IF_USE_MUNMAP("+ %lu KiB unmapped ")
9717                      "+ %lu KiB internal)\n",
9718                      (unsigned long)GC_gc_no, (long)GC_bytes_found,
9719                      TO_KiB_UL(GC_heapsize - GC_unmapped_bytes)
9720                      COMMA_IF_USE_MUNMAP(TO_KiB_UL(GC_unmapped_bytes)),
9721                      TO_KiB_UL(GC_our_mem_bytes - GC_heapsize));
9722     if (GC_debugging_started) {
9723       (*GC_check_heap)();
9724     }
9725     if (GC_on_collection_event) {
9726       GC_on_collection_event(GC_EVENT_MARK_END);
9727 #ifdef THREADS
9728         GC_on_collection_event(GC_EVENT_PRE_START_WORLD);
9729 #endif
9730     }
9731 #ifdef THREAD_LOCAL_ALLOC
9732       GC_world_stopped = FALSE;
9733 #endif
9734     START_WORLD();
9735 #ifdef THREADS
9736       if (GC_on_collection_event)
9737         GC_on_collection_event(GC_EVENT_POST_START_WORLD);
9738 #endif
9739 #ifndef NO_CLOCK
9740       if (GC_PRINT_STATS_FLAG) {
9741         unsigned long time_diff;
9742         unsigned total_time, divisor;
9743         CLOCK_TYPE current_time;
9744         GET_TIME(current_time);
9745         time_diff = MS_TIME_DIFF(current_time,start_time);
9746         total_time = world_stopped_total_time;
9747         divisor = world_stopped_total_divisor;
9748         if ((int)total_time < 0 || divisor >= MAX_TOTAL_TIME_DIVISOR) {
9749           total_time >>= 1;
9750           divisor >>= 1;
9751         }
9752         total_time += time_diff < (((unsigned)-1) >> 1) ?
9753                         (unsigned)time_diff : ((unsigned)-1) >> 1;
9754         world_stopped_total_time = total_time;
9755         world_stopped_total_divisor = ++divisor;
9756         GC_ASSERT(divisor != 0);
9757         GC_log_printf("World-stopped marking took %lu ms %lu ns"
9758                       " (%u ms in average)\n",
9759                       time_diff, NS_FRAC_TIME_DIFF(current_time, start_time),
9760                       total_time / divisor);
9761       }
9762 #endif
9763     return(TRUE);
9764 }
9765 GC_INNER void GC_set_fl_marks(ptr_t q)
9766 {
9767     if (q ) {
9768       struct hblk *h = HBLKPTR(q);
9769       struct hblk *last_h = h;
9770       hdr *hhdr = HDR(h);
9771       IF_PER_OBJ(word sz = hhdr->hb_sz;)
9772       for (;;) {
9773         word bit_no = MARK_BIT_NO((ptr_t)q - (ptr_t)h, sz);
9774         if (!mark_bit_from_hdr(hhdr, bit_no)) {
9775           set_mark_bit_from_hdr(hhdr, bit_no);
9776           ++hhdr -> hb_n_marks;
9777         }
9778         q = (ptr_t)obj_link(q);
9779         if (q == NULL)
9780           break;
9781         h = HBLKPTR(q);
9782         if (h != last_h) {
9783           last_h = h;
9784           hhdr = HDR(h);
9785           IF_PER_OBJ(sz = hhdr->hb_sz;)
9786         }
9787       }
9788     }
9789 }
9790 #if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
9791   void GC_check_fl_marks(void **pfreelist)
9792   {
9793 #if defined(AO_HAVE_load_acquire_read) && !defined(THREAD_SANITIZER)
9794       AO_t *list = (AO_t *)AO_load_acquire_read((AO_t *)pfreelist);
9795       AO_t *prev;
9796       AO_t *p;
9797       if ((word)list <= HBLKSIZE) return;
9798       prev = (AO_t *)pfreelist;
9799       for (p = list; p != NULL;) {
9800         AO_t *next;
9801         if (!GC_is_marked(p)) {
9802           ABORT_ARG2("Unmarked local free list entry",
9803                      ": object %p on list %p", (void *)p, (void *)list);
9804         }
9805         next = (AO_t *)AO_load_acquire_read(p);
9806         if (AO_load(prev) != (AO_t)p)
9807           break;
9808         prev = p;
9809         p = next;
9810       }
9811 #else
9812       (void)pfreelist;
9813 #endif
9814   }
9815 #endif
9816 STATIC void GC_clear_fl_marks(ptr_t q)
9817 {
9818       struct hblk *h = HBLKPTR(q);
9819       struct hblk *last_h = h;
9820       hdr *hhdr = HDR(h);
9821       word sz = hhdr->hb_sz;
9822       for (;;) {
9823         word bit_no = MARK_BIT_NO((ptr_t)q - (ptr_t)h, sz);
9824         if (mark_bit_from_hdr(hhdr, bit_no)) {
9825           size_t n_marks = hhdr -> hb_n_marks;
9826           GC_ASSERT(n_marks != 0);
9827           clear_mark_bit_from_hdr(hhdr, bit_no);
9828           n_marks--;
9829 #ifdef PARALLEL_MARK
9830             if (0 != n_marks || !GC_parallel) {
9831               hhdr -> hb_n_marks = n_marks;
9832             }
9833 #else
9834             hhdr -> hb_n_marks = n_marks;
9835 #endif
9836         }
9837         GC_bytes_found -= sz;
9838         q = (ptr_t)obj_link(q);
9839         if (q == NULL)
9840           break;
9841         h = HBLKPTR(q);
9842         if (h != last_h) {
9843           last_h = h;
9844           hhdr = HDR(h);
9845           sz = hhdr->hb_sz;
9846         }
9847       }
9848 }
9849 #if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
9850   void GC_check_tls(void);
9851 #endif
9852 GC_on_heap_resize_proc GC_on_heap_resize = 0;
9853 GC_INLINE int GC_compute_heap_usage_percent(void)
9854 {
9855   word used = GC_composite_in_use + GC_atomic_in_use;
9856   word heap_sz = GC_heapsize - GC_unmapped_bytes;
9857 #if defined(CPPCHECK)
9858     word limit = (GC_WORD_MAX >> 1) / 50;
9859 #else
9860     const word limit = GC_WORD_MAX / 100;
9861 #endif
9862   return used >= heap_sz ? 0 : used < limit ?
9863                 (int)((used * 100) / heap_sz) : (int)(used / (heap_sz / 100));
9864 }
9865 STATIC void GC_finish_collection(void)
9866 {
9867 #ifndef NO_CLOCK
9868       CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
9869       CLOCK_TYPE finalize_time = CLOCK_TYPE_INITIALIZER;
9870 #endif
9871     GC_ASSERT(I_HOLD_LOCK());
9872 #if defined(GC_ASSERTIONS) \
9873        && defined(THREAD_LOCAL_ALLOC) && !defined(DBG_HDRS_ALL)
9874         GC_check_tls();
9875 #endif
9876 #ifndef NO_CLOCK
9877       if (GC_print_stats)
9878         GET_TIME(start_time);
9879 #endif
9880     if (GC_on_collection_event)
9881       GC_on_collection_event(GC_EVENT_RECLAIM_START);
9882 #ifndef GC_GET_HEAP_USAGE_NOT_NEEDED
9883       if (GC_bytes_found > 0)
9884         GC_reclaimed_bytes_before_gc += (word)GC_bytes_found;
9885 #endif
9886     GC_bytes_found = 0;
9887 #if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
9888         if (GETENV("GC_PRINT_ADDRESS_MAP") != 0) {
9889           GC_print_address_map();
9890         }
9891 #endif
9892     COND_DUMP;
9893     if (GC_find_leak) {
9894       word size;
9895       unsigned kind;
9896       ptr_t q;
9897       for (kind = 0; kind < GC_n_kinds; kind++) {
9898         for (size = 1; size <= MAXOBJGRANULES; size++) {
9899           q = (ptr_t)GC_obj_kinds[kind].ok_freelist[size];
9900           if (q != NULL)
9901             GC_set_fl_marks(q);
9902         }
9903       }
9904       GC_start_reclaim(TRUE);
9905     }
9906 #ifndef GC_NO_FINALIZATION
9907       GC_finalize();
9908 #endif
9909 #ifndef NO_CLOCK
9910       if (GC_print_stats)
9911         GET_TIME(finalize_time);
9912 #endif
9913     if (GC_print_back_height) {
9914 #ifdef MAKE_BACK_GRAPH
9915         GC_traverse_back_graph();
9916 #elif !defined(SMALL_CONFIG)
9917         GC_err_printf("Back height not available: "
9918                       "Rebuild collector with -DMAKE_BACK_GRAPH\n");
9919 #endif
9920     }
9921     {
9922       word size;
9923       ptr_t q;
9924       unsigned kind;
9925       for (kind = 0; kind < GC_n_kinds; kind++) {
9926         for (size = 1; size <= MAXOBJGRANULES; size++) {
9927           q = (ptr_t)GC_obj_kinds[kind].ok_freelist[size];
9928           if (q != NULL)
9929             GC_clear_fl_marks(q);
9930         }
9931       }
9932     }
9933     GC_VERBOSE_LOG_PRINTF("Bytes recovered before sweep - f.l. count = %ld\n",
9934                           (long)GC_bytes_found);
9935     GC_start_reclaim(FALSE);
9936     GC_DBGLOG_PRINTF("In-use heap: %d%% (%lu KiB pointers + %lu KiB other)\n",
9937                      GC_compute_heap_usage_percent(),
9938                      TO_KiB_UL(GC_composite_in_use),
9939                      TO_KiB_UL(GC_atomic_in_use));
9940     if (GC_is_full_gc) {
9941         GC_used_heap_size_after_full = USED_HEAP_SIZE;
9942         GC_need_full_gc = FALSE;
9943     } else {
9944         GC_need_full_gc = USED_HEAP_SIZE - GC_used_heap_size_after_full
9945                             > min_bytes_allocd();
9946     }
9947     GC_VERBOSE_LOG_PRINTF("Immediately reclaimed %ld bytes, heapsize:"
9948                           " %lu bytes" IF_USE_MUNMAP(" (%lu unmapped)") "\n",
9949                           (long)GC_bytes_found,
9950                           (unsigned long)GC_heapsize
9951                           COMMA_IF_USE_MUNMAP((unsigned long)
9952                                               GC_unmapped_bytes));
9953     GC_n_attempts = 0;
9954     GC_is_full_gc = FALSE;
9955     GC_bytes_allocd_before_gc += GC_bytes_allocd;
9956     GC_non_gc_bytes_at_gc = GC_non_gc_bytes;
9957     GC_bytes_allocd = 0;
9958     GC_bytes_dropped = 0;
9959     GC_bytes_freed = 0;
9960     GC_finalizer_bytes_freed = 0;
9961     IF_USE_MUNMAP(GC_unmap_old());
9962     if (GC_on_collection_event)
9963       GC_on_collection_event(GC_EVENT_RECLAIM_END);
9964 #ifndef NO_CLOCK
9965       if (GC_print_stats) {
9966         CLOCK_TYPE done_time;
9967         GET_TIME(done_time);
9968 #if !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION)
9969           GC_print_finalization_stats();
9970 #endif
9971         GC_log_printf("Finalize and initiate sweep took %lu ms %lu ns"
9972                       " + %lu ms %lu ns\n",
9973                       MS_TIME_DIFF(finalize_time, start_time),
9974                       NS_FRAC_TIME_DIFF(finalize_time, start_time),
9975                       MS_TIME_DIFF(done_time, finalize_time),
9976                       NS_FRAC_TIME_DIFF(done_time, finalize_time));
9977       }
9978 #elif !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION)
9979       if (GC_print_stats)
9980         GC_print_finalization_stats();
9981 #endif
9982 }
9983 STATIC GC_bool GC_try_to_collect_general(GC_stop_func stop_func,
9984                                          GC_bool force_unmap GC_ATTR_UNUSED)
9985 {
9986     GC_bool result;
9987     IF_USE_MUNMAP(int old_unmap_threshold;)
9988     IF_CANCEL(int cancel_state;)
9989     DCL_LOCK_STATE;
9990     if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
9991     if (GC_debugging_started) GC_print_all_smashed();
9992     GC_INVOKE_FINALIZERS();
9993     LOCK();
9994     DISABLE_CANCEL(cancel_state);
9995 #ifdef USE_MUNMAP
9996       old_unmap_threshold = GC_unmap_threshold;
9997       if (force_unmap ||
9998           (GC_force_unmap_on_gcollect && old_unmap_threshold > 0))
9999         GC_unmap_threshold = 1;
10000 #endif
10001     ENTER_GC();
10002       GC_noop6(0,0,0,0,0,0);
10003     result = GC_try_to_collect_inner(stop_func != 0 ? stop_func :
10004                                      GC_default_stop_func);
10005     EXIT_GC();
10006     IF_USE_MUNMAP(GC_unmap_threshold = old_unmap_threshold);
10007     RESTORE_CANCEL(cancel_state);
10008     UNLOCK();
10009     if (result) {
10010         if (GC_debugging_started) GC_print_all_smashed();
10011         GC_INVOKE_FINALIZERS();
10012     }
10013     return(result);
10014 }
10015 GC_API int GC_CALL GC_try_to_collect(GC_stop_func stop_func)
10016 {
10017     GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func));
10018     return (int)GC_try_to_collect_general(stop_func, FALSE);
10019 }
10020 GC_API void GC_CALL GC_gcollect(void)
10021 {
10022     (void)GC_try_to_collect_general(0, FALSE);
10023     if (GC_have_errors) GC_print_all_errors();
10024 }
10025 STATIC word GC_heapsize_at_forced_unmap = 0;
10026 GC_API void GC_CALL GC_gcollect_and_unmap(void)
10027 {
10028     GC_heapsize_at_forced_unmap = GC_heapsize;
10029     (void)GC_try_to_collect_general(GC_never_stop_func, TRUE);
10030 }
10031 #ifdef USE_PROC_FOR_LIBRARIES
10032   GC_INNER void GC_add_to_our_memory(ptr_t p, size_t bytes)
10033   {
10034     GC_ASSERT(p != NULL);
10035     if (GC_n_memory >= MAX_HEAP_SECTS)
10036       ABORT("Too many GC-allocated memory sections: Increase MAX_HEAP_SECTS");
10037     GC_our_memory[GC_n_memory].hs_start = p;
10038     GC_our_memory[GC_n_memory].hs_bytes = bytes;
10039     GC_n_memory++;
10040     GC_our_mem_bytes += bytes;
10041   }
10042 #endif
10043 STATIC void GC_add_to_heap(struct hblk *p, size_t bytes)
10044 {
10045     hdr * phdr;
10046     word endp;
10047     size_t old_capacity = 0;
10048     void *old_heap_sects = NULL;
10049 #ifdef GC_ASSERTIONS
10050       unsigned i;
10051 #endif
10052     GC_ASSERT((word)p % HBLKSIZE == 0);
10053     GC_ASSERT(bytes % HBLKSIZE == 0);
10054     GC_ASSERT(bytes > 0);
10055     GC_ASSERT(GC_all_nils != NULL);
10056     if (GC_n_heap_sects == GC_capacity_heap_sects) {
10057 #ifndef INITIAL_HEAP_SECTS
10058 #define INITIAL_HEAP_SECTS 32
10059 #endif
10060       size_t new_capacity = GC_n_heap_sects > 0 ?
10061                 (size_t)GC_n_heap_sects * 2 : INITIAL_HEAP_SECTS;
10062       void *new_heap_sects =
10063                 GC_scratch_alloc(new_capacity * sizeof(struct HeapSect));
10064       if (EXPECT(NULL == new_heap_sects, FALSE)) {
10065         new_capacity = (size_t)GC_n_heap_sects + INITIAL_HEAP_SECTS;
10066         new_heap_sects =
10067                 GC_scratch_alloc(new_capacity * sizeof(struct HeapSect));
10068         if (NULL == new_heap_sects)
10069           ABORT("Insufficient memory for heap sections");
10070       }
10071       old_capacity = GC_capacity_heap_sects;
10072       old_heap_sects = GC_heap_sects;
10073       if (GC_n_heap_sects > 0)
10074         BCOPY(old_heap_sects, new_heap_sects,
10075               GC_n_heap_sects * sizeof(struct HeapSect));
10076       GC_capacity_heap_sects = new_capacity;
10077       GC_heap_sects = (struct HeapSect *)new_heap_sects;
10078       GC_COND_LOG_PRINTF("Grew heap sections array to %lu elements\n",
10079                          (unsigned long)new_capacity);
10080     }
10081     while ((word)p <= HBLKSIZE) {
10082         ++p;
10083         bytes -= HBLKSIZE;
10084         if (0 == bytes) return;
10085     }
10086     endp = (word)p + bytes;
10087     if (endp <= (word)p) {
10088         bytes -= HBLKSIZE;
10089         if (0 == bytes) return;
10090         endp -= HBLKSIZE;
10091     }
10092     phdr = GC_install_header(p);
10093     if (0 == phdr) {
10094         return;
10095     }
10096     GC_ASSERT(endp > (word)p && endp == (word)p + bytes);
10097 #ifdef GC_ASSERTIONS
10098       for (i = 0; i < GC_n_heap_sects; i++) {
10099         word hs_start = (word)GC_heap_sects[i].hs_start;
10100         word hs_end = hs_start + GC_heap_sects[i].hs_bytes;
10101         word p_e = (word)p + bytes;
10102         GC_ASSERT(!((hs_start <= (word)p && (word)p < hs_end)
10103                     || (hs_start < p_e && p_e <= hs_end)
10104                     || ((word)p < hs_start && hs_end < p_e)));
10105       }
10106 #endif
10107     GC_heap_sects[GC_n_heap_sects].hs_start = (ptr_t)p;
10108     GC_heap_sects[GC_n_heap_sects].hs_bytes = bytes;
10109     GC_n_heap_sects++;
10110     phdr -> hb_sz = bytes;
10111     phdr -> hb_flags = 0;
10112     GC_freehblk(p);
10113     GC_heapsize += bytes;
10114     GC_collect_at_heapsize += bytes;
10115     if (GC_collect_at_heapsize < GC_heapsize )
10116        GC_collect_at_heapsize = GC_WORD_MAX;
10117     if ((word)p <= (word)GC_least_plausible_heap_addr
10118         || GC_least_plausible_heap_addr == 0) {
10119         GC_least_plausible_heap_addr = (void *)((ptr_t)p - sizeof(word));
10120     }
10121     if ((word)p + bytes >= (word)GC_greatest_plausible_heap_addr) {
10122         GC_greatest_plausible_heap_addr = (void *)endp;
10123     }
10124     if (old_capacity > 0) {
10125 #ifndef GWW_VDB
10126         GC_scratch_recycle_no_gww(old_heap_sects,
10127                                   old_capacity * sizeof(struct HeapSect));
10128 #else
10129         GC_noop1((word)old_heap_sects);
10130 #endif
10131     }
10132 }
10133 #if !defined(NO_DEBUGGING)
10134   void GC_print_heap_sects(void)
10135   {
10136     unsigned i;
10137     GC_printf("Total heap size: %lu" IF_USE_MUNMAP(" (%lu unmapped)") "\n",
10138               (unsigned long)GC_heapsize
10139               COMMA_IF_USE_MUNMAP((unsigned long)GC_unmapped_bytes));
10140     for (i = 0; i < GC_n_heap_sects; i++) {
10141       ptr_t start = GC_heap_sects[i].hs_start;
10142       size_t len = GC_heap_sects[i].hs_bytes;
10143       struct hblk *h;
10144       unsigned nbl = 0;
10145       for (h = (struct hblk *)start; (word)h < (word)(start + len); h++) {
10146         if (GC_is_black_listed(h, HBLKSIZE)) nbl++;
10147       }
10148       GC_printf("Section %d from %p to %p %u/%lu blacklisted\n",
10149                 i, (void *)start, (void *)&start[len],
10150                 nbl, (unsigned long)divHBLKSZ(len));
10151     }
10152   }
10153 #endif
10154 void * GC_least_plausible_heap_addr = (void *)GC_WORD_MAX;
10155 void * GC_greatest_plausible_heap_addr = 0;
10156 GC_INLINE word GC_max(word x, word y)
10157 {
10158     return(x > y? x : y);
10159 }
10160 GC_INLINE word GC_min(word x, word y)
10161 {
10162     return(x < y? x : y);
10163 }
10164 STATIC word GC_max_heapsize = 0;
10165 GC_API void GC_CALL GC_set_max_heap_size(GC_word n)
10166 {
10167     GC_max_heapsize = n;
10168 }
10169 GC_word GC_max_retries = 0;
10170 GC_INNER void GC_scratch_recycle_inner(void *ptr, size_t bytes)
10171 {
10172   size_t page_offset;
10173   size_t displ = 0;
10174   size_t recycled_bytes;
10175   if (NULL == ptr) return;
10176   GC_ASSERT(bytes != 0);
10177   GC_ASSERT(GC_page_size != 0);
10178   page_offset = (word)ptr & (GC_page_size - 1);
10179   if (page_offset != 0)
10180     displ = GC_page_size - page_offset;
10181   recycled_bytes = bytes > displ ? (bytes - displ) & ~(GC_page_size - 1) : 0;
10182   GC_COND_LOG_PRINTF("Recycle %lu/%lu scratch-allocated bytes at %p\n",
10183                 (unsigned long)recycled_bytes, (unsigned long)bytes, ptr);
10184   if (recycled_bytes > 0)
10185     GC_add_to_heap((struct hblk *)((word)ptr + displ), recycled_bytes);
10186 }
10187 GC_INNER GC_bool GC_expand_hp_inner(word n)
10188 {
10189     size_t bytes;
10190     struct hblk * space;
10191     word expansion_slop;
10192     GC_ASSERT(I_HOLD_LOCK());
10193     GC_ASSERT(GC_page_size != 0);
10194     if (n < MINHINCR) n = MINHINCR;
10195     bytes = ROUNDUP_PAGESIZE((size_t)n * HBLKSIZE);
10196     if (GC_max_heapsize != 0
10197         && (GC_max_heapsize < (word)bytes
10198             || GC_heapsize > GC_max_heapsize - (word)bytes)) {
10199         return(FALSE);
10200     }
10201     space = GET_MEM(bytes);
10202     if (EXPECT(NULL == space, FALSE)) {
10203         WARN("Failed to expand heap by %" WARN_PRIdPTR " bytes\n",
10204              (word)bytes);
10205         return(FALSE);
10206     }
10207     GC_add_to_our_memory((ptr_t)space, bytes);
10208     GC_INFOLOG_PRINTF("Grow heap to %lu KiB after %lu bytes allocated\n",
10209                       TO_KiB_UL(GC_heapsize + (word)bytes),
10210                       (unsigned long)GC_bytes_allocd);
10211     expansion_slop = min_bytes_allocd() + 4 * MAXHINCR * HBLKSIZE;
10212     if ((GC_last_heap_addr == 0 && !((word)space & SIGNB))
10213         || (GC_last_heap_addr != 0
10214             && (word)GC_last_heap_addr < (word)space)) {
10215         word new_limit = (word)space + (word)bytes + expansion_slop;
10216         if (new_limit > (word)space) {
10217           GC_greatest_plausible_heap_addr =
10218             (void *)GC_max((word)GC_greatest_plausible_heap_addr,
10219                            (word)new_limit);
10220         }
10221     } else {
10222         word new_limit = (word)space - expansion_slop;
10223         if (new_limit < (word)space) {
10224           GC_least_plausible_heap_addr =
10225             (void *)GC_min((word)GC_least_plausible_heap_addr,
10226                            (word)space - expansion_slop);
10227         }
10228     }
10229     GC_last_heap_addr = (ptr_t)space;
10230     GC_add_to_heap(space, bytes);
10231     GC_collect_at_heapsize =
10232         GC_heapsize + expansion_slop - 2 * MAXHINCR * HBLKSIZE;
10233     if (GC_collect_at_heapsize < GC_heapsize )
10234         GC_collect_at_heapsize = GC_WORD_MAX;
10235     if (GC_on_heap_resize)
10236         (*GC_on_heap_resize)(GC_heapsize);
10237     return(TRUE);
10238 }
10239 GC_API int GC_CALL GC_expand_hp(size_t bytes)
10240 {
10241     int result;
10242     DCL_LOCK_STATE;
10243     if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
10244     LOCK();
10245     result = (int)GC_expand_hp_inner(divHBLKSZ((word)bytes));
10246     if (result) GC_requested_heapsize += bytes;
10247     UNLOCK();
10248     return(result);
10249 }
10250 GC_INNER unsigned GC_fail_count = 0;
10251 #if defined(GC_ALLOCD_BYTES_PER_FINALIZER) && !defined(CPPCHECK)
10252   STATIC word GC_allocd_bytes_per_finalizer = GC_ALLOCD_BYTES_PER_FINALIZER;
10253 #else
10254   STATIC word GC_allocd_bytes_per_finalizer = 10000;
10255 #endif
10256 GC_API void GC_CALL GC_set_allocd_bytes_per_finalizer(GC_word value)
10257 {
10258   GC_allocd_bytes_per_finalizer = value;
10259 }
10260 GC_API GC_word GC_CALL GC_get_allocd_bytes_per_finalizer(void)
10261 {
10262   return GC_allocd_bytes_per_finalizer;
10263 }
10264 static word last_fo_entries = 0;
10265 static word last_bytes_finalized = 0;
10266 GC_INNER GC_bool GC_collect_or_expand(word needed_blocks,
10267                                       GC_bool ignore_off_page,
10268                                       GC_bool retry)
10269 {
10270     GC_bool gc_not_stopped = TRUE;
10271     word blocks_to_get;
10272     IF_CANCEL(int cancel_state;)
10273     GC_ASSERT(I_HOLD_LOCK());
10274     DISABLE_CANCEL(cancel_state);
10275     if (!GC_incremental && !GC_dont_gc &&
10276         ((GC_dont_expand && GC_bytes_allocd > 0)
10277          || (GC_fo_entries > last_fo_entries
10278              && (last_bytes_finalized | GC_bytes_finalized) != 0
10279              && (GC_fo_entries - last_fo_entries)
10280                 * GC_allocd_bytes_per_finalizer > GC_bytes_allocd)
10281          || GC_should_collect())) {
10282       gc_not_stopped = GC_try_to_collect_inner(
10283                         GC_bytes_allocd > 0 && (!GC_dont_expand || !retry) ?
10284                         GC_default_stop_func : GC_never_stop_func);
10285       if (gc_not_stopped == TRUE || !retry) {
10286         last_fo_entries = GC_fo_entries;
10287         last_bytes_finalized = GC_bytes_finalized;
10288         RESTORE_CANCEL(cancel_state);
10289         return(TRUE);
10290       }
10291     }
10292     blocks_to_get = (GC_heapsize - GC_heapsize_at_forced_unmap)
10293                         / (HBLKSIZE * GC_free_space_divisor)
10294                     + needed_blocks;
10295     if (blocks_to_get > MAXHINCR) {
10296       word slop;
10297       if (ignore_off_page) {
10298         slop = 4;
10299       } else {
10300         slop = 2 * divHBLKSZ(BL_LIMIT);
10301         if (slop > needed_blocks) slop = needed_blocks;
10302       }
10303       if (needed_blocks + slop > MAXHINCR) {
10304         blocks_to_get = needed_blocks + slop;
10305       } else {
10306         blocks_to_get = MAXHINCR;
10307       }
10308       if (blocks_to_get > divHBLKSZ(GC_WORD_MAX))
10309         blocks_to_get = divHBLKSZ(GC_WORD_MAX);
10310     }
10311     if (!GC_expand_hp_inner(blocks_to_get)
10312         && (blocks_to_get == needed_blocks
10313             || !GC_expand_hp_inner(needed_blocks))) {
10314       if (gc_not_stopped == FALSE) {
10315         GC_gcollect_inner();
10316         GC_ASSERT(GC_bytes_allocd == 0);
10317       } else if (GC_fail_count++ < GC_max_retries) {
10318         WARN("Out of Memory!  Trying to continue...\n", 0);
10319         GC_gcollect_inner();
10320       } else {
10321 #if !defined(AMIGA) || !defined(GC_AMIGA_FASTALLOC)
10322           WARN("Out of Memory! Heap size: %" WARN_PRIdPTR " MiB."
10323                " Returning NULL!\n", (GC_heapsize - GC_unmapped_bytes) >> 20);
10324 #endif
10325         RESTORE_CANCEL(cancel_state);
10326         return(FALSE);
10327       }
10328     } else if (GC_fail_count) {
10329       GC_COND_LOG_PRINTF("Memory available again...\n");
10330     }
10331     RESTORE_CANCEL(cancel_state);
10332     return(TRUE);
10333 }
10334 GC_INNER ptr_t GC_allocobj(size_t gran, int kind)
10335 {
10336     void ** flh = &(GC_obj_kinds[kind].ok_freelist[gran]);
10337     GC_bool tried_minor = FALSE;
10338     GC_bool retry = FALSE;
10339     GC_ASSERT(I_HOLD_LOCK());
10340     if (gran == 0) return(0);
10341     while (*flh == 0) {
10342       ENTER_GC();
10343 #ifndef GC_DISABLE_INCREMENTAL
10344         if (GC_incremental && GC_time_limit != GC_TIME_UNLIMITED) {
10345           GC_collect_a_little_inner(1);
10346         }
10347 #endif
10348         GC_ASSERT(!GC_is_full_gc
10349                   || NULL == GC_obj_kinds[kind].ok_reclaim_list
10350                   || NULL == GC_obj_kinds[kind].ok_reclaim_list[gran]);
10351         GC_continue_reclaim(gran, kind);
10352       EXIT_GC();
10353 #if defined(CPPCHECK)
10354         GC_noop1((word)&flh);
10355 #endif
10356       if (NULL == *flh) {
10357         GC_new_hblk(gran, kind);
10358 #if defined(CPPCHECK)
10359           GC_noop1((word)&flh);
10360 #endif
10361         if (NULL == *flh) {
10362           ENTER_GC();
10363           if (GC_incremental && GC_time_limit == GC_TIME_UNLIMITED
10364               && !tried_minor) {
10365             GC_collect_a_little_inner(1);
10366             tried_minor = TRUE;
10367           } else {
10368             if (!GC_collect_or_expand(1, FALSE, retry)) {
10369               EXIT_GC();
10370               return(0);
10371             }
10372             retry = TRUE;
10373           }
10374           EXIT_GC();
10375         }
10376       }
10377     }
10378     GC_fail_count = 0;
10379     return (ptr_t)(*flh);
10380 }
10381 #ifndef MSWINCE
10382 #include <errno.h>
10383 #endif
10384 #include <string.h>
10385 #ifndef SHORT_DBG_HDRS
10386   GC_INNER int GC_has_other_debug_info(ptr_t p)
10387   {
10388     ptr_t body = (ptr_t)((oh *)p + 1);
10389     word sz = GC_size(p);
10390     if (HBLKPTR(p) != HBLKPTR((ptr_t)body)
10391         || sz < DEBUG_BYTES + EXTRA_BYTES) {
10392       return 0;
10393     }
10394     if (((oh *)p) -> oh_sf != (START_FLAG ^ (word)body)
10395         && ((word *)p)[BYTES_TO_WORDS(sz)-1] != (END_FLAG ^ (word)body)) {
10396       return 0;
10397     }
10398     if (((oh *)p)->oh_sz == sz) {
10399       return -1;
10400     }
10401     return 1;
10402   }
10403 #endif
10404 #ifdef LINT2
10405   long GC_random(void)
10406   {
10407     static unsigned seed = 1;
10408     seed = (seed * 1103515245U + 12345) & GC_RAND_MAX;
10409     return (long)seed;
10410   }
10411 #endif
10412 #ifdef KEEP_BACK_PTRS
10413 #ifdef LINT2
10414 #define RANDOM() GC_random()
10415 #else
10416 #include <stdlib.h>
10417 #define GC_RAND_MAX RAND_MAX
10418 #if defined(__GLIBC__) || defined(SOLARIS) \
10419      || defined(HPUX) || defined(IRIX5) || defined(OSF1)
10420 #define RANDOM() random()
10421 #else
10422 #define RANDOM() (long)rand()
10423 #endif
10424 #endif
10425   GC_INNER void GC_store_back_pointer(ptr_t source, ptr_t dest)
10426   {
10427     if (GC_HAS_DEBUG_INFO(dest)) {
10428 #ifdef PARALLEL_MARK
10429         AO_store((volatile AO_t *)&((oh *)dest)->oh_back_ptr,
10430                  (AO_t)HIDE_BACK_PTR(source));
10431 #else
10432         ((oh *)dest) -> oh_back_ptr = HIDE_BACK_PTR(source);
10433 #endif
10434     }
10435   }
10436   GC_INNER void GC_marked_for_finalization(ptr_t dest)
10437   {
10438     GC_store_back_pointer(MARKED_FOR_FINALIZATION, dest);
10439   }
10440   GC_API GC_ref_kind GC_CALL GC_get_back_ptr_info(void *dest, void **base_p,
10441                                                   size_t *offset_p)
10442   {
10443     oh * hdr = (oh *)GC_base(dest);
10444     ptr_t bp;
10445     ptr_t bp_base;
10446 #ifdef LINT2
10447       if (!hdr) ABORT("Invalid GC_get_back_ptr_info argument");
10448 #endif
10449     if (!GC_HAS_DEBUG_INFO((ptr_t) hdr)) return GC_NO_SPACE;
10450     bp = (ptr_t)GC_REVEAL_POINTER(hdr -> oh_back_ptr);
10451     if (MARKED_FOR_FINALIZATION == bp) return GC_FINALIZER_REFD;
10452     if (MARKED_FROM_REGISTER == bp) return GC_REFD_FROM_REG;
10453     if (NOT_MARKED == bp) return GC_UNREFERENCED;
10454 #if ALIGNMENT == 1
10455       {
10456         ptr_t alternate_ptr = bp + 1;
10457         ptr_t target = *(ptr_t *)bp;
10458         ptr_t alternate_target = *(ptr_t *)alternate_ptr;
10459         if ((word)alternate_target >= (word)GC_least_plausible_heap_addr
10460             && (word)alternate_target <= (word)GC_greatest_plausible_heap_addr
10461             && ((word)target < (word)GC_least_plausible_heap_addr
10462                 || (word)target > (word)GC_greatest_plausible_heap_addr)) {
10463             bp = alternate_ptr;
10464         }
10465       }
10466 #endif
10467     bp_base = (ptr_t)GC_base(bp);
10468     if (NULL == bp_base) {
10469       *base_p = bp;
10470       *offset_p = 0;
10471       return GC_REFD_FROM_ROOT;
10472     } else {
10473       if (GC_HAS_DEBUG_INFO(bp_base)) bp_base += sizeof(oh);
10474       *base_p = bp_base;
10475       *offset_p = bp - bp_base;
10476       return GC_REFD_FROM_HEAP;
10477     }
10478   }
10479   GC_API void * GC_CALL GC_generate_random_heap_address(void)
10480   {
10481     size_t i;
10482     word heap_offset = RANDOM();
10483     if (GC_heapsize > GC_RAND_MAX) {
10484         heap_offset *= GC_RAND_MAX;
10485         heap_offset += RANDOM();
10486     }
10487     heap_offset %= GC_heapsize;
10488     for (i = 0;; ++i) {
10489         size_t size;
10490         if (i >= GC_n_heap_sects)
10491           ABORT("GC_generate_random_heap_address: size inconsistency");
10492         size = GC_heap_sects[i].hs_bytes;
10493         if (heap_offset < size) {
10494             break;
10495         } else {
10496             heap_offset -= size;
10497         }
10498     }
10499     return GC_heap_sects[i].hs_start + heap_offset;
10500   }
10501   GC_API void * GC_CALL GC_generate_random_valid_address(void)
10502   {
10503     ptr_t result;
10504     ptr_t base;
10505     do {
10506       result = (ptr_t)GC_generate_random_heap_address();
10507       base = (ptr_t)GC_base(result);
10508     } while (NULL == base || !GC_is_marked(base));
10509     return result;
10510   }
10511   GC_API void GC_CALL GC_print_backtrace(void *p)
10512   {
10513     void *current = p;
10514     int i;
10515     GC_ref_kind source;
10516     size_t offset;
10517     void *base;
10518     GC_print_heap_obj((ptr_t)GC_base(current));
10519     for (i = 0; ; ++i) {
10520       source = GC_get_back_ptr_info(current, &base, &offset);
10521       if (GC_UNREFERENCED == source) {
10522         GC_err_printf("Reference could not be found\n");
10523         goto out;
10524       }
10525       if (GC_NO_SPACE == source) {
10526         GC_err_printf("No debug info in object: Can't find reference\n");
10527         goto out;
10528       }
10529       GC_err_printf("Reachable via %d levels of pointers from ", i);
10530       switch(source) {
10531         case GC_REFD_FROM_ROOT:
10532           GC_err_printf("root at %p\n\n", base);
10533           goto out;
10534         case GC_REFD_FROM_REG:
10535           GC_err_printf("root in register\n\n");
10536           goto out;
10537         case GC_FINALIZER_REFD:
10538           GC_err_printf("list of finalizable objects\n\n");
10539           goto out;
10540         case GC_REFD_FROM_HEAP:
10541           GC_err_printf("offset %ld in object:\n", (long)offset);
10542           GC_print_heap_obj((ptr_t)GC_base(base));
10543           break;
10544         default:
10545           GC_err_printf("INTERNAL ERROR: UNEXPECTED SOURCE!!!!\n");
10546           goto out;
10547       }
10548       current = base;
10549     }
10550     out:;
10551   }
10552   GC_INNER void GC_generate_random_backtrace_no_gc(void)
10553   {
10554     void * current;
10555     current = GC_generate_random_valid_address();
10556     GC_printf("\n****Chosen address %p in object\n", current);
10557     GC_print_backtrace(current);
10558   }
10559   GC_API void GC_CALL GC_generate_random_backtrace(void)
10560   {
10561     if (GC_try_to_collect(GC_never_stop_func) == 0) {
10562       GC_err_printf("Cannot generate a backtrace: "
10563                     "garbage collection is disabled!\n");
10564       return;
10565     }
10566     GC_generate_random_backtrace_no_gc();
10567   }
10568 #endif
10569 #define CROSSES_HBLK(p, sz) \
10570         (((word)((p) + sizeof(oh) + (sz) - 1) ^ (word)(p)) >= HBLKSIZE)
10571 GC_INNER void *GC_store_debug_info_inner(void *p, word sz GC_ATTR_UNUSED,
10572                                          const char *string, int linenum)
10573 {
10574     word * result = (word *)((oh *)p + 1);
10575     GC_ASSERT(I_HOLD_LOCK());
10576     GC_ASSERT(GC_size(p) >= sizeof(oh) + sz);
10577     GC_ASSERT(!(SMALL_OBJ(sz) && CROSSES_HBLK((ptr_t)p, sz)));
10578 #ifdef KEEP_BACK_PTRS
10579       ((oh *)p) -> oh_back_ptr = HIDE_BACK_PTR(NOT_MARKED);
10580 #endif
10581 #ifdef MAKE_BACK_GRAPH
10582       ((oh *)p) -> oh_bg_ptr = HIDE_BACK_PTR((ptr_t)0);
10583 #endif
10584     ((oh *)p) -> oh_string = string;
10585     ((oh *)p) -> oh_int = linenum;
10586 #ifndef SHORT_DBG_HDRS
10587       ((oh *)p) -> oh_sz = sz;
10588       ((oh *)p) -> oh_sf = START_FLAG ^ (word)result;
10589       ((word *)p)[BYTES_TO_WORDS(GC_size(p))-1] =
10590          result[SIMPLE_ROUNDED_UP_WORDS(sz)] = END_FLAG ^ (word)result;
10591 #endif
10592     return result;
10593 }
10594 static void *store_debug_info(void *p, size_t lb,
10595                               const char *fn, GC_EXTRA_PARAMS)
10596 {
10597     void *result;
10598     DCL_LOCK_STATE;
10599     if (NULL == p) {
10600         GC_err_printf("%s(%lu) returning NULL (%s:%d)\n",
10601                       fn, (unsigned long)lb, s, i);
10602         return NULL;
10603     }
10604     LOCK();
10605     if (!GC_debugging_started)
10606         GC_start_debugging_inner();
10607     ADD_CALL_CHAIN(p, ra);
10608     result = GC_store_debug_info_inner(p, (word)lb, s, i);
10609     UNLOCK();
10610     return result;
10611 }
10612 #ifndef SHORT_DBG_HDRS
10613   STATIC ptr_t GC_check_annotated_obj(oh *ohdr)
10614   {
10615     ptr_t body = (ptr_t)(ohdr + 1);
10616     word gc_sz = GC_size((ptr_t)ohdr);
10617     if (ohdr -> oh_sz + DEBUG_BYTES > gc_sz) {
10618         return((ptr_t)(&(ohdr -> oh_sz)));
10619     }
10620     if (ohdr -> oh_sf != (START_FLAG ^ (word)body)) {
10621         return((ptr_t)(&(ohdr -> oh_sf)));
10622     }
10623     if (((word *)ohdr)[BYTES_TO_WORDS(gc_sz)-1] != (END_FLAG ^ (word)body)) {
10624         return (ptr_t)(&((word *)ohdr)[BYTES_TO_WORDS(gc_sz)-1]);
10625     }
10626     if (((word *)body)[SIMPLE_ROUNDED_UP_WORDS(ohdr -> oh_sz)]
10627         != (END_FLAG ^ (word)body)) {
10628         return (ptr_t)(&((word *)body)[SIMPLE_ROUNDED_UP_WORDS(ohdr->oh_sz)]);
10629     }
10630     return(0);
10631   }
10632 #endif
10633 STATIC GC_describe_type_fn GC_describe_type_fns[MAXOBJKINDS] = {0};
10634 GC_API void GC_CALL GC_register_describe_type_fn(int kind,
10635                                                  GC_describe_type_fn fn)
10636 {
10637   GC_describe_type_fns[kind] = fn;
10638 }
10639 #define GET_OH_LINENUM(ohdr) ((int)(ohdr)->oh_int)
10640 #ifndef SHORT_DBG_HDRS
10641 #define IF_NOT_SHORTDBG_HDRS(x) x
10642 #define COMMA_IFNOT_SHORTDBG_HDRS(x) , x
10643 #else
10644 #define IF_NOT_SHORTDBG_HDRS(x)
10645 #define COMMA_IFNOT_SHORTDBG_HDRS(x)
10646 #endif
10647 STATIC void GC_print_obj(ptr_t p)
10648 {
10649     oh * ohdr = (oh *)GC_base(p);
10650     ptr_t q;
10651     hdr * hhdr;
10652     int kind;
10653     const char *kind_str;
10654     char buffer[GC_TYPE_DESCR_LEN + 1];
10655     GC_ASSERT(I_DONT_HOLD_LOCK());
10656 #ifdef LINT2
10657       if (!ohdr) ABORT("Invalid GC_print_obj argument");
10658 #endif
10659     q = (ptr_t)(ohdr + 1);
10660     hhdr = GC_find_header(q);
10661     kind = hhdr -> hb_obj_kind;
10662     if (0 != GC_describe_type_fns[kind] && GC_is_marked(ohdr)) {
10663         buffer[GC_TYPE_DESCR_LEN] = 0;
10664         (GC_describe_type_fns[kind])(q, buffer);
10665         GC_ASSERT(buffer[GC_TYPE_DESCR_LEN] == 0);
10666         kind_str = buffer;
10667     } else {
10668         switch(kind) {
10669           case PTRFREE:
10670             kind_str = "PTRFREE";
10671             break;
10672           case NORMAL:
10673             kind_str = "NORMAL";
10674             break;
10675           case UNCOLLECTABLE:
10676             kind_str = "UNCOLLECTABLE";
10677             break;
10678 #ifdef GC_ATOMIC_UNCOLLECTABLE
10679             case AUNCOLLECTABLE:
10680               kind_str = "ATOMIC_UNCOLLECTABLE";
10681               break;
10682 #endif
10683           default:
10684             kind_str = NULL;
10685         }
10686     }
10687     if (NULL != kind_str) {
10688         GC_err_printf("%p (%s:%d," IF_NOT_SHORTDBG_HDRS(" sz= %lu,") " %s)\n",
10689                       (void *)((ptr_t)ohdr + sizeof(oh)),
10690                       ohdr->oh_string, GET_OH_LINENUM(ohdr)
10691                       COMMA_IFNOT_SHORTDBG_HDRS((unsigned long)ohdr->oh_sz),
10692                       kind_str);
10693     } else {
10694         GC_err_printf("%p (%s:%d," IF_NOT_SHORTDBG_HDRS(" sz= %lu,")
10695                       " kind= %d, descr= 0x%lx)\n",
10696                       (void *)((ptr_t)ohdr + sizeof(oh)),
10697                       ohdr->oh_string, GET_OH_LINENUM(ohdr)
10698                       COMMA_IFNOT_SHORTDBG_HDRS((unsigned long)ohdr->oh_sz),
10699                       kind, (unsigned long)hhdr->hb_descr);
10700     }
10701     PRINT_CALL_CHAIN(ohdr);
10702 }
10703 STATIC void GC_debug_print_heap_obj_proc(ptr_t p)
10704 {
10705     GC_ASSERT(I_DONT_HOLD_LOCK());
10706     if (GC_HAS_DEBUG_INFO(p)) {
10707         GC_print_obj(p);
10708     } else {
10709         GC_default_print_heap_obj_proc(p);
10710     }
10711 }
10712 #ifndef SHORT_DBG_HDRS
10713   STATIC void GC_print_smashed_obj(const char *msg, void *p,
10714                                    ptr_t clobbered_addr)
10715   {
10716     oh * ohdr = (oh *)GC_base(p);
10717     GC_ASSERT(I_DONT_HOLD_LOCK());
10718 #ifdef LINT2
10719       if (!ohdr) ABORT("Invalid GC_print_smashed_obj argument");
10720 #endif
10721     if ((word)clobbered_addr <= (word)(&ohdr->oh_sz)
10722         || ohdr -> oh_string == 0) {
10723         GC_err_printf(
10724                 "%s %p in or near object at %p(<smashed>, appr. sz= %lu)\n",
10725                 msg, (void *)clobbered_addr, p,
10726                 (unsigned long)(GC_size((ptr_t)ohdr) - DEBUG_BYTES));
10727     } else {
10728         GC_err_printf("%s %p in or near object at %p (%s:%d, sz= %lu)\n",
10729                 msg, (void *)clobbered_addr, p,
10730                 (word)(ohdr -> oh_string) < HBLKSIZE ? "(smashed string)" :
10731                 ohdr -> oh_string[0] == '\0' ? "EMPTY(smashed?)" :
10732                                                 ohdr -> oh_string,
10733                 GET_OH_LINENUM(ohdr), (unsigned long)(ohdr -> oh_sz));
10734         PRINT_CALL_CHAIN(ohdr);
10735     }
10736   }
10737   STATIC void GC_check_heap_proc (void);
10738   STATIC void GC_print_all_smashed_proc (void);
10739 #else
10740   STATIC void GC_do_nothing(void) {}
10741 #endif
10742 GC_INNER void GC_start_debugging_inner(void)
10743 {
10744   GC_ASSERT(I_HOLD_LOCK());
10745 #ifndef SHORT_DBG_HDRS
10746     GC_check_heap = GC_check_heap_proc;
10747     GC_print_all_smashed = GC_print_all_smashed_proc;
10748 #else
10749     GC_check_heap = GC_do_nothing;
10750     GC_print_all_smashed = GC_do_nothing;
10751 #endif
10752   GC_print_heap_obj = GC_debug_print_heap_obj_proc;
10753   GC_debugging_started = TRUE;
10754   GC_register_displacement_inner((word)sizeof(oh));
10755 #if defined(CPPCHECK)
10756     GC_noop1(GC_debug_header_size);
10757 #endif
10758 }
10759 const size_t GC_debug_header_size = sizeof(oh);
10760 GC_API size_t GC_CALL GC_get_debug_header_size(void) {
10761   return sizeof(oh);
10762 }
10763 GC_API void GC_CALL GC_debug_register_displacement(size_t offset)
10764 {
10765   DCL_LOCK_STATE;
10766   LOCK();
10767   GC_register_displacement_inner(offset);
10768   GC_register_displacement_inner((word)sizeof(oh) + offset);
10769   UNLOCK();
10770 }
10771 #ifdef GC_ADD_CALLER
10772 #if defined(HAVE_DLADDR) && defined(GC_HAVE_RETURN_ADDR_PARENT)
10773 #include <dlfcn.h>
10774     STATIC void GC_caller_func_offset(word ad, const char **symp, int *offp)
10775     {
10776       Dl_info caller;
10777       if (ad && dladdr((void *)ad, &caller) && caller.dli_sname != NULL) {
10778         *symp = caller.dli_sname;
10779         *offp = (int)((char *)ad - (char *)caller.dli_saddr);
10780       }
10781       if (NULL == *symp) {
10782         *symp = "unknown";
10783       }
10784     }
10785 #else
10786 #define GC_caller_func_offset(ad, symp, offp) (void)(*(symp) = "unknown")
10787 #endif
10788 #endif
10789 GC_API GC_ATTR_MALLOC void * GC_CALL GC_debug_malloc(size_t lb,
10790                                                      GC_EXTRA_PARAMS)
10791 {
10792     void * result;
10793     result = GC_malloc(SIZET_SAT_ADD(lb, DEBUG_BYTES));
10794 #ifdef GC_ADD_CALLER
10795       if (s == NULL) {
10796         GC_caller_func_offset(ra, &s, &i);
10797       }
10798 #endif
10799     return store_debug_info(result, lb, "GC_debug_malloc", OPT_RA s, i);
10800 }
10801 GC_API GC_ATTR_MALLOC void * GC_CALL
10802     GC_debug_malloc_ignore_off_page(size_t lb, GC_EXTRA_PARAMS)
10803 {
10804     void * result = GC_malloc_ignore_off_page(SIZET_SAT_ADD(lb, DEBUG_BYTES));
10805     return store_debug_info(result, lb, "GC_debug_malloc_ignore_off_page",
10806                             OPT_RA s, i);
10807 }
10808 GC_API GC_ATTR_MALLOC void * GC_CALL
10809     GC_debug_malloc_atomic_ignore_off_page(size_t lb, GC_EXTRA_PARAMS)
10810 {
10811     void * result = GC_malloc_atomic_ignore_off_page(
10812                                 SIZET_SAT_ADD(lb, DEBUG_BYTES));
10813     return store_debug_info(result, lb,
10814                             "GC_debug_malloc_atomic_ignore_off_page",
10815                             OPT_RA s, i);
10816 }
10817 STATIC void * GC_debug_generic_malloc(size_t lb, int knd, GC_EXTRA_PARAMS)
10818 {
10819     void * result = GC_generic_malloc(SIZET_SAT_ADD(lb, DEBUG_BYTES), knd);
10820     return store_debug_info(result, lb, "GC_debug_generic_malloc",
10821                             OPT_RA s, i);
10822 }
10823 #ifdef DBG_HDRS_ALL
10824   GC_INNER void * GC_debug_generic_malloc_inner(size_t lb, int k)
10825   {
10826     void * result;
10827     GC_ASSERT(I_HOLD_LOCK());
10828     result = GC_generic_malloc_inner(SIZET_SAT_ADD(lb, DEBUG_BYTES), k);
10829     if (NULL == result) {
10830         GC_err_printf("GC internal allocation (%lu bytes) returning NULL\n",
10831                        (unsigned long) lb);
10832         return(0);
10833     }
10834     if (!GC_debugging_started) {
10835         GC_start_debugging_inner();
10836     }
10837     ADD_CALL_CHAIN(result, GC_RETURN_ADDR);
10838     return (GC_store_debug_info_inner(result, (word)lb, "INTERNAL", 0));
10839   }
10840   GC_INNER void * GC_debug_generic_malloc_inner_ignore_off_page(size_t lb,
10841                                                                 int k)
10842   {
10843     void * result;
10844     GC_ASSERT(I_HOLD_LOCK());
10845     result = GC_generic_malloc_inner_ignore_off_page(
10846                                 SIZET_SAT_ADD(lb, DEBUG_BYTES), k);
10847     if (NULL == result) {
10848         GC_err_printf("GC internal allocation (%lu bytes) returning NULL\n",
10849                        (unsigned long) lb);
10850         return(0);
10851     }
10852     if (!GC_debugging_started) {
10853         GC_start_debugging_inner();
10854     }
10855     ADD_CALL_CHAIN(result, GC_RETURN_ADDR);
10856     return (GC_store_debug_info_inner(result, (word)lb, "INTERNAL", 0));
10857   }
10858 #endif
10859 #ifndef CPPCHECK
10860   GC_API void * GC_CALL GC_debug_malloc_stubborn(size_t lb, GC_EXTRA_PARAMS)
10861   {
10862     return GC_debug_malloc(lb, OPT_RA s, i);
10863   }
10864   GC_API void GC_CALL GC_debug_change_stubborn(
10865                                 const void * p GC_ATTR_UNUSED) {}
10866 #endif
10867 GC_API void GC_CALL GC_debug_end_stubborn_change(const void *p)
10868 {
10869     const void * q = GC_base_C(p);
10870     if (NULL == q) {
10871         ABORT_ARG1("GC_debug_end_stubborn_change: bad arg", ": %p", p);
10872     }
10873     GC_end_stubborn_change(q);
10874 }
10875 GC_API void GC_CALL GC_debug_ptr_store_and_dirty(void *p, const void *q)
10876 {
10877     *(void **)GC_is_visible(p) = GC_is_valid_displacement((void *)q);
10878     GC_debug_end_stubborn_change(p);
10879     REACHABLE_AFTER_DIRTY(q);
10880 }
10881 GC_API GC_ATTR_MALLOC void * GC_CALL GC_debug_malloc_atomic(size_t lb,
10882                                                             GC_EXTRA_PARAMS)
10883 {
10884     void * result = GC_malloc_atomic(SIZET_SAT_ADD(lb, DEBUG_BYTES));
10885     return store_debug_info(result, lb, "GC_debug_malloc_atomic",
10886                             OPT_RA s, i);
10887 }
10888 GC_API GC_ATTR_MALLOC char * GC_CALL GC_debug_strdup(const char *str,
10889                                                      GC_EXTRA_PARAMS)
10890 {
10891   char *copy;
10892   size_t lb;
10893   if (str == NULL) {
10894     if (GC_find_leak)
10895       GC_err_printf("strdup(NULL) behavior is undefined\n");
10896     return NULL;
10897   }
10898   lb = strlen(str) + 1;
10899   copy = (char *)GC_debug_malloc_atomic(lb, OPT_RA s, i);
10900   if (copy == NULL) {
10901 #ifndef MSWINCE
10902       errno = ENOMEM;
10903 #endif
10904     return NULL;
10905   }
10906   BCOPY(str, copy, lb);
10907   return copy;
10908 }
10909 GC_API GC_ATTR_MALLOC char * GC_CALL GC_debug_strndup(const char *str,
10910                                                 size_t size, GC_EXTRA_PARAMS)
10911 {
10912   char *copy;
10913   size_t len = strlen(str);
10914   if (len > size)
10915     len = size;
10916   copy = (char *)GC_debug_malloc_atomic(len + 1, OPT_RA s, i);
10917   if (copy == NULL) {
10918 #ifndef MSWINCE
10919       errno = ENOMEM;
10920 #endif
10921     return NULL;
10922   }
10923   if (len > 0)
10924     BCOPY(str, copy, len);
10925   copy[len] = '\0';
10926   return copy;
10927 }
10928 #ifdef GC_REQUIRE_WCSDUP
10929 #include <wchar.h>
10930   GC_API GC_ATTR_MALLOC wchar_t * GC_CALL GC_debug_wcsdup(const wchar_t *str,
10931                                                           GC_EXTRA_PARAMS)
10932   {
10933     size_t lb = (wcslen(str) + 1) * sizeof(wchar_t);
10934     wchar_t *copy = (wchar_t *)GC_debug_malloc_atomic(lb, OPT_RA s, i);
10935     if (copy == NULL) {
10936 #ifndef MSWINCE
10937         errno = ENOMEM;
10938 #endif
10939       return NULL;
10940     }
10941     BCOPY(str, copy, lb);
10942     return copy;
10943   }
10944 #endif
10945 GC_API GC_ATTR_MALLOC void * GC_CALL GC_debug_malloc_uncollectable(size_t lb,
10946                                                         GC_EXTRA_PARAMS)
10947 {
10948     void * result = GC_malloc_uncollectable(
10949                                 SIZET_SAT_ADD(lb, UNCOLLECTABLE_DEBUG_BYTES));
10950     return store_debug_info(result, lb, "GC_debug_malloc_uncollectable",
10951                             OPT_RA s, i);
10952 }
10953 #ifdef GC_ATOMIC_UNCOLLECTABLE
10954   GC_API GC_ATTR_MALLOC void * GC_CALL
10955         GC_debug_malloc_atomic_uncollectable(size_t lb, GC_EXTRA_PARAMS)
10956   {
10957     void * result = GC_malloc_atomic_uncollectable(
10958                                 SIZET_SAT_ADD(lb, UNCOLLECTABLE_DEBUG_BYTES));
10959     return store_debug_info(result, lb,
10960                             "GC_debug_malloc_atomic_uncollectable",
10961                             OPT_RA s, i);
10962   }
10963 #endif
10964 #ifndef GC_FREED_MEM_MARKER
10965 #if CPP_WORDSZ == 32
10966 #define GC_FREED_MEM_MARKER 0xdeadbeef
10967 #else
10968 #define GC_FREED_MEM_MARKER GC_WORD_C(0xEFBEADDEdeadbeef)
10969 #endif
10970 #endif
10971 GC_API void GC_CALL GC_debug_free(void * p)
10972 {
10973     ptr_t base;
10974     if (0 == p) return;
10975     base = (ptr_t)GC_base(p);
10976     if (NULL == base) {
10977 #if defined(REDIRECT_MALLOC) \
10978          && ((defined(NEED_CALLINFO) && defined(GC_HAVE_BUILTIN_BACKTRACE)) \
10979              || defined(GC_LINUX_THREADS) || defined(GC_SOLARIS_THREADS) \
10980              || defined(MSWIN32))
10981         if (!GC_is_heap_ptr(p)) return;
10982 #endif
10983       ABORT_ARG1("Invalid pointer passed to free()", ": %p", p);
10984     }
10985     if ((ptr_t)p - (ptr_t)base != sizeof(oh)) {
10986 #if defined(REDIRECT_FREE) && defined(USE_PROC_FOR_LIBRARIES)
10987 #endif
10988       GC_err_printf(
10989                "GC_debug_free called on pointer %p w/o debugging info\n", p);
10990     } else {
10991 #ifndef SHORT_DBG_HDRS
10992         ptr_t clobbered = GC_check_annotated_obj((oh *)base);
10993         word sz = GC_size(base);
10994         if (clobbered != 0) {
10995           GC_have_errors = TRUE;
10996           if (((oh *)base) -> oh_sz == sz) {
10997             GC_print_smashed_obj(
10998                   "GC_debug_free: found previously deallocated (?) object at",
10999                   p, clobbered);
11000             return;
11001           } else {
11002             GC_print_smashed_obj("GC_debug_free: found smashed location at",
11003                                  p, clobbered);
11004           }
11005         }
11006         ((oh *)base) -> oh_sz = sz;
11007 #endif
11008     }
11009     if (GC_find_leak
11010 #ifndef SHORT_DBG_HDRS
11011           && ((ptr_t)p - (ptr_t)base != sizeof(oh) || !GC_findleak_delay_free)
11012 #endif
11013         ) {
11014       GC_free(base);
11015     } else {
11016       hdr * hhdr = HDR(p);
11017       if (hhdr -> hb_obj_kind == UNCOLLECTABLE
11018 #ifdef GC_ATOMIC_UNCOLLECTABLE
11019             || hhdr -> hb_obj_kind == AUNCOLLECTABLE
11020 #endif
11021           ) {
11022         GC_free(base);
11023       } else {
11024         word i;
11025         word sz = hhdr -> hb_sz;
11026         word obj_sz = BYTES_TO_WORDS(sz - sizeof(oh));
11027         for (i = 0; i < obj_sz; ++i)
11028           ((word *)p)[i] = GC_FREED_MEM_MARKER;
11029         GC_ASSERT((word *)p + i == (word *)(base + sz));
11030         LOCK();
11031         GC_bytes_freed += sz;
11032         UNLOCK();
11033       }
11034     }
11035 }
11036 #if defined(THREADS) && defined(DBG_HDRS_ALL)
11037   GC_INNER void GC_debug_free_inner(void * p)
11038   {
11039     ptr_t base = (ptr_t)GC_base(p);
11040     GC_ASSERT((ptr_t)p - (ptr_t)base == sizeof(oh));
11041 #ifdef LINT2
11042       if (!base) ABORT("Invalid GC_debug_free_inner argument");
11043 #endif
11044 #ifndef SHORT_DBG_HDRS
11045       ((oh *)base) -> oh_sz = GC_size(base);
11046 #endif
11047     GC_free_inner(base);
11048   }
11049 #endif
11050 GC_API void * GC_CALL GC_debug_realloc(void * p, size_t lb, GC_EXTRA_PARAMS)
11051 {
11052     void * base;
11053     void * result;
11054     hdr * hhdr;
11055     if (p == 0) {
11056       return GC_debug_malloc(lb, OPT_RA s, i);
11057     }
11058     if (0 == lb)  {
11059       GC_debug_free(p);
11060       return NULL;
11061     }
11062 #ifdef GC_ADD_CALLER
11063       if (s == NULL) {
11064         GC_caller_func_offset(ra, &s, &i);
11065       }
11066 #endif
11067     base = GC_base(p);
11068     if (base == 0) {
11069         ABORT_ARG1("Invalid pointer passed to realloc()", ": %p", p);
11070     }
11071     if ((ptr_t)p - (ptr_t)base != sizeof(oh)) {
11072         GC_err_printf(
11073               "GC_debug_realloc called on pointer %p w/o debugging info\n", p);
11074         return(GC_realloc(p, lb));
11075     }
11076     hhdr = HDR(base);
11077     switch (hhdr -> hb_obj_kind) {
11078       case NORMAL:
11079         result = GC_debug_malloc(lb, OPT_RA s, i);
11080         break;
11081       case PTRFREE:
11082         result = GC_debug_malloc_atomic(lb, OPT_RA s, i);
11083         break;
11084       case UNCOLLECTABLE:
11085         result = GC_debug_malloc_uncollectable(lb, OPT_RA s, i);
11086         break;
11087 #ifdef GC_ATOMIC_UNCOLLECTABLE
11088       case AUNCOLLECTABLE:
11089         result = GC_debug_malloc_atomic_uncollectable(lb, OPT_RA s, i);
11090         break;
11091 #endif
11092       default:
11093         result = NULL;
11094         ABORT_RET("GC_debug_realloc: encountered bad kind");
11095     }
11096     if (result != NULL) {
11097       size_t old_sz;
11098 #ifdef SHORT_DBG_HDRS
11099         old_sz = GC_size(base) - sizeof(oh);
11100 #else
11101         old_sz = ((oh *)base) -> oh_sz;
11102 #endif
11103       if (old_sz > 0)
11104         BCOPY(p, result, old_sz < lb ? old_sz : lb);
11105       GC_debug_free(p);
11106     }
11107     return(result);
11108 }
11109 GC_API GC_ATTR_MALLOC void * GC_CALL
11110     GC_debug_generic_or_special_malloc(size_t lb, int knd, GC_EXTRA_PARAMS)
11111 {
11112     switch (knd) {
11113         case PTRFREE:
11114             return GC_debug_malloc_atomic(lb, OPT_RA s, i);
11115         case NORMAL:
11116             return GC_debug_malloc(lb, OPT_RA s, i);
11117         case UNCOLLECTABLE:
11118             return GC_debug_malloc_uncollectable(lb, OPT_RA s, i);
11119 #ifdef GC_ATOMIC_UNCOLLECTABLE
11120         case AUNCOLLECTABLE:
11121             return GC_debug_malloc_atomic_uncollectable(lb, OPT_RA s, i);
11122 #endif
11123         default:
11124             return GC_debug_generic_malloc(lb, knd, OPT_RA s, i);
11125     }
11126 }
11127 #ifndef SHORT_DBG_HDRS
11128 #ifndef MAX_SMASHED
11129 #define MAX_SMASHED 20
11130 #endif
11131 STATIC ptr_t GC_smashed[MAX_SMASHED] = {0};
11132 STATIC unsigned GC_n_smashed = 0;
11133 STATIC void GC_add_smashed(ptr_t smashed)
11134 {
11135     GC_ASSERT(GC_is_marked(GC_base(smashed)));
11136     GC_smashed[GC_n_smashed] = smashed;
11137     if (GC_n_smashed < MAX_SMASHED - 1) ++GC_n_smashed;
11138     GC_have_errors = TRUE;
11139 }
11140 STATIC void GC_print_all_smashed_proc(void)
11141 {
11142     unsigned i;
11143     GC_ASSERT(I_DONT_HOLD_LOCK());
11144     if (GC_n_smashed == 0) return;
11145     GC_err_printf("GC_check_heap_block: found %u smashed heap objects:\n",
11146                   GC_n_smashed);
11147     for (i = 0; i < GC_n_smashed; ++i) {
11148         ptr_t base = (ptr_t)GC_base(GC_smashed[i]);
11149 #ifdef LINT2
11150           if (!base) ABORT("Invalid GC_smashed element");
11151 #endif
11152         GC_print_smashed_obj("", base + sizeof(oh), GC_smashed[i]);
11153         GC_smashed[i] = 0;
11154     }
11155     GC_n_smashed = 0;
11156 }
11157 STATIC void GC_check_heap_block(struct hblk *hbp, word dummy GC_ATTR_UNUSED)
11158 {
11159     struct hblkhdr * hhdr = HDR(hbp);
11160     word sz = hhdr -> hb_sz;
11161     word bit_no;
11162     char *p, *plim;
11163     p = hbp->hb_body;
11164     if (sz > MAXOBJBYTES) {
11165       plim = p;
11166     } else {
11167       plim = hbp->hb_body + HBLKSIZE - sz;
11168     }
11169     for (bit_no = 0; (word)p <= (word)plim;
11170          bit_no += MARK_BIT_OFFSET(sz), p += sz) {
11171       if (mark_bit_from_hdr(hhdr, bit_no) && GC_HAS_DEBUG_INFO((ptr_t)p)) {
11172         ptr_t clobbered = GC_check_annotated_obj((oh *)p);
11173         if (clobbered != 0)
11174           GC_add_smashed(clobbered);
11175       }
11176     }
11177 }
11178 STATIC void GC_check_heap_proc(void)
11179 {
11180   GC_STATIC_ASSERT((sizeof(oh) & (GRANULE_BYTES - 1)) == 0);
11181   GC_apply_to_all_blocks(GC_check_heap_block, 0);
11182 }
11183 GC_INNER GC_bool GC_check_leaked(ptr_t base)
11184 {
11185   word i;
11186   word obj_sz;
11187   word *p;
11188   if (
11189 #if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
11190         (*(word *)base & 1) != 0 &&
11191 #endif
11192       GC_has_other_debug_info(base) >= 0)
11193     return TRUE;
11194   p = (word *)(base + sizeof(oh));
11195   obj_sz = BYTES_TO_WORDS(HDR(base)->hb_sz - sizeof(oh));
11196   for (i = 0; i < obj_sz; ++i)
11197     if (p[i] != GC_FREED_MEM_MARKER) {
11198         GC_set_mark_bit(base);
11199         GC_add_smashed((ptr_t)(&p[i]));
11200         break;
11201     }
11202   return FALSE;
11203 }
11204 #endif
11205 #ifndef GC_NO_FINALIZATION
11206 struct closure {
11207     GC_finalization_proc cl_fn;
11208     void * cl_data;
11209 };
11210 STATIC void * GC_make_closure(GC_finalization_proc fn, void * data)
11211 {
11212     struct closure * result =
11213 #ifdef DBG_HDRS_ALL
11214       (struct closure *) GC_debug_malloc(sizeof (struct closure),
11215                                          GC_EXTRAS);
11216 #else
11217       (struct closure *) GC_malloc(sizeof (struct closure));
11218 #endif
11219     if (result != 0) {
11220       result -> cl_fn = fn;
11221       result -> cl_data = data;
11222     }
11223     return((void *)result);
11224 }
11225 STATIC void GC_CALLBACK GC_debug_invoke_finalizer(void * obj, void * data)
11226 {
11227     struct closure * cl = (struct closure *) data;
11228     (*(cl -> cl_fn))((void *)((char *)obj + sizeof(oh)), cl -> cl_data);
11229 }
11230 #define OFN_UNSET ((GC_finalization_proc)~(signed_word)0)
11231 static void store_old(void *obj, GC_finalization_proc my_old_fn,
11232                       struct closure *my_old_cd, GC_finalization_proc *ofn,
11233                       void **ocd)
11234 {
11235     if (0 != my_old_fn) {
11236       if (my_old_fn == OFN_UNSET) {
11237         return;
11238       }
11239       if (my_old_fn != GC_debug_invoke_finalizer) {
11240         GC_err_printf("Debuggable object at %p had a non-debug finalizer\n",
11241                       obj);
11242       } else {
11243         if (ofn) *ofn = my_old_cd -> cl_fn;
11244         if (ocd) *ocd = my_old_cd -> cl_data;
11245       }
11246     } else {
11247       if (ofn) *ofn = 0;
11248       if (ocd) *ocd = 0;
11249     }
11250 }
11251 GC_API void GC_CALL GC_debug_register_finalizer(void * obj,
11252                                         GC_finalization_proc fn,
11253                                         void * cd, GC_finalization_proc *ofn,
11254                                         void * *ocd)
11255 {
11256     GC_finalization_proc my_old_fn = OFN_UNSET;
11257     void * my_old_cd;
11258     ptr_t base = (ptr_t)GC_base(obj);
11259     if (NULL == base) {
11260         if (ocd) *ocd = 0;
11261         if (ofn) *ofn = 0;
11262         return;
11263     }
11264     if ((ptr_t)obj - base != sizeof(oh)) {
11265         GC_err_printf("GC_debug_register_finalizer called with"
11266                       " non-base-pointer %p\n", obj);
11267     }
11268     if (0 == fn) {
11269       GC_register_finalizer(base, 0, 0, &my_old_fn, &my_old_cd);
11270     } else {
11271       cd = GC_make_closure(fn, cd);
11272       if (cd == 0) return;
11273       GC_register_finalizer(base, GC_debug_invoke_finalizer,
11274                             cd, &my_old_fn, &my_old_cd);
11275     }
11276     store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
11277 }
11278 GC_API void GC_CALL GC_debug_register_finalizer_no_order
11279                                     (void * obj, GC_finalization_proc fn,
11280                                      void * cd, GC_finalization_proc *ofn,
11281                                      void * *ocd)
11282 {
11283     GC_finalization_proc my_old_fn = OFN_UNSET;
11284     void * my_old_cd;
11285     ptr_t base = (ptr_t)GC_base(obj);
11286     if (NULL == base) {
11287         if (ocd) *ocd = 0;
11288         if (ofn) *ofn = 0;
11289         return;
11290     }
11291     if ((ptr_t)obj - base != sizeof(oh)) {
11292         GC_err_printf("GC_debug_register_finalizer_no_order called with"
11293                       " non-base-pointer %p\n", obj);
11294     }
11295     if (0 == fn) {
11296       GC_register_finalizer_no_order(base, 0, 0, &my_old_fn, &my_old_cd);
11297     } else {
11298       cd = GC_make_closure(fn, cd);
11299       if (cd == 0) return;
11300       GC_register_finalizer_no_order(base, GC_debug_invoke_finalizer,
11301                                      cd, &my_old_fn, &my_old_cd);
11302     }
11303     store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
11304 }
11305 GC_API void GC_CALL GC_debug_register_finalizer_unreachable
11306                                     (void * obj, GC_finalization_proc fn,
11307                                      void * cd, GC_finalization_proc *ofn,
11308                                      void * *ocd)
11309 {
11310     GC_finalization_proc my_old_fn = OFN_UNSET;
11311     void * my_old_cd;
11312     ptr_t base = (ptr_t)GC_base(obj);
11313     if (NULL == base) {
11314         if (ocd) *ocd = 0;
11315         if (ofn) *ofn = 0;
11316         return;
11317     }
11318     if ((ptr_t)obj - base != sizeof(oh)) {
11319         GC_err_printf("GC_debug_register_finalizer_unreachable called with"
11320                       " non-base-pointer %p\n", obj);
11321     }
11322     if (0 == fn) {
11323       GC_register_finalizer_unreachable(base, 0, 0, &my_old_fn, &my_old_cd);
11324     } else {
11325       cd = GC_make_closure(fn, cd);
11326       if (cd == 0) return;
11327       GC_register_finalizer_unreachable(base, GC_debug_invoke_finalizer,
11328                                         cd, &my_old_fn, &my_old_cd);
11329     }
11330     store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
11331 }
11332 GC_API void GC_CALL GC_debug_register_finalizer_ignore_self
11333                                     (void * obj, GC_finalization_proc fn,
11334                                      void * cd, GC_finalization_proc *ofn,
11335                                      void * *ocd)
11336 {
11337     GC_finalization_proc my_old_fn = OFN_UNSET;
11338     void * my_old_cd;
11339     ptr_t base = (ptr_t)GC_base(obj);
11340     if (NULL == base) {
11341         if (ocd) *ocd = 0;
11342         if (ofn) *ofn = 0;
11343         return;
11344     }
11345     if ((ptr_t)obj - base != sizeof(oh)) {
11346         GC_err_printf("GC_debug_register_finalizer_ignore_self called with"
11347                       " non-base-pointer %p\n", obj);
11348     }
11349     if (0 == fn) {
11350       GC_register_finalizer_ignore_self(base, 0, 0, &my_old_fn, &my_old_cd);
11351     } else {
11352       cd = GC_make_closure(fn, cd);
11353       if (cd == 0) return;
11354       GC_register_finalizer_ignore_self(base, GC_debug_invoke_finalizer,
11355                                         cd, &my_old_fn, &my_old_cd);
11356     }
11357     store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
11358 }
11359 #endif
11360 GC_API GC_ATTR_MALLOC void * GC_CALL GC_debug_malloc_replacement(size_t lb)
11361 {
11362     return GC_debug_malloc(lb, GC_DBG_EXTRAS);
11363 }
11364 GC_API void * GC_CALL GC_debug_realloc_replacement(void *p, size_t lb)
11365 {
11366     return GC_debug_realloc(p, lb, GC_DBG_EXTRAS);
11367 }
11368 #ifndef GC_NO_FINALIZATION
11369 #ifndef GC_JAVAXFC_H
11370 #define GC_JAVAXFC_H
11371 #ifndef GC_H
11372 #endif
11373 #ifdef __cplusplus
11374   extern "C" {
11375 #endif
11376 GC_API void GC_CALL GC_finalize_all(void);
11377 #ifdef GC_THREADS
11378 #ifndef GC_SUSPEND_THREAD_ID
11379 #define GC_SUSPEND_THREAD_ID void*
11380 #endif
11381   GC_API void GC_CALL GC_suspend_thread(GC_SUSPEND_THREAD_ID);
11382   GC_API void GC_CALL GC_resume_thread(GC_SUSPEND_THREAD_ID);
11383   GC_API int GC_CALL GC_is_thread_suspended(GC_SUSPEND_THREAD_ID);
11384 #endif
11385 #ifdef __cplusplus
11386   }
11387 #endif
11388 #endif
11389 typedef void (* finalization_mark_proc)(ptr_t );
11390 #define HASH3(addr,size,log_size) \
11391         ((((word)(addr) >> 3) ^ ((word)(addr) >> (3 + (log_size)))) \
11392          & ((size) - 1))
11393 #define HASH2(addr,log_size) HASH3(addr, (word)1 << (log_size), log_size)
11394 struct hash_chain_entry {
11395     word hidden_key;
11396     struct hash_chain_entry * next;
11397 };
11398 struct disappearing_link {
11399     struct hash_chain_entry prolog;
11400 #define dl_hidden_link prolog.hidden_key
11401 #define dl_next(x) (struct disappearing_link *)((x) -> prolog.next)
11402 #define dl_set_next(x, y) \
11403                 (void)((x)->prolog.next = (struct hash_chain_entry *)(y))
11404     word dl_hidden_obj;
11405 };
11406 struct finalizable_object {
11407     struct hash_chain_entry prolog;
11408 #define fo_hidden_base prolog.hidden_key
11409 #define fo_next(x) (struct finalizable_object *)((x) -> prolog.next)
11410 #define fo_set_next(x,y) ((x)->prolog.next = (struct hash_chain_entry *)(y))
11411     GC_finalization_proc fo_fn;
11412     ptr_t fo_client_data;
11413     word fo_object_size;
11414     finalization_mark_proc fo_mark_proc;
11415 };
11416 #ifdef AO_HAVE_store
11417 #define SET_FINALIZE_NOW(fo) \
11418             AO_store((volatile AO_t *)&GC_fnlz_roots.finalize_now, (AO_t)(fo))
11419 #else
11420 #define SET_FINALIZE_NOW(fo) (void)(GC_fnlz_roots.finalize_now = (fo))
11421 #endif
11422 GC_API void GC_CALL GC_push_finalizer_structures(void)
11423 {
11424   GC_ASSERT((word)(&GC_dl_hashtbl.head) % sizeof(word) == 0);
11425   GC_ASSERT((word)(&GC_fnlz_roots) % sizeof(word) == 0);
11426 #ifndef GC_LONG_REFS_NOT_NEEDED
11427     GC_ASSERT((word)(&GC_ll_hashtbl.head) % sizeof(word) == 0);
11428     GC_PUSH_ALL_SYM(GC_ll_hashtbl.head);
11429 #endif
11430   GC_PUSH_ALL_SYM(GC_dl_hashtbl.head);
11431   GC_PUSH_ALL_SYM(GC_fnlz_roots);
11432 }
11433 #ifndef GC_ON_GROW_LOG_SIZE_MIN
11434 #define GC_ON_GROW_LOG_SIZE_MIN CPP_LOG_HBLKSIZE
11435 #endif
11436 STATIC void GC_grow_table(struct hash_chain_entry ***table,
11437                           unsigned *log_size_ptr, word *entries_ptr)
11438 {
11439     word i;
11440     struct hash_chain_entry *p;
11441     unsigned log_old_size = *log_size_ptr;
11442     unsigned log_new_size = log_old_size + 1;
11443     word old_size = *table == NULL ? 0 : (word)1 << log_old_size;
11444     word new_size = (word)1 << log_new_size;
11445     struct hash_chain_entry **new_table;
11446     GC_ASSERT(I_HOLD_LOCK());
11447     if (log_old_size >= GC_ON_GROW_LOG_SIZE_MIN && !GC_incremental) {
11448       IF_CANCEL(int cancel_state;)
11449       DISABLE_CANCEL(cancel_state);
11450       (void)GC_try_to_collect_inner(GC_never_stop_func);
11451       RESTORE_CANCEL(cancel_state);
11452       if (*entries_ptr < ((word)1 << log_old_size) - (*entries_ptr >> 2))
11453         return;
11454     }
11455     new_table = (struct hash_chain_entry **)
11456                     GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(
11457                         (size_t)new_size * sizeof(struct hash_chain_entry *),
11458                         NORMAL);
11459     if (new_table == 0) {
11460         if (*table == 0) {
11461             ABORT("Insufficient space for initial table allocation");
11462         } else {
11463             return;
11464         }
11465     }
11466     for (i = 0; i < old_size; i++) {
11467       p = (*table)[i];
11468       while (p != 0) {
11469         ptr_t real_key = (ptr_t)GC_REVEAL_POINTER(p->hidden_key);
11470         struct hash_chain_entry *next = p -> next;
11471         size_t new_hash = HASH3(real_key, new_size, log_new_size);
11472         p -> next = new_table[new_hash];
11473         GC_dirty(p);
11474         new_table[new_hash] = p;
11475         p = next;
11476       }
11477     }
11478     *log_size_ptr = log_new_size;
11479     *table = new_table;
11480     GC_dirty(new_table);
11481 }
11482 GC_API int GC_CALL GC_register_disappearing_link(void * * link)
11483 {
11484     ptr_t base;
11485     base = (ptr_t)GC_base(link);
11486     if (base == 0)
11487         ABORT("Bad arg to GC_register_disappearing_link");
11488     return(GC_general_register_disappearing_link(link, base));
11489 }
11490 STATIC int GC_register_disappearing_link_inner(
11491                         struct dl_hashtbl_s *dl_hashtbl, void **link,
11492                         const void *obj, const char *tbl_log_name)
11493 {
11494     struct disappearing_link *curr_dl;
11495     size_t index;
11496     struct disappearing_link * new_dl;
11497     DCL_LOCK_STATE;
11498     if (EXPECT(GC_find_leak, FALSE)) return GC_UNIMPLEMENTED;
11499     LOCK();
11500     GC_ASSERT(obj != NULL && GC_base_C(obj) == obj);
11501     if (EXPECT(NULL == dl_hashtbl -> head, FALSE)
11502         || EXPECT(dl_hashtbl -> entries
11503                   > ((word)1 << dl_hashtbl -> log_size), FALSE)) {
11504         GC_grow_table((struct hash_chain_entry ***)&dl_hashtbl -> head,
11505                       &dl_hashtbl -> log_size, &dl_hashtbl -> entries);
11506         GC_COND_LOG_PRINTF("Grew %s table to %u entries\n", tbl_log_name,
11507                            1U << dl_hashtbl -> log_size);
11508     }
11509     index = HASH2(link, dl_hashtbl -> log_size);
11510     for (curr_dl = dl_hashtbl -> head[index]; curr_dl != 0;
11511          curr_dl = dl_next(curr_dl)) {
11512         if (curr_dl -> dl_hidden_link == GC_HIDE_POINTER(link)) {
11513             curr_dl -> dl_hidden_obj = GC_HIDE_POINTER(obj);
11514             UNLOCK();
11515             return GC_DUPLICATE;
11516         }
11517     }
11518     new_dl = (struct disappearing_link *)
11519         GC_INTERNAL_MALLOC(sizeof(struct disappearing_link),NORMAL);
11520     if (0 == new_dl) {
11521       GC_oom_func oom_fn = GC_oom_fn;
11522       UNLOCK();
11523       new_dl = (struct disappearing_link *)
11524                 (*oom_fn)(sizeof(struct disappearing_link));
11525       if (0 == new_dl) {
11526         return GC_NO_MEMORY;
11527       }
11528       LOCK();
11529       index = HASH2(link, dl_hashtbl -> log_size);
11530       for (curr_dl = dl_hashtbl -> head[index]; curr_dl != 0;
11531            curr_dl = dl_next(curr_dl)) {
11532         if (curr_dl -> dl_hidden_link == GC_HIDE_POINTER(link)) {
11533           curr_dl -> dl_hidden_obj = GC_HIDE_POINTER(obj);
11534           UNLOCK();
11535 #ifndef DBG_HDRS_ALL
11536             GC_free((void *)new_dl);
11537 #endif
11538           return GC_DUPLICATE;
11539         }
11540       }
11541     }
11542     new_dl -> dl_hidden_obj = GC_HIDE_POINTER(obj);
11543     new_dl -> dl_hidden_link = GC_HIDE_POINTER(link);
11544     dl_set_next(new_dl, dl_hashtbl -> head[index]);
11545     GC_dirty(new_dl);
11546     dl_hashtbl -> head[index] = new_dl;
11547     dl_hashtbl -> entries++;
11548     GC_dirty(dl_hashtbl->head + index);
11549     UNLOCK();
11550     return GC_SUCCESS;
11551 }
11552 GC_API int GC_CALL GC_general_register_disappearing_link(void * * link,
11553                                                          const void * obj)
11554 {
11555     if (((word)link & (ALIGNMENT-1)) != 0 || !NONNULL_ARG_NOT_NULL(link))
11556         ABORT("Bad arg to GC_general_register_disappearing_link");
11557     return GC_register_disappearing_link_inner(&GC_dl_hashtbl, link, obj,
11558                                                "dl");
11559 }
11560 #ifdef DBG_HDRS_ALL
11561 #define FREE_DL_ENTRY(curr_dl) dl_set_next(curr_dl, NULL)
11562 #else
11563 #define FREE_DL_ENTRY(curr_dl) GC_free(curr_dl)
11564 #endif
11565 GC_INLINE struct disappearing_link *GC_unregister_disappearing_link_inner(
11566                                 struct dl_hashtbl_s *dl_hashtbl, void **link)
11567 {
11568     struct disappearing_link *curr_dl;
11569     struct disappearing_link *prev_dl = NULL;
11570     size_t index;
11571     GC_ASSERT(I_HOLD_LOCK());
11572     if (EXPECT(NULL == dl_hashtbl -> head, FALSE)) return NULL;
11573     index = HASH2(link, dl_hashtbl -> log_size);
11574     for (curr_dl = dl_hashtbl -> head[index]; curr_dl;
11575          curr_dl = dl_next(curr_dl)) {
11576         if (curr_dl -> dl_hidden_link == GC_HIDE_POINTER(link)) {
11577             if (NULL == prev_dl) {
11578                 dl_hashtbl -> head[index] = dl_next(curr_dl);
11579                 GC_dirty(dl_hashtbl->head + index);
11580             } else {
11581                 dl_set_next(prev_dl, dl_next(curr_dl));
11582                 GC_dirty(prev_dl);
11583             }
11584             dl_hashtbl -> entries--;
11585             break;
11586         }
11587         prev_dl = curr_dl;
11588     }
11589     return curr_dl;
11590 }
11591 GC_API int GC_CALL GC_unregister_disappearing_link(void * * link)
11592 {
11593     struct disappearing_link *curr_dl;
11594     DCL_LOCK_STATE;
11595     if (((word)link & (ALIGNMENT-1)) != 0) return(0);
11596     LOCK();
11597     curr_dl = GC_unregister_disappearing_link_inner(&GC_dl_hashtbl, link);
11598     UNLOCK();
11599     if (NULL == curr_dl) return 0;
11600     FREE_DL_ENTRY(curr_dl);
11601     return 1;
11602 }
11603 #ifndef GC_TOGGLE_REFS_NOT_NEEDED
11604   typedef union toggle_ref_u GCToggleRef;
11605   STATIC GC_toggleref_func GC_toggleref_callback = 0;
11606   GC_INNER void GC_process_togglerefs(void)
11607   {
11608     size_t i;
11609     size_t new_size = 0;
11610     GC_bool needs_barrier = FALSE;
11611     GC_ASSERT(I_HOLD_LOCK());
11612     for (i = 0; i < GC_toggleref_array_size; ++i) {
11613       GCToggleRef r = GC_toggleref_arr[i];
11614       void *obj = r.strong_ref;
11615       if (((word)obj & 1) != 0) {
11616         obj = GC_REVEAL_POINTER(r.weak_ref);
11617       }
11618       if (NULL == obj) {
11619         continue;
11620       }
11621       switch (GC_toggleref_callback(obj)) {
11622       case GC_TOGGLE_REF_DROP:
11623         break;
11624       case GC_TOGGLE_REF_STRONG:
11625         GC_toggleref_arr[new_size++].strong_ref = obj;
11626         needs_barrier = TRUE;
11627         break;
11628       case GC_TOGGLE_REF_WEAK:
11629         GC_toggleref_arr[new_size++].weak_ref = GC_HIDE_POINTER(obj);
11630         break;
11631       default:
11632         ABORT("Bad toggle-ref status returned by callback");
11633       }
11634     }
11635     if (new_size < GC_toggleref_array_size) {
11636       BZERO(&GC_toggleref_arr[new_size],
11637             (GC_toggleref_array_size - new_size) * sizeof(GCToggleRef));
11638       GC_toggleref_array_size = new_size;
11639     }
11640     if (needs_barrier)
11641       GC_dirty(GC_toggleref_arr);
11642   }
11643   STATIC void GC_normal_finalize_mark_proc(ptr_t);
11644   static void push_and_mark_object(void *p)
11645   {
11646     GC_normal_finalize_mark_proc((ptr_t)p);
11647     while (!GC_mark_stack_empty()) {
11648       MARK_FROM_MARK_STACK();
11649     }
11650     GC_set_mark_bit(p);
11651     if (GC_mark_state != MS_NONE) {
11652       while (!GC_mark_some(0)) {
11653       }
11654     }
11655   }
11656   STATIC void GC_mark_togglerefs(void)
11657   {
11658     size_t i;
11659     if (NULL == GC_toggleref_arr)
11660       return;
11661     GC_set_mark_bit(GC_toggleref_arr);
11662     for (i = 0; i < GC_toggleref_array_size; ++i) {
11663       void *obj = GC_toggleref_arr[i].strong_ref;
11664       if (obj != NULL && ((word)obj & 1) == 0) {
11665         push_and_mark_object(obj);
11666       }
11667     }
11668   }
11669   STATIC void GC_clear_togglerefs(void)
11670   {
11671     size_t i;
11672     for (i = 0; i < GC_toggleref_array_size; ++i) {
11673       if ((GC_toggleref_arr[i].weak_ref & 1) != 0) {
11674         if (!GC_is_marked(GC_REVEAL_POINTER(GC_toggleref_arr[i].weak_ref))) {
11675           GC_toggleref_arr[i].weak_ref = 0;
11676         } else {
11677         }
11678       }
11679     }
11680   }
11681   GC_API void GC_CALL GC_set_toggleref_func(GC_toggleref_func fn)
11682   {
11683     DCL_LOCK_STATE;
11684     LOCK();
11685     GC_toggleref_callback = fn;
11686     UNLOCK();
11687   }
11688   GC_API GC_toggleref_func GC_CALL GC_get_toggleref_func(void)
11689   {
11690     GC_toggleref_func fn;
11691     DCL_LOCK_STATE;
11692     LOCK();
11693     fn = GC_toggleref_callback;
11694     UNLOCK();
11695     return fn;
11696   }
11697   static GC_bool ensure_toggleref_capacity(size_t capacity_inc)
11698   {
11699     GC_ASSERT(I_HOLD_LOCK());
11700     if (NULL == GC_toggleref_arr) {
11701       GC_toggleref_array_capacity = 32;
11702       GC_toggleref_arr = (GCToggleRef *)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(
11703                         GC_toggleref_array_capacity * sizeof(GCToggleRef),
11704                         NORMAL);
11705       if (NULL == GC_toggleref_arr)
11706         return FALSE;
11707     }
11708     if (GC_toggleref_array_size + capacity_inc
11709         >= GC_toggleref_array_capacity) {
11710       GCToggleRef *new_array;
11711       while (GC_toggleref_array_capacity
11712               < GC_toggleref_array_size + capacity_inc) {
11713         GC_toggleref_array_capacity *= 2;
11714         if ((GC_toggleref_array_capacity
11715              & ((size_t)1 << (sizeof(size_t) * 8 - 1))) != 0)
11716           return FALSE;
11717       }
11718       new_array = (GCToggleRef *)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(
11719                         GC_toggleref_array_capacity * sizeof(GCToggleRef),
11720                         NORMAL);
11721       if (NULL == new_array)
11722         return FALSE;
11723       if (EXPECT(GC_toggleref_array_size > 0, TRUE))
11724         BCOPY(GC_toggleref_arr, new_array,
11725               GC_toggleref_array_size * sizeof(GCToggleRef));
11726       GC_INTERNAL_FREE(GC_toggleref_arr);
11727       GC_toggleref_arr = new_array;
11728     }
11729     return TRUE;
11730   }
11731   GC_API int GC_CALL GC_toggleref_add(void *obj, int is_strong_ref)
11732   {
11733     int res = GC_SUCCESS;
11734     DCL_LOCK_STATE;
11735     GC_ASSERT(NONNULL_ARG_NOT_NULL(obj));
11736     LOCK();
11737     if (GC_toggleref_callback != 0) {
11738       if (!ensure_toggleref_capacity(1)) {
11739         res = GC_NO_MEMORY;
11740       } else {
11741         GC_toggleref_arr[GC_toggleref_array_size].strong_ref =
11742                         is_strong_ref ? obj : (void *)GC_HIDE_POINTER(obj);
11743         if (is_strong_ref)
11744           GC_dirty(GC_toggleref_arr + GC_toggleref_array_size);
11745         GC_toggleref_array_size++;
11746       }
11747     }
11748     UNLOCK();
11749     return res;
11750   }
11751 #endif
11752 STATIC GC_await_finalize_proc GC_object_finalized_proc = 0;
11753 GC_API void GC_CALL GC_set_await_finalize_proc(GC_await_finalize_proc fn)
11754 {
11755   DCL_LOCK_STATE;
11756   LOCK();
11757   GC_object_finalized_proc = fn;
11758   UNLOCK();
11759 }
11760 GC_API GC_await_finalize_proc GC_CALL GC_get_await_finalize_proc(void)
11761 {
11762   GC_await_finalize_proc fn;
11763   DCL_LOCK_STATE;
11764   LOCK();
11765   fn = GC_object_finalized_proc;
11766   UNLOCK();
11767   return fn;
11768 }
11769 #ifndef GC_LONG_REFS_NOT_NEEDED
11770   GC_API int GC_CALL GC_register_long_link(void * * link, const void * obj)
11771   {
11772     if (((word)link & (ALIGNMENT-1)) != 0 || !NONNULL_ARG_NOT_NULL(link))
11773         ABORT("Bad arg to GC_register_long_link");
11774     return GC_register_disappearing_link_inner(&GC_ll_hashtbl, link, obj,
11775                                                "long dl");
11776   }
11777   GC_API int GC_CALL GC_unregister_long_link(void * * link)
11778   {
11779     struct disappearing_link *curr_dl;
11780     DCL_LOCK_STATE;
11781     if (((word)link & (ALIGNMENT-1)) != 0) return(0);
11782     LOCK();
11783     curr_dl = GC_unregister_disappearing_link_inner(&GC_ll_hashtbl, link);
11784     UNLOCK();
11785     if (NULL == curr_dl) return 0;
11786     FREE_DL_ENTRY(curr_dl);
11787     return 1;
11788   }
11789 #endif
11790 #ifndef GC_MOVE_DISAPPEARING_LINK_NOT_NEEDED
11791   STATIC int GC_move_disappearing_link_inner(
11792                                 struct dl_hashtbl_s *dl_hashtbl,
11793                                 void **link, void **new_link)
11794   {
11795     struct disappearing_link *curr_dl, *new_dl;
11796     struct disappearing_link *prev_dl = NULL;
11797     size_t curr_index, new_index;
11798     word curr_hidden_link, new_hidden_link;
11799     GC_ASSERT(I_HOLD_LOCK());
11800     if (EXPECT(NULL == dl_hashtbl -> head, FALSE)) return GC_NOT_FOUND;
11801     curr_index = HASH2(link, dl_hashtbl -> log_size);
11802     curr_hidden_link = GC_HIDE_POINTER(link);
11803     for (curr_dl = dl_hashtbl -> head[curr_index]; curr_dl;
11804          curr_dl = dl_next(curr_dl)) {
11805       if (curr_dl -> dl_hidden_link == curr_hidden_link)
11806         break;
11807       prev_dl = curr_dl;
11808     }
11809     if (EXPECT(NULL == curr_dl, FALSE)) {
11810       return GC_NOT_FOUND;
11811     } else if (link == new_link) {
11812       return GC_SUCCESS;
11813     }
11814     new_index = HASH2(new_link, dl_hashtbl -> log_size);
11815     new_hidden_link = GC_HIDE_POINTER(new_link);
11816     for (new_dl = dl_hashtbl -> head[new_index]; new_dl;
11817          new_dl = dl_next(new_dl)) {
11818       if (new_dl -> dl_hidden_link == new_hidden_link) {
11819         return GC_DUPLICATE;
11820       }
11821     }
11822     if (NULL == prev_dl) {
11823       dl_hashtbl -> head[curr_index] = dl_next(curr_dl);
11824     } else {
11825       dl_set_next(prev_dl, dl_next(curr_dl));
11826       GC_dirty(prev_dl);
11827     }
11828     curr_dl -> dl_hidden_link = new_hidden_link;
11829     dl_set_next(curr_dl, dl_hashtbl -> head[new_index]);
11830     dl_hashtbl -> head[new_index] = curr_dl;
11831     GC_dirty(curr_dl);
11832     GC_dirty(dl_hashtbl->head);
11833     return GC_SUCCESS;
11834   }
11835   GC_API int GC_CALL GC_move_disappearing_link(void **link, void **new_link)
11836   {
11837     int result;
11838     DCL_LOCK_STATE;
11839     if (((word)new_link & (ALIGNMENT-1)) != 0
11840         || !NONNULL_ARG_NOT_NULL(new_link))
11841       ABORT("Bad new_link arg to GC_move_disappearing_link");
11842     if (((word)link & (ALIGNMENT-1)) != 0)
11843       return GC_NOT_FOUND;
11844     LOCK();
11845     result = GC_move_disappearing_link_inner(&GC_dl_hashtbl, link, new_link);
11846     UNLOCK();
11847     return result;
11848   }
11849 #ifndef GC_LONG_REFS_NOT_NEEDED
11850     GC_API int GC_CALL GC_move_long_link(void **link, void **new_link)
11851     {
11852       int result;
11853       DCL_LOCK_STATE;
11854       if (((word)new_link & (ALIGNMENT-1)) != 0
11855           || !NONNULL_ARG_NOT_NULL(new_link))
11856         ABORT("Bad new_link arg to GC_move_long_link");
11857       if (((word)link & (ALIGNMENT-1)) != 0)
11858         return GC_NOT_FOUND;
11859       LOCK();
11860       result = GC_move_disappearing_link_inner(&GC_ll_hashtbl, link, new_link);
11861       UNLOCK();
11862       return result;
11863     }
11864 #endif
11865 #endif
11866 #if defined(_MSC_VER) && defined(I386)
11867   GC_ATTR_NOINLINE
11868 #endif
11869 STATIC void GC_normal_finalize_mark_proc(ptr_t p)
11870 {
11871     GC_mark_stack_top = GC_push_obj(p, HDR(p), GC_mark_stack_top,
11872                                     GC_mark_stack + GC_mark_stack_size);
11873 }
11874 STATIC void GC_ignore_self_finalize_mark_proc(ptr_t p)
11875 {
11876     hdr * hhdr = HDR(p);
11877     word descr = hhdr -> hb_descr;
11878     ptr_t q;
11879     ptr_t scan_limit;
11880     ptr_t target_limit = p + hhdr -> hb_sz - 1;
11881     if ((descr & GC_DS_TAGS) == GC_DS_LENGTH) {
11882        scan_limit = p + descr - sizeof(word);
11883     } else {
11884        scan_limit = target_limit + 1 - sizeof(word);
11885     }
11886     for (q = p; (word)q <= (word)scan_limit; q += ALIGNMENT) {
11887         word r = *(word *)q;
11888         if (r < (word)p || r > (word)target_limit) {
11889             GC_PUSH_ONE_HEAP(r, q, GC_mark_stack_top);
11890         }
11891     }
11892 }
11893 STATIC void GC_null_finalize_mark_proc(ptr_t p GC_ATTR_UNUSED) {}
11894 STATIC void GC_unreachable_finalize_mark_proc(ptr_t p)
11895 {
11896     GC_normal_finalize_mark_proc(p);
11897 }
11898 STATIC void GC_register_finalizer_inner(void * obj,
11899                                         GC_finalization_proc fn, void *cd,
11900                                         GC_finalization_proc *ofn, void **ocd,
11901                                         finalization_mark_proc mp)
11902 {
11903     struct finalizable_object * curr_fo;
11904     size_t index;
11905     struct finalizable_object *new_fo = 0;
11906     hdr *hhdr = NULL;
11907     DCL_LOCK_STATE;
11908     if (EXPECT(GC_find_leak, FALSE)) return;
11909     LOCK();
11910     if (EXPECT(NULL == GC_fnlz_roots.fo_head, FALSE)
11911         || EXPECT(GC_fo_entries > ((word)1 << GC_log_fo_table_size), FALSE)) {
11912         GC_grow_table((struct hash_chain_entry ***)&GC_fnlz_roots.fo_head,
11913                       &GC_log_fo_table_size, &GC_fo_entries);
11914         GC_COND_LOG_PRINTF("Grew fo table to %u entries\n",
11915                            1U << GC_log_fo_table_size);
11916     }
11917     for (;;) {
11918       struct finalizable_object *prev_fo = NULL;
11919       GC_oom_func oom_fn;
11920       index = HASH2(obj, GC_log_fo_table_size);
11921       curr_fo = GC_fnlz_roots.fo_head[index];
11922       while (curr_fo != 0) {
11923         GC_ASSERT(GC_size(curr_fo) >= sizeof(struct finalizable_object));
11924         if (curr_fo -> fo_hidden_base == GC_HIDE_POINTER(obj)) {
11925           if (ocd) *ocd = (void *) (curr_fo -> fo_client_data);
11926           if (ofn) *ofn = curr_fo -> fo_fn;
11927           if (prev_fo == 0) {
11928             GC_fnlz_roots.fo_head[index] = fo_next(curr_fo);
11929           } else {
11930             fo_set_next(prev_fo, fo_next(curr_fo));
11931             GC_dirty(prev_fo);
11932           }
11933           if (fn == 0) {
11934             GC_fo_entries--;
11935 #if !defined(THREADS) && !defined(DBG_HDRS_ALL)
11936               GC_free((void *)curr_fo);
11937 #endif
11938           } else {
11939             curr_fo -> fo_fn = fn;
11940             curr_fo -> fo_client_data = (ptr_t)cd;
11941             curr_fo -> fo_mark_proc = mp;
11942             GC_dirty(curr_fo);
11943             if (prev_fo == 0) {
11944               GC_fnlz_roots.fo_head[index] = curr_fo;
11945             } else {
11946               fo_set_next(prev_fo, curr_fo);
11947               GC_dirty(prev_fo);
11948             }
11949           }
11950           if (NULL == prev_fo)
11951             GC_dirty(GC_fnlz_roots.fo_head + index);
11952           UNLOCK();
11953 #ifndef DBG_HDRS_ALL
11954               GC_free((void *)new_fo);
11955 #endif
11956           return;
11957         }
11958         prev_fo = curr_fo;
11959         curr_fo = fo_next(curr_fo);
11960       }
11961       if (EXPECT(new_fo != 0, FALSE)) {
11962         GC_ASSERT(fn != 0);
11963 #ifdef LINT2
11964           if (NULL == hhdr) ABORT("Bad hhdr in GC_register_finalizer_inner");
11965 #endif
11966         break;
11967       }
11968       if (fn == 0) {
11969         if (ocd) *ocd = 0;
11970         if (ofn) *ofn = 0;
11971         UNLOCK();
11972         return;
11973       }
11974       GET_HDR(obj, hhdr);
11975       if (EXPECT(0 == hhdr, FALSE)) {
11976         if (ocd) *ocd = 0;
11977         if (ofn) *ofn = 0;
11978         UNLOCK();
11979         return;
11980       }
11981       new_fo = (struct finalizable_object *)
11982         GC_INTERNAL_MALLOC(sizeof(struct finalizable_object),NORMAL);
11983       if (EXPECT(new_fo != 0, TRUE))
11984         break;
11985       oom_fn = GC_oom_fn;
11986       UNLOCK();
11987       new_fo = (struct finalizable_object *)
11988                 (*oom_fn)(sizeof(struct finalizable_object));
11989       if (0 == new_fo) {
11990         return;
11991       }
11992       LOCK();
11993     }
11994     GC_ASSERT(GC_size(new_fo) >= sizeof(struct finalizable_object));
11995     if (ocd) *ocd = 0;
11996     if (ofn) *ofn = 0;
11997     new_fo -> fo_hidden_base = GC_HIDE_POINTER(obj);
11998     new_fo -> fo_fn = fn;
11999     new_fo -> fo_client_data = (ptr_t)cd;
12000     new_fo -> fo_object_size = hhdr -> hb_sz;
12001     new_fo -> fo_mark_proc = mp;
12002     fo_set_next(new_fo, GC_fnlz_roots.fo_head[index]);
12003     GC_dirty(new_fo);
12004     GC_fo_entries++;
12005     GC_fnlz_roots.fo_head[index] = new_fo;
12006     GC_dirty(GC_fnlz_roots.fo_head + index);
12007     UNLOCK();
12008 }
12009 GC_API void GC_CALL GC_register_finalizer(void * obj,
12010                                   GC_finalization_proc fn, void * cd,
12011                                   GC_finalization_proc *ofn, void ** ocd)
12012 {
12013     GC_register_finalizer_inner(obj, fn, cd, ofn,
12014                                 ocd, GC_normal_finalize_mark_proc);
12015 }
12016 GC_API void GC_CALL GC_register_finalizer_ignore_self(void * obj,
12017                                GC_finalization_proc fn, void * cd,
12018                                GC_finalization_proc *ofn, void ** ocd)
12019 {
12020     GC_register_finalizer_inner(obj, fn, cd, ofn,
12021                                 ocd, GC_ignore_self_finalize_mark_proc);
12022 }
12023 GC_API void GC_CALL GC_register_finalizer_no_order(void * obj,
12024                                GC_finalization_proc fn, void * cd,
12025                                GC_finalization_proc *ofn, void ** ocd)
12026 {
12027     GC_register_finalizer_inner(obj, fn, cd, ofn,
12028                                 ocd, GC_null_finalize_mark_proc);
12029 }
12030 static GC_bool need_unreachable_finalization = FALSE;
12031 GC_API void GC_CALL GC_register_finalizer_unreachable(void * obj,
12032                                GC_finalization_proc fn, void * cd,
12033                                GC_finalization_proc *ofn, void ** ocd)
12034 {
12035     need_unreachable_finalization = TRUE;
12036     GC_ASSERT(GC_java_finalization);
12037     GC_register_finalizer_inner(obj, fn, cd, ofn,
12038                                 ocd, GC_unreachable_finalize_mark_proc);
12039 }
12040 #ifndef NO_DEBUGGING
12041   STATIC void GC_dump_finalization_links(
12042                                 const struct dl_hashtbl_s *dl_hashtbl)
12043   {
12044     size_t dl_size = (size_t)1 << dl_hashtbl -> log_size;
12045     size_t i;
12046     if (NULL == dl_hashtbl -> head) return;
12047     for (i = 0; i < dl_size; i++) {
12048       struct disappearing_link *curr_dl;
12049       for (curr_dl = dl_hashtbl -> head[i]; curr_dl != 0;
12050            curr_dl = dl_next(curr_dl)) {
12051         ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_dl->dl_hidden_obj);
12052         ptr_t real_link = (ptr_t)GC_REVEAL_POINTER(curr_dl->dl_hidden_link);
12053         GC_printf("Object: %p, link: %p\n",
12054                   (void *)real_ptr, (void *)real_link);
12055       }
12056     }
12057   }
12058   GC_API void GC_CALL GC_dump_finalization(void)
12059   {
12060     struct finalizable_object * curr_fo;
12061     size_t i;
12062     size_t fo_size = GC_fnlz_roots.fo_head == NULL ? 0 :
12063                                 (size_t)1 << GC_log_fo_table_size;
12064     GC_printf("Disappearing (short) links:\n");
12065     GC_dump_finalization_links(&GC_dl_hashtbl);
12066 #ifndef GC_LONG_REFS_NOT_NEEDED
12067       GC_printf("Disappearing long links:\n");
12068       GC_dump_finalization_links(&GC_ll_hashtbl);
12069 #endif
12070     GC_printf("Finalizers:\n");
12071     for (i = 0; i < fo_size; i++) {
12072       for (curr_fo = GC_fnlz_roots.fo_head[i];
12073            curr_fo != NULL; curr_fo = fo_next(curr_fo)) {
12074         ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base);
12075         GC_printf("Finalizable object: %p\n", (void *)real_ptr);
12076       }
12077     }
12078   }
12079 #endif
12080 #ifndef SMALL_CONFIG
12081   STATIC word GC_old_dl_entries = 0;
12082 #ifndef GC_LONG_REFS_NOT_NEEDED
12083     STATIC word GC_old_ll_entries = 0;
12084 #endif
12085 #endif
12086 #ifndef THREADS
12087   STATIC int GC_finalizer_nested = 0;
12088   STATIC unsigned GC_finalizer_skipped = 0;
12089   STATIC unsigned char *GC_check_finalizer_nested(void)
12090   {
12091     unsigned nesting_level = *(unsigned char *)&GC_finalizer_nested;
12092     if (nesting_level) {
12093       if (++GC_finalizer_skipped < (1U << nesting_level)) return NULL;
12094       GC_finalizer_skipped = 0;
12095     }
12096     *(char *)&GC_finalizer_nested = (char)(nesting_level + 1);
12097     return (unsigned char *)&GC_finalizer_nested;
12098   }
12099 #endif
12100 GC_INLINE void GC_make_disappearing_links_disappear(
12101                                         struct dl_hashtbl_s* dl_hashtbl,
12102                                         GC_bool is_remove_dangling)
12103 {
12104   size_t i;
12105   size_t dl_size = (size_t)1 << dl_hashtbl -> log_size;
12106   GC_bool needs_barrier = FALSE;
12107   GC_ASSERT(I_HOLD_LOCK());
12108   if (NULL == dl_hashtbl -> head) return;
12109   for (i = 0; i < dl_size; i++) {
12110     struct disappearing_link *curr_dl, *next_dl;
12111     struct disappearing_link *prev_dl = NULL;
12112     for (curr_dl = dl_hashtbl->head[i]; curr_dl != NULL; curr_dl = next_dl) {
12113       next_dl = dl_next(curr_dl);
12114       if (is_remove_dangling) {
12115         ptr_t real_link = (ptr_t)GC_base(GC_REVEAL_POINTER(
12116                                                 curr_dl->dl_hidden_link));
12117         if (NULL == real_link || EXPECT(GC_is_marked(real_link), TRUE)) {
12118           prev_dl = curr_dl;
12119           continue;
12120         }
12121       } else {
12122         if (EXPECT(GC_is_marked((ptr_t)GC_REVEAL_POINTER(
12123                                         curr_dl->dl_hidden_obj)), TRUE)) {
12124           prev_dl = curr_dl;
12125           continue;
12126         }
12127         *(ptr_t *)GC_REVEAL_POINTER(curr_dl->dl_hidden_link) = NULL;
12128       }
12129       if (NULL == prev_dl) {
12130         dl_hashtbl -> head[i] = next_dl;
12131         needs_barrier = TRUE;
12132       } else {
12133         dl_set_next(prev_dl, next_dl);
12134         GC_dirty(prev_dl);
12135       }
12136       GC_clear_mark_bit(curr_dl);
12137       dl_hashtbl -> entries--;
12138     }
12139   }
12140   if (needs_barrier)
12141     GC_dirty(dl_hashtbl -> head);
12142 }
12143 GC_INNER void GC_finalize(void)
12144 {
12145     struct finalizable_object * curr_fo, * prev_fo, * next_fo;
12146     ptr_t real_ptr;
12147     size_t i;
12148     size_t fo_size = GC_fnlz_roots.fo_head == NULL ? 0 :
12149                                 (size_t)1 << GC_log_fo_table_size;
12150     GC_bool needs_barrier = FALSE;
12151     GC_ASSERT(I_HOLD_LOCK());
12152 #ifndef SMALL_CONFIG
12153       GC_old_dl_entries = GC_dl_hashtbl.entries;
12154 #ifndef GC_LONG_REFS_NOT_NEEDED
12155         GC_old_ll_entries = GC_ll_hashtbl.entries;
12156 #endif
12157 #endif
12158 #ifndef GC_TOGGLE_REFS_NOT_NEEDED
12159       GC_mark_togglerefs();
12160 #endif
12161     GC_make_disappearing_links_disappear(&GC_dl_hashtbl, FALSE);
12162     GC_ASSERT(GC_mark_state == MS_NONE);
12163     for (i = 0; i < fo_size; i++) {
12164       for (curr_fo = GC_fnlz_roots.fo_head[i];
12165            curr_fo != NULL; curr_fo = fo_next(curr_fo)) {
12166         GC_ASSERT(GC_size(curr_fo) >= sizeof(struct finalizable_object));
12167         real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base);
12168         if (!GC_is_marked(real_ptr)) {
12169             GC_MARKED_FOR_FINALIZATION(real_ptr);
12170             GC_MARK_FO(real_ptr, curr_fo -> fo_mark_proc);
12171             if (GC_is_marked(real_ptr)) {
12172                 WARN("Finalization cycle involving %p\n", real_ptr);
12173             }
12174         }
12175       }
12176     }
12177     GC_bytes_finalized = 0;
12178     for (i = 0; i < fo_size; i++) {
12179       curr_fo = GC_fnlz_roots.fo_head[i];
12180       prev_fo = 0;
12181       while (curr_fo != 0) {
12182         real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base);
12183         if (!GC_is_marked(real_ptr)) {
12184             if (!GC_java_finalization) {
12185               GC_set_mark_bit(real_ptr);
12186             }
12187               next_fo = fo_next(curr_fo);
12188               if (NULL == prev_fo) {
12189                 GC_fnlz_roots.fo_head[i] = next_fo;
12190                 if (GC_object_finalized_proc) {
12191                   GC_dirty(GC_fnlz_roots.fo_head + i);
12192                 } else {
12193                   needs_barrier = TRUE;
12194                 }
12195               } else {
12196                 fo_set_next(prev_fo, next_fo);
12197                 GC_dirty(prev_fo);
12198               }
12199               GC_fo_entries--;
12200               if (GC_object_finalized_proc)
12201                 GC_object_finalized_proc(real_ptr);
12202               fo_set_next(curr_fo, GC_fnlz_roots.finalize_now);
12203               GC_dirty(curr_fo);
12204               SET_FINALIZE_NOW(curr_fo);
12205               curr_fo -> fo_hidden_base =
12206                         (word)GC_REVEAL_POINTER(curr_fo -> fo_hidden_base);
12207               GC_bytes_finalized +=
12208                         curr_fo -> fo_object_size
12209                         + sizeof(struct finalizable_object);
12210             GC_ASSERT(GC_is_marked(GC_base(curr_fo)));
12211             curr_fo = next_fo;
12212         } else {
12213             prev_fo = curr_fo;
12214             curr_fo = fo_next(curr_fo);
12215         }
12216       }
12217     }
12218   if (GC_java_finalization) {
12219       for (curr_fo = GC_fnlz_roots.finalize_now;
12220            curr_fo != NULL; curr_fo = fo_next(curr_fo)) {
12221         real_ptr = (ptr_t)curr_fo -> fo_hidden_base;
12222         if (!GC_is_marked(real_ptr)) {
12223             if (curr_fo -> fo_mark_proc == GC_null_finalize_mark_proc) {
12224                 GC_MARK_FO(real_ptr, GC_normal_finalize_mark_proc);
12225             }
12226             if (curr_fo -> fo_mark_proc != GC_unreachable_finalize_mark_proc) {
12227                 GC_set_mark_bit(real_ptr);
12228             }
12229         }
12230       }
12231       if (need_unreachable_finalization) {
12232         curr_fo = GC_fnlz_roots.finalize_now;
12233         GC_ASSERT(NULL == curr_fo || GC_fnlz_roots.fo_head != NULL);
12234         prev_fo = NULL;
12235         while (curr_fo != NULL) {
12236           next_fo = fo_next(curr_fo);
12237           if (curr_fo -> fo_mark_proc == GC_unreachable_finalize_mark_proc) {
12238             real_ptr = (ptr_t)curr_fo -> fo_hidden_base;
12239             if (!GC_is_marked(real_ptr)) {
12240               GC_set_mark_bit(real_ptr);
12241             } else {
12242               if (NULL == prev_fo) {
12243                 SET_FINALIZE_NOW(next_fo);
12244               } else {
12245                 fo_set_next(prev_fo, next_fo);
12246                 GC_dirty(prev_fo);
12247               }
12248               curr_fo -> fo_hidden_base =
12249                                 GC_HIDE_POINTER(curr_fo -> fo_hidden_base);
12250               GC_bytes_finalized -=
12251                   curr_fo->fo_object_size + sizeof(struct finalizable_object);
12252               i = HASH2(real_ptr, GC_log_fo_table_size);
12253               fo_set_next(curr_fo, GC_fnlz_roots.fo_head[i]);
12254               GC_dirty(curr_fo);
12255               GC_fo_entries++;
12256               GC_fnlz_roots.fo_head[i] = curr_fo;
12257               curr_fo = prev_fo;
12258               needs_barrier = TRUE;
12259             }
12260           }
12261           prev_fo = curr_fo;
12262           curr_fo = next_fo;
12263         }
12264       }
12265   }
12266   if (needs_barrier)
12267     GC_dirty(GC_fnlz_roots.fo_head);
12268   GC_make_disappearing_links_disappear(&GC_dl_hashtbl, TRUE);
12269 #ifndef GC_TOGGLE_REFS_NOT_NEEDED
12270     GC_clear_togglerefs();
12271 #endif
12272 #ifndef GC_LONG_REFS_NOT_NEEDED
12273     GC_make_disappearing_links_disappear(&GC_ll_hashtbl, FALSE);
12274     GC_make_disappearing_links_disappear(&GC_ll_hashtbl, TRUE);
12275 #endif
12276   if (GC_fail_count) {
12277 #ifdef THREADS
12278       GC_reset_finalizer_nested();
12279 #else
12280       GC_finalizer_nested = 0;
12281 #endif
12282   }
12283 }
12284 #ifndef JAVA_FINALIZATION_NOT_NEEDED
12285   STATIC void GC_enqueue_all_finalizers(void)
12286   {
12287     struct finalizable_object * next_fo;
12288     size_t i;
12289     size_t fo_size = GC_fnlz_roots.fo_head == NULL ? 0 :
12290                                 (size_t)1 << GC_log_fo_table_size;
12291     GC_ASSERT(I_HOLD_LOCK());
12292     GC_bytes_finalized = 0;
12293     for (i = 0; i < fo_size; i++) {
12294       struct finalizable_object * curr_fo = GC_fnlz_roots.fo_head[i];
12295       GC_fnlz_roots.fo_head[i] = NULL;
12296       while (curr_fo != NULL) {
12297           ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base);
12298           GC_MARK_FO(real_ptr, GC_normal_finalize_mark_proc);
12299           GC_set_mark_bit(real_ptr);
12300           next_fo = fo_next(curr_fo);
12301           fo_set_next(curr_fo, GC_fnlz_roots.finalize_now);
12302           GC_dirty(curr_fo);
12303           SET_FINALIZE_NOW(curr_fo);
12304           curr_fo -> fo_hidden_base =
12305                         (word)GC_REVEAL_POINTER(curr_fo -> fo_hidden_base);
12306           GC_bytes_finalized +=
12307                 curr_fo -> fo_object_size + sizeof(struct finalizable_object);
12308           curr_fo = next_fo;
12309       }
12310     }
12311     GC_fo_entries = 0;
12312   }
12313   GC_API void GC_CALL GC_finalize_all(void)
12314   {
12315     DCL_LOCK_STATE;
12316     LOCK();
12317     while (GC_fo_entries > 0) {
12318       GC_enqueue_all_finalizers();
12319       UNLOCK();
12320       GC_invoke_finalizers();
12321       LOCK();
12322     }
12323     UNLOCK();
12324   }
12325 #endif
12326 GC_API int GC_CALL GC_should_invoke_finalizers(void)
12327 {
12328 #ifdef AO_HAVE_load
12329     return AO_load((volatile AO_t *)&GC_fnlz_roots.finalize_now) != 0;
12330 #else
12331     return GC_fnlz_roots.finalize_now != NULL;
12332 #endif
12333 }
12334 GC_API int GC_CALL GC_invoke_finalizers(void)
12335 {
12336     int count = 0;
12337     word bytes_freed_before = 0;
12338     DCL_LOCK_STATE;
12339     while (GC_should_invoke_finalizers()) {
12340         struct finalizable_object * curr_fo;
12341 #ifdef THREADS
12342             LOCK();
12343 #endif
12344         if (count == 0) {
12345             bytes_freed_before = GC_bytes_freed;
12346         }
12347         curr_fo = GC_fnlz_roots.finalize_now;
12348 #ifdef THREADS
12349             if (curr_fo != NULL)
12350                 SET_FINALIZE_NOW(fo_next(curr_fo));
12351             UNLOCK();
12352             if (curr_fo == 0) break;
12353 #else
12354             GC_fnlz_roots.finalize_now = fo_next(curr_fo);
12355 #endif
12356         fo_set_next(curr_fo, 0);
12357         (*(curr_fo -> fo_fn))((ptr_t)(curr_fo -> fo_hidden_base),
12358                               curr_fo -> fo_client_data);
12359         curr_fo -> fo_client_data = 0;
12360         ++count;
12361     }
12362     if (count != 0
12363 #if defined(THREADS) && !defined(THREAD_SANITIZER)
12364             && bytes_freed_before != GC_bytes_freed
12365 #endif
12366        ) {
12367         LOCK();
12368         GC_finalizer_bytes_freed += (GC_bytes_freed - bytes_freed_before);
12369         UNLOCK();
12370     }
12371     return count;
12372 }
12373 static word last_finalizer_notification = 0;
12374 GC_INNER void GC_notify_or_invoke_finalizers(void)
12375 {
12376     GC_finalizer_notifier_proc notifier_fn = 0;
12377 #if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
12378       static word last_back_trace_gc_no = 1;
12379 #endif
12380     DCL_LOCK_STATE;
12381 #if defined(THREADS) && !defined(KEEP_BACK_PTRS) \
12382        && !defined(MAKE_BACK_GRAPH)
12383       if (!GC_should_invoke_finalizers())
12384         return;
12385 #endif
12386     LOCK();
12387 #if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
12388       if (GC_gc_no > last_back_trace_gc_no) {
12389 #ifdef KEEP_BACK_PTRS
12390           long i;
12391           last_back_trace_gc_no = GC_WORD_MAX;
12392           for (i = 0; i < GC_backtraces; ++i) {
12393               UNLOCK();
12394               GC_generate_random_backtrace_no_gc();
12395               LOCK();
12396           }
12397           last_back_trace_gc_no = GC_gc_no;
12398 #endif
12399 #ifdef MAKE_BACK_GRAPH
12400           if (GC_print_back_height) {
12401             GC_print_back_graph_stats();
12402           }
12403 #endif
12404       }
12405 #endif
12406     if (NULL == GC_fnlz_roots.finalize_now) {
12407       UNLOCK();
12408       return;
12409     }
12410     if (!GC_finalize_on_demand) {
12411       unsigned char *pnested = GC_check_finalizer_nested();
12412       UNLOCK();
12413       if (pnested != NULL) {
12414         (void) GC_invoke_finalizers();
12415         *pnested = 0;
12416 #ifndef THREADS
12417           GC_ASSERT(NULL == GC_fnlz_roots.finalize_now);
12418 #endif
12419       }
12420       return;
12421     }
12422     if (last_finalizer_notification != GC_gc_no) {
12423         notifier_fn = GC_finalizer_notifier;
12424         last_finalizer_notification = GC_gc_no;
12425     }
12426     UNLOCK();
12427     if (notifier_fn != 0)
12428         (*notifier_fn)();
12429 }
12430 #ifndef SMALL_CONFIG
12431 #ifndef GC_LONG_REFS_NOT_NEEDED
12432 #define IF_LONG_REFS_PRESENT_ELSE(x,y) (x)
12433 #else
12434 #define IF_LONG_REFS_PRESENT_ELSE(x,y) (y)
12435 #endif
12436   GC_INNER void GC_print_finalization_stats(void)
12437   {
12438     struct finalizable_object *fo;
12439     unsigned long ready = 0;
12440     GC_log_printf("%lu finalization entries;"
12441                   " %lu/%lu short/long disappearing links alive\n",
12442                   (unsigned long)GC_fo_entries,
12443                   (unsigned long)GC_dl_hashtbl.entries,
12444                   (unsigned long)IF_LONG_REFS_PRESENT_ELSE(
12445                                                 GC_ll_hashtbl.entries, 0));
12446     for (fo = GC_fnlz_roots.finalize_now; fo != NULL; fo = fo_next(fo))
12447       ++ready;
12448     GC_log_printf("%lu finalization-ready objects;"
12449                   " %ld/%ld short/long links cleared\n",
12450                   ready,
12451                   (long)GC_old_dl_entries - (long)GC_dl_hashtbl.entries,
12452                   (long)IF_LONG_REFS_PRESENT_ELSE(
12453                               GC_old_ll_entries - GC_ll_hashtbl.entries, 0));
12454   }
12455 #endif
12456 #endif
12457 #ifdef ENABLE_DISCLAIM
12458 #ifndef GC_DISCLAIM_H
12459 #define GC_DISCLAIM_H
12460 #ifdef __cplusplus
12461   extern "C" {
12462 #endif
12463 GC_API void GC_CALL GC_init_finalized_malloc(void);
12464 typedef int (GC_CALLBACK * GC_disclaim_proc)(void * );
12465 GC_API void GC_CALL GC_register_disclaim_proc(int ,
12466                                 GC_disclaim_proc ,
12467                                 int ) GC_ATTR_NONNULL(2);
12468 struct GC_finalizer_closure {
12469     GC_finalization_proc proc;
12470     void *cd;
12471 };
12472 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
12473         GC_finalized_malloc(size_t ,
12474                 const struct GC_finalizer_closure * ) GC_ATTR_NONNULL(2);
12475 #ifdef __cplusplus
12476   }
12477 #endif
12478 #endif
12479 #if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
12480 #define FINALIZER_CLOSURE_FLAG 0x2
12481 #else
12482 #define FINALIZER_CLOSURE_FLAG 0x1
12483 #endif
12484 STATIC int GC_CALLBACK GC_finalized_disclaim(void *obj)
12485 {
12486     word fc_word = *(word *)obj;
12487     if ((fc_word & FINALIZER_CLOSURE_FLAG) != 0) {
12488         const struct GC_finalizer_closure *fc
12489                         = (struct GC_finalizer_closure *)(fc_word
12490                                         & ~(word)FINALIZER_CLOSURE_FLAG);
12491         GC_ASSERT(!GC_find_leak);
12492         (*fc->proc)((word *)obj + 1, fc->cd);
12493     }
12494     return 0;
12495 }
12496 GC_API void GC_CALL GC_init_finalized_malloc(void)
12497 {
12498     DCL_LOCK_STATE;
12499     GC_init();
12500     LOCK();
12501     if (GC_finalized_kind != 0) {
12502         UNLOCK();
12503         return;
12504     }
12505     GC_register_displacement_inner(sizeof(word));
12506     GC_register_displacement_inner(FINALIZER_CLOSURE_FLAG);
12507     GC_register_displacement_inner(sizeof(oh) + FINALIZER_CLOSURE_FLAG);
12508     GC_finalized_kind = GC_new_kind_inner(GC_new_free_list_inner(),
12509                                           GC_DS_LENGTH, TRUE, TRUE);
12510     GC_ASSERT(GC_finalized_kind != 0);
12511     GC_register_disclaim_proc(GC_finalized_kind, GC_finalized_disclaim, TRUE);
12512     UNLOCK();
12513 }
12514 GC_API void GC_CALL GC_register_disclaim_proc(int kind, GC_disclaim_proc proc,
12515                                               int mark_unconditionally)
12516 {
12517     GC_ASSERT((unsigned)kind < MAXOBJKINDS);
12518     GC_ASSERT(NONNULL_ARG_NOT_NULL(proc));
12519     if (!EXPECT(GC_find_leak, FALSE)) {
12520         GC_obj_kinds[kind].ok_disclaim_proc = proc;
12521         GC_obj_kinds[kind].ok_mark_unconditionally =
12522                                         (GC_bool)mark_unconditionally;
12523     }
12524 }
12525 GC_API GC_ATTR_MALLOC void * GC_CALL GC_finalized_malloc(size_t lb,
12526                                 const struct GC_finalizer_closure *fclos)
12527 {
12528     word *op;
12529     GC_ASSERT(GC_finalized_kind != 0);
12530     GC_ASSERT(NONNULL_ARG_NOT_NULL(fclos));
12531     GC_ASSERT(((word)fclos & FINALIZER_CLOSURE_FLAG) == 0);
12532     op = (word *)GC_malloc_kind(SIZET_SAT_ADD(lb, sizeof(word)),
12533                                 GC_finalized_kind);
12534     if (EXPECT(NULL == op, FALSE))
12535         return NULL;
12536     *op = (word)fclos | FINALIZER_CLOSURE_FLAG;
12537     GC_dirty(op);
12538     REACHABLE_AFTER_DIRTY(fclos);
12539     return op + 1;
12540 }
12541 #endif
12542 #include <stdio.h>
12543 #include <string.h>
12544 STATIC GC_bool GC_alloc_reclaim_list(struct obj_kind *kind)
12545 {
12546     struct hblk ** result = (struct hblk **)
12547                 GC_scratch_alloc((MAXOBJGRANULES+1) * sizeof(struct hblk *));
12548     if (result == 0) return(FALSE);
12549     BZERO(result, (MAXOBJGRANULES+1)*sizeof(struct hblk *));
12550     kind -> ok_reclaim_list = result;
12551     return(TRUE);
12552 }
12553 GC_INNER ptr_t GC_alloc_large(size_t lb, int k, unsigned flags)
12554 {
12555     struct hblk * h;
12556     word n_blocks;
12557     ptr_t result;
12558     GC_bool retry = FALSE;
12559     GC_ASSERT(I_HOLD_LOCK());
12560     lb = ROUNDUP_GRANULE_SIZE(lb);
12561     n_blocks = OBJ_SZ_TO_BLOCKS_CHECKED(lb);
12562     if (!EXPECT(GC_is_initialized, TRUE)) {
12563       DCL_LOCK_STATE;
12564       UNLOCK();
12565       GC_init();
12566       LOCK();
12567     }
12568         if (GC_incremental && !GC_dont_gc) {
12569             ENTER_GC();
12570             GC_collect_a_little_inner((int)n_blocks);
12571             EXIT_GC();
12572         }
12573     h = GC_allochblk(lb, k, flags);
12574 #ifdef USE_MUNMAP
12575         if (0 == h) {
12576             GC_merge_unmapped();
12577             h = GC_allochblk(lb, k, flags);
12578         }
12579 #endif
12580     while (0 == h && GC_collect_or_expand(n_blocks, flags != 0, retry)) {
12581         h = GC_allochblk(lb, k, flags);
12582         retry = TRUE;
12583     }
12584     if (h == 0) {
12585         result = 0;
12586     } else {
12587         size_t total_bytes = n_blocks * HBLKSIZE;
12588         if (n_blocks > 1) {
12589             GC_large_allocd_bytes += total_bytes;
12590             if (GC_large_allocd_bytes > GC_max_large_allocd_bytes)
12591                 GC_max_large_allocd_bytes = GC_large_allocd_bytes;
12592         }
12593         result = h -> hb_body;
12594     }
12595     return result;
12596 }
12597 STATIC ptr_t GC_alloc_large_and_clear(size_t lb, int k, unsigned flags)
12598 {
12599     ptr_t result;
12600     GC_ASSERT(I_HOLD_LOCK());
12601     result = GC_alloc_large(lb, k, flags);
12602     if (result != NULL
12603           && (GC_debugging_started || GC_obj_kinds[k].ok_init)) {
12604         word n_blocks = OBJ_SZ_TO_BLOCKS(lb);
12605         BZERO(result, n_blocks * HBLKSIZE);
12606     }
12607     return result;
12608 }
12609 STATIC void GC_extend_size_map(size_t i)
12610 {
12611   size_t orig_granule_sz = ROUNDED_UP_GRANULES(i);
12612   size_t granule_sz;
12613   size_t byte_sz = GRANULES_TO_BYTES(orig_granule_sz);
12614   size_t smaller_than_i = byte_sz - (byte_sz >> 3);
12615   size_t low_limit;
12616   size_t number_of_objs;
12617   GC_ASSERT(I_HOLD_LOCK());
12618   GC_ASSERT(0 == GC_size_map[i]);
12619   if (0 == GC_size_map[smaller_than_i]) {
12620     low_limit = byte_sz - (byte_sz >> 2);
12621     granule_sz = orig_granule_sz;
12622     while (GC_size_map[low_limit] != 0)
12623       low_limit++;
12624   } else {
12625     low_limit = smaller_than_i + 1;
12626     while (GC_size_map[low_limit] != 0)
12627       low_limit++;
12628     granule_sz = ROUNDED_UP_GRANULES(low_limit);
12629     granule_sz += granule_sz >> 3;
12630     if (granule_sz < orig_granule_sz)
12631       granule_sz = orig_granule_sz;
12632   }
12633   granule_sz = (granule_sz + 1) & ~1;
12634   if (granule_sz > MAXOBJGRANULES)
12635     granule_sz = MAXOBJGRANULES;
12636   number_of_objs = HBLK_GRANULES / granule_sz;
12637   GC_ASSERT(number_of_objs != 0);
12638   granule_sz = (HBLK_GRANULES / number_of_objs) & ~1;
12639   byte_sz = GRANULES_TO_BYTES(granule_sz) - EXTRA_BYTES;
12640   for (; low_limit <= byte_sz; low_limit++)
12641     GC_size_map[low_limit] = granule_sz;
12642 }
12643 GC_INNER void * GC_generic_malloc_inner(size_t lb, int k)
12644 {
12645     void *op;
12646     GC_ASSERT(I_HOLD_LOCK());
12647     GC_ASSERT(k < MAXOBJKINDS);
12648     if (SMALL_OBJ(lb)) {
12649         struct obj_kind * kind = GC_obj_kinds + k;
12650         size_t lg = GC_size_map[lb];
12651         void ** opp = &(kind -> ok_freelist[lg]);
12652         op = *opp;
12653         if (EXPECT(0 == op, FALSE)) {
12654           if (lg == 0) {
12655             if (!EXPECT(GC_is_initialized, TRUE)) {
12656               DCL_LOCK_STATE;
12657               UNLOCK();
12658               GC_init();
12659               LOCK();
12660               lg = GC_size_map[lb];
12661             }
12662             if (0 == lg) {
12663               GC_extend_size_map(lb);
12664               lg = GC_size_map[lb];
12665               GC_ASSERT(lg != 0);
12666             }
12667             opp = &(kind -> ok_freelist[lg]);
12668             op = *opp;
12669           }
12670           if (0 == op) {
12671             if (0 == kind -> ok_reclaim_list &&
12672                 !GC_alloc_reclaim_list(kind))
12673               return NULL;
12674             op = GC_allocobj(lg, k);
12675             if (0 == op)
12676               return NULL;
12677           }
12678         }
12679         *opp = obj_link(op);
12680         obj_link(op) = 0;
12681         GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
12682     } else {
12683         op = (ptr_t)GC_alloc_large_and_clear(ADD_SLOP(lb), k, 0);
12684         if (op != NULL)
12685             GC_bytes_allocd += lb;
12686     }
12687     return op;
12688 }
12689 #if defined(DBG_HDRS_ALL) || defined(GC_GCJ_SUPPORT) \
12690     || !defined(GC_NO_FINALIZATION)
12691   GC_INNER void * GC_generic_malloc_inner_ignore_off_page(size_t lb, int k)
12692   {
12693     word lb_adjusted;
12694     void * op;
12695     GC_ASSERT(I_HOLD_LOCK());
12696     if (lb <= HBLKSIZE)
12697         return GC_generic_malloc_inner(lb, k);
12698     GC_ASSERT(k < MAXOBJKINDS);
12699     lb_adjusted = ADD_SLOP(lb);
12700     op = GC_alloc_large_and_clear(lb_adjusted, k, IGNORE_OFF_PAGE);
12701     if (op != NULL)
12702         GC_bytes_allocd += lb_adjusted;
12703     return op;
12704   }
12705 #endif
12706 #ifdef GC_COLLECT_AT_MALLOC
12707 #if defined(CPPCHECK)
12708     size_t GC_dbg_collect_at_malloc_min_lb = 16*1024;
12709 #else
12710     size_t GC_dbg_collect_at_malloc_min_lb = (GC_COLLECT_AT_MALLOC);
12711 #endif
12712 #endif
12713 GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_malloc(size_t lb, int k)
12714 {
12715     void * result;
12716     DCL_LOCK_STATE;
12717     GC_ASSERT(k < MAXOBJKINDS);
12718     if (EXPECT(GC_have_errors, FALSE))
12719       GC_print_all_errors();
12720     GC_INVOKE_FINALIZERS();
12721     GC_DBG_COLLECT_AT_MALLOC(lb);
12722     if (SMALL_OBJ(lb)) {
12723         LOCK();
12724         result = GC_generic_malloc_inner(lb, k);
12725         UNLOCK();
12726     } else {
12727         size_t lg;
12728         size_t lb_rounded;
12729         word n_blocks;
12730         GC_bool init;
12731         lg = ROUNDED_UP_GRANULES(lb);
12732         lb_rounded = GRANULES_TO_BYTES(lg);
12733         n_blocks = OBJ_SZ_TO_BLOCKS(lb_rounded);
12734         init = GC_obj_kinds[k].ok_init;
12735         LOCK();
12736         result = (ptr_t)GC_alloc_large(lb_rounded, k, 0);
12737         if (0 != result) {
12738           if (GC_debugging_started) {
12739             BZERO(result, n_blocks * HBLKSIZE);
12740           } else {
12741 #ifdef THREADS
12742                 ((word *)result)[0] = 0;
12743                 ((word *)result)[1] = 0;
12744                 ((word *)result)[GRANULES_TO_WORDS(lg)-1] = 0;
12745                 ((word *)result)[GRANULES_TO_WORDS(lg)-2] = 0;
12746 #endif
12747           }
12748           GC_bytes_allocd += lb_rounded;
12749         }
12750         UNLOCK();
12751         if (init && !GC_debugging_started && 0 != result) {
12752             BZERO(result, n_blocks * HBLKSIZE);
12753         }
12754     }
12755     if (0 == result) {
12756         return((*GC_get_oom_fn())(lb));
12757     } else {
12758         return(result);
12759     }
12760 }
12761 GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind_global(size_t lb, int k)
12762 {
12763     GC_ASSERT(k < MAXOBJKINDS);
12764     if (SMALL_OBJ(lb)) {
12765         void *op;
12766         void **opp;
12767         size_t lg;
12768         DCL_LOCK_STATE;
12769         GC_DBG_COLLECT_AT_MALLOC(lb);
12770         LOCK();
12771         lg = GC_size_map[lb];
12772         opp = &GC_obj_kinds[k].ok_freelist[lg];
12773         op = *opp;
12774         if (EXPECT(op != NULL, TRUE)) {
12775             if (k == PTRFREE) {
12776                 *opp = obj_link(op);
12777             } else {
12778                 GC_ASSERT(0 == obj_link(op)
12779                           || ((word)obj_link(op)
12780                                 <= (word)GC_greatest_plausible_heap_addr
12781                               && (word)obj_link(op)
12782                                 >= (word)GC_least_plausible_heap_addr));
12783                 *opp = obj_link(op);
12784                 obj_link(op) = 0;
12785             }
12786             GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
12787             UNLOCK();
12788             return op;
12789         }
12790         UNLOCK();
12791     }
12792     return GC_clear_stack(GC_generic_malloc(lb, k));
12793 }
12794 #if defined(THREADS) && !defined(THREAD_LOCAL_ALLOC)
12795   GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind(size_t lb, int k)
12796   {
12797     return GC_malloc_kind_global(lb, k);
12798   }
12799 #endif
12800 GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_atomic(size_t lb)
12801 {
12802     return GC_malloc_kind(lb, PTRFREE);
12803 }
12804 GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc(size_t lb)
12805 {
12806     return GC_malloc_kind(lb, NORMAL);
12807 }
12808 GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_malloc_uncollectable(
12809                                                         size_t lb, int k)
12810 {
12811     void *op;
12812     DCL_LOCK_STATE;
12813     GC_ASSERT(k < MAXOBJKINDS);
12814     if (SMALL_OBJ(lb)) {
12815         void **opp;
12816         size_t lg;
12817         GC_DBG_COLLECT_AT_MALLOC(lb);
12818         if (EXTRA_BYTES != 0 && lb != 0) lb--;
12819         LOCK();
12820         lg = GC_size_map[lb];
12821         opp = &GC_obj_kinds[k].ok_freelist[lg];
12822         op = *opp;
12823         if (EXPECT(op != NULL, TRUE)) {
12824             *opp = obj_link(op);
12825             obj_link(op) = 0;
12826             GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
12827             GC_non_gc_bytes += GRANULES_TO_BYTES((word)lg);
12828             UNLOCK();
12829         } else {
12830             UNLOCK();
12831             op = GC_generic_malloc(lb, k);
12832         }
12833         GC_ASSERT(0 == op || GC_is_marked(op));
12834     } else {
12835       op = GC_generic_malloc(lb, k);
12836       if (op ) {
12837         hdr * hhdr = HDR(op);
12838         GC_ASSERT(((word)op & (HBLKSIZE - 1)) == 0);
12839         LOCK();
12840         set_mark_bit_from_hdr(hhdr, 0);
12841 #ifndef THREADS
12842           GC_ASSERT(hhdr -> hb_n_marks == 0);
12843 #endif
12844         hhdr -> hb_n_marks = 1;
12845         UNLOCK();
12846       }
12847     }
12848     return op;
12849 }
12850 GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_uncollectable(size_t lb)
12851 {
12852   return GC_generic_malloc_uncollectable(lb, UNCOLLECTABLE);
12853 }
12854 #ifdef GC_ATOMIC_UNCOLLECTABLE
12855   GC_API GC_ATTR_MALLOC void * GC_CALL
12856         GC_malloc_atomic_uncollectable(size_t lb)
12857   {
12858     return GC_generic_malloc_uncollectable(lb, AUNCOLLECTABLE);
12859   }
12860 #endif
12861 #if defined(REDIRECT_MALLOC) && !defined(REDIRECT_MALLOC_IN_HEADER)
12862 #ifndef MSWINCE
12863 #include <errno.h>
12864 #endif
12865 #define GC_debug_malloc_replacement(lb) GC_debug_malloc(lb, GC_DBG_EXTRAS)
12866 #if defined(CPPCHECK)
12867 #define REDIRECT_MALLOC_F GC_malloc
12868 #else
12869 #define REDIRECT_MALLOC_F REDIRECT_MALLOC
12870 #endif
12871   void * malloc(size_t lb)
12872   {
12873 #if defined(I386) && defined(GC_SOLARIS_THREADS)
12874       if (!EXPECT(GC_is_initialized, TRUE)) return sbrk(lb);
12875 #endif
12876     return (void *)REDIRECT_MALLOC_F(lb);
12877   }
12878 #if defined(GC_LINUX_THREADS)
12879     STATIC ptr_t GC_libpthread_start = 0;
12880     STATIC ptr_t GC_libpthread_end = 0;
12881     STATIC ptr_t GC_libld_start = 0;
12882     STATIC ptr_t GC_libld_end = 0;
12883     STATIC void GC_init_lib_bounds(void)
12884     {
12885       IF_CANCEL(int cancel_state;)
12886       if (GC_libpthread_start != 0) return;
12887       DISABLE_CANCEL(cancel_state);
12888       GC_init();
12889       if (!GC_text_mapping("libpthread-",
12890                            &GC_libpthread_start, &GC_libpthread_end)) {
12891           WARN("Failed to find libpthread.so text mapping: Expect crash\n", 0);
12892             GC_libpthread_start = (ptr_t)1;
12893       }
12894       if (!GC_text_mapping("ld-", &GC_libld_start, &GC_libld_end)) {
12895           WARN("Failed to find ld.so text mapping: Expect crash\n", 0);
12896       }
12897       RESTORE_CANCEL(cancel_state);
12898     }
12899 #endif
12900   void * calloc(size_t n, size_t lb)
12901   {
12902     if ((lb | n) > GC_SQRT_SIZE_MAX
12903         && lb && n > GC_SIZE_MAX / lb)
12904       return (*GC_get_oom_fn())(GC_SIZE_MAX);
12905 #if defined(GC_LINUX_THREADS)
12906       {
12907         static GC_bool lib_bounds_set = FALSE;
12908         ptr_t caller = (ptr_t)__builtin_return_address(0);
12909         if (!EXPECT(lib_bounds_set, TRUE)) {
12910           GC_init_lib_bounds();
12911           lib_bounds_set = TRUE;
12912         }
12913         if (((word)caller >= (word)GC_libpthread_start
12914              && (word)caller < (word)GC_libpthread_end)
12915             || ((word)caller >= (word)GC_libld_start
12916                 && (word)caller < (word)GC_libld_end))
12917           return GC_generic_malloc_uncollectable(n * lb, UNCOLLECTABLE);
12918       }
12919 #endif
12920     return (void *)REDIRECT_MALLOC_F(n * lb);
12921   }
12922 #ifndef strdup
12923     char *strdup(const char *s)
12924     {
12925       size_t lb = strlen(s) + 1;
12926       char *result = (char *)REDIRECT_MALLOC_F(lb);
12927       if (result == 0) {
12928         errno = ENOMEM;
12929         return 0;
12930       }
12931       BCOPY(s, result, lb);
12932       return result;
12933     }
12934 #endif
12935 #ifndef strndup
12936     char *strndup(const char *str, size_t size)
12937     {
12938       char *copy;
12939       size_t len = strlen(str);
12940       if (len > size)
12941         len = size;
12942       copy = (char *)REDIRECT_MALLOC_F(len + 1);
12943       if (copy == NULL) {
12944         errno = ENOMEM;
12945         return NULL;
12946       }
12947       if (EXPECT(len > 0, TRUE))
12948         BCOPY(str, copy, len);
12949       copy[len] = '\0';
12950       return copy;
12951     }
12952 #endif
12953 #undef GC_debug_malloc_replacement
12954 #endif
12955 GC_API void GC_CALL GC_free(void * p)
12956 {
12957     struct hblk *h;
12958     hdr *hhdr;
12959     size_t sz;
12960     size_t ngranules;
12961     int knd;
12962     struct obj_kind * ok;
12963     DCL_LOCK_STATE;
12964     if (p ) {
12965     } else {
12966         return;
12967     }
12968 #ifdef LOG_ALLOCS
12969       GC_log_printf("GC_free(%p) after GC #%lu\n",
12970                     p, (unsigned long)GC_gc_no);
12971 #endif
12972     h = HBLKPTR(p);
12973     hhdr = HDR(h);
12974 #if defined(REDIRECT_MALLOC) && \
12975         ((defined(NEED_CALLINFO) && defined(GC_HAVE_BUILTIN_BACKTRACE)) \
12976          || defined(GC_SOLARIS_THREADS) || defined(GC_LINUX_THREADS) \
12977          || defined(MSWIN32))
12978         if (0 == hhdr) return;
12979 #endif
12980     GC_ASSERT(GC_base(p) == p);
12981     sz = (size_t)hhdr->hb_sz;
12982     ngranules = BYTES_TO_GRANULES(sz);
12983     knd = hhdr -> hb_obj_kind;
12984     ok = &GC_obj_kinds[knd];
12985     if (EXPECT(ngranules <= MAXOBJGRANULES, TRUE)) {
12986         void **flh;
12987         LOCK();
12988         GC_bytes_freed += sz;
12989         if (IS_UNCOLLECTABLE(knd)) GC_non_gc_bytes -= sz;
12990         if (ok -> ok_init && EXPECT(sz > sizeof(word), TRUE)) {
12991             BZERO((word *)p + 1, sz-sizeof(word));
12992         }
12993         flh = &(ok -> ok_freelist[ngranules]);
12994         obj_link(p) = *flh;
12995         *flh = (ptr_t)p;
12996         UNLOCK();
12997     } else {
12998         size_t nblocks = OBJ_SZ_TO_BLOCKS(sz);
12999         LOCK();
13000         GC_bytes_freed += sz;
13001         if (IS_UNCOLLECTABLE(knd)) GC_non_gc_bytes -= sz;
13002         if (nblocks > 1) {
13003           GC_large_allocd_bytes -= nblocks * HBLKSIZE;
13004         }
13005         GC_freehblk(h);
13006         UNLOCK();
13007     }
13008 }
13009 #ifdef THREADS
13010   GC_INNER void GC_free_inner(void * p)
13011   {
13012     struct hblk *h;
13013     hdr *hhdr;
13014     size_t sz;
13015     size_t ngranules;
13016     int knd;
13017     struct obj_kind * ok;
13018     h = HBLKPTR(p);
13019     hhdr = HDR(h);
13020     knd = hhdr -> hb_obj_kind;
13021     sz = (size_t)hhdr->hb_sz;
13022     ngranules = BYTES_TO_GRANULES(sz);
13023     ok = &GC_obj_kinds[knd];
13024     if (ngranules <= MAXOBJGRANULES) {
13025         void ** flh;
13026         GC_bytes_freed += sz;
13027         if (IS_UNCOLLECTABLE(knd)) GC_non_gc_bytes -= sz;
13028         if (ok -> ok_init && EXPECT(sz > sizeof(word), TRUE)) {
13029             BZERO((word *)p + 1, sz-sizeof(word));
13030         }
13031         flh = &(ok -> ok_freelist[ngranules]);
13032         obj_link(p) = *flh;
13033         *flh = (ptr_t)p;
13034     } else {
13035         size_t nblocks = OBJ_SZ_TO_BLOCKS(sz);
13036         GC_bytes_freed += sz;
13037         if (IS_UNCOLLECTABLE(knd)) GC_non_gc_bytes -= sz;
13038         if (nblocks > 1) {
13039           GC_large_allocd_bytes -= nblocks * HBLKSIZE;
13040         }
13041         GC_freehblk(h);
13042     }
13043   }
13044 #endif
13045 #if defined(REDIRECT_MALLOC) && !defined(REDIRECT_FREE)
13046 #define REDIRECT_FREE GC_free
13047 #endif
13048 #if defined(REDIRECT_FREE) && !defined(REDIRECT_MALLOC_IN_HEADER)
13049 #if defined(CPPCHECK)
13050 #define REDIRECT_FREE_F GC_free
13051 #else
13052 #define REDIRECT_FREE_F REDIRECT_FREE
13053 #endif
13054   void free(void * p)
13055   {
13056 #ifndef IGNORE_FREE
13057 #if defined(GC_LINUX_THREADS) && !defined(USE_PROC_FOR_LIBRARIES)
13058         ptr_t caller = (ptr_t)__builtin_return_address(0);
13059         if (((word)caller >= (word)GC_libpthread_start
13060              && (word)caller < (word)GC_libpthread_end)
13061             || ((word)caller >= (word)GC_libld_start
13062                 && (word)caller < (word)GC_libld_end)) {
13063           GC_free(p);
13064           return;
13065         }
13066 #endif
13067       REDIRECT_FREE_F(p);
13068 #endif
13069   }
13070 #endif
13071 #include <stdio.h>
13072 #include <string.h>
13073 #ifndef MSWINCE
13074 #include <errno.h>
13075 #endif
13076 #ifndef GC_ALLOC_PTRS_H
13077 #define GC_ALLOC_PTRS_H
13078 #ifdef __cplusplus
13079   extern "C" {
13080 #endif
13081 #ifndef GC_API_PRIV
13082 #define GC_API_PRIV GC_API
13083 #endif
13084 #ifndef GC_APIVAR_CONST
13085 #if defined(GC_BUILD) || !defined(GC_DLL)
13086 #define GC_APIVAR_CONST const
13087 #else
13088 #define GC_APIVAR_CONST
13089 #endif
13090 #endif
13091 GC_API_PRIV void ** GC_APIVAR_CONST GC_objfreelist_ptr;
13092 GC_API_PRIV void ** GC_APIVAR_CONST GC_aobjfreelist_ptr;
13093 GC_API_PRIV void ** GC_APIVAR_CONST GC_uobjfreelist_ptr;
13094 #ifdef GC_ATOMIC_UNCOLLECTABLE
13095   GC_API_PRIV void ** GC_APIVAR_CONST GC_auobjfreelist_ptr;
13096 #endif
13097 GC_API_PRIV void GC_CALL GC_incr_bytes_allocd(size_t );
13098 GC_API_PRIV void GC_CALL GC_incr_bytes_freed(size_t );
13099 #ifdef __cplusplus
13100   }
13101 #endif
13102 #endif
13103 void ** const GC_objfreelist_ptr = GC_objfreelist;
13104 void ** const GC_aobjfreelist_ptr = GC_aobjfreelist;
13105 void ** const GC_uobjfreelist_ptr = GC_uobjfreelist;
13106 #ifdef GC_ATOMIC_UNCOLLECTABLE
13107     void ** const GC_auobjfreelist_ptr = GC_auobjfreelist;
13108 #endif
13109 GC_API int GC_CALL GC_get_kind_and_size(const void * p, size_t * psize)
13110 {
13111     hdr * hhdr = HDR(p);
13112     if (psize != NULL) {
13113         *psize = (size_t)hhdr->hb_sz;
13114     }
13115     return hhdr -> hb_obj_kind;
13116 }
13117 GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_or_special_malloc(size_t lb,
13118                                                                   int knd)
13119 {
13120     switch(knd) {
13121         case PTRFREE:
13122         case NORMAL:
13123             return GC_malloc_kind(lb, knd);
13124         case UNCOLLECTABLE:
13125 #ifdef GC_ATOMIC_UNCOLLECTABLE
13126           case AUNCOLLECTABLE:
13127 #endif
13128             return GC_generic_malloc_uncollectable(lb, knd);
13129         default:
13130             return GC_generic_malloc(lb, knd);
13131     }
13132 }
13133 GC_API void * GC_CALL GC_realloc(void * p, size_t lb)
13134 {
13135     struct hblk * h;
13136     hdr * hhdr;
13137     void * result;
13138     size_t sz;
13139     size_t orig_sz;
13140     int obj_kind;
13141     if (p == 0) return(GC_malloc(lb));
13142     if (0 == lb)  {
13143 #ifndef IGNORE_FREE
13144         GC_free(p);
13145 #endif
13146       return NULL;
13147     }
13148     h = HBLKPTR(p);
13149     hhdr = HDR(h);
13150     sz = (size_t)hhdr->hb_sz;
13151     obj_kind = hhdr -> hb_obj_kind;
13152     orig_sz = sz;
13153     if (sz > MAXOBJBYTES) {
13154         word descr = GC_obj_kinds[obj_kind].ok_descriptor;
13155         sz = (sz + HBLKSIZE-1) & ~HBLKMASK;
13156         if (GC_obj_kinds[obj_kind].ok_relocate_descr)
13157           descr += sz;
13158 #ifdef AO_HAVE_store
13159           GC_STATIC_ASSERT(sizeof(hhdr->hb_sz) == sizeof(AO_t));
13160           AO_store((volatile AO_t *)&hhdr->hb_sz, (AO_t)sz);
13161           AO_store((volatile AO_t *)&hhdr->hb_descr, (AO_t)descr);
13162 #else
13163           {
13164             DCL_LOCK_STATE;
13165             LOCK();
13166             hhdr -> hb_sz = sz;
13167             hhdr -> hb_descr = descr;
13168             UNLOCK();
13169           }
13170 #endif
13171 #ifdef MARK_BIT_PER_OBJ
13172             GC_ASSERT(hhdr -> hb_inv_sz == LARGE_INV_SZ);
13173 #endif
13174 #ifdef MARK_BIT_PER_GRANULE
13175             GC_ASSERT((hhdr -> hb_flags & LARGE_BLOCK) != 0
13176                         && hhdr -> hb_map[ANY_INDEX] == 1);
13177 #endif
13178           if (IS_UNCOLLECTABLE(obj_kind)) GC_non_gc_bytes += (sz - orig_sz);
13179     }
13180     if (ADD_SLOP(lb) <= sz) {
13181         if (lb >= (sz >> 1)) {
13182             if (orig_sz > lb) {
13183                 BZERO(((ptr_t)p) + lb, orig_sz - lb);
13184             }
13185             return(p);
13186         }
13187         sz = lb;
13188     }
13189     result = GC_generic_or_special_malloc((word)lb, obj_kind);
13190     if (result != NULL) {
13191       BCOPY(p, result, sz);
13192 #ifndef IGNORE_FREE
13193         GC_free(p);
13194 #endif
13195     }
13196     return result;
13197 }
13198 #if defined(REDIRECT_MALLOC) && !defined(REDIRECT_REALLOC)
13199 #define REDIRECT_REALLOC GC_realloc
13200 #endif
13201 #ifdef REDIRECT_REALLOC
13202 #define GC_debug_realloc_replacement(p, lb) \
13203         GC_debug_realloc(p, lb, GC_DBG_EXTRAS)
13204 #if !defined(REDIRECT_MALLOC_IN_HEADER)
13205     void * realloc(void * p, size_t lb)
13206     {
13207       return(REDIRECT_REALLOC(p, lb));
13208     }
13209 #endif
13210 #undef GC_debug_realloc_replacement
13211 #endif
13212 GC_API GC_ATTR_MALLOC void * GC_CALL
13213     GC_generic_malloc_ignore_off_page(size_t lb, int k)
13214 {
13215     void *result;
13216     size_t lg;
13217     size_t lb_rounded;
13218     word n_blocks;
13219     GC_bool init;
13220     DCL_LOCK_STATE;
13221     if (SMALL_OBJ(lb))
13222         return GC_generic_malloc(lb, k);
13223     GC_ASSERT(k < MAXOBJKINDS);
13224     lg = ROUNDED_UP_GRANULES(lb);
13225     lb_rounded = GRANULES_TO_BYTES(lg);
13226     n_blocks = OBJ_SZ_TO_BLOCKS(lb_rounded);
13227     init = GC_obj_kinds[k].ok_init;
13228     if (EXPECT(GC_have_errors, FALSE))
13229       GC_print_all_errors();
13230     GC_INVOKE_FINALIZERS();
13231     GC_DBG_COLLECT_AT_MALLOC(lb);
13232     LOCK();
13233     result = (ptr_t)GC_alloc_large(ADD_SLOP(lb), k, IGNORE_OFF_PAGE);
13234     if (NULL == result) {
13235         GC_oom_func oom_fn = GC_oom_fn;
13236         UNLOCK();
13237         return (*oom_fn)(lb);
13238     }
13239     if (GC_debugging_started) {
13240         BZERO(result, n_blocks * HBLKSIZE);
13241     } else {
13242 #ifdef THREADS
13243             ((word *)result)[0] = 0;
13244             ((word *)result)[1] = 0;
13245             ((word *)result)[GRANULES_TO_WORDS(lg)-1] = 0;
13246             ((word *)result)[GRANULES_TO_WORDS(lg)-2] = 0;
13247 #endif
13248     }
13249     GC_bytes_allocd += lb_rounded;
13250     UNLOCK();
13251     if (init && !GC_debugging_started) {
13252         BZERO(result, n_blocks * HBLKSIZE);
13253     }
13254     return(result);
13255 }
13256 GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_ignore_off_page(size_t lb)
13257 {
13258     return GC_generic_malloc_ignore_off_page(lb, NORMAL);
13259 }
13260 GC_API GC_ATTR_MALLOC void * GC_CALL
13261     GC_malloc_atomic_ignore_off_page(size_t lb)
13262 {
13263     return GC_generic_malloc_ignore_off_page(lb, PTRFREE);
13264 }
13265 void GC_CALL GC_incr_bytes_allocd(size_t n)
13266 {
13267     GC_bytes_allocd += n;
13268 }
13269 void GC_CALL GC_incr_bytes_freed(size_t n)
13270 {
13271     GC_bytes_freed += n;
13272 }
13273 GC_API size_t GC_CALL GC_get_expl_freed_bytes_since_gc(void)
13274 {
13275     return (size_t)GC_bytes_freed;
13276 }
13277 #ifdef PARALLEL_MARK
13278     STATIC volatile AO_t GC_bytes_allocd_tmp = 0;
13279 #endif
13280 GC_API void GC_CALL GC_generic_malloc_many(size_t lb, int k, void **result)
13281 {
13282     void *op;
13283     void *p;
13284     void **opp;
13285     size_t lw;
13286     size_t lg;
13287     signed_word my_bytes_allocd = 0;
13288     struct obj_kind * ok = &(GC_obj_kinds[k]);
13289     struct hblk ** rlh;
13290     DCL_LOCK_STATE;
13291     GC_ASSERT(lb != 0 && (lb & (GRANULE_BYTES-1)) == 0);
13292     if (!SMALL_OBJ(lb) || GC_manual_vdb) {
13293         op = GC_generic_malloc(lb, k);
13294         if (EXPECT(0 != op, TRUE))
13295             obj_link(op) = 0;
13296         *result = op;
13297 #ifndef GC_DISABLE_INCREMENTAL
13298           if (GC_manual_vdb && GC_is_heap_ptr(result)) {
13299             GC_dirty_inner(result);
13300             REACHABLE_AFTER_DIRTY(op);
13301           }
13302 #endif
13303         return;
13304     }
13305     GC_ASSERT(k < MAXOBJKINDS);
13306     lw = BYTES_TO_WORDS(lb);
13307     lg = BYTES_TO_GRANULES(lb);
13308     if (EXPECT(GC_have_errors, FALSE))
13309       GC_print_all_errors();
13310     GC_INVOKE_FINALIZERS();
13311     GC_DBG_COLLECT_AT_MALLOC(lb);
13312     if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
13313     LOCK();
13314       if (GC_incremental && !GC_dont_gc) {
13315         ENTER_GC();
13316         GC_collect_a_little_inner(1);
13317         EXIT_GC();
13318       }
13319     rlh = ok -> ok_reclaim_list;
13320     if (rlh != NULL) {
13321         struct hblk * hbp;
13322         hdr * hhdr;
13323         while ((hbp = rlh[lg]) != NULL) {
13324             hhdr = HDR(hbp);
13325             rlh[lg] = hhdr -> hb_next;
13326             GC_ASSERT(hhdr -> hb_sz == lb);
13327             hhdr -> hb_last_reclaimed = (unsigned short) GC_gc_no;
13328 #ifdef PARALLEL_MARK
13329               if (GC_parallel) {
13330                   signed_word my_bytes_allocd_tmp =
13331                                 (signed_word)AO_load(&GC_bytes_allocd_tmp);
13332                   GC_ASSERT(my_bytes_allocd_tmp >= 0);
13333                   if (my_bytes_allocd_tmp != 0) {
13334                     (void)AO_fetch_and_add(&GC_bytes_allocd_tmp,
13335                                            (AO_t)(-my_bytes_allocd_tmp));
13336                     GC_bytes_allocd += my_bytes_allocd_tmp;
13337                   }
13338                   GC_acquire_mark_lock();
13339                   ++ GC_fl_builder_count;
13340                   UNLOCK();
13341                   GC_release_mark_lock();
13342               }
13343 #endif
13344             op = GC_reclaim_generic(hbp, hhdr, lb,
13345                                     ok -> ok_init, 0, &my_bytes_allocd);
13346             if (op != 0) {
13347 #ifdef PARALLEL_MARK
13348                 if (GC_parallel) {
13349                   *result = op;
13350                   (void)AO_fetch_and_add(&GC_bytes_allocd_tmp,
13351                                          (AO_t)my_bytes_allocd);
13352                   GC_acquire_mark_lock();
13353                   -- GC_fl_builder_count;
13354                   if (GC_fl_builder_count == 0) GC_notify_all_builder();
13355 #ifdef THREAD_SANITIZER
13356                     GC_release_mark_lock();
13357                     LOCK();
13358                     GC_bytes_found += my_bytes_allocd;
13359                     UNLOCK();
13360 #else
13361                     GC_bytes_found += my_bytes_allocd;
13362                     GC_release_mark_lock();
13363 #endif
13364                   (void) GC_clear_stack(0);
13365                   return;
13366                 }
13367 #endif
13368               GC_bytes_found += my_bytes_allocd;
13369               GC_bytes_allocd += my_bytes_allocd;
13370               goto out;
13371             }
13372 #ifdef PARALLEL_MARK
13373               if (GC_parallel) {
13374                 GC_acquire_mark_lock();
13375                 -- GC_fl_builder_count;
13376                 if (GC_fl_builder_count == 0) GC_notify_all_builder();
13377                 GC_release_mark_lock();
13378                 LOCK();
13379                 rlh = ok -> ok_reclaim_list;
13380                 if (NULL == rlh) break;
13381               }
13382 #endif
13383         }
13384     }
13385       opp = &(GC_obj_kinds[k].ok_freelist[lg]);
13386       if ( (op = *opp) != 0 ) {
13387         *opp = 0;
13388         my_bytes_allocd = 0;
13389         for (p = op; p != 0; p = obj_link(p)) {
13390           my_bytes_allocd += lb;
13391           if ((word)my_bytes_allocd >= HBLKSIZE) {
13392             *opp = obj_link(p);
13393             obj_link(p) = 0;
13394             break;
13395           }
13396         }
13397         GC_bytes_allocd += my_bytes_allocd;
13398         goto out;
13399       }
13400     {
13401         struct hblk *h = GC_allochblk(lb, k, 0);
13402         if (h ) {
13403           if (IS_UNCOLLECTABLE(k)) GC_set_hdr_marks(HDR(h));
13404           GC_bytes_allocd += HBLKSIZE - HBLKSIZE % lb;
13405 #ifdef PARALLEL_MARK
13406             if (GC_parallel) {
13407               GC_acquire_mark_lock();
13408               ++ GC_fl_builder_count;
13409               UNLOCK();
13410               GC_release_mark_lock();
13411               op = GC_build_fl(h, lw,
13412                         (ok -> ok_init || GC_debugging_started), 0);
13413               *result = op;
13414               GC_acquire_mark_lock();
13415               -- GC_fl_builder_count;
13416               if (GC_fl_builder_count == 0) GC_notify_all_builder();
13417               GC_release_mark_lock();
13418               (void) GC_clear_stack(0);
13419               return;
13420             }
13421 #endif
13422           op = GC_build_fl(h, lw, (ok -> ok_init || GC_debugging_started), 0);
13423           goto out;
13424         }
13425     }
13426       op = GC_generic_malloc_inner(lb, k);
13427       if (0 != op) obj_link(op) = 0;
13428   out:
13429     *result = op;
13430     UNLOCK();
13431     (void) GC_clear_stack(0);
13432 }
13433 GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_many(size_t lb)
13434 {
13435     void *result;
13436     lb = SIZET_SAT_ADD(lb, EXTRA_BYTES + GRANULE_BYTES - 1)
13437             & ~(GRANULE_BYTES - 1);
13438     GC_generic_malloc_many(lb, NORMAL, &result);
13439     return result;
13440 }
13441 #include <limits.h>
13442 GC_API GC_ATTR_MALLOC void * GC_CALL GC_memalign(size_t align, size_t lb)
13443 {
13444     size_t new_lb;
13445     size_t offset;
13446     ptr_t result;
13447     if (align <= GRANULE_BYTES) return GC_malloc(lb);
13448     if (align >= HBLKSIZE/2 || lb >= HBLKSIZE/2) {
13449         if (align > HBLKSIZE) {
13450           return (*GC_get_oom_fn())(LONG_MAX-1024);
13451         }
13452         return GC_malloc(lb <= HBLKSIZE? HBLKSIZE : lb);
13453     }
13454     new_lb = SIZET_SAT_ADD(lb, align - 1);
13455     result = (ptr_t)GC_malloc(new_lb);
13456     offset = (word)result % align;
13457     if (offset != 0) {
13458         offset = align - offset;
13459         if (!GC_all_interior_pointers) {
13460             GC_STATIC_ASSERT(VALID_OFFSET_SZ <= HBLKSIZE);
13461             GC_ASSERT(offset < VALID_OFFSET_SZ);
13462             GC_register_displacement(offset);
13463         }
13464     }
13465     result += offset;
13466     GC_ASSERT((word)result % align == 0);
13467     return result;
13468 }
13469 GC_API int GC_CALL GC_posix_memalign(void **memptr, size_t align, size_t lb)
13470 {
13471   size_t align_minus_one = align - 1;
13472   if (align < sizeof(void *) || (align_minus_one & align) != 0) {
13473 #ifdef MSWINCE
13474       return ERROR_INVALID_PARAMETER;
13475 #else
13476       return EINVAL;
13477 #endif
13478   }
13479   if ((*memptr = GC_memalign(align, lb)) == NULL) {
13480 #ifdef MSWINCE
13481       return ERROR_NOT_ENOUGH_MEMORY;
13482 #else
13483       return ENOMEM;
13484 #endif
13485   }
13486   return 0;
13487 }
13488 GC_API GC_ATTR_MALLOC char * GC_CALL GC_strdup(const char *s)
13489 {
13490   char *copy;
13491   size_t lb;
13492   if (s == NULL) return NULL;
13493   lb = strlen(s) + 1;
13494   copy = (char *)GC_malloc_atomic(lb);
13495   if (NULL == copy) {
13496 #ifndef MSWINCE
13497       errno = ENOMEM;
13498 #endif
13499     return NULL;
13500   }
13501   BCOPY(s, copy, lb);
13502   return copy;
13503 }
13504 GC_API GC_ATTR_MALLOC char * GC_CALL GC_strndup(const char *str, size_t size)
13505 {
13506   char *copy;
13507   size_t len = strlen(str);
13508   if (len > size)
13509     len = size;
13510   copy = (char *)GC_malloc_atomic(len + 1);
13511   if (copy == NULL) {
13512 #ifndef MSWINCE
13513       errno = ENOMEM;
13514 #endif
13515     return NULL;
13516   }
13517   if (EXPECT(len > 0, TRUE))
13518     BCOPY(str, copy, len);
13519   copy[len] = '\0';
13520   return copy;
13521 }
13522 #ifdef GC_REQUIRE_WCSDUP
13523 #include <wchar.h>
13524   GC_API GC_ATTR_MALLOC wchar_t * GC_CALL GC_wcsdup(const wchar_t *str)
13525   {
13526     size_t lb = (wcslen(str) + 1) * sizeof(wchar_t);
13527     wchar_t *copy = (wchar_t *)GC_malloc_atomic(lb);
13528     if (copy == NULL) {
13529 #ifndef MSWINCE
13530         errno = ENOMEM;
13531 #endif
13532       return NULL;
13533     }
13534     BCOPY(str, copy, lb);
13535     return copy;
13536   }
13537 #endif
13538 #ifndef CPPCHECK
13539   GC_API void * GC_CALL GC_malloc_stubborn(size_t lb)
13540   {
13541     return GC_malloc(lb);
13542   }
13543   GC_API void GC_CALL GC_change_stubborn(const void *p GC_ATTR_UNUSED)
13544   {
13545   }
13546 #endif
13547 GC_API void GC_CALL GC_end_stubborn_change(const void *p)
13548 {
13549   GC_dirty(p);
13550 }
13551 GC_API void GC_CALL GC_ptr_store_and_dirty(void *p, const void *q)
13552 {
13553   *(const void **)p = q;
13554   GC_dirty(p);
13555   REACHABLE_AFTER_DIRTY(q);
13556 }
13557 #if defined(__MINGW32__) && !defined(__MINGW_EXCPT_DEFINE_PSDK) \
13558     && defined(__i386__)
13559 #define __MINGW_EXCPT_DEFINE_PSDK 1
13560 #endif
13561 #include <stdio.h>
13562 #if defined(MSWIN32) && defined(__GNUC__)
13563 #include <excpt.h>
13564 #endif
13565 GC_ATTR_NOINLINE
13566 void GC_noop6(word arg1 GC_ATTR_UNUSED, word arg2 GC_ATTR_UNUSED,
13567               word arg3 GC_ATTR_UNUSED, word arg4 GC_ATTR_UNUSED,
13568               word arg5 GC_ATTR_UNUSED, word arg6 GC_ATTR_UNUSED)
13569 {
13570 #if defined(AO_HAVE_compiler_barrier) && !defined(BASE_ATOMIC_OPS_EMULATED)
13571     AO_compiler_barrier();
13572 #else
13573     GC_noop1(0);
13574 #endif
13575 }
13576 volatile word GC_noop_sink;
13577 GC_ATTR_NO_SANITIZE_THREAD
13578 GC_API void GC_CALL GC_noop1(word x)
13579 {
13580     GC_noop_sink = x;
13581 }
13582 GC_INNER struct obj_kind GC_obj_kinds[MAXOBJKINDS] = {
13583  { &GC_aobjfreelist[0], 0 ,
13584                  GC_DS_LENGTH, FALSE, FALSE
13585                  OK_DISCLAIM_INITZ },
13586   { &GC_objfreelist[0], 0,
13587                  GC_DS_LENGTH,
13588                 TRUE , TRUE
13589                  OK_DISCLAIM_INITZ },
13590               { &GC_uobjfreelist[0], 0,
13591                  GC_DS_LENGTH, TRUE , TRUE
13592                  OK_DISCLAIM_INITZ },
13593 #ifdef GC_ATOMIC_UNCOLLECTABLE
13594               { &GC_auobjfreelist[0], 0,
13595                  GC_DS_LENGTH, FALSE , FALSE
13596                  OK_DISCLAIM_INITZ },
13597 #endif
13598 };
13599 #ifndef INITIAL_MARK_STACK_SIZE
13600 #define INITIAL_MARK_STACK_SIZE (1*HBLKSIZE)
13601 #endif
13602 #if !defined(GC_DISABLE_INCREMENTAL)
13603   STATIC word GC_n_rescuing_pages = 0;
13604 #endif
13605 #ifdef PARALLEL_MARK
13606   GC_INNER GC_bool GC_parallel_mark_disabled = FALSE;
13607 #endif
13608 GC_INNER GC_bool GC_collection_in_progress(void)
13609 {
13610     return(GC_mark_state != MS_NONE);
13611 }
13612 GC_INNER void GC_clear_hdr_marks(hdr *hhdr)
13613 {
13614   size_t last_bit;
13615 #ifdef AO_HAVE_load
13616     last_bit = FINAL_MARK_BIT((size_t)AO_load((volatile AO_t *)&hhdr->hb_sz));
13617 #else
13618     last_bit = FINAL_MARK_BIT((size_t)hhdr->hb_sz);
13619 #endif
13620     BZERO(hhdr -> hb_marks, sizeof(hhdr->hb_marks));
13621     set_mark_bit_from_hdr(hhdr, last_bit);
13622     hhdr -> hb_n_marks = 0;
13623 }
13624 GC_INNER void GC_set_hdr_marks(hdr *hhdr)
13625 {
13626     unsigned i;
13627     size_t sz = (size_t)hhdr->hb_sz;
13628     unsigned n_marks = (unsigned)FINAL_MARK_BIT(sz);
13629 #ifdef USE_MARK_BYTES
13630       for (i = 0; i <= n_marks; i += (unsigned)MARK_BIT_OFFSET(sz)) {
13631         hhdr -> hb_marks[i] = 1;
13632       }
13633 #else
13634       for (i = 0; i < divWORDSZ(n_marks + WORDSZ); ++i) {
13635         hhdr -> hb_marks[i] = GC_WORD_MAX;
13636       }
13637 #endif
13638 #ifdef MARK_BIT_PER_OBJ
13639       hhdr -> hb_n_marks = n_marks;
13640 #else
13641       hhdr -> hb_n_marks = HBLK_OBJS(sz);
13642 #endif
13643 }
13644 static void clear_marks_for_block(struct hblk *h, word dummy GC_ATTR_UNUSED)
13645 {
13646     hdr * hhdr = HDR(h);
13647     if (IS_UNCOLLECTABLE(hhdr -> hb_obj_kind)) return;
13648     GC_clear_hdr_marks(hhdr);
13649 }
13650 GC_API void GC_CALL GC_set_mark_bit(const void *p)
13651 {
13652     struct hblk *h = HBLKPTR(p);
13653     hdr * hhdr = HDR(h);
13654     word bit_no = MARK_BIT_NO((ptr_t)p - (ptr_t)h, hhdr -> hb_sz);
13655     if (!mark_bit_from_hdr(hhdr, bit_no)) {
13656       set_mark_bit_from_hdr(hhdr, bit_no);
13657       ++hhdr -> hb_n_marks;
13658     }
13659 }
13660 GC_API void GC_CALL GC_clear_mark_bit(const void *p)
13661 {
13662     struct hblk *h = HBLKPTR(p);
13663     hdr * hhdr = HDR(h);
13664     word bit_no = MARK_BIT_NO((ptr_t)p - (ptr_t)h, hhdr -> hb_sz);
13665     if (mark_bit_from_hdr(hhdr, bit_no)) {
13666       size_t n_marks = hhdr -> hb_n_marks;
13667       GC_ASSERT(n_marks != 0);
13668       clear_mark_bit_from_hdr(hhdr, bit_no);
13669       n_marks--;
13670 #ifdef PARALLEL_MARK
13671         if (n_marks != 0 || !GC_parallel)
13672           hhdr -> hb_n_marks = n_marks;
13673 #else
13674           hhdr -> hb_n_marks = n_marks;
13675 #endif
13676     }
13677 }
13678 GC_API int GC_CALL GC_is_marked(const void *p)
13679 {
13680     struct hblk *h = HBLKPTR(p);
13681     hdr * hhdr = HDR(h);
13682     word bit_no = MARK_BIT_NO((ptr_t)p - (ptr_t)h, hhdr -> hb_sz);
13683     return (int)mark_bit_from_hdr(hhdr, bit_no);
13684 }
13685 GC_INNER void GC_clear_marks(void)
13686 {
13687     GC_apply_to_all_blocks(clear_marks_for_block, (word)0);
13688     GC_objects_are_marked = FALSE;
13689     GC_mark_state = MS_INVALID;
13690     GC_scan_ptr = NULL;
13691 }
13692 GC_INNER void GC_initiate_gc(void)
13693 {
13694     GC_ASSERT(I_HOLD_LOCK());
13695 #ifndef GC_DISABLE_INCREMENTAL
13696         if (GC_incremental) {
13697 #ifdef CHECKSUMS
13698             GC_read_dirty(FALSE);
13699             GC_check_dirty();
13700 #else
13701             GC_read_dirty(GC_mark_state == MS_INVALID);
13702 #endif
13703         }
13704         GC_n_rescuing_pages = 0;
13705 #endif
13706     if (GC_mark_state == MS_NONE) {
13707         GC_mark_state = MS_PUSH_RESCUERS;
13708     } else if (GC_mark_state != MS_INVALID) {
13709         ABORT("Unexpected state");
13710     }
13711     GC_scan_ptr = NULL;
13712 }
13713 #ifdef PARALLEL_MARK
13714     STATIC void GC_do_parallel_mark(void);
13715 #endif
13716 #ifdef GC_DISABLE_INCREMENTAL
13717 #define GC_push_next_marked_dirty(h) GC_push_next_marked(h)
13718 #else
13719   STATIC struct hblk * GC_push_next_marked_dirty(struct hblk *h);
13720 #endif
13721 STATIC struct hblk * GC_push_next_marked(struct hblk *h);
13722 STATIC struct hblk * GC_push_next_marked_uncollectable(struct hblk *h);
13723 static void alloc_mark_stack(size_t);
13724 #ifdef WRAP_MARK_SOME
13725   STATIC GC_bool GC_mark_some_inner(ptr_t cold_gc_frame)
13726 #else
13727   GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame)
13728 #endif
13729 {
13730     switch(GC_mark_state) {
13731         case MS_NONE:
13732             break;
13733         case MS_PUSH_RESCUERS:
13734             if ((word)GC_mark_stack_top
13735                 >= (word)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE/2)) {
13736                 GC_mark_stack_too_small = TRUE;
13737                 MARK_FROM_MARK_STACK();
13738                 break;
13739             } else {
13740                 GC_scan_ptr = GC_push_next_marked_dirty(GC_scan_ptr);
13741                 if (NULL == GC_scan_ptr) {
13742 #if !defined(GC_DISABLE_INCREMENTAL)
13743                     GC_COND_LOG_PRINTF("Marked from %lu dirty pages\n",
13744                                        (unsigned long)GC_n_rescuing_pages);
13745 #endif
13746                     GC_push_roots(FALSE, cold_gc_frame);
13747                     GC_objects_are_marked = TRUE;
13748                     if (GC_mark_state != MS_INVALID) {
13749                         GC_mark_state = MS_ROOTS_PUSHED;
13750                     }
13751                 }
13752             }
13753             break;
13754         case MS_PUSH_UNCOLLECTABLE:
13755             if ((word)GC_mark_stack_top
13756                 >= (word)(GC_mark_stack + GC_mark_stack_size/4)) {
13757 #ifdef PARALLEL_MARK
13758                   if (GC_parallel) GC_mark_stack_too_small = TRUE;
13759 #endif
13760                 MARK_FROM_MARK_STACK();
13761                 break;
13762             } else {
13763                 GC_scan_ptr = GC_push_next_marked_uncollectable(GC_scan_ptr);
13764                 if (NULL == GC_scan_ptr) {
13765                     GC_push_roots(TRUE, cold_gc_frame);
13766                     GC_objects_are_marked = TRUE;
13767                     if (GC_mark_state != MS_INVALID) {
13768                         GC_mark_state = MS_ROOTS_PUSHED;
13769                     }
13770                 }
13771             }
13772             break;
13773         case MS_ROOTS_PUSHED:
13774 #ifdef PARALLEL_MARK
13775                 if (GC_parallel && !GC_parallel_mark_disabled) {
13776                   GC_do_parallel_mark();
13777                   GC_ASSERT((word)GC_mark_stack_top < (word)GC_first_nonempty);
13778                   GC_mark_stack_top = GC_mark_stack - 1;
13779                   if (GC_mark_stack_too_small) {
13780                     alloc_mark_stack(2*GC_mark_stack_size);
13781                   }
13782                   if (GC_mark_state == MS_ROOTS_PUSHED) {
13783                     GC_mark_state = MS_NONE;
13784                     return(TRUE);
13785                   }
13786                   break;
13787                 }
13788 #endif
13789             if ((word)GC_mark_stack_top >= (word)GC_mark_stack) {
13790                 MARK_FROM_MARK_STACK();
13791                 break;
13792             } else {
13793                 GC_mark_state = MS_NONE;
13794                 if (GC_mark_stack_too_small) {
13795                     alloc_mark_stack(2*GC_mark_stack_size);
13796                 }
13797                 return(TRUE);
13798             }
13799         case MS_INVALID:
13800         case MS_PARTIALLY_INVALID:
13801             if (!GC_objects_are_marked) {
13802                 GC_mark_state = MS_PUSH_UNCOLLECTABLE;
13803                 break;
13804             }
13805             if ((word)GC_mark_stack_top >= (word)GC_mark_stack) {
13806                 MARK_FROM_MARK_STACK();
13807                 break;
13808             }
13809             if (NULL == GC_scan_ptr && GC_mark_state == MS_INVALID) {
13810                 if (GC_mark_stack_too_small) {
13811                     alloc_mark_stack(2*GC_mark_stack_size);
13812                 }
13813                 GC_mark_state = MS_PARTIALLY_INVALID;
13814             }
13815             GC_scan_ptr = GC_push_next_marked(GC_scan_ptr);
13816             if (NULL == GC_scan_ptr && GC_mark_state == MS_PARTIALLY_INVALID) {
13817                 GC_push_roots(TRUE, cold_gc_frame);
13818                 GC_objects_are_marked = TRUE;
13819                 if (GC_mark_state != MS_INVALID) {
13820                     GC_mark_state = MS_ROOTS_PUSHED;
13821                 }
13822             }
13823             break;
13824         default:
13825             ABORT("GC_mark_some: bad state");
13826     }
13827     return(FALSE);
13828 }
13829 #ifdef WRAP_MARK_SOME
13830 #if (defined(MSWIN32) || defined(MSWINCE)) && defined(__GNUC__)
13831     typedef struct {
13832       EXCEPTION_REGISTRATION ex_reg;
13833       void *alt_path;
13834     } ext_ex_regn;
13835     static EXCEPTION_DISPOSITION mark_ex_handler(
13836         struct _EXCEPTION_RECORD *ex_rec,
13837         void *est_frame,
13838         struct _CONTEXT *context,
13839         void *disp_ctxt GC_ATTR_UNUSED)
13840     {
13841         if (ex_rec->ExceptionCode == STATUS_ACCESS_VIOLATION) {
13842           ext_ex_regn *xer = (ext_ex_regn *)est_frame;
13843           context->Esp = context->Ebp;
13844           context->Ebp = *((DWORD *)context->Esp);
13845           context->Esp = context->Esp - 8;
13846           context->Eip = (DWORD )(xer->alt_path);
13847           return ExceptionContinueExecution;
13848         } else {
13849             return ExceptionContinueSearch;
13850         }
13851     }
13852 #endif
13853   GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame)
13854   {
13855       GC_bool ret_val;
13856 #if defined(MSWIN32) || defined(MSWINCE)
13857 #ifndef __GNUC__
13858       __try {
13859           ret_val = GC_mark_some_inner(cold_gc_frame);
13860       } __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ?
13861                 EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
13862           goto handle_ex;
13863       }
13864 #if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
13865         if (GC_started_thread_while_stopped())
13866           goto handle_thr_start;
13867 #endif
13868      rm_handler:
13869       return ret_val;
13870 #else
13871       ext_ex_regn er;
13872 #if GC_GNUC_PREREQ(4, 7) || GC_CLANG_PREREQ(3, 3)
13873 #pragma GCC diagnostic push
13874 #if defined(__clang__) || GC_GNUC_PREREQ(6, 4)
13875 #pragma GCC diagnostic ignored "-Wpedantic"
13876 #else
13877 #pragma GCC diagnostic ignored "-pedantic"
13878 #endif
13879         er.alt_path = &&handle_ex;
13880 #pragma GCC diagnostic pop
13881 #elif !defined(CPPCHECK)
13882         er.alt_path = &&handle_ex;
13883 #endif
13884       er.ex_reg.handler = mark_ex_handler;
13885       __asm__ __volatile__ ("movl %%fs:0, %0" : "=r" (er.ex_reg.prev));
13886       __asm__ __volatile__ ("movl %0, %%fs:0" : : "r" (&er));
13887       ret_val = GC_mark_some_inner(cold_gc_frame);
13888         if (er.alt_path == 0)
13889           goto handle_ex;
13890 #if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
13891         if (GC_started_thread_while_stopped())
13892           goto handle_thr_start;
13893 #endif
13894     rm_handler:
13895       __asm__ __volatile__ ("mov %0, %%fs:0" : : "r" (er.ex_reg.prev));
13896       return ret_val;
13897 #endif
13898 #else
13899 #ifndef DEFAULT_VDB
13900         if (GC_auto_incremental) {
13901           static GC_bool is_warned = FALSE;
13902           if (!is_warned) {
13903             is_warned = TRUE;
13904             WARN("Incremental GC incompatible with /proc roots\n", 0);
13905           }
13906         }
13907 #endif
13908       GC_setup_temporary_fault_handler();
13909       if(SETJMP(GC_jmp_buf) != 0) goto handle_ex;
13910       ret_val = GC_mark_some_inner(cold_gc_frame);
13911     rm_handler:
13912       GC_reset_fault_handler();
13913       return ret_val;
13914 #endif
13915 handle_ex:
13916       {
13917         static word warned_gc_no;
13918         if (warned_gc_no != GC_gc_no) {
13919           WARN("Caught ACCESS_VIOLATION in marker;"
13920                " memory mapping disappeared\n", 0);
13921           warned_gc_no = GC_gc_no;
13922         }
13923       }
13924 #if (defined(MSWIN32) || defined(MSWINCE)) && defined(GC_WIN32_THREADS) \
13925        && !defined(GC_PTHREADS)
13926       handle_thr_start:
13927 #endif
13928 #ifdef REGISTER_LIBRARIES_EARLY
13929         START_WORLD();
13930         GC_cond_register_dynamic_libraries();
13931         STOP_WORLD();
13932 #endif
13933       GC_invalidate_mark_state();
13934       GC_scan_ptr = NULL;
13935       ret_val = FALSE;
13936       goto rm_handler;
13937   }
13938 #endif
13939 GC_INNER void GC_invalidate_mark_state(void)
13940 {
13941     GC_mark_state = MS_INVALID;
13942     GC_mark_stack_top = GC_mark_stack-1;
13943 }
13944 GC_INNER mse * GC_signal_mark_stack_overflow(mse *msp)
13945 {
13946     GC_mark_state = MS_INVALID;
13947 #ifdef PARALLEL_MARK
13948       if (!GC_parallel)
13949         GC_mark_stack_too_small = TRUE;
13950 #else
13951       GC_mark_stack_too_small = TRUE;
13952 #endif
13953     GC_COND_LOG_PRINTF("Mark stack overflow; current size: %lu entries\n",
13954                        (unsigned long)GC_mark_stack_size);
13955     return(msp - GC_MARK_STACK_DISCARDS);
13956 }
13957 GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY GC_ATTR_NO_SANITIZE_THREAD
13958 GC_INNER mse * GC_mark_from(mse *mark_stack_top, mse *mark_stack,
13959                             mse *mark_stack_limit)
13960 {
13961   signed_word credit = HBLKSIZE;
13962   ptr_t current_p;
13963   word current;
13964   ptr_t limit = 0;
13965   word descr;
13966   ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
13967   ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
13968   DECLARE_HDR_CACHE;
13969 #define SPLIT_RANGE_WORDS 128
13970   GC_objects_are_marked = TRUE;
13971   INIT_HDR_CACHE;
13972 #ifdef OS2
13973     while ((word)mark_stack_top >= (word)mark_stack && credit >= 0)
13974 #else
13975     while (((((word)mark_stack_top - (word)mark_stack) | (word)credit)
13976             & SIGNB) == 0)
13977 #endif
13978   {
13979     current_p = mark_stack_top -> mse_start;
13980     descr = mark_stack_top -> mse_descr.w;
13981   retry:
13982     if (descr & ((~(WORDS_TO_BYTES(SPLIT_RANGE_WORDS) - 1)) | GC_DS_TAGS)) {
13983       word tag = descr & GC_DS_TAGS;
13984       GC_STATIC_ASSERT(GC_DS_TAGS == 0x3);
13985       switch(tag) {
13986         case GC_DS_LENGTH:
13987           GC_ASSERT(descr < (word)GC_greatest_plausible_heap_addr
13988                             - (word)GC_least_plausible_heap_addr
13989                 || (word)(current_p + descr)
13990                             <= (word)GC_least_plausible_heap_addr
13991                 || (word)current_p >= (word)GC_greatest_plausible_heap_addr);
13992 #ifdef PARALLEL_MARK
13993 #define SHARE_BYTES 2048
13994             if (descr > SHARE_BYTES && GC_parallel
13995                 && (word)mark_stack_top < (word)(mark_stack_limit - 1)) {
13996               word new_size = (descr/2) & ~(word)(sizeof(word)-1);
13997               mark_stack_top -> mse_start = current_p;
13998               mark_stack_top -> mse_descr.w = new_size + sizeof(word);
13999               mark_stack_top++;
14000 #ifdef ENABLE_TRACE
14001                 if ((word)GC_trace_addr >= (word)current_p
14002                     && (word)GC_trace_addr < (word)(current_p + descr)) {
14003                   GC_log_printf("GC #%lu: large section; start %p, len %lu,"
14004                                 " splitting (parallel) at %p\n",
14005                                 (unsigned long)GC_gc_no, (void *)current_p,
14006                                 (unsigned long)descr,
14007                                 (void *)(current_p + new_size));
14008                 }
14009 #endif
14010               current_p += new_size;
14011               descr -= new_size;
14012               goto retry;
14013             }
14014 #endif
14015           mark_stack_top -> mse_start =
14016                 limit = current_p + WORDS_TO_BYTES(SPLIT_RANGE_WORDS-1);
14017           mark_stack_top -> mse_descr.w =
14018                                 descr - WORDS_TO_BYTES(SPLIT_RANGE_WORDS-1);
14019 #ifdef ENABLE_TRACE
14020             if ((word)GC_trace_addr >= (word)current_p
14021                 && (word)GC_trace_addr < (word)(current_p + descr)) {
14022               GC_log_printf("GC #%lu: large section; start %p, len %lu,"
14023                             " splitting at %p\n",
14024                             (unsigned long)GC_gc_no, (void *)current_p,
14025                             (unsigned long)descr, (void *)limit);
14026             }
14027 #endif
14028           limit += sizeof(word) - ALIGNMENT;
14029           break;
14030         case GC_DS_BITMAP:
14031           mark_stack_top--;
14032 #ifdef ENABLE_TRACE
14033             if ((word)GC_trace_addr >= (word)current_p
14034                 && (word)GC_trace_addr < (word)(current_p
14035                                                 + WORDS_TO_BYTES(WORDSZ-2))) {
14036               GC_log_printf("GC #%lu: tracing from %p bitmap descr %lu\n",
14037                             (unsigned long)GC_gc_no, (void *)current_p,
14038                             (unsigned long)descr);
14039             }
14040 #endif
14041           descr &= ~GC_DS_TAGS;
14042           credit -= WORDS_TO_BYTES(WORDSZ/2);
14043           while (descr != 0) {
14044             if ((descr & SIGNB) != 0) {
14045               current = *(word *)current_p;
14046               FIXUP_POINTER(current);
14047               if (current >= (word)least_ha && current < (word)greatest_ha) {
14048                 PREFETCH((ptr_t)current);
14049 #ifdef ENABLE_TRACE
14050                   if (GC_trace_addr == current_p) {
14051                     GC_log_printf("GC #%lu: considering(3) %p -> %p\n",
14052                                   (unsigned long)GC_gc_no, (void *)current_p,
14053                                   (void *)current);
14054                   }
14055 #endif
14056                 PUSH_CONTENTS((ptr_t)current, mark_stack_top,
14057                               mark_stack_limit, current_p);
14058               }
14059             }
14060             descr <<= 1;
14061             current_p += sizeof(word);
14062           }
14063           continue;
14064         case GC_DS_PROC:
14065           mark_stack_top--;
14066 #ifdef ENABLE_TRACE
14067             if ((word)GC_trace_addr >= (word)current_p
14068                 && GC_base(current_p) != 0
14069                 && GC_base(current_p) == GC_base(GC_trace_addr)) {
14070               GC_log_printf("GC #%lu: tracing from %p, proc descr %lu\n",
14071                             (unsigned long)GC_gc_no, (void *)current_p,
14072                             (unsigned long)descr);
14073             }
14074 #endif
14075           credit -= GC_PROC_BYTES;
14076           mark_stack_top = (*PROC(descr))((word *)current_p, mark_stack_top,
14077                                           mark_stack_limit, ENV(descr));
14078           continue;
14079         case GC_DS_PER_OBJECT:
14080           if ((signed_word)descr >= 0) {
14081             descr = *(word *)(current_p + descr - GC_DS_PER_OBJECT);
14082           } else {
14083             ptr_t type_descr = *(ptr_t *)current_p;
14084             if (EXPECT(0 == type_descr, FALSE)) {
14085                 mark_stack_top--;
14086                 continue;
14087             }
14088             descr = *(word *)(type_descr
14089                               - ((signed_word)descr + (GC_INDIR_PER_OBJ_BIAS
14090                                                        - GC_DS_PER_OBJECT)));
14091           }
14092           if (0 == descr) {
14093               mark_stack_top--;
14094               continue;
14095           }
14096           goto retry;
14097       }
14098     } else {
14099       mark_stack_top--;
14100 #ifndef SMALL_CONFIG
14101         if (descr < sizeof(word))
14102           continue;
14103 #endif
14104 #ifdef ENABLE_TRACE
14105         if ((word)GC_trace_addr >= (word)current_p
14106             && (word)GC_trace_addr < (word)(current_p + descr)) {
14107           GC_log_printf("GC #%lu: small object; start %p, len %lu\n",
14108                         (unsigned long)GC_gc_no, (void *)current_p,
14109                         (unsigned long)descr);
14110         }
14111 #endif
14112       limit = current_p + (word)descr;
14113     }
14114     GC_ASSERT(!((word)current_p & (ALIGNMENT-1)));
14115     credit -= limit - current_p;
14116     limit -= sizeof(word);
14117     {
14118 #define PREF_DIST 4
14119 #ifndef SMALL_CONFIG
14120         word deferred;
14121         for(;;) {
14122           PREFETCH(limit - PREF_DIST*CACHE_LINE_SIZE);
14123           GC_ASSERT((word)limit >= (word)current_p);
14124           deferred = *(word *)limit;
14125           FIXUP_POINTER(deferred);
14126           limit -= ALIGNMENT;
14127           if (deferred >= (word)least_ha && deferred < (word)greatest_ha) {
14128             PREFETCH((ptr_t)deferred);
14129             break;
14130           }
14131           if ((word)current_p > (word)limit) goto next_object;
14132           deferred = *(word *)limit;
14133           FIXUP_POINTER(deferred);
14134           limit -= ALIGNMENT;
14135           if (deferred >= (word)least_ha && deferred < (word)greatest_ha) {
14136             PREFETCH((ptr_t)deferred);
14137             break;
14138           }
14139           if ((word)current_p > (word)limit) goto next_object;
14140         }
14141 #endif
14142       while ((word)current_p <= (word)limit) {
14143         current = *(word *)current_p;
14144         FIXUP_POINTER(current);
14145         PREFETCH(current_p + PREF_DIST*CACHE_LINE_SIZE);
14146         if (current >= (word)least_ha && current < (word)greatest_ha) {
14147           PREFETCH((ptr_t)current);
14148 #ifdef ENABLE_TRACE
14149             if (GC_trace_addr == current_p) {
14150               GC_log_printf("GC #%lu: considering(1) %p -> %p\n",
14151                             (unsigned long)GC_gc_no, (void *)current_p,
14152                             (void *)current);
14153             }
14154 #endif
14155           PUSH_CONTENTS((ptr_t)current, mark_stack_top,
14156                         mark_stack_limit, current_p);
14157         }
14158         current_p += ALIGNMENT;
14159       }
14160 #ifndef SMALL_CONFIG
14161 #ifdef ENABLE_TRACE
14162             if (GC_trace_addr == current_p) {
14163               GC_log_printf("GC #%lu: considering(2) %p -> %p\n",
14164                             (unsigned long)GC_gc_no, (void *)current_p,
14165                             (void *)deferred);
14166             }
14167 #endif
14168         PUSH_CONTENTS((ptr_t)deferred, mark_stack_top,
14169                       mark_stack_limit, current_p);
14170         next_object:;
14171 #endif
14172     }
14173   }
14174   return mark_stack_top;
14175 }
14176 #ifdef PARALLEL_MARK
14177 STATIC GC_bool GC_help_wanted = FALSE;
14178 STATIC unsigned GC_helper_count = 0;
14179 STATIC unsigned GC_active_count = 0;
14180 GC_INNER word GC_mark_no = 0;
14181 #ifdef LINT2
14182 #define LOCAL_MARK_STACK_SIZE (HBLKSIZE / 8)
14183 #else
14184 #define LOCAL_MARK_STACK_SIZE HBLKSIZE
14185 #endif
14186 GC_INNER void GC_wait_for_markers_init(void)
14187 {
14188   signed_word count;
14189   if (GC_markers_m1 == 0)
14190     return;
14191 #ifndef CAN_HANDLE_FORK
14192     GC_ASSERT(NULL == GC_main_local_mark_stack);
14193 #else
14194     if (NULL == GC_main_local_mark_stack)
14195 #endif
14196   {
14197     size_t bytes_to_get =
14198                 ROUNDUP_PAGESIZE_IF_MMAP(LOCAL_MARK_STACK_SIZE * sizeof(mse));
14199     GC_ASSERT(GC_page_size != 0);
14200     GC_main_local_mark_stack = (mse *)GET_MEM(bytes_to_get);
14201     if (NULL == GC_main_local_mark_stack)
14202       ABORT("Insufficient memory for main local_mark_stack");
14203     GC_add_to_our_memory((ptr_t)GC_main_local_mark_stack, bytes_to_get);
14204   }
14205   GC_acquire_mark_lock();
14206   GC_fl_builder_count += GC_markers_m1;
14207   count = GC_fl_builder_count;
14208   GC_release_mark_lock();
14209   if (count != 0) {
14210     GC_ASSERT(count > 0);
14211     GC_wait_for_reclaim();
14212   }
14213 }
14214 STATIC mse * GC_steal_mark_stack(mse * low, mse * high, mse * local,
14215                                  unsigned max, mse **next)
14216 {
14217     mse *p;
14218     mse *top = local - 1;
14219     unsigned i = 0;
14220     GC_ASSERT((word)high >= (word)(low - 1)
14221               && (word)(high - low + 1) <= GC_mark_stack_size);
14222     for (p = low; (word)p <= (word)high && i <= max; ++p) {
14223         word descr = (word)AO_load(&p->mse_descr.ao);
14224         if (descr != 0) {
14225             AO_store_release_write(&p->mse_descr.ao, 0);
14226             ++top;
14227             top -> mse_descr.w = descr;
14228             top -> mse_start = p -> mse_start;
14229             GC_ASSERT((descr & GC_DS_TAGS) != GC_DS_LENGTH
14230                       || descr < (word)GC_greatest_plausible_heap_addr
14231                                         - (word)GC_least_plausible_heap_addr
14232                       || (word)(p->mse_start + descr)
14233                             <= (word)GC_least_plausible_heap_addr
14234                       || (word)p->mse_start
14235                             >= (word)GC_greatest_plausible_heap_addr);
14236             ++i;
14237             if ((descr & GC_DS_TAGS) == GC_DS_LENGTH) i += (int)(descr >> 8);
14238         }
14239     }
14240     *next = p;
14241     return top;
14242 }
14243 STATIC void GC_return_mark_stack(mse * low, mse * high)
14244 {
14245     mse * my_top;
14246     mse * my_start;
14247     size_t stack_size;
14248     if ((word)high < (word)low) return;
14249     stack_size = high - low + 1;
14250     GC_acquire_mark_lock();
14251     my_top = GC_mark_stack_top;
14252     my_start = my_top + 1;
14253     if ((word)(my_start - GC_mark_stack + stack_size)
14254                 > (word)GC_mark_stack_size) {
14255       GC_COND_LOG_PRINTF("No room to copy back mark stack\n");
14256       GC_mark_state = MS_INVALID;
14257       GC_mark_stack_too_small = TRUE;
14258     } else {
14259       BCOPY(low, my_start, stack_size * sizeof(mse));
14260       GC_ASSERT((mse *)AO_load((volatile AO_t *)(&GC_mark_stack_top))
14261                 == my_top);
14262       AO_store_release_write((volatile AO_t *)(&GC_mark_stack_top),
14263                              (AO_t)(my_top + stack_size));
14264     }
14265     GC_release_mark_lock();
14266     GC_notify_all_marker();
14267 }
14268 #ifndef N_LOCAL_ITERS
14269 #define N_LOCAL_ITERS 1
14270 #endif
14271 static GC_bool has_inactive_helpers(void)
14272 {
14273   GC_bool res;
14274   GC_acquire_mark_lock();
14275   res = GC_active_count < GC_helper_count;
14276   GC_release_mark_lock();
14277   return res;
14278 }
14279 STATIC void GC_do_local_mark(mse *local_mark_stack, mse *local_top)
14280 {
14281     unsigned n;
14282     for (;;) {
14283         for (n = 0; n < N_LOCAL_ITERS; ++n) {
14284             local_top = GC_mark_from(local_top, local_mark_stack,
14285                                      local_mark_stack + LOCAL_MARK_STACK_SIZE);
14286             if ((word)local_top < (word)local_mark_stack) return;
14287             if ((word)(local_top - local_mark_stack)
14288                         >= LOCAL_MARK_STACK_SIZE / 2) {
14289                 GC_return_mark_stack(local_mark_stack, local_top);
14290                 return;
14291             }
14292         }
14293         if ((word)AO_load((volatile AO_t *)&GC_mark_stack_top)
14294             < (word)AO_load(&GC_first_nonempty)
14295             && (word)local_top > (word)(local_mark_stack + 1)
14296             && has_inactive_helpers()) {
14297             mse * new_bottom = local_mark_stack
14298                                 + (local_top - local_mark_stack)/2;
14299             GC_ASSERT((word)new_bottom > (word)local_mark_stack
14300                       && (word)new_bottom < (word)local_top);
14301             GC_return_mark_stack(local_mark_stack, new_bottom - 1);
14302             memmove(local_mark_stack, new_bottom,
14303                     (local_top - new_bottom + 1) * sizeof(mse));
14304             local_top -= (new_bottom - local_mark_stack);
14305         }
14306     }
14307 }
14308 #ifndef ENTRIES_TO_GET
14309 #define ENTRIES_TO_GET 5
14310 #endif
14311 STATIC void GC_mark_local(mse *local_mark_stack, int id)
14312 {
14313     mse * my_first_nonempty;
14314     GC_active_count++;
14315     my_first_nonempty = (mse *)AO_load(&GC_first_nonempty);
14316     GC_ASSERT((word)GC_mark_stack <= (word)my_first_nonempty);
14317     GC_ASSERT((word)my_first_nonempty
14318         <= (word)AO_load((volatile AO_t *)&GC_mark_stack_top) + sizeof(mse));
14319     GC_VERBOSE_LOG_PRINTF("Starting mark helper %d\n", id);
14320     GC_release_mark_lock();
14321     for (;;) {
14322         size_t n_on_stack;
14323         unsigned n_to_get;
14324         mse * my_top;
14325         mse * local_top;
14326         mse * global_first_nonempty = (mse *)AO_load(&GC_first_nonempty);
14327         GC_ASSERT((word)my_first_nonempty >= (word)GC_mark_stack &&
14328                   (word)my_first_nonempty <=
14329                         (word)AO_load((volatile AO_t *)&GC_mark_stack_top)
14330                         + sizeof(mse));
14331         GC_ASSERT((word)global_first_nonempty >= (word)GC_mark_stack);
14332         if ((word)my_first_nonempty < (word)global_first_nonempty) {
14333             my_first_nonempty = global_first_nonempty;
14334         } else if ((word)global_first_nonempty < (word)my_first_nonempty) {
14335             (void)AO_compare_and_swap(&GC_first_nonempty,
14336                                       (AO_t)global_first_nonempty,
14337                                       (AO_t)my_first_nonempty);
14338         }
14339         my_top = (mse *)AO_load_acquire((volatile AO_t *)(&GC_mark_stack_top));
14340         if ((word)my_top < (word)my_first_nonempty) {
14341             GC_acquire_mark_lock();
14342             my_top = GC_mark_stack_top;
14343             n_on_stack = my_top - my_first_nonempty + 1;
14344             if (0 == n_on_stack) {
14345                 GC_active_count--;
14346                 GC_ASSERT(GC_active_count <= GC_helper_count);
14347                 if (0 == GC_active_count) GC_notify_all_marker();
14348                 while (GC_active_count > 0
14349                        && (word)AO_load(&GC_first_nonempty)
14350                                 > (word)GC_mark_stack_top) {
14351                     GC_wait_marker();
14352                 }
14353                 if (GC_active_count == 0
14354                     && (word)AO_load(&GC_first_nonempty)
14355                         > (word)GC_mark_stack_top) {
14356                     GC_bool need_to_notify = FALSE;
14357                     GC_helper_count--;
14358                     if (0 == GC_helper_count) need_to_notify = TRUE;
14359                     GC_VERBOSE_LOG_PRINTF("Finished mark helper %d\n", id);
14360                     if (need_to_notify) GC_notify_all_marker();
14361                     return;
14362                 }
14363                 GC_active_count++;
14364                 GC_ASSERT(GC_active_count > 0);
14365                 GC_release_mark_lock();
14366                 continue;
14367             } else {
14368                 GC_release_mark_lock();
14369             }
14370         } else {
14371             n_on_stack = my_top - my_first_nonempty + 1;
14372         }
14373         n_to_get = ENTRIES_TO_GET;
14374         if (n_on_stack < 2 * ENTRIES_TO_GET) n_to_get = 1;
14375         local_top = GC_steal_mark_stack(my_first_nonempty, my_top,
14376                                         local_mark_stack, n_to_get,
14377                                         &my_first_nonempty);
14378         GC_ASSERT((word)my_first_nonempty >= (word)GC_mark_stack &&
14379                   (word)my_first_nonempty <=
14380                         (word)AO_load((volatile AO_t *)&GC_mark_stack_top)
14381                         + sizeof(mse));
14382         GC_do_local_mark(local_mark_stack, local_top);
14383     }
14384 }
14385 STATIC void GC_do_parallel_mark(void)
14386 {
14387     GC_acquire_mark_lock();
14388     GC_ASSERT(I_HOLD_LOCK());
14389     if (GC_help_wanted || GC_active_count != 0 || GC_helper_count != 0)
14390         ABORT("Tried to start parallel mark in bad state");
14391     GC_VERBOSE_LOG_PRINTF("Starting marking for mark phase number %lu\n",
14392                           (unsigned long)GC_mark_no);
14393     GC_first_nonempty = (AO_t)GC_mark_stack;
14394     GC_active_count = 0;
14395     GC_helper_count = 1;
14396     GC_help_wanted = TRUE;
14397     GC_notify_all_marker();
14398     GC_mark_local(GC_main_local_mark_stack, 0);
14399     GC_help_wanted = FALSE;
14400     while (GC_helper_count > 0) {
14401       GC_wait_marker();
14402     }
14403     GC_VERBOSE_LOG_PRINTF("Finished marking for mark phase number %lu\n",
14404                           (unsigned long)GC_mark_no);
14405     GC_mark_no++;
14406     GC_release_mark_lock();
14407     GC_notify_all_marker();
14408 }
14409 GC_INNER void GC_help_marker(word my_mark_no)
14410 {
14411 #define my_id my_id_mse.mse_descr.w
14412     mse my_id_mse;
14413     mse local_mark_stack[LOCAL_MARK_STACK_SIZE];
14414     GC_ASSERT(GC_parallel);
14415     while (GC_mark_no < my_mark_no
14416            || (!GC_help_wanted && GC_mark_no == my_mark_no)) {
14417       GC_wait_marker();
14418     }
14419     my_id = GC_helper_count;
14420     if (GC_mark_no != my_mark_no || my_id > (unsigned)GC_markers_m1) {
14421       return;
14422     }
14423     GC_helper_count = (unsigned)my_id + 1;
14424     GC_mark_local(local_mark_stack, (int)my_id);
14425 #undef my_id
14426 }
14427 #endif
14428 static void alloc_mark_stack(size_t n)
14429 {
14430     mse * new_stack = (mse *)GC_scratch_alloc(n * sizeof(struct GC_ms_entry));
14431 #ifdef GWW_VDB
14432       static GC_bool GC_incremental_at_stack_alloc = FALSE;
14433       GC_bool recycle_old = !GC_auto_incremental
14434                             || GC_incremental_at_stack_alloc;
14435       GC_incremental_at_stack_alloc = GC_auto_incremental;
14436 #else
14437 #define recycle_old TRUE
14438 #endif
14439     GC_mark_stack_too_small = FALSE;
14440     if (GC_mark_stack != NULL) {
14441         if (new_stack != 0) {
14442           if (recycle_old) {
14443             GC_scratch_recycle_inner(GC_mark_stack,
14444                         GC_mark_stack_size * sizeof(struct GC_ms_entry));
14445           }
14446           GC_mark_stack = new_stack;
14447           GC_mark_stack_size = n;
14448           GC_mark_stack_limit = new_stack + n;
14449           GC_COND_LOG_PRINTF("Grew mark stack to %lu frames\n",
14450                              (unsigned long)GC_mark_stack_size);
14451         } else {
14452           WARN("Failed to grow mark stack to %" WARN_PRIdPTR " frames\n", n);
14453         }
14454     } else if (NULL == new_stack) {
14455         GC_err_printf("No space for mark stack\n");
14456         EXIT();
14457     } else {
14458         GC_mark_stack = new_stack;
14459         GC_mark_stack_size = n;
14460         GC_mark_stack_limit = new_stack + n;
14461     }
14462     GC_mark_stack_top = GC_mark_stack-1;
14463 }
14464 GC_INNER void GC_mark_init(void)
14465 {
14466     alloc_mark_stack(INITIAL_MARK_STACK_SIZE);
14467 }
14468 GC_API void GC_CALL GC_push_all(void *bottom, void *top)
14469 {
14470     word length;
14471     bottom = (void *)(((word)bottom + ALIGNMENT-1) & ~(ALIGNMENT-1));
14472     top = (void *)((word)top & ~(ALIGNMENT-1));
14473     if ((word)bottom >= (word)top) return;
14474     GC_mark_stack_top++;
14475     if ((word)GC_mark_stack_top >= (word)GC_mark_stack_limit) {
14476         ABORT("Unexpected mark stack overflow");
14477     }
14478     length = (word)top - (word)bottom;
14479 #if GC_DS_TAGS > ALIGNMENT - 1
14480         length += GC_DS_TAGS;
14481         length &= ~GC_DS_TAGS;
14482 #endif
14483     GC_mark_stack_top -> mse_start = (ptr_t)bottom;
14484     GC_mark_stack_top -> mse_descr.w = length;
14485 }
14486 #ifndef GC_DISABLE_INCREMENTAL
14487   STATIC void GC_push_selected(ptr_t bottom, ptr_t top,
14488                                GC_bool (*dirty_fn)(struct hblk *))
14489   {
14490     struct hblk * h;
14491     bottom = (ptr_t)(((word) bottom + ALIGNMENT-1) & ~(ALIGNMENT-1));
14492     top = (ptr_t)(((word) top) & ~(ALIGNMENT-1));
14493     if ((word)bottom >= (word)top) return;
14494     h = HBLKPTR(bottom + HBLKSIZE);
14495     if ((word)top <= (word)h) {
14496         if ((*dirty_fn)(h-1)) {
14497             GC_push_all(bottom, top);
14498         }
14499         return;
14500     }
14501     if ((*dirty_fn)(h-1)) {
14502         if ((word)(GC_mark_stack_top - GC_mark_stack)
14503             > 3 * GC_mark_stack_size / 4) {
14504             GC_push_all(bottom, top);
14505             return;
14506         }
14507         GC_push_all(bottom, h);
14508     }
14509     while ((word)(h+1) <= (word)top) {
14510         if ((*dirty_fn)(h)) {
14511             if ((word)(GC_mark_stack_top - GC_mark_stack)
14512                 > 3 * GC_mark_stack_size / 4) {
14513                 GC_push_all(h, top);
14514                 return;
14515             } else {
14516                 GC_push_all(h, h + 1);
14517             }
14518         }
14519         h++;
14520     }
14521     if ((ptr_t)h != top && (*dirty_fn)(h)) {
14522        GC_push_all(h, top);
14523     }
14524   }
14525   GC_API void GC_CALL GC_push_conditional(void *bottom, void *top, int all)
14526   {
14527     if (!all) {
14528       GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_page_was_dirty);
14529     } else {
14530 #ifdef PROC_VDB
14531         if (GC_auto_incremental) {
14532           GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_page_was_ever_dirty);
14533         } else
14534 #endif
14535        {
14536         GC_push_all(bottom, top);
14537       }
14538     }
14539   }
14540 #ifndef NO_VDB_FOR_STATIC_ROOTS
14541 #ifndef PROC_VDB
14542       STATIC GC_bool GC_static_page_was_dirty(struct hblk *h)
14543       {
14544         return get_pht_entry_from_index(GC_grungy_pages, PHT_HASH(h));
14545       }
14546 #endif
14547     GC_INNER void GC_push_conditional_static(void *bottom, void *top,
14548                                              GC_bool all)
14549     {
14550 #ifdef PROC_VDB
14551         GC_push_conditional(bottom, top, all);
14552 #else
14553         if (all || !GC_is_vdb_for_static_roots()) {
14554           GC_push_all(bottom, top);
14555         } else {
14556           GC_push_selected((ptr_t)bottom, (ptr_t)top,
14557                            GC_static_page_was_dirty);
14558         }
14559 #endif
14560     }
14561 #endif
14562 #else
14563   GC_API void GC_CALL GC_push_conditional(void *bottom, void *top,
14564                                           int all GC_ATTR_UNUSED)
14565   {
14566     GC_push_all(bottom, top);
14567   }
14568 #endif
14569 #if defined(AMIGA) || defined(MACOS) || defined(GC_DARWIN_THREADS)
14570   void GC_push_one(word p)
14571   {
14572     GC_PUSH_ONE_STACK(p, MARKED_FROM_REGISTER);
14573   }
14574 #endif
14575 #ifdef GC_WIN32_THREADS
14576   GC_INNER void GC_push_many_regs(const word *regs, unsigned count)
14577   {
14578     unsigned i;
14579     for (i = 0; i < count; i++)
14580       GC_PUSH_ONE_STACK(regs[i], MARKED_FROM_REGISTER);
14581   }
14582 #endif
14583 GC_API struct GC_ms_entry * GC_CALL GC_mark_and_push(void *obj,
14584                                                 mse *mark_stack_ptr,
14585                                                 mse *mark_stack_limit,
14586                                                 void ** src GC_ATTR_UNUSED)
14587 {
14588     hdr * hhdr;
14589     PREFETCH(obj);
14590     GET_HDR(obj, hhdr);
14591     if ((EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr), FALSE)
14592          && (!GC_all_interior_pointers
14593              || NULL == (hhdr = GC_find_header((ptr_t)GC_base(obj)))))
14594         || EXPECT(HBLK_IS_FREE(hhdr), FALSE)) {
14595       GC_ADD_TO_BLACK_LIST_NORMAL(obj, (ptr_t)src);
14596       return mark_stack_ptr;
14597     }
14598     return GC_push_contents_hdr((ptr_t)obj, mark_stack_ptr, mark_stack_limit,
14599                                 (ptr_t)src, hhdr, TRUE);
14600 }
14601 #if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
14602     GC_INNER void GC_mark_and_push_stack(ptr_t p, ptr_t source)
14603 #else
14604     GC_INNER void GC_mark_and_push_stack(ptr_t p)
14605 #define source ((ptr_t)0)
14606 #endif
14607 {
14608     hdr * hhdr;
14609     ptr_t r = p;
14610     PREFETCH(p);
14611     GET_HDR(p, hhdr);
14612     if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr), FALSE)) {
14613       if (NULL == hhdr
14614             || (r = (ptr_t)GC_base(p)) == NULL
14615             || (hhdr = HDR(r)) == NULL) {
14616         GC_ADD_TO_BLACK_LIST_STACK(p, source);
14617         return;
14618       }
14619     }
14620     if (EXPECT(HBLK_IS_FREE(hhdr), FALSE)) {
14621         GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
14622         return;
14623     }
14624 #ifdef THREADS
14625       GC_dirty(p);
14626 #endif
14627     GC_mark_stack_top = GC_push_contents_hdr(r, GC_mark_stack_top,
14628                                              GC_mark_stack_limit,
14629                                              source, hhdr, FALSE);
14630 }
14631 #undef source
14632 #ifdef TRACE_BUF
14633 #ifndef TRACE_ENTRIES
14634 #define TRACE_ENTRIES 1000
14635 #endif
14636 struct trace_entry {
14637     char * kind;
14638     word gc_no;
14639     word bytes_allocd;
14640     word arg1;
14641     word arg2;
14642 } GC_trace_buf[TRACE_ENTRIES] = { { NULL, 0, 0, 0, 0 } };
14643 void GC_add_trace_entry(char *kind, word arg1, word arg2)
14644 {
14645     GC_trace_buf[GC_trace_buf_ptr].kind = kind;
14646     GC_trace_buf[GC_trace_buf_ptr].gc_no = GC_gc_no;
14647     GC_trace_buf[GC_trace_buf_ptr].bytes_allocd = GC_bytes_allocd;
14648     GC_trace_buf[GC_trace_buf_ptr].arg1 = arg1 ^ 0x80000000;
14649     GC_trace_buf[GC_trace_buf_ptr].arg2 = arg2 ^ 0x80000000;
14650     GC_trace_buf_ptr++;
14651     if (GC_trace_buf_ptr >= TRACE_ENTRIES) GC_trace_buf_ptr = 0;
14652 }
14653 GC_API void GC_CALL GC_print_trace_inner(word gc_no)
14654 {
14655     int i;
14656     for (i = GC_trace_buf_ptr-1; i != GC_trace_buf_ptr; i--) {
14657         struct trace_entry *p;
14658         if (i < 0) i = TRACE_ENTRIES-1;
14659         p = GC_trace_buf + i;
14660         if (p -> gc_no < gc_no || p -> kind == 0) {
14661             return;
14662         }
14663         GC_printf("Trace:%s (gc:%u, bytes:%lu) 0x%lX, 0x%lX\n",
14664                   p -> kind, (unsigned)p -> gc_no,
14665                   (unsigned long)p -> bytes_allocd,
14666                   (long)p->arg1 ^ 0x80000000L, (long)p->arg2 ^ 0x80000000L);
14667     }
14668     GC_printf("Trace incomplete\n");
14669 }
14670 GC_API void GC_CALL GC_print_trace(word gc_no)
14671 {
14672     DCL_LOCK_STATE;
14673     LOCK();
14674     GC_print_trace_inner(gc_no);
14675     UNLOCK();
14676 }
14677 #endif
14678 GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY GC_ATTR_NO_SANITIZE_THREAD
14679 GC_API void GC_CALL GC_push_all_eager(void *bottom, void *top)
14680 {
14681     word * b = (word *)(((word) bottom + ALIGNMENT-1) & ~(ALIGNMENT-1));
14682     word * t = (word *)(((word) top) & ~(ALIGNMENT-1));
14683     REGISTER word *p;
14684     REGISTER word *lim;
14685     REGISTER ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
14686     REGISTER ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
14687 #define GC_greatest_plausible_heap_addr greatest_ha
14688 #define GC_least_plausible_heap_addr least_ha
14689     if (top == 0) return;
14690       lim = t - 1 ;
14691       for (p = b; (word)p <= (word)lim;
14692            p = (word *)(((ptr_t)p) + ALIGNMENT)) {
14693         REGISTER word q = *p;
14694         GC_PUSH_ONE_STACK(q, p);
14695       }
14696 #undef GC_greatest_plausible_heap_addr
14697 #undef GC_least_plausible_heap_addr
14698 }
14699 GC_INNER void GC_push_all_stack(ptr_t bottom, ptr_t top)
14700 {
14701 #ifndef NEED_FIXUP_POINTER
14702       if (GC_all_interior_pointers
14703 #if defined(THREADS) && defined(MPROTECT_VDB)
14704             && !GC_auto_incremental
14705 #endif
14706           && (word)GC_mark_stack_top
14707              < (word)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE/8)) {
14708         GC_push_all(bottom, top);
14709       } else
14710 #endif
14711      {
14712       GC_push_all_eager(bottom, top);
14713     }
14714 }
14715 #if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK)
14716   GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY
14717   GC_ATTR_NO_SANITIZE_THREAD
14718   GC_INNER void GC_push_conditional_eager(void *bottom, void *top,
14719                                           GC_bool all)
14720   {
14721     word * b = (word *)(((word) bottom + ALIGNMENT-1) & ~(ALIGNMENT-1));
14722     word * t = (word *)(((word) top) & ~(ALIGNMENT-1));
14723     REGISTER word *p;
14724     REGISTER word *lim;
14725     REGISTER ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
14726     REGISTER ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
14727 #define GC_greatest_plausible_heap_addr greatest_ha
14728 #define GC_least_plausible_heap_addr least_ha
14729     if (top == NULL)
14730       return;
14731     (void)all;
14732     lim = t - 1;
14733     for (p = b; (word)p <= (word)lim; p = (word *)((ptr_t)p + ALIGNMENT)) {
14734       REGISTER word q = *p;
14735       GC_PUSH_ONE_HEAP(q, p, GC_mark_stack_top);
14736     }
14737 #undef GC_greatest_plausible_heap_addr
14738 #undef GC_least_plausible_heap_addr
14739   }
14740 #endif
14741 #if !defined(SMALL_CONFIG) && !defined(USE_MARK_BYTES) && \
14742     defined(MARK_BIT_PER_GRANULE)
14743 #if GC_GRANULE_WORDS == 1
14744 #define USE_PUSH_MARKED_ACCELERATORS
14745 #define PUSH_GRANULE(q) \
14746                 do { \
14747                   word qcontents = (q)[0]; \
14748                   GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \
14749                 } while (0)
14750 #elif GC_GRANULE_WORDS == 2
14751 #define USE_PUSH_MARKED_ACCELERATORS
14752 #define PUSH_GRANULE(q) \
14753                 do { \
14754                   word qcontents = (q)[0]; \
14755                   GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \
14756                   qcontents = (q)[1]; \
14757                   GC_PUSH_ONE_HEAP(qcontents, (q)+1, GC_mark_stack_top); \
14758                 } while (0)
14759 #elif GC_GRANULE_WORDS == 4
14760 #define USE_PUSH_MARKED_ACCELERATORS
14761 #define PUSH_GRANULE(q) \
14762                 do { \
14763                   word qcontents = (q)[0]; \
14764                   GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \
14765                   qcontents = (q)[1]; \
14766                   GC_PUSH_ONE_HEAP(qcontents, (q)+1, GC_mark_stack_top); \
14767                   qcontents = (q)[2]; \
14768                   GC_PUSH_ONE_HEAP(qcontents, (q)+2, GC_mark_stack_top); \
14769                   qcontents = (q)[3]; \
14770                   GC_PUSH_ONE_HEAP(qcontents, (q)+3, GC_mark_stack_top); \
14771                 } while (0)
14772 #endif
14773 #endif
14774 #ifdef USE_PUSH_MARKED_ACCELERATORS
14775 STATIC void GC_push_marked1(struct hblk *h, hdr *hhdr)
14776 {
14777     word * mark_word_addr = &(hhdr->hb_marks[0]);
14778     word *p;
14779     word *plim;
14780     ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
14781     ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
14782     mse * mark_stack_top = GC_mark_stack_top;
14783     mse * mark_stack_limit = GC_mark_stack_limit;
14784 #undef GC_mark_stack_top
14785 #undef GC_mark_stack_limit
14786 #define GC_mark_stack_top mark_stack_top
14787 #define GC_mark_stack_limit mark_stack_limit
14788 #define GC_greatest_plausible_heap_addr greatest_ha
14789 #define GC_least_plausible_heap_addr least_ha
14790     p = (word *)(h->hb_body);
14791     plim = (word *)(((word)h) + HBLKSIZE);
14792         while ((word)p < (word)plim) {
14793             word mark_word = *mark_word_addr++;
14794             word *q = p;
14795             while(mark_word != 0) {
14796               if (mark_word & 1) {
14797                   PUSH_GRANULE(q);
14798               }
14799               q += GC_GRANULE_WORDS;
14800               mark_word >>= 1;
14801             }
14802             p += WORDSZ*GC_GRANULE_WORDS;
14803         }
14804 #undef GC_greatest_plausible_heap_addr
14805 #undef GC_least_plausible_heap_addr
14806 #undef GC_mark_stack_top
14807 #undef GC_mark_stack_limit
14808 #define GC_mark_stack_limit GC_arrays._mark_stack_limit
14809 #define GC_mark_stack_top GC_arrays._mark_stack_top
14810     GC_mark_stack_top = mark_stack_top;
14811 }
14812 #ifndef UNALIGNED_PTRS
14813 STATIC void GC_push_marked2(struct hblk *h, hdr *hhdr)
14814 {
14815     word * mark_word_addr = &(hhdr->hb_marks[0]);
14816     word *p;
14817     word *plim;
14818     ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
14819     ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
14820     mse * mark_stack_top = GC_mark_stack_top;
14821     mse * mark_stack_limit = GC_mark_stack_limit;
14822 #undef GC_mark_stack_top
14823 #undef GC_mark_stack_limit
14824 #define GC_mark_stack_top mark_stack_top
14825 #define GC_mark_stack_limit mark_stack_limit
14826 #define GC_greatest_plausible_heap_addr greatest_ha
14827 #define GC_least_plausible_heap_addr least_ha
14828     p = (word *)(h->hb_body);
14829     plim = (word *)(((word)h) + HBLKSIZE);
14830         while ((word)p < (word)plim) {
14831             word mark_word = *mark_word_addr++;
14832             word *q = p;
14833             while(mark_word != 0) {
14834               if (mark_word & 1) {
14835                   PUSH_GRANULE(q);
14836                   PUSH_GRANULE(q + GC_GRANULE_WORDS);
14837               }
14838               q += 2 * GC_GRANULE_WORDS;
14839               mark_word >>= 2;
14840             }
14841             p += WORDSZ*GC_GRANULE_WORDS;
14842         }
14843 #undef GC_greatest_plausible_heap_addr
14844 #undef GC_least_plausible_heap_addr
14845 #undef GC_mark_stack_top
14846 #undef GC_mark_stack_limit
14847 #define GC_mark_stack_limit GC_arrays._mark_stack_limit
14848 #define GC_mark_stack_top GC_arrays._mark_stack_top
14849     GC_mark_stack_top = mark_stack_top;
14850 }
14851 #if GC_GRANULE_WORDS < 4
14852 STATIC void GC_push_marked4(struct hblk *h, hdr *hhdr)
14853 {
14854     word * mark_word_addr = &(hhdr->hb_marks[0]);
14855     word *p;
14856     word *plim;
14857     ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
14858     ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
14859     mse * mark_stack_top = GC_mark_stack_top;
14860     mse * mark_stack_limit = GC_mark_stack_limit;
14861 #undef GC_mark_stack_top
14862 #undef GC_mark_stack_limit
14863 #define GC_mark_stack_top mark_stack_top
14864 #define GC_mark_stack_limit mark_stack_limit
14865 #define GC_greatest_plausible_heap_addr greatest_ha
14866 #define GC_least_plausible_heap_addr least_ha
14867     p = (word *)(h->hb_body);
14868     plim = (word *)(((word)h) + HBLKSIZE);
14869         while ((word)p < (word)plim) {
14870             word mark_word = *mark_word_addr++;
14871             word *q = p;
14872             while(mark_word != 0) {
14873               if (mark_word & 1) {
14874                   PUSH_GRANULE(q);
14875                   PUSH_GRANULE(q + GC_GRANULE_WORDS);
14876                   PUSH_GRANULE(q + 2*GC_GRANULE_WORDS);
14877                   PUSH_GRANULE(q + 3*GC_GRANULE_WORDS);
14878               }
14879               q += 4 * GC_GRANULE_WORDS;
14880               mark_word >>= 4;
14881             }
14882             p += WORDSZ*GC_GRANULE_WORDS;
14883         }
14884 #undef GC_greatest_plausible_heap_addr
14885 #undef GC_least_plausible_heap_addr
14886 #undef GC_mark_stack_top
14887 #undef GC_mark_stack_limit
14888 #define GC_mark_stack_limit GC_arrays._mark_stack_limit
14889 #define GC_mark_stack_top GC_arrays._mark_stack_top
14890     GC_mark_stack_top = mark_stack_top;
14891 }
14892 #endif
14893 #endif
14894 #endif
14895 STATIC void GC_push_marked(struct hblk *h, hdr *hhdr)
14896 {
14897     word sz = hhdr -> hb_sz;
14898     word descr = hhdr -> hb_descr;
14899     ptr_t p;
14900     word bit_no;
14901     ptr_t lim;
14902     mse * GC_mark_stack_top_reg;
14903     mse * mark_stack_limit = GC_mark_stack_limit;
14904         if (( GC_DS_LENGTH) == descr) return;
14905         if (GC_block_empty(hhdr)) return;
14906 #if !defined(GC_DISABLE_INCREMENTAL)
14907       GC_n_rescuing_pages++;
14908 #endif
14909     GC_objects_are_marked = TRUE;
14910     if (sz > MAXOBJBYTES) {
14911         lim = h -> hb_body;
14912     } else {
14913         lim = (ptr_t)((word)(h + 1)->hb_body - sz);
14914     }
14915     switch(BYTES_TO_GRANULES(sz)) {
14916 #if defined(USE_PUSH_MARKED_ACCELERATORS)
14917      case 1:
14918        GC_push_marked1(h, hhdr);
14919        break;
14920 #if !defined(UNALIGNED_PTRS)
14921        case 2:
14922          GC_push_marked2(h, hhdr);
14923          break;
14924 #if GC_GRANULE_WORDS < 4
14925        case 4:
14926          GC_push_marked4(h, hhdr);
14927          break;
14928 #endif
14929 #endif
14930 #else
14931      case 1:
14932 #endif
14933      default:
14934       GC_mark_stack_top_reg = GC_mark_stack_top;
14935       for (p = h -> hb_body, bit_no = 0; (word)p <= (word)lim;
14936            p += sz, bit_no += MARK_BIT_OFFSET(sz)) {
14937         if (mark_bit_from_hdr(hhdr, bit_no)) {
14938           GC_mark_stack_top_reg = GC_push_obj(p, hhdr, GC_mark_stack_top_reg,
14939                                               mark_stack_limit);
14940         }
14941       }
14942       GC_mark_stack_top = GC_mark_stack_top_reg;
14943     }
14944 }
14945 #ifdef ENABLE_DISCLAIM
14946  STATIC void GC_push_unconditionally(struct hblk *h, hdr *hhdr)
14947  {
14948     word sz = hhdr -> hb_sz;
14949     word descr = hhdr -> hb_descr;
14950     ptr_t p;
14951     ptr_t lim;
14952     mse * GC_mark_stack_top_reg;
14953     mse * mark_stack_limit = GC_mark_stack_limit;
14954     if (( GC_DS_LENGTH) == descr)
14955         return;
14956 #if !defined(GC_DISABLE_INCREMENTAL)
14957       GC_n_rescuing_pages++;
14958 #endif
14959     GC_objects_are_marked = TRUE;
14960     if (sz > MAXOBJBYTES)
14961         lim = h -> hb_body;
14962     else
14963         lim = (ptr_t)((word)(h + 1)->hb_body - sz);
14964     GC_mark_stack_top_reg = GC_mark_stack_top;
14965     for (p = h -> hb_body; (word)p <= (word)lim; p += sz)
14966       if ((*(word *)p & 0x3) != 0)
14967         GC_mark_stack_top_reg = GC_push_obj(p, hhdr, GC_mark_stack_top_reg,
14968                                             mark_stack_limit);
14969     GC_mark_stack_top = GC_mark_stack_top_reg;
14970   }
14971 #endif
14972 #ifndef GC_DISABLE_INCREMENTAL
14973   STATIC GC_bool GC_block_was_dirty(struct hblk *h, hdr *hhdr)
14974   {
14975     word sz = hhdr -> hb_sz;
14976     if (sz <= MAXOBJBYTES) {
14977          return(GC_page_was_dirty(h));
14978     } else {
14979          ptr_t p = (ptr_t)h;
14980          while ((word)p < (word)h + sz) {
14981              if (GC_page_was_dirty((struct hblk *)p)) return(TRUE);
14982              p += HBLKSIZE;
14983          }
14984          return(FALSE);
14985     }
14986   }
14987 #endif
14988 STATIC struct hblk * GC_push_next_marked(struct hblk *h)
14989 {
14990     hdr * hhdr = HDR(h);
14991     if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr) || HBLK_IS_FREE(hhdr), FALSE)) {
14992       h = GC_next_block(h, FALSE);
14993       if (NULL == h) return NULL;
14994       hhdr = GC_find_header((ptr_t)h);
14995     } else {
14996 #ifdef LINT2
14997         if (NULL == h) ABORT("Bad HDR() definition");
14998 #endif
14999     }
15000     GC_push_marked(h, hhdr);
15001     return(h + OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz));
15002 }
15003 #ifndef GC_DISABLE_INCREMENTAL
15004   STATIC struct hblk * GC_push_next_marked_dirty(struct hblk *h)
15005   {
15006     hdr * hhdr = HDR(h);
15007     if (!GC_incremental) ABORT("Dirty bits not set up");
15008     for (;;) {
15009         if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr)
15010                    || HBLK_IS_FREE(hhdr), FALSE)) {
15011           h = GC_next_block(h, FALSE);
15012           if (NULL == h) return NULL;
15013           hhdr = GC_find_header((ptr_t)h);
15014         } else {
15015 #ifdef LINT2
15016             if (NULL == h) ABORT("Bad HDR() definition");
15017 #endif
15018         }
15019         if (GC_block_was_dirty(h, hhdr))
15020           break;
15021         h += OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
15022         hhdr = HDR(h);
15023     }
15024 #ifdef ENABLE_DISCLAIM
15025       if ((hhdr -> hb_flags & MARK_UNCONDITIONALLY) != 0) {
15026         GC_push_unconditionally(h, hhdr);
15027       } else
15028 #endif
15029      {
15030       GC_push_marked(h, hhdr);
15031     }
15032     return(h + OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz));
15033   }
15034 #endif
15035 STATIC struct hblk * GC_push_next_marked_uncollectable(struct hblk *h)
15036 {
15037     hdr * hhdr = HDR(h);
15038     for (;;) {
15039         if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr)
15040                    || HBLK_IS_FREE(hhdr), FALSE)) {
15041           h = GC_next_block(h, FALSE);
15042           if (NULL == h) return NULL;
15043           hhdr = GC_find_header((ptr_t)h);
15044         } else {
15045 #ifdef LINT2
15046             if (NULL == h) ABORT("Bad HDR() definition");
15047 #endif
15048         }
15049         if (hhdr -> hb_obj_kind == UNCOLLECTABLE) {
15050             GC_push_marked(h, hhdr);
15051             break;
15052         }
15053 #ifdef ENABLE_DISCLAIM
15054             if ((hhdr -> hb_flags & MARK_UNCONDITIONALLY) != 0) {
15055                 GC_push_unconditionally(h, hhdr);
15056                 break;
15057             }
15058 #endif
15059         h += OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
15060         hhdr = HDR(h);
15061     }
15062     return(h + OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz));
15063 }
15064 #include <stdio.h>
15065 int GC_no_dls = 0;
15066 #if !defined(NO_DEBUGGING) || defined(GC_ASSERTIONS)
15067   GC_INNER word GC_compute_root_size(void)
15068   {
15069     int i;
15070     word size = 0;
15071     for (i = 0; i < n_root_sets; i++) {
15072       size += GC_static_roots[i].r_end - GC_static_roots[i].r_start;
15073     }
15074     return size;
15075   }
15076 #endif
15077 #if !defined(NO_DEBUGGING)
15078   void GC_print_static_roots(void)
15079   {
15080     int i;
15081     word size;
15082     for (i = 0; i < n_root_sets; i++) {
15083         GC_printf("From %p to %p%s\n",
15084                   (void *)GC_static_roots[i].r_start,
15085                   (void *)GC_static_roots[i].r_end,
15086                   GC_static_roots[i].r_tmp ? " (temporary)" : "");
15087     }
15088     GC_printf("GC_root_size= %lu\n", (unsigned long)GC_root_size);
15089     if ((size = GC_compute_root_size()) != GC_root_size)
15090       GC_err_printf("GC_root_size incorrect!! Should be: %lu\n",
15091                     (unsigned long)size);
15092   }
15093 #endif
15094 #ifndef THREADS
15095   GC_INNER GC_bool GC_is_static_root(void *p)
15096   {
15097     static int last_root_set = MAX_ROOT_SETS;
15098     int i;
15099     if (last_root_set < n_root_sets
15100         && (word)p >= (word)GC_static_roots[last_root_set].r_start
15101         && (word)p < (word)GC_static_roots[last_root_set].r_end)
15102       return(TRUE);
15103     for (i = 0; i < n_root_sets; i++) {
15104         if ((word)p >= (word)GC_static_roots[i].r_start
15105             && (word)p < (word)GC_static_roots[i].r_end) {
15106           last_root_set = i;
15107           return(TRUE);
15108         }
15109     }
15110     return(FALSE);
15111   }
15112 #endif
15113 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
15114   GC_INLINE int rt_hash(ptr_t addr)
15115   {
15116     word result = (word) addr;
15117 #if CPP_WORDSZ > 8*LOG_RT_SIZE
15118         result ^= result >> 8*LOG_RT_SIZE;
15119 #endif
15120 #if CPP_WORDSZ > 4*LOG_RT_SIZE
15121         result ^= result >> 4*LOG_RT_SIZE;
15122 #endif
15123     result ^= result >> 2*LOG_RT_SIZE;
15124     result ^= result >> LOG_RT_SIZE;
15125     result &= (RT_SIZE-1);
15126     return(result);
15127   }
15128   GC_INNER void * GC_roots_present(ptr_t b)
15129   {
15130     int h = rt_hash(b);
15131     struct roots *p = GC_root_index[h];
15132     while (p != 0) {
15133         if (p -> r_start == (ptr_t)b) return(p);
15134         p = p -> r_next;
15135     }
15136     return NULL;
15137   }
15138   GC_INLINE void add_roots_to_index(struct roots *p)
15139   {
15140     int h = rt_hash(p -> r_start);
15141     p -> r_next = GC_root_index[h];
15142     GC_root_index[h] = p;
15143   }
15144 #endif
15145 GC_INNER word GC_root_size = 0;
15146 GC_API void GC_CALL GC_add_roots(void *b, void *e)
15147 {
15148     DCL_LOCK_STATE;
15149     if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
15150     LOCK();
15151     GC_add_roots_inner((ptr_t)b, (ptr_t)e, FALSE);
15152     UNLOCK();
15153 }
15154 void GC_add_roots_inner(ptr_t b, ptr_t e, GC_bool tmp)
15155 {
15156     GC_ASSERT((word)b <= (word)e);
15157     b = (ptr_t)(((word)b + (sizeof(word) - 1)) & ~(word)(sizeof(word) - 1));
15158     e = (ptr_t)((word)e & ~(word)(sizeof(word) - 1));
15159     if ((word)b >= (word)e) return;
15160 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
15161       {
15162         int i;
15163         struct roots * old = NULL;
15164         for (i = 0; i < n_root_sets; i++) {
15165             old = GC_static_roots + i;
15166             if ((word)b <= (word)old->r_end
15167                  && (word)e >= (word)old->r_start) {
15168                 if ((word)b < (word)old->r_start) {
15169                     GC_root_size += old->r_start - b;
15170                     old -> r_start = b;
15171                 }
15172                 if ((word)e > (word)old->r_end) {
15173                     GC_root_size += e - old->r_end;
15174                     old -> r_end = e;
15175                 }
15176                 old -> r_tmp &= tmp;
15177                 break;
15178             }
15179         }
15180         if (i < n_root_sets) {
15181             struct roots *other;
15182             for (i++; i < n_root_sets; i++) {
15183               other = GC_static_roots + i;
15184               b = other -> r_start;
15185               e = other -> r_end;
15186               if ((word)b <= (word)old->r_end
15187                   && (word)e >= (word)old->r_start) {
15188                 if ((word)b < (word)old->r_start) {
15189                     GC_root_size += old->r_start - b;
15190                     old -> r_start = b;
15191                 }
15192                 if ((word)e > (word)old->r_end) {
15193                     GC_root_size += e - old->r_end;
15194                     old -> r_end = e;
15195                 }
15196                 old -> r_tmp &= other -> r_tmp;
15197                   GC_root_size -= (other -> r_end - other -> r_start);
15198                   other -> r_start = GC_static_roots[n_root_sets-1].r_start;
15199                   other -> r_end = GC_static_roots[n_root_sets-1].r_end;
15200                   n_root_sets--;
15201               }
15202             }
15203           return;
15204         }
15205       }
15206 #else
15207       {
15208         struct roots * old = (struct roots *)GC_roots_present(b);
15209         if (old != 0) {
15210           if ((word)e <= (word)old->r_end) {
15211             old -> r_tmp &= tmp;
15212             return;
15213           }
15214           if (old -> r_tmp == tmp || !tmp) {
15215             GC_root_size += e - old -> r_end;
15216             old -> r_end = e;
15217             old -> r_tmp = tmp;
15218             return;
15219           }
15220           b = old -> r_end;
15221         }
15222       }
15223 #endif
15224     if (n_root_sets == MAX_ROOT_SETS) {
15225         ABORT("Too many root sets");
15226     }
15227 #ifdef DEBUG_ADD_DEL_ROOTS
15228       GC_log_printf("Adding data root section %d: %p .. %p%s\n",
15229                     n_root_sets, (void *)b, (void *)e,
15230                     tmp ? " (temporary)" : "");
15231 #endif
15232     GC_static_roots[n_root_sets].r_start = (ptr_t)b;
15233     GC_static_roots[n_root_sets].r_end = (ptr_t)e;
15234     GC_static_roots[n_root_sets].r_tmp = tmp;
15235 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
15236       GC_static_roots[n_root_sets].r_next = 0;
15237       add_roots_to_index(GC_static_roots + n_root_sets);
15238 #endif
15239     GC_root_size += e - b;
15240     n_root_sets++;
15241 }
15242 GC_API void GC_CALL GC_clear_roots(void)
15243 {
15244     DCL_LOCK_STATE;
15245     if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
15246     LOCK();
15247 #ifdef THREADS
15248       GC_roots_were_cleared = TRUE;
15249 #endif
15250     n_root_sets = 0;
15251     GC_root_size = 0;
15252 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
15253       BZERO(GC_root_index, RT_SIZE * sizeof(void *));
15254 #endif
15255 #ifdef DEBUG_ADD_DEL_ROOTS
15256       GC_log_printf("Clear all data root sections\n");
15257 #endif
15258     UNLOCK();
15259 }
15260 STATIC void GC_remove_root_at_pos(int i)
15261 {
15262 #ifdef DEBUG_ADD_DEL_ROOTS
15263       GC_log_printf("Remove data root section at %d: %p .. %p%s\n",
15264                     i, (void *)GC_static_roots[i].r_start,
15265                     (void *)GC_static_roots[i].r_end,
15266                     GC_static_roots[i].r_tmp ? " (temporary)" : "");
15267 #endif
15268     GC_root_size -= (GC_static_roots[i].r_end - GC_static_roots[i].r_start);
15269     GC_static_roots[i].r_start = GC_static_roots[n_root_sets-1].r_start;
15270     GC_static_roots[i].r_end = GC_static_roots[n_root_sets-1].r_end;
15271     GC_static_roots[i].r_tmp = GC_static_roots[n_root_sets-1].r_tmp;
15272     n_root_sets--;
15273 }
15274 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
15275   STATIC void GC_rebuild_root_index(void)
15276   {
15277     int i;
15278     BZERO(GC_root_index, RT_SIZE * sizeof(void *));
15279     for (i = 0; i < n_root_sets; i++)
15280         add_roots_to_index(GC_static_roots + i);
15281   }
15282 #endif
15283 #if defined(DYNAMIC_LOADING) || defined(MSWIN32) || defined(MSWINCE) \
15284      || defined(PCR) || defined(CYGWIN32)
15285 STATIC void GC_remove_tmp_roots(void)
15286 {
15287     int i;
15288 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
15289       int old_n_roots = n_root_sets;
15290 #endif
15291     for (i = 0; i < n_root_sets; ) {
15292         if (GC_static_roots[i].r_tmp) {
15293             GC_remove_root_at_pos(i);
15294         } else {
15295             i++;
15296         }
15297     }
15298 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
15299       if (n_root_sets < old_n_roots)
15300         GC_rebuild_root_index();
15301 #endif
15302 }
15303 #endif
15304 #if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32)
15305   STATIC void GC_remove_roots_inner(ptr_t b, ptr_t e);
15306   GC_API void GC_CALL GC_remove_roots(void *b, void *e)
15307   {
15308     DCL_LOCK_STATE;
15309     if ((((word)b + (sizeof(word) - 1)) & ~(word)(sizeof(word) - 1)) >=
15310         ((word)e & ~(word)(sizeof(word) - 1)))
15311       return;
15312     LOCK();
15313     GC_remove_roots_inner((ptr_t)b, (ptr_t)e);
15314     UNLOCK();
15315   }
15316   STATIC void GC_remove_roots_inner(ptr_t b, ptr_t e)
15317   {
15318     int i;
15319     GC_bool rebuild = FALSE;
15320     for (i = 0; i < n_root_sets; ) {
15321         if ((word)GC_static_roots[i].r_start >= (word)b
15322             && (word)GC_static_roots[i].r_end <= (word)e) {
15323             GC_remove_root_at_pos(i);
15324             rebuild = TRUE;
15325         } else {
15326             i++;
15327         }
15328     }
15329     if (rebuild)
15330         GC_rebuild_root_index();
15331   }
15332 #endif
15333 #ifdef USE_PROC_FOR_LIBRARIES
15334   GC_INLINE void swap_static_roots(int i, int j)
15335   {
15336     ptr_t r_start = GC_static_roots[i].r_start;
15337     ptr_t r_end = GC_static_roots[i].r_end;
15338     GC_bool r_tmp = GC_static_roots[i].r_tmp;
15339     GC_static_roots[i].r_start = GC_static_roots[j].r_start;
15340     GC_static_roots[i].r_end = GC_static_roots[j].r_end;
15341     GC_static_roots[i].r_tmp = GC_static_roots[j].r_tmp;
15342     GC_static_roots[j].r_start = r_start;
15343     GC_static_roots[j].r_end = r_end;
15344     GC_static_roots[j].r_tmp = r_tmp;
15345   }
15346   GC_INNER void GC_remove_roots_subregion(ptr_t b, ptr_t e)
15347   {
15348     int i;
15349     GC_bool rebuild = FALSE;
15350     GC_ASSERT(I_HOLD_LOCK());
15351     GC_ASSERT((word)b % sizeof(word) == 0 && (word)e % sizeof(word) == 0);
15352     for (i = 0; i < n_root_sets; i++) {
15353       ptr_t r_start, r_end;
15354       if (GC_static_roots[i].r_tmp) {
15355 #ifdef GC_ASSERTIONS
15356           int j;
15357           for (j = i + 1; j < n_root_sets; j++) {
15358             GC_ASSERT(GC_static_roots[j].r_tmp);
15359           }
15360 #endif
15361         break;
15362       }
15363       r_start = GC_static_roots[i].r_start;
15364       r_end = GC_static_roots[i].r_end;
15365       if (!EXPECT((word)e <= (word)r_start || (word)r_end <= (word)b, TRUE)) {
15366 #ifdef DEBUG_ADD_DEL_ROOTS
15367           GC_log_printf("Removing %p .. %p from root section %d (%p .. %p)\n",
15368                         (void *)b, (void *)e,
15369                         i, (void *)r_start, (void *)r_end);
15370 #endif
15371         if ((word)r_start < (word)b) {
15372           GC_root_size -= r_end - b;
15373           GC_static_roots[i].r_end = b;
15374           if ((word)e < (word)r_end) {
15375             int j;
15376             if (rebuild) {
15377               GC_rebuild_root_index();
15378               rebuild = FALSE;
15379             }
15380             GC_add_roots_inner(e, r_end, FALSE);
15381             for (j = i + 1; j < n_root_sets; j++)
15382               if (GC_static_roots[j].r_tmp)
15383                 break;
15384             if (j < n_root_sets-1 && !GC_static_roots[n_root_sets-1].r_tmp) {
15385               swap_static_roots(j, n_root_sets - 1);
15386               rebuild = TRUE;
15387             }
15388           }
15389         } else {
15390           if ((word)e < (word)r_end) {
15391             GC_root_size -= e - r_start;
15392             GC_static_roots[i].r_start = e;
15393           } else {
15394             GC_remove_root_at_pos(i);
15395             if (i < n_root_sets - 1 && GC_static_roots[i].r_tmp
15396                 && !GC_static_roots[i + 1].r_tmp) {
15397               int j;
15398               for (j = i + 2; j < n_root_sets; j++)
15399                 if (GC_static_roots[j].r_tmp)
15400                   break;
15401               swap_static_roots(i, j - 1);
15402             }
15403             i--;
15404           }
15405           rebuild = TRUE;
15406         }
15407       }
15408     }
15409     if (rebuild)
15410       GC_rebuild_root_index();
15411   }
15412 #endif
15413 #if !defined(NO_DEBUGGING)
15414   GC_API int GC_CALL GC_is_tmp_root(void *p)
15415   {
15416     static int last_root_set = MAX_ROOT_SETS;
15417     int i;
15418     if (last_root_set < n_root_sets
15419         && (word)p >= (word)GC_static_roots[last_root_set].r_start
15420         && (word)p < (word)GC_static_roots[last_root_set].r_end)
15421         return GC_static_roots[last_root_set].r_tmp;
15422     for (i = 0; i < n_root_sets; i++) {
15423         if ((word)p >= (word)GC_static_roots[i].r_start
15424             && (word)p < (word)GC_static_roots[i].r_end) {
15425             last_root_set = i;
15426             return GC_static_roots[i].r_tmp;
15427         }
15428     }
15429     return(FALSE);
15430   }
15431 #endif
15432 GC_INNER ptr_t GC_approx_sp(void)
15433 {
15434     volatile word sp;
15435 #if defined(S390) && !defined(CPPCHECK) && (__clang_major__ < 8)
15436         sp = (word)&sp;
15437 #elif defined(CPPCHECK) || (__GNUC__ >= 4  \
15438                                && !defined(STACK_NOT_SCANNED))
15439         sp = (word)__builtin_frame_address(0);
15440 #else
15441         sp = (word)&sp;
15442 #endif
15443     return((ptr_t)sp);
15444 }
15445 GC_API void GC_CALL GC_clear_exclusion_table(void)
15446 {
15447     GC_excl_table_entries = 0;
15448 }
15449 STATIC struct exclusion * GC_next_exclusion(ptr_t start_addr)
15450 {
15451     size_t low = 0;
15452     size_t high;
15453     GC_ASSERT(GC_excl_table_entries > 0);
15454     high = GC_excl_table_entries - 1;
15455     while (high > low) {
15456         size_t mid = (low + high) >> 1;
15457         if ((word) GC_excl_table[mid].e_end <= (word) start_addr) {
15458             low = mid + 1;
15459         } else {
15460             high = mid;
15461         }
15462     }
15463     if ((word) GC_excl_table[low].e_end <= (word) start_addr) return 0;
15464     return GC_excl_table + low;
15465 }
15466 GC_INNER void GC_exclude_static_roots_inner(void *start, void *finish)
15467 {
15468     struct exclusion * next;
15469     size_t next_index;
15470     GC_ASSERT((word)start % sizeof(word) == 0);
15471     GC_ASSERT((word)start < (word)finish);
15472     if (0 == GC_excl_table_entries) {
15473         next = 0;
15474     } else {
15475         next = GC_next_exclusion((ptr_t)start);
15476     }
15477     if (0 != next) {
15478       size_t i;
15479       if ((word)(next -> e_start) < (word) finish) {
15480         ABORT("Exclusion ranges overlap");
15481       }
15482       if ((word)(next -> e_start) == (word) finish) {
15483           next -> e_start = (ptr_t)start;
15484           return;
15485       }
15486       next_index = next - GC_excl_table;
15487       for (i = GC_excl_table_entries; i > next_index; --i) {
15488         GC_excl_table[i] = GC_excl_table[i-1];
15489       }
15490     } else {
15491       next_index = GC_excl_table_entries;
15492     }
15493     if (GC_excl_table_entries == MAX_EXCLUSIONS) ABORT("Too many exclusions");
15494     GC_excl_table[next_index].e_start = (ptr_t)start;
15495     GC_excl_table[next_index].e_end = (ptr_t)finish;
15496     ++GC_excl_table_entries;
15497 }
15498 GC_API void GC_CALL GC_exclude_static_roots(void *b, void *e)
15499 {
15500     DCL_LOCK_STATE;
15501     if (b == e) return;
15502     b = (void *)((word)b & ~(word)(sizeof(word) - 1));
15503     e = (void *)(((word)e + (sizeof(word) - 1)) & ~(word)(sizeof(word) - 1));
15504     if (NULL == e)
15505       e = (void *)(~(word)(sizeof(word) - 1));
15506     LOCK();
15507     GC_exclude_static_roots_inner(b, e);
15508     UNLOCK();
15509 }
15510 #if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK)
15511 #define GC_PUSH_CONDITIONAL(b, t, all) \
15512                 (GC_parallel \
15513                     ? GC_push_conditional_eager(b, t, all) \
15514                     : GC_push_conditional_static(b, t, all))
15515 #else
15516 #define GC_PUSH_CONDITIONAL(b, t, all) GC_push_conditional_static(b, t, all)
15517 #endif
15518 STATIC void GC_push_conditional_with_exclusions(ptr_t bottom, ptr_t top,
15519                                                 GC_bool all)
15520 {
15521     while ((word)bottom < (word)top) {
15522         struct exclusion *next = GC_next_exclusion(bottom);
15523         ptr_t excl_start;
15524         if (0 == next
15525             || (word)(excl_start = next -> e_start) >= (word)top) {
15526           GC_PUSH_CONDITIONAL(bottom, top, all);
15527           break;
15528         }
15529         if ((word)excl_start > (word)bottom)
15530           GC_PUSH_CONDITIONAL(bottom, excl_start, all);
15531         bottom = next -> e_end;
15532     }
15533 }
15534 #ifdef IA64
15535   GC_INNER void GC_push_all_register_sections(ptr_t bs_lo, ptr_t bs_hi,
15536                   int eager, struct GC_traced_stack_sect_s *traced_stack_sect)
15537   {
15538     while (traced_stack_sect != NULL) {
15539         ptr_t frame_bs_lo = traced_stack_sect -> backing_store_end;
15540         GC_ASSERT((word)frame_bs_lo <= (word)bs_hi);
15541         if (eager) {
15542             GC_push_all_eager(frame_bs_lo, bs_hi);
15543         } else {
15544             GC_push_all_stack(frame_bs_lo, bs_hi);
15545         }
15546         bs_hi = traced_stack_sect -> saved_backing_store_ptr;
15547         traced_stack_sect = traced_stack_sect -> prev;
15548     }
15549     GC_ASSERT((word)bs_lo <= (word)bs_hi);
15550     if (eager) {
15551         GC_push_all_eager(bs_lo, bs_hi);
15552     } else {
15553         GC_push_all_stack(bs_lo, bs_hi);
15554     }
15555   }
15556 #endif
15557 #ifdef THREADS
15558 GC_INNER void GC_push_all_stack_sections(ptr_t lo, ptr_t hi,
15559                         struct GC_traced_stack_sect_s *traced_stack_sect)
15560 {
15561     while (traced_stack_sect != NULL) {
15562         GC_ASSERT((word)lo HOTTER_THAN (word)traced_stack_sect);
15563 #ifdef STACK_GROWS_UP
15564             GC_push_all_stack((ptr_t)traced_stack_sect, lo);
15565 #else
15566             GC_push_all_stack(lo, (ptr_t)traced_stack_sect);
15567 #endif
15568         lo = traced_stack_sect -> saved_stack_ptr;
15569         GC_ASSERT(lo != NULL);
15570         traced_stack_sect = traced_stack_sect -> prev;
15571     }
15572     GC_ASSERT(!((word)hi HOTTER_THAN (word)lo));
15573 #ifdef STACK_GROWS_UP
15574         GC_push_all_stack(hi, lo);
15575 #else
15576         GC_push_all_stack(lo, hi);
15577 #endif
15578 }
15579 #else
15580 STATIC void GC_push_all_stack_partially_eager(ptr_t bottom, ptr_t top,
15581                                               ptr_t cold_gc_frame)
15582 {
15583 #ifndef NEED_FIXUP_POINTER
15584   if (GC_all_interior_pointers) {
15585     if (0 == cold_gc_frame) {
15586         GC_push_all_stack(bottom, top);
15587         return;
15588     }
15589     GC_ASSERT((word)bottom <= (word)cold_gc_frame
15590               && (word)cold_gc_frame <= (word)top);
15591 #ifdef STACK_GROWS_DOWN
15592         GC_push_all(cold_gc_frame - sizeof(ptr_t), top);
15593         GC_push_all_eager(bottom, cold_gc_frame);
15594 #else
15595         GC_push_all(bottom, cold_gc_frame + sizeof(ptr_t));
15596         GC_push_all_eager(cold_gc_frame, top);
15597 #endif
15598   } else
15599 #endif
15600    {
15601     GC_push_all_eager(bottom, top);
15602   }
15603 #ifdef TRACE_BUF
15604     GC_add_trace_entry("GC_push_all_stack", (word)bottom, (word)top);
15605 #endif
15606 }
15607 STATIC void GC_push_all_stack_part_eager_sections(ptr_t lo, ptr_t hi,
15608         ptr_t cold_gc_frame, struct GC_traced_stack_sect_s *traced_stack_sect)
15609 {
15610     GC_ASSERT(traced_stack_sect == NULL || cold_gc_frame == NULL ||
15611               (word)cold_gc_frame HOTTER_THAN (word)traced_stack_sect);
15612     while (traced_stack_sect != NULL) {
15613         GC_ASSERT((word)lo HOTTER_THAN (word)traced_stack_sect);
15614 #ifdef STACK_GROWS_UP
15615             GC_push_all_stack_partially_eager((ptr_t)traced_stack_sect, lo,
15616                                               cold_gc_frame);
15617 #else
15618             GC_push_all_stack_partially_eager(lo, (ptr_t)traced_stack_sect,
15619                                               cold_gc_frame);
15620 #endif
15621         lo = traced_stack_sect -> saved_stack_ptr;
15622         GC_ASSERT(lo != NULL);
15623         traced_stack_sect = traced_stack_sect -> prev;
15624         cold_gc_frame = NULL;
15625     }
15626     GC_ASSERT(!((word)hi HOTTER_THAN (word)lo));
15627 #ifdef STACK_GROWS_UP
15628         GC_push_all_stack_partially_eager(hi, lo, cold_gc_frame);
15629 #else
15630         GC_push_all_stack_partially_eager(lo, hi, cold_gc_frame);
15631 #endif
15632 }
15633 #endif
15634 STATIC void GC_push_current_stack(ptr_t cold_gc_frame,
15635                                   void * context GC_ATTR_UNUSED)
15636 {
15637 #if defined(THREADS)
15638 #ifdef STACK_GROWS_DOWN
15639           GC_push_all_eager(GC_approx_sp(), cold_gc_frame);
15640 #else
15641           GC_push_all_eager(cold_gc_frame, GC_approx_sp());
15642 #endif
15643 #else
15644         GC_push_all_stack_part_eager_sections(GC_approx_sp(), GC_stackbottom,
15645                                         cold_gc_frame, GC_traced_stack_sect);
15646 #ifdef IA64
15647               {
15648                 ptr_t bsp = GC_save_regs_ret_val;
15649                 ptr_t cold_gc_bs_pointer = bsp - 2048;
15650                 if (GC_all_interior_pointers
15651                     && (word)cold_gc_bs_pointer > (word)BACKING_STORE_BASE) {
15652                   if (GC_traced_stack_sect != NULL
15653                       && (word)cold_gc_bs_pointer
15654                           < (word)GC_traced_stack_sect->backing_store_end)
15655                     cold_gc_bs_pointer =
15656                                 GC_traced_stack_sect->backing_store_end;
15657                   GC_push_all_register_sections(BACKING_STORE_BASE,
15658                         cold_gc_bs_pointer, FALSE, GC_traced_stack_sect);
15659                   GC_push_all_eager(cold_gc_bs_pointer, bsp);
15660                 } else {
15661                   GC_push_all_register_sections(BACKING_STORE_BASE, bsp,
15662                                 TRUE , GC_traced_stack_sect);
15663                 }
15664               }
15665 #endif
15666 #endif
15667 }
15668 GC_INNER void (*GC_push_typed_structures)(void) = 0;
15669 GC_INNER void GC_cond_register_dynamic_libraries(void)
15670 {
15671 #if (defined(DYNAMIC_LOADING) && !defined(MSWIN_XBOX1)) \
15672      || defined(CYGWIN32) || defined(MSWIN32) || defined(MSWINCE) \
15673      || defined(PCR)
15674     GC_remove_tmp_roots();
15675     if (!GC_no_dls) GC_register_dynamic_libraries();
15676 #else
15677     GC_no_dls = TRUE;
15678 #endif
15679 }
15680 STATIC void GC_push_regs_and_stack(ptr_t cold_gc_frame)
15681 {
15682 #ifdef THREADS
15683       if (NULL == cold_gc_frame)
15684         return;
15685 #endif
15686     GC_with_callee_saves_pushed(GC_push_current_stack, cold_gc_frame);
15687 }
15688 GC_INNER void GC_push_roots(GC_bool all, ptr_t cold_gc_frame GC_ATTR_UNUSED)
15689 {
15690     int i;
15691     unsigned kind;
15692 #if !defined(REGISTER_LIBRARIES_EARLY)
15693         GC_cond_register_dynamic_libraries();
15694 #endif
15695     for (i = 0; i < n_root_sets; i++) {
15696         GC_push_conditional_with_exclusions(
15697                              GC_static_roots[i].r_start,
15698                              GC_static_roots[i].r_end, all);
15699     }
15700     for (kind = 0; kind < GC_n_kinds; kind++) {
15701         void *base = GC_base(GC_obj_kinds[kind].ok_freelist);
15702         if (base != NULL) {
15703             GC_set_mark_bit(base);
15704         }
15705     }
15706 #ifndef GC_NO_FINALIZATION
15707         GC_push_finalizer_structures();
15708 #endif
15709 #ifdef THREADS
15710         if (GC_no_dls || GC_roots_were_cleared)
15711             GC_push_thread_structures();
15712 #endif
15713     if (GC_push_typed_structures)
15714         GC_push_typed_structures();
15715 #if defined(THREAD_LOCAL_ALLOC)
15716         if (GC_world_stopped)
15717             GC_mark_thread_local_free_lists();
15718 #endif
15719 #ifndef STACK_NOT_SCANNED
15720         GC_push_regs_and_stack(cold_gc_frame);
15721 #endif
15722     if (GC_push_other_roots != 0) {
15723         (*GC_push_other_roots)();
15724     }
15725 }
15726 #ifdef ENABLE_DISCLAIM
15727 #endif
15728 #include <stdio.h>
15729 GC_INNER signed_word GC_bytes_found = 0;
15730 #if defined(PARALLEL_MARK)
15731   GC_INNER signed_word GC_fl_builder_count = 0;
15732 #endif
15733 #ifndef MAX_LEAKED
15734 #define MAX_LEAKED 40
15735 #endif
15736 STATIC ptr_t GC_leaked[MAX_LEAKED] = { NULL };
15737 STATIC unsigned GC_n_leaked = 0;
15738 GC_INNER GC_bool GC_have_errors = FALSE;
15739 #if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM)
15740   STATIC void GC_reclaim_unconditionally_marked(void);
15741 #endif
15742 GC_INLINE void GC_add_leaked(ptr_t leaked)
15743 {
15744 #ifndef SHORT_DBG_HDRS
15745      if (GC_findleak_delay_free && !GC_check_leaked(leaked))
15746        return;
15747 #endif
15748     GC_have_errors = TRUE;
15749     if (GC_n_leaked < MAX_LEAKED) {
15750       GC_leaked[GC_n_leaked++] = leaked;
15751       GC_set_mark_bit(leaked);
15752     }
15753 }
15754 GC_INNER void GC_print_all_errors(void)
15755 {
15756     static GC_bool printing_errors = FALSE;
15757     GC_bool have_errors;
15758     unsigned i, n_leaked;
15759     ptr_t leaked[MAX_LEAKED];
15760     DCL_LOCK_STATE;
15761     LOCK();
15762     if (printing_errors) {
15763         UNLOCK();
15764         return;
15765     }
15766     have_errors = GC_have_errors;
15767     printing_errors = TRUE;
15768     n_leaked = GC_n_leaked;
15769     if (n_leaked > 0) {
15770       GC_ASSERT(n_leaked <= MAX_LEAKED);
15771       BCOPY(GC_leaked, leaked, n_leaked * sizeof(ptr_t));
15772       GC_n_leaked = 0;
15773       BZERO(GC_leaked, n_leaked * sizeof(ptr_t));
15774     }
15775     UNLOCK();
15776     if (GC_debugging_started) {
15777       GC_print_all_smashed();
15778     } else {
15779       have_errors = FALSE;
15780     }
15781     if (n_leaked > 0) {
15782         GC_err_printf("Found %u leaked objects:\n", n_leaked);
15783         have_errors = TRUE;
15784     }
15785     for (i = 0; i < n_leaked; i++) {
15786         ptr_t p = leaked[i];
15787 #ifndef SKIP_LEAKED_OBJECTS_PRINTING
15788           GC_print_heap_obj(p);
15789 #endif
15790         GC_free(p);
15791     }
15792     if (have_errors
15793 #ifndef GC_ABORT_ON_LEAK
15794           && GETENV("GC_ABORT_ON_LEAK") != NULL
15795 #endif
15796         ) {
15797       ABORT("Leaked or smashed objects encountered");
15798     }
15799     LOCK();
15800     printing_errors = FALSE;
15801     UNLOCK();
15802 }
15803 GC_INNER GC_bool GC_block_empty(hdr *hhdr)
15804 {
15805     return (hhdr -> hb_n_marks == 0);
15806 }
15807 STATIC GC_bool GC_block_nearly_full(hdr *hhdr, word sz)
15808 {
15809     return hhdr -> hb_n_marks > HBLK_OBJS(sz) * 7 / 8;
15810 }
15811 GC_INLINE word *GC_clear_block(word *p, word sz, signed_word *count)
15812 {
15813   word *q = (word *)((ptr_t)p + sz);
15814 #ifdef USE_MARK_BYTES
15815     GC_ASSERT((sz & 1) == 0);
15816     GC_ASSERT(((word)p & (2 * sizeof(word) - 1)) == 0);
15817     p[1] = 0;
15818     p += 2;
15819     while ((word)p < (word)q) {
15820       CLEAR_DOUBLE(p);
15821       p += 2;
15822     }
15823 #else
15824     p++;
15825     while ((word)p < (word)q) {
15826       *p++ = 0;
15827     }
15828 #endif
15829   *count += sz;
15830   return p;
15831 }
15832 STATIC ptr_t GC_reclaim_clear(struct hblk *hbp, hdr *hhdr, word sz,
15833                               ptr_t list, signed_word *count)
15834 {
15835     word bit_no = 0;
15836     ptr_t p, plim;
15837     GC_ASSERT(hhdr == GC_find_header((ptr_t)hbp));
15838 #ifndef THREADS
15839       GC_ASSERT(sz == hhdr -> hb_sz);
15840 #else
15841 #endif
15842     GC_ASSERT((sz & (BYTES_PER_WORD-1)) == 0);
15843     p = hbp->hb_body;
15844     plim = p + HBLKSIZE - sz;
15845         while ((word)p <= (word)plim) {
15846             if (mark_bit_from_hdr(hhdr, bit_no)) {
15847                 p += sz;
15848             } else {
15849                 obj_link(p) = list;
15850                 list = p;
15851                 p = (ptr_t)GC_clear_block((word *)p, sz, count);
15852             }
15853             bit_no += MARK_BIT_OFFSET(sz);
15854         }
15855     return list;
15856 }
15857 STATIC ptr_t GC_reclaim_uninit(struct hblk *hbp, hdr *hhdr, word sz,
15858                                ptr_t list, signed_word *count)
15859 {
15860     word bit_no = 0;
15861     word *p, *plim;
15862     signed_word n_bytes_found = 0;
15863 #ifndef THREADS
15864       GC_ASSERT(sz == hhdr -> hb_sz);
15865 #endif
15866     p = (word *)(hbp->hb_body);
15867     plim = (word *)((ptr_t)hbp + HBLKSIZE - sz);
15868         while ((word)p <= (word)plim) {
15869             if (!mark_bit_from_hdr(hhdr, bit_no)) {
15870                 n_bytes_found += sz;
15871                     obj_link(p) = list;
15872                     list = ((ptr_t)p);
15873             }
15874             p = (word *)((ptr_t)p + sz);
15875             bit_no += MARK_BIT_OFFSET(sz);
15876         }
15877     *count += n_bytes_found;
15878     return(list);
15879 }
15880 #ifdef ENABLE_DISCLAIM
15881   STATIC ptr_t GC_disclaim_and_reclaim(struct hblk *hbp, hdr *hhdr, word sz,
15882                                        ptr_t list, signed_word *count)
15883   {
15884     word bit_no = 0;
15885     ptr_t p, plim;
15886     struct obj_kind *ok = &GC_obj_kinds[hhdr->hb_obj_kind];
15887     int (GC_CALLBACK *disclaim)(void *) = ok->ok_disclaim_proc;
15888 #ifndef THREADS
15889       GC_ASSERT(sz == hhdr -> hb_sz);
15890 #endif
15891     p = hbp->hb_body;
15892     plim = p + HBLKSIZE - sz;
15893     for (; (word)p <= (word)plim; bit_no += MARK_BIT_OFFSET(sz)) {
15894         if (mark_bit_from_hdr(hhdr, bit_no)) {
15895             p += sz;
15896         } else if ((*disclaim)(p)) {
15897             set_mark_bit_from_hdr(hhdr, bit_no);
15898             hhdr -> hb_n_marks++;
15899             p += sz;
15900         } else {
15901             obj_link(p) = list;
15902             list = p;
15903             p = (ptr_t)GC_clear_block((word *)p, sz, count);
15904         }
15905     }
15906     return list;
15907   }
15908 #endif
15909 STATIC void GC_reclaim_check(struct hblk *hbp, hdr *hhdr, word sz)
15910 {
15911     word bit_no;
15912     ptr_t p, plim;
15913 #ifndef THREADS
15914       GC_ASSERT(sz == hhdr -> hb_sz);
15915 #endif
15916     p = hbp->hb_body;
15917     plim = p + HBLKSIZE - sz;
15918     for (bit_no = 0; (word)p <= (word)plim;
15919          p += sz, bit_no += MARK_BIT_OFFSET(sz)) {
15920       if (!mark_bit_from_hdr(hhdr, bit_no)) {
15921         GC_add_leaked(p);
15922       }
15923     }
15924 }
15925 #ifdef AO_HAVE_load
15926 #define IS_PTRFREE_SAFE(hhdr) \
15927                 (AO_load((volatile AO_t *)&(hhdr)->hb_descr) == 0)
15928 #else
15929 #define IS_PTRFREE_SAFE(hhdr) ((hhdr)->hb_descr == 0)
15930 #endif
15931 GC_INNER ptr_t GC_reclaim_generic(struct hblk * hbp, hdr *hhdr, size_t sz,
15932                                   GC_bool init, ptr_t list,
15933                                   signed_word *count)
15934 {
15935     ptr_t result;
15936     GC_ASSERT(GC_find_header((ptr_t)hbp) == hhdr);
15937 #ifndef GC_DISABLE_INCREMENTAL
15938       GC_remove_protection(hbp, 1, IS_PTRFREE_SAFE(hhdr));
15939 #endif
15940 #ifdef ENABLE_DISCLAIM
15941       if ((hhdr -> hb_flags & HAS_DISCLAIM) != 0) {
15942         result = GC_disclaim_and_reclaim(hbp, hhdr, sz, list, count);
15943       } else
15944 #endif
15945      if (init || GC_debugging_started) {
15946       result = GC_reclaim_clear(hbp, hhdr, sz, list, count);
15947     } else {
15948       GC_ASSERT(IS_PTRFREE_SAFE(hhdr));
15949       result = GC_reclaim_uninit(hbp, hhdr, sz, list, count);
15950     }
15951     if (IS_UNCOLLECTABLE(hhdr -> hb_obj_kind)) GC_set_hdr_marks(hhdr);
15952     return result;
15953 }
15954 STATIC void GC_reclaim_small_nonempty_block(struct hblk *hbp, word sz,
15955                                             GC_bool report_if_found)
15956 {
15957     hdr *hhdr = HDR(hbp);
15958     struct obj_kind * ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
15959     void **flh = &(ok -> ok_freelist[BYTES_TO_GRANULES(sz)]);
15960     hhdr -> hb_last_reclaimed = (unsigned short) GC_gc_no;
15961     if (report_if_found) {
15962         GC_reclaim_check(hbp, hhdr, sz);
15963     } else {
15964         *flh = GC_reclaim_generic(hbp, hhdr, sz, ok -> ok_init,
15965                                   (ptr_t)(*flh), &GC_bytes_found);
15966     }
15967 }
15968 #ifdef ENABLE_DISCLAIM
15969   STATIC void GC_disclaim_and_reclaim_or_free_small_block(struct hblk *hbp)
15970   {
15971     hdr *hhdr = HDR(hbp);
15972     word sz = hhdr -> hb_sz;
15973     struct obj_kind * ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
15974     void **flh = &(ok -> ok_freelist[BYTES_TO_GRANULES(sz)]);
15975     void *flh_next;
15976     hhdr -> hb_last_reclaimed = (unsigned short) GC_gc_no;
15977     flh_next = GC_reclaim_generic(hbp, hhdr, sz, ok -> ok_init,
15978                                   (ptr_t)(*flh), &GC_bytes_found);
15979     if (hhdr -> hb_n_marks)
15980         *flh = flh_next;
15981     else {
15982         GC_bytes_found += HBLKSIZE;
15983         GC_freehblk(hbp);
15984     }
15985   }
15986 #endif
15987 STATIC void GC_reclaim_block(struct hblk *hbp, word report_if_found)
15988 {
15989     hdr * hhdr = HDR(hbp);
15990     word sz;
15991     struct obj_kind * ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
15992 #ifdef AO_HAVE_load
15993         sz = (word)AO_load((volatile AO_t *)&hhdr->hb_sz);
15994 #else
15995         sz = hhdr -> hb_sz;
15996 #endif
15997     if( sz > MAXOBJBYTES ) {
15998         if( !mark_bit_from_hdr(hhdr, 0) ) {
15999             if (report_if_found) {
16000               GC_add_leaked((ptr_t)hbp);
16001             } else {
16002               word blocks;
16003 #ifdef ENABLE_DISCLAIM
16004                 if (EXPECT(hhdr->hb_flags & HAS_DISCLAIM, 0)) {
16005                   if ((*ok->ok_disclaim_proc)(hbp)) {
16006                     set_mark_bit_from_hdr(hhdr, 0);
16007                     goto in_use;
16008                   }
16009                 }
16010 #endif
16011               blocks = OBJ_SZ_TO_BLOCKS(sz);
16012 #if defined(CPPCHECK)
16013                 GC_noop1((word)&blocks);
16014 #endif
16015               if (blocks > 1) {
16016                 GC_large_allocd_bytes -= blocks * HBLKSIZE;
16017               }
16018               GC_bytes_found += sz;
16019               GC_freehblk(hbp);
16020             }
16021         } else {
16022 #ifdef ENABLE_DISCLAIM
16023            in_use:
16024 #endif
16025             if (IS_PTRFREE_SAFE(hhdr)) {
16026               GC_atomic_in_use += sz;
16027             } else {
16028               GC_composite_in_use += sz;
16029             }
16030         }
16031     } else {
16032         GC_bool empty = GC_block_empty(hhdr);
16033 #ifdef PARALLEL_MARK
16034           GC_ASSERT(hhdr -> hb_n_marks <= 2 * (HBLKSIZE/sz + 1) + 16);
16035 #else
16036           GC_ASSERT(sz * hhdr -> hb_n_marks <= HBLKSIZE);
16037 #endif
16038         if (report_if_found) {
16039           GC_reclaim_small_nonempty_block(hbp, sz,
16040                                           TRUE );
16041         } else if (empty) {
16042 #ifdef ENABLE_DISCLAIM
16043           if ((hhdr -> hb_flags & HAS_DISCLAIM) != 0) {
16044             GC_disclaim_and_reclaim_or_free_small_block(hbp);
16045           } else
16046 #endif
16047            {
16048             GC_bytes_found += HBLKSIZE;
16049             GC_freehblk(hbp);
16050           }
16051         } else if (GC_find_leak || !GC_block_nearly_full(hhdr, sz)) {
16052           struct hblk **rlh = ok -> ok_reclaim_list;
16053           if (rlh != NULL) {
16054             rlh += BYTES_TO_GRANULES(sz);
16055             hhdr -> hb_next = *rlh;
16056             *rlh = hbp;
16057           }
16058         }
16059         if (IS_PTRFREE_SAFE(hhdr)) {
16060           GC_atomic_in_use += sz * hhdr -> hb_n_marks;
16061         } else {
16062           GC_composite_in_use += sz * hhdr -> hb_n_marks;
16063         }
16064     }
16065 }
16066 #if !defined(NO_DEBUGGING)
16067 struct Print_stats
16068 {
16069         size_t number_of_blocks;
16070         size_t total_bytes;
16071 };
16072 #ifdef USE_MARK_BYTES
16073 unsigned GC_n_set_marks(hdr *hhdr)
16074 {
16075     unsigned result = 0;
16076     word i;
16077     word sz = hhdr -> hb_sz;
16078     word offset = MARK_BIT_OFFSET(sz);
16079     word limit = FINAL_MARK_BIT(sz);
16080     for (i = 0; i < limit; i += offset) {
16081         result += hhdr -> hb_marks[i];
16082     }
16083     GC_ASSERT(hhdr -> hb_marks[limit]);
16084     return(result);
16085 }
16086 #else
16087 static unsigned set_bits(word n)
16088 {
16089     word m = n;
16090     unsigned result = 0;
16091     while (m > 0) {
16092         if (m & 1) result++;
16093         m >>= 1;
16094     }
16095     return(result);
16096 }
16097 unsigned GC_n_set_marks(hdr *hhdr)
16098 {
16099     unsigned result = 0;
16100     word i;
16101     word n_mark_words;
16102 #ifdef MARK_BIT_PER_OBJ
16103       word n_objs = HBLK_OBJS(hhdr -> hb_sz);
16104       if (0 == n_objs) n_objs = 1;
16105       n_mark_words = divWORDSZ(n_objs + WORDSZ - 1);
16106 #else
16107       n_mark_words = MARK_BITS_SZ;
16108 #endif
16109     for (i = 0; i < n_mark_words - 1; i++) {
16110         result += set_bits(hhdr -> hb_marks[i]);
16111     }
16112 #ifdef MARK_BIT_PER_OBJ
16113       result += set_bits((hhdr -> hb_marks[n_mark_words - 1])
16114                          << (n_mark_words * WORDSZ - n_objs));
16115 #else
16116       result += set_bits(hhdr -> hb_marks[n_mark_words - 1]);
16117 #endif
16118     return result;
16119 }
16120 #endif
16121 STATIC void GC_print_block_descr(struct hblk *h,
16122                                  word  raw_ps)
16123 {
16124     hdr * hhdr = HDR(h);
16125     size_t bytes = hhdr -> hb_sz;
16126     struct Print_stats *ps;
16127     unsigned n_marks = GC_n_set_marks(hhdr);
16128     unsigned n_objs = (unsigned)HBLK_OBJS(bytes);
16129     if (0 == n_objs) n_objs = 1;
16130     if (hhdr -> hb_n_marks != n_marks) {
16131       GC_printf("%u,%u,%u!=%u,%u\n", hhdr->hb_obj_kind, (unsigned)bytes,
16132                 (unsigned)hhdr->hb_n_marks, n_marks, n_objs);
16133     } else {
16134       GC_printf("%u,%u,%u,%u\n", hhdr->hb_obj_kind, (unsigned)bytes,
16135                 n_marks, n_objs);
16136     }
16137     ps = (struct Print_stats *)raw_ps;
16138     ps->total_bytes += (bytes + (HBLKSIZE-1)) & ~(HBLKSIZE-1);
16139     ps->number_of_blocks++;
16140 }
16141 void GC_print_block_list(void)
16142 {
16143     struct Print_stats pstats;
16144     GC_printf("kind(0=ptrfree,1=normal,2=unc.),"
16145               "size_in_bytes,#_marks_set,#objs\n");
16146     pstats.number_of_blocks = 0;
16147     pstats.total_bytes = 0;
16148     GC_apply_to_all_blocks(GC_print_block_descr, (word)&pstats);
16149     GC_printf("blocks= %lu, bytes= %lu\n",
16150               (unsigned long)pstats.number_of_blocks,
16151               (unsigned long)pstats.total_bytes);
16152 }
16153 GC_API void GC_CALL GC_print_free_list(int kind, size_t sz_in_granules)
16154 {
16155     void *flh_next;
16156     int n;
16157     GC_ASSERT(kind < MAXOBJKINDS);
16158     GC_ASSERT(sz_in_granules <= MAXOBJGRANULES);
16159     flh_next = GC_obj_kinds[kind].ok_freelist[sz_in_granules];
16160     for (n = 0; flh_next; n++) {
16161         GC_printf("Free object in heap block %p [%d]: %p\n",
16162                   (void *)HBLKPTR(flh_next), n, flh_next);
16163         flh_next = obj_link(flh_next);
16164     }
16165 }
16166 #endif
16167 STATIC void GC_clear_fl_links(void **flp)
16168 {
16169     void *next = *flp;
16170     while (0 != next) {
16171        *flp = 0;
16172        flp = &(obj_link(next));
16173        next = *flp;
16174     }
16175 }
16176 GC_INNER void GC_start_reclaim(GC_bool report_if_found)
16177 {
16178     unsigned kind;
16179 #if defined(PARALLEL_MARK)
16180       GC_ASSERT(0 == GC_fl_builder_count);
16181 #endif
16182       GC_composite_in_use = 0;
16183       GC_atomic_in_use = 0;
16184       for (kind = 0; kind < GC_n_kinds; kind++) {
16185         struct hblk ** rlist = GC_obj_kinds[kind].ok_reclaim_list;
16186         GC_bool should_clobber = (GC_obj_kinds[kind].ok_descriptor != 0);
16187         if (rlist == 0) continue;
16188         if (!report_if_found) {
16189             void **fop;
16190             void **lim = &(GC_obj_kinds[kind].ok_freelist[MAXOBJGRANULES+1]);
16191             for (fop = GC_obj_kinds[kind].ok_freelist;
16192                  (word)fop < (word)lim; (*(word **)&fop)++) {
16193               if (*fop != 0) {
16194                 if (should_clobber) {
16195                   GC_clear_fl_links(fop);
16196                 } else {
16197                   *fop = 0;
16198                 }
16199               }
16200             }
16201         }
16202         BZERO(rlist, (MAXOBJGRANULES + 1) * sizeof(void *));
16203       }
16204     GC_apply_to_all_blocks(GC_reclaim_block, (word)report_if_found);
16205 #ifdef EAGER_SWEEP
16206     GC_reclaim_all((GC_stop_func)0, FALSE);
16207 #elif defined(ENABLE_DISCLAIM)
16208     GC_reclaim_unconditionally_marked();
16209 #endif
16210 #if defined(PARALLEL_MARK)
16211     GC_ASSERT(0 == GC_fl_builder_count);
16212 #endif
16213 }
16214 GC_INNER void GC_continue_reclaim(word sz , int kind)
16215 {
16216     hdr * hhdr;
16217     struct hblk * hbp;
16218     struct obj_kind * ok = &(GC_obj_kinds[kind]);
16219     struct hblk ** rlh = ok -> ok_reclaim_list;
16220     void **flh = &(ok -> ok_freelist[sz]);
16221     if (NULL == rlh)
16222         return;
16223     for (rlh += sz; (hbp = *rlh) != NULL; ) {
16224         hhdr = HDR(hbp);
16225         *rlh = hhdr -> hb_next;
16226         GC_reclaim_small_nonempty_block(hbp, hhdr -> hb_sz, FALSE);
16227         if (*flh != 0)
16228             break;
16229     }
16230 }
16231 GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old)
16232 {
16233     word sz;
16234     unsigned kind;
16235     hdr * hhdr;
16236     struct hblk * hbp;
16237     struct obj_kind * ok;
16238     struct hblk ** rlp;
16239     struct hblk ** rlh;
16240 #ifndef NO_CLOCK
16241       CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
16242       if (GC_print_stats == VERBOSE)
16243         GET_TIME(start_time);
16244 #endif
16245     for (kind = 0; kind < GC_n_kinds; kind++) {
16246         ok = &(GC_obj_kinds[kind]);
16247         rlp = ok -> ok_reclaim_list;
16248         if (rlp == 0) continue;
16249         for (sz = 1; sz <= MAXOBJGRANULES; sz++) {
16250             for (rlh = rlp + sz; (hbp = *rlh) != NULL; ) {
16251                 if (stop_func != (GC_stop_func)0 && (*stop_func)()) {
16252                     return(FALSE);
16253                 }
16254                 hhdr = HDR(hbp);
16255                 *rlh = hhdr -> hb_next;
16256                 if (!ignore_old
16257                     || (word)hhdr->hb_last_reclaimed == GC_gc_no - 1) {
16258                     GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE);
16259                 }
16260             }
16261         }
16262     }
16263 #ifndef NO_CLOCK
16264       if (GC_print_stats == VERBOSE) {
16265         CLOCK_TYPE done_time;
16266         GET_TIME(done_time);
16267         GC_verbose_log_printf(
16268                         "Disposing of reclaim lists took %lu ms %lu ns\n",
16269                         MS_TIME_DIFF(done_time, start_time),
16270                         NS_FRAC_TIME_DIFF(done_time, start_time));
16271       }
16272 #endif
16273     return(TRUE);
16274 }
16275 #if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM)
16276   STATIC void GC_reclaim_unconditionally_marked(void)
16277   {
16278     word sz;
16279     unsigned kind;
16280     hdr * hhdr;
16281     struct hblk * hbp;
16282     struct obj_kind * ok;
16283     struct hblk ** rlp;
16284     struct hblk ** rlh;
16285     for (kind = 0; kind < GC_n_kinds; kind++) {
16286         ok = &(GC_obj_kinds[kind]);
16287         if (!ok->ok_mark_unconditionally)
16288           continue;
16289         rlp = ok->ok_reclaim_list;
16290         if (rlp == 0)
16291           continue;
16292         for (sz = 1; sz <= MAXOBJGRANULES; sz++) {
16293             rlh = rlp + sz;
16294             while ((hbp = *rlh) != 0) {
16295                 hhdr = HDR(hbp);
16296                 *rlh = hhdr->hb_next;
16297                 GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE);
16298             }
16299         }
16300     }
16301   }
16302 #endif
16303 struct enumerate_reachable_s {
16304   GC_reachable_object_proc proc;
16305   void *client_data;
16306 };
16307 STATIC void GC_do_enumerate_reachable_objects(struct hblk *hbp, word ped)
16308 {
16309   struct hblkhdr *hhdr = HDR(hbp);
16310   size_t sz = (size_t)hhdr->hb_sz;
16311   size_t bit_no;
16312   char *p, *plim;
16313   if (GC_block_empty(hhdr)) {
16314     return;
16315   }
16316   p = hbp->hb_body;
16317   if (sz > MAXOBJBYTES) {
16318     plim = p;
16319   } else {
16320     plim = hbp->hb_body + HBLKSIZE - sz;
16321   }
16322   for (bit_no = 0; p <= plim; bit_no += MARK_BIT_OFFSET(sz), p += sz) {
16323     if (mark_bit_from_hdr(hhdr, bit_no)) {
16324       ((struct enumerate_reachable_s *)ped)->proc(p, sz,
16325                         ((struct enumerate_reachable_s *)ped)->client_data);
16326     }
16327   }
16328 }
16329 GC_API void GC_CALL GC_enumerate_reachable_objects_inner(
16330                                                 GC_reachable_object_proc proc,
16331                                                 void *client_data)
16332 {
16333   struct enumerate_reachable_s ed;
16334   GC_ASSERT(I_HOLD_LOCK());
16335   ed.proc = proc;
16336   ed.client_data = client_data;
16337   GC_apply_to_all_blocks(GC_do_enumerate_reachable_objects, (word)&ed);
16338 }
16339 #ifndef GC_TYPED_H
16340 #define GC_TYPED_H
16341 #ifndef GC_H
16342 #endif
16343 #ifdef __cplusplus
16344   extern "C" {
16345 #endif
16346 typedef GC_word * GC_bitmap;
16347 #define GC_WORDSZ (8 * sizeof(GC_word))
16348 #define GC_get_bit(bm, index) \
16349             (((bm)[(index) / GC_WORDSZ] >> ((index) % GC_WORDSZ)) & 1)
16350 #define GC_set_bit(bm, index) \
16351             ((bm)[(index) / GC_WORDSZ] |= (GC_word)1 << ((index) % GC_WORDSZ))
16352 #define GC_WORD_OFFSET(t, f) (offsetof(t,f) / sizeof(GC_word))
16353 #define GC_WORD_LEN(t) (sizeof(t) / sizeof(GC_word))
16354 #define GC_BITMAP_SIZE(t) ((GC_WORD_LEN(t) + GC_WORDSZ - 1) / GC_WORDSZ)
16355 typedef GC_word GC_descr;
16356 GC_API GC_descr GC_CALL GC_make_descriptor(const GC_word * ,
16357                                 size_t );
16358 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
16359         GC_malloc_explicitly_typed(size_t ,
16360                                    GC_descr );
16361 GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void * GC_CALL
16362         GC_malloc_explicitly_typed_ignore_off_page(size_t ,
16363                                                    GC_descr );
16364 GC_API GC_ATTR_MALLOC GC_ATTR_CALLOC_SIZE(1, 2) void * GC_CALL
16365         GC_calloc_explicitly_typed(size_t ,
16366                                    size_t ,
16367                                    GC_descr );
16368 #ifdef GC_DEBUG
16369 #define GC_MALLOC_EXPLICITLY_TYPED(bytes, d) ((void)(d), GC_MALLOC(bytes))
16370 #define GC_CALLOC_EXPLICITLY_TYPED(n, bytes, d) \
16371                         ((void)(d), GC_MALLOC((n) * (bytes)))
16372 #else
16373 #define GC_MALLOC_EXPLICITLY_TYPED(bytes, d) \
16374                         GC_malloc_explicitly_typed(bytes, d)
16375 #define GC_CALLOC_EXPLICITLY_TYPED(n, bytes, d) \
16376                         GC_calloc_explicitly_typed(n, bytes, d)
16377 #endif
16378 #ifdef __cplusplus
16379   }
16380 #endif
16381 #endif
16382 #define TYPD_EXTRA_BYTES (sizeof(word) - EXTRA_BYTES)
16383 STATIC int GC_explicit_kind = 0;
16384 STATIC int GC_array_kind = 0;
16385 struct LeafDescriptor {
16386         word ld_tag;
16387 #define LEAF_TAG 1
16388         size_t ld_size;
16389         size_t ld_nelements;
16390         GC_descr ld_descriptor;
16391 };
16392 struct ComplexArrayDescriptor {
16393         word ad_tag;
16394 #define ARRAY_TAG 2
16395         size_t ad_nelements;
16396         union ComplexDescriptor * ad_element_descr;
16397 };
16398 struct SequenceDescriptor {
16399         word sd_tag;
16400 #define SEQUENCE_TAG 3
16401         union ComplexDescriptor * sd_first;
16402         union ComplexDescriptor * sd_second;
16403 };
16404 typedef union ComplexDescriptor {
16405     struct LeafDescriptor ld;
16406     struct ComplexArrayDescriptor ad;
16407     struct SequenceDescriptor sd;
16408 } complex_descriptor;
16409 #define TAG ad.ad_tag
16410 #define ED_INITIAL_SIZE 100
16411 STATIC int GC_typed_mark_proc_index = 0;
16412 STATIC int GC_array_mark_proc_index = 0;
16413 STATIC void GC_push_typed_structures_proc(void)
16414 {
16415   GC_PUSH_ALL_SYM(GC_ext_descriptors);
16416 }
16417 STATIC signed_word GC_add_ext_descriptor(const word * bm, word nbits)
16418 {
16419     size_t nwords = divWORDSZ(nbits + WORDSZ-1);
16420     signed_word result;
16421     size_t i;
16422     word last_part;
16423     size_t extra_bits;
16424     DCL_LOCK_STATE;
16425     LOCK();
16426     while (GC_avail_descr + nwords >= GC_ed_size) {
16427         typed_ext_descr_t *newExtD;
16428         size_t new_size;
16429         word ed_size = GC_ed_size;
16430         if (ed_size == 0) {
16431             GC_ASSERT((word)(&GC_ext_descriptors) % sizeof(word) == 0);
16432             GC_push_typed_structures = GC_push_typed_structures_proc;
16433             UNLOCK();
16434             new_size = ED_INITIAL_SIZE;
16435         } else {
16436             UNLOCK();
16437             new_size = 2 * ed_size;
16438             if (new_size > MAX_ENV) return(-1);
16439         }
16440         newExtD = (typed_ext_descr_t*)GC_malloc_atomic(new_size
16441                                                 * sizeof(typed_ext_descr_t));
16442         if (NULL == newExtD)
16443             return -1;
16444         LOCK();
16445         if (ed_size == GC_ed_size) {
16446             if (GC_avail_descr != 0) {
16447                 BCOPY(GC_ext_descriptors, newExtD,
16448                       GC_avail_descr * sizeof(typed_ext_descr_t));
16449             }
16450             GC_ed_size = new_size;
16451             GC_ext_descriptors = newExtD;
16452         }
16453     }
16454     result = GC_avail_descr;
16455     for (i = 0; i < nwords-1; i++) {
16456         GC_ext_descriptors[result + i].ed_bitmap = bm[i];
16457         GC_ext_descriptors[result + i].ed_continued = TRUE;
16458     }
16459     last_part = bm[i];
16460     extra_bits = nwords * WORDSZ - nbits;
16461     last_part <<= extra_bits;
16462     last_part >>= extra_bits;
16463     GC_ext_descriptors[result + i].ed_bitmap = last_part;
16464     GC_ext_descriptors[result + i].ed_continued = FALSE;
16465     GC_avail_descr += nwords;
16466     UNLOCK();
16467     return(result);
16468 }
16469 STATIC GC_descr GC_bm_table[WORDSZ/2];
16470 STATIC GC_descr GC_double_descr(GC_descr descriptor, word nwords)
16471 {
16472     if ((descriptor & GC_DS_TAGS) == GC_DS_LENGTH) {
16473         descriptor = GC_bm_table[BYTES_TO_WORDS((word)descriptor)];
16474     }
16475     descriptor |= (descriptor & ~GC_DS_TAGS) >> nwords;
16476     return(descriptor);
16477 }
16478 STATIC complex_descriptor *
16479 GC_make_sequence_descriptor(complex_descriptor *first,
16480                             complex_descriptor *second);
16481 #define COMPLEX 2
16482 #define LEAF    1
16483 #define SIMPLE  0
16484 #define NO_MEM  (-1)
16485 STATIC int GC_make_array_descriptor(size_t nelements, size_t size,
16486                                     GC_descr descriptor, GC_descr *simple_d,
16487                                     complex_descriptor **complex_d,
16488                                     struct LeafDescriptor * leaf)
16489 {
16490 #define OPT_THRESHOLD 50
16491     if ((descriptor & GC_DS_TAGS) == GC_DS_LENGTH) {
16492       if (descriptor == (GC_descr)size) {
16493         *simple_d = nelements * descriptor;
16494         return(SIMPLE);
16495       } else if ((word)descriptor == 0) {
16496         *simple_d = (GC_descr)0;
16497         return(SIMPLE);
16498       }
16499     }
16500     if (nelements <= OPT_THRESHOLD) {
16501       if (nelements <= 1) {
16502         if (nelements == 1) {
16503             *simple_d = descriptor;
16504             return(SIMPLE);
16505         } else {
16506             *simple_d = (GC_descr)0;
16507             return(SIMPLE);
16508         }
16509       }
16510     } else if (size <= BITMAP_BITS/2
16511                && (descriptor & GC_DS_TAGS) != GC_DS_PROC
16512                && (size & (sizeof(word)-1)) == 0) {
16513       int result =
16514           GC_make_array_descriptor(nelements/2, 2*size,
16515                                    GC_double_descr(descriptor,
16516                                                    BYTES_TO_WORDS(size)),
16517                                    simple_d, complex_d, leaf);
16518       if ((nelements & 1) == 0) {
16519           return(result);
16520       } else {
16521           struct LeafDescriptor * one_element =
16522               (struct LeafDescriptor *)
16523                 GC_malloc_atomic(sizeof(struct LeafDescriptor));
16524           if (result == NO_MEM || one_element == 0) return(NO_MEM);
16525           one_element -> ld_tag = LEAF_TAG;
16526           one_element -> ld_size = size;
16527           one_element -> ld_nelements = 1;
16528           one_element -> ld_descriptor = descriptor;
16529           switch(result) {
16530             case SIMPLE:
16531             {
16532               struct LeafDescriptor * beginning =
16533                 (struct LeafDescriptor *)
16534                   GC_malloc_atomic(sizeof(struct LeafDescriptor));
16535               if (beginning == 0) return(NO_MEM);
16536               beginning -> ld_tag = LEAF_TAG;
16537               beginning -> ld_size = size;
16538               beginning -> ld_nelements = 1;
16539               beginning -> ld_descriptor = *simple_d;
16540               *complex_d = GC_make_sequence_descriptor(
16541                                 (complex_descriptor *)beginning,
16542                                 (complex_descriptor *)one_element);
16543               break;
16544             }
16545             case LEAF:
16546             {
16547               struct LeafDescriptor * beginning =
16548                 (struct LeafDescriptor *)
16549                   GC_malloc_atomic(sizeof(struct LeafDescriptor));
16550               if (beginning == 0) return(NO_MEM);
16551               beginning -> ld_tag = LEAF_TAG;
16552               beginning -> ld_size = leaf -> ld_size;
16553               beginning -> ld_nelements = leaf -> ld_nelements;
16554               beginning -> ld_descriptor = leaf -> ld_descriptor;
16555               *complex_d = GC_make_sequence_descriptor(
16556                                 (complex_descriptor *)beginning,
16557                                 (complex_descriptor *)one_element);
16558               break;
16559             }
16560             case COMPLEX:
16561               *complex_d = GC_make_sequence_descriptor(
16562                                 *complex_d,
16563                                 (complex_descriptor *)one_element);
16564               break;
16565           }
16566           return(COMPLEX);
16567       }
16568     }
16569     leaf -> ld_size = size;
16570     leaf -> ld_nelements = nelements;
16571     leaf -> ld_descriptor = descriptor;
16572     return(LEAF);
16573 }
16574 STATIC complex_descriptor *
16575 GC_make_sequence_descriptor(complex_descriptor *first,
16576                             complex_descriptor *second)
16577 {
16578     struct SequenceDescriptor * result =
16579         (struct SequenceDescriptor *)
16580                 GC_malloc(sizeof(struct SequenceDescriptor));
16581     if (result != 0) {
16582         result -> sd_tag = SEQUENCE_TAG;
16583         result -> sd_first = first;
16584         result -> sd_second = second;
16585         GC_dirty(result);
16586         REACHABLE_AFTER_DIRTY(first);
16587         REACHABLE_AFTER_DIRTY(second);
16588     }
16589     return((complex_descriptor *)result);
16590 }
16591 STATIC mse * GC_typed_mark_proc(word * addr, mse * mark_stack_ptr,
16592                                 mse * mark_stack_limit, word env);
16593 STATIC mse * GC_array_mark_proc(word * addr, mse * mark_stack_ptr,
16594                                 mse * mark_stack_limit, word env);
16595 STATIC void GC_init_explicit_typing(void)
16596 {
16597     unsigned i;
16598     GC_STATIC_ASSERT(sizeof(struct LeafDescriptor) % sizeof(word) == 0);
16599       GC_explicit_kind = GC_new_kind_inner(GC_new_free_list_inner(),
16600                             (WORDS_TO_BYTES((word)-1) | GC_DS_PER_OBJECT),
16601                             TRUE, TRUE);
16602       GC_typed_mark_proc_index = GC_new_proc_inner(GC_typed_mark_proc);
16603       GC_array_mark_proc_index = GC_new_proc_inner(GC_array_mark_proc);
16604       GC_array_kind = GC_new_kind_inner(GC_new_free_list_inner(),
16605                             GC_MAKE_PROC(GC_array_mark_proc_index, 0),
16606                             FALSE, TRUE);
16607       GC_bm_table[0] = GC_DS_BITMAP;
16608       for (i = 1; i < WORDSZ/2; i++) {
16609           GC_bm_table[i] = (((word)-1) << (WORDSZ - i)) | GC_DS_BITMAP;
16610       }
16611 }
16612 STATIC mse * GC_typed_mark_proc(word * addr, mse * mark_stack_ptr,
16613                                 mse * mark_stack_limit, word env)
16614 {
16615     word bm = GC_ext_descriptors[env].ed_bitmap;
16616     word * current_p = addr;
16617     word current;
16618     ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
16619     ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
16620     DECLARE_HDR_CACHE;
16621     INIT_HDR_CACHE;
16622     for (; bm != 0; bm >>= 1, current_p++) {
16623         if (bm & 1) {
16624             current = *current_p;
16625             FIXUP_POINTER(current);
16626             if (current >= (word)least_ha && current <= (word)greatest_ha) {
16627                 PUSH_CONTENTS((ptr_t)current, mark_stack_ptr,
16628                               mark_stack_limit, (ptr_t)current_p);
16629             }
16630         }
16631     }
16632     if (GC_ext_descriptors[env].ed_continued) {
16633         mark_stack_ptr++;
16634         if ((word)mark_stack_ptr >= (word)mark_stack_limit) {
16635             mark_stack_ptr = GC_signal_mark_stack_overflow(mark_stack_ptr);
16636         }
16637         mark_stack_ptr -> mse_start = (ptr_t)(addr + WORDSZ);
16638         mark_stack_ptr -> mse_descr.w =
16639                         GC_MAKE_PROC(GC_typed_mark_proc_index, env + 1);
16640     }
16641     return(mark_stack_ptr);
16642 }
16643 STATIC word GC_descr_obj_size(complex_descriptor *d)
16644 {
16645     switch(d -> TAG) {
16646       case LEAF_TAG:
16647         return(d -> ld.ld_nelements * d -> ld.ld_size);
16648       case ARRAY_TAG:
16649         return(d -> ad.ad_nelements
16650                * GC_descr_obj_size(d -> ad.ad_element_descr));
16651       case SEQUENCE_TAG:
16652         return(GC_descr_obj_size(d -> sd.sd_first)
16653                + GC_descr_obj_size(d -> sd.sd_second));
16654       default:
16655         ABORT_RET("Bad complex descriptor");
16656         return 0;
16657     }
16658 }
16659 STATIC mse * GC_push_complex_descriptor(word *addr, complex_descriptor *d,
16660                                         mse *msp, mse *msl)
16661 {
16662     ptr_t current = (ptr_t)addr;
16663     word nelements;
16664     word sz;
16665     word i;
16666     switch(d -> TAG) {
16667       case LEAF_TAG:
16668         {
16669           GC_descr descr = d -> ld.ld_descriptor;
16670           nelements = d -> ld.ld_nelements;
16671           if (msl - msp <= (ptrdiff_t)nelements) return(0);
16672           sz = d -> ld.ld_size;
16673           for (i = 0; i < nelements; i++) {
16674               msp++;
16675               msp -> mse_start = current;
16676               msp -> mse_descr.w = descr;
16677               current += sz;
16678           }
16679           return(msp);
16680         }
16681       case ARRAY_TAG:
16682         {
16683           complex_descriptor *descr = d -> ad.ad_element_descr;
16684           nelements = d -> ad.ad_nelements;
16685           sz = GC_descr_obj_size(descr);
16686           for (i = 0; i < nelements; i++) {
16687               msp = GC_push_complex_descriptor((word *)current, descr,
16688                                                 msp, msl);
16689               if (msp == 0) return(0);
16690               current += sz;
16691           }
16692           return(msp);
16693         }
16694       case SEQUENCE_TAG:
16695         {
16696           sz = GC_descr_obj_size(d -> sd.sd_first);
16697           msp = GC_push_complex_descriptor((word *)current, d -> sd.sd_first,
16698                                            msp, msl);
16699           if (msp == 0) return(0);
16700           current += sz;
16701           msp = GC_push_complex_descriptor((word *)current, d -> sd.sd_second,
16702                                            msp, msl);
16703           return(msp);
16704         }
16705       default:
16706         ABORT_RET("Bad complex descriptor");
16707         return 0;
16708    }
16709 }
16710 STATIC mse * GC_array_mark_proc(word * addr, mse * mark_stack_ptr,
16711                                 mse * mark_stack_limit,
16712                                 word env GC_ATTR_UNUSED)
16713 {
16714     hdr * hhdr = HDR(addr);
16715     word sz = hhdr -> hb_sz;
16716     word nwords = BYTES_TO_WORDS(sz);
16717     complex_descriptor * descr = (complex_descriptor *)(addr[nwords-1]);
16718     mse * orig_mark_stack_ptr = mark_stack_ptr;
16719     mse * new_mark_stack_ptr;
16720     if (descr == 0) {
16721         return(orig_mark_stack_ptr);
16722     }
16723     new_mark_stack_ptr = GC_push_complex_descriptor(addr, descr,
16724                                                     mark_stack_ptr,
16725                                                     mark_stack_limit-1);
16726     if (new_mark_stack_ptr == 0) {
16727         if (NULL == mark_stack_ptr) ABORT("Bad mark_stack_ptr");
16728 #ifdef PARALLEL_MARK
16729             if (GC_mark_stack + GC_mark_stack_size == mark_stack_limit)
16730 #endif
16731         {
16732             GC_mark_stack_too_small = TRUE;
16733         }
16734         new_mark_stack_ptr = orig_mark_stack_ptr + 1;
16735         new_mark_stack_ptr -> mse_start = (ptr_t)addr;
16736         new_mark_stack_ptr -> mse_descr.w = sz | GC_DS_LENGTH;
16737     } else {
16738         new_mark_stack_ptr++;
16739         new_mark_stack_ptr -> mse_start = (ptr_t)(addr + nwords - 1);
16740         new_mark_stack_ptr -> mse_descr.w = sizeof(word) | GC_DS_LENGTH;
16741     }
16742     return new_mark_stack_ptr;
16743 }
16744 GC_API GC_descr GC_CALL GC_make_descriptor(const GC_word * bm, size_t len)
16745 {
16746     signed_word last_set_bit = len - 1;
16747     GC_descr result;
16748     DCL_LOCK_STATE;
16749 #if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_store_release)
16750       if (!EXPECT(AO_load_acquire(&GC_explicit_typing_initialized), TRUE)) {
16751         LOCK();
16752         if (!GC_explicit_typing_initialized) {
16753           GC_init_explicit_typing();
16754           AO_store_release(&GC_explicit_typing_initialized, TRUE);
16755         }
16756         UNLOCK();
16757       }
16758 #else
16759       LOCK();
16760       if (!EXPECT(GC_explicit_typing_initialized, TRUE)) {
16761         GC_init_explicit_typing();
16762         GC_explicit_typing_initialized = TRUE;
16763       }
16764       UNLOCK();
16765 #endif
16766     while (last_set_bit >= 0 && !GC_get_bit(bm, last_set_bit))
16767       last_set_bit--;
16768     if (last_set_bit < 0) return(0 );
16769 #if ALIGNMENT == CPP_WORDSZ/8
16770     {
16771       signed_word i;
16772       for (i = 0; i < last_set_bit; i++) {
16773         if (!GC_get_bit(bm, i)) {
16774           break;
16775         }
16776       }
16777       if (i == last_set_bit) {
16778         return (WORDS_TO_BYTES(last_set_bit+1) | GC_DS_LENGTH);
16779       }
16780     }
16781 #endif
16782     if ((word)last_set_bit < BITMAP_BITS) {
16783         signed_word i;
16784         result = SIGNB;
16785         for (i = last_set_bit - 1; i >= 0; i--) {
16786             result >>= 1;
16787             if (GC_get_bit(bm, i)) result |= SIGNB;
16788         }
16789         result |= GC_DS_BITMAP;
16790     } else {
16791         signed_word index = GC_add_ext_descriptor(bm, (word)last_set_bit + 1);
16792         if (index == -1) return(WORDS_TO_BYTES(last_set_bit+1) | GC_DS_LENGTH);
16793         result = GC_MAKE_PROC(GC_typed_mark_proc_index, (word)index);
16794     }
16795     return result;
16796 }
16797 GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_explicitly_typed(size_t lb,
16798                                                                 GC_descr d)
16799 {
16800     word *op;
16801     size_t lg;
16802     GC_ASSERT(GC_explicit_typing_initialized);
16803     lb = SIZET_SAT_ADD(lb, TYPD_EXTRA_BYTES);
16804     op = (word *)GC_malloc_kind(lb, GC_explicit_kind);
16805     if (EXPECT(NULL == op, FALSE))
16806         return NULL;
16807     lg = BYTES_TO_GRANULES(GC_size(op));
16808     op[GRANULES_TO_WORDS(lg) - 1] = d;
16809     GC_dirty(op + GRANULES_TO_WORDS(lg) - 1);
16810     REACHABLE_AFTER_DIRTY(d);
16811     return op;
16812 }
16813 #define GENERAL_MALLOC_IOP(lb, k) \
16814                 GC_clear_stack(GC_generic_malloc_ignore_off_page(lb, k))
16815 GC_API GC_ATTR_MALLOC void * GC_CALL
16816     GC_malloc_explicitly_typed_ignore_off_page(size_t lb, GC_descr d)
16817 {
16818     ptr_t op;
16819     size_t lg;
16820     DCL_LOCK_STATE;
16821     GC_ASSERT(GC_explicit_typing_initialized);
16822     lb = SIZET_SAT_ADD(lb, TYPD_EXTRA_BYTES);
16823     if (SMALL_OBJ(lb)) {
16824         void **opp;
16825         GC_DBG_COLLECT_AT_MALLOC(lb);
16826         LOCK();
16827         lg = GC_size_map[lb];
16828         opp = &GC_obj_kinds[GC_explicit_kind].ok_freelist[lg];
16829         op = (ptr_t)(*opp);
16830         if (EXPECT(0 == op, FALSE)) {
16831             UNLOCK();
16832             op = (ptr_t)GENERAL_MALLOC_IOP(lb, GC_explicit_kind);
16833             if (0 == op) return 0;
16834             lg = BYTES_TO_GRANULES(GC_size(op));
16835         } else {
16836             *opp = obj_link(op);
16837             obj_link(op) = 0;
16838             GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
16839             UNLOCK();
16840         }
16841     } else {
16842         op = (ptr_t)GENERAL_MALLOC_IOP(lb, GC_explicit_kind);
16843         if (NULL == op) return NULL;
16844         lg = BYTES_TO_GRANULES(GC_size(op));
16845     }
16846     ((word *)op)[GRANULES_TO_WORDS(lg) - 1] = d;
16847     GC_dirty(op + GRANULES_TO_WORDS(lg) - 1);
16848     REACHABLE_AFTER_DIRTY(d);
16849     return op;
16850 }
16851 GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n,
16852                                                         size_t lb, GC_descr d)
16853 {
16854     word *op;
16855     size_t lg;
16856     GC_descr simple_descr;
16857     complex_descriptor *complex_descr;
16858     int descr_type;
16859     struct LeafDescriptor leaf;
16860     GC_ASSERT(GC_explicit_typing_initialized);
16861     descr_type = GC_make_array_descriptor((word)n, (word)lb, d, &simple_descr,
16862                                           &complex_descr, &leaf);
16863     if ((lb | n) > GC_SQRT_SIZE_MAX
16864         && lb > 0 && n > GC_SIZE_MAX / lb)
16865       return (*GC_get_oom_fn())(GC_SIZE_MAX);
16866     lb *= n;
16867     switch(descr_type) {
16868         case NO_MEM: return(0);
16869         case SIMPLE:
16870             return GC_malloc_explicitly_typed(lb, simple_descr);
16871         case LEAF:
16872             lb = SIZET_SAT_ADD(lb,
16873                         sizeof(struct LeafDescriptor) + TYPD_EXTRA_BYTES);
16874             break;
16875         case COMPLEX:
16876             lb = SIZET_SAT_ADD(lb, TYPD_EXTRA_BYTES);
16877             break;
16878     }
16879     op = (word *)GC_malloc_kind(lb, GC_array_kind);
16880     if (EXPECT(NULL == op, FALSE))
16881         return NULL;
16882     lg = BYTES_TO_GRANULES(GC_size(op));
16883     if (descr_type == LEAF) {
16884         volatile struct LeafDescriptor * lp =
16885             (struct LeafDescriptor *)
16886                 (op + GRANULES_TO_WORDS(lg)
16887                     - (BYTES_TO_WORDS(sizeof(struct LeafDescriptor)) + 1));
16888         lp -> ld_tag = LEAF_TAG;
16889         lp -> ld_size = leaf.ld_size;
16890         lp -> ld_nelements = leaf.ld_nelements;
16891         lp -> ld_descriptor = leaf.ld_descriptor;
16892         ((volatile word *)op)[GRANULES_TO_WORDS(lg) - 1] = (word)lp;
16893     } else {
16894 #ifndef GC_NO_FINALIZATION
16895         size_t lw = GRANULES_TO_WORDS(lg);
16896         op[lw - 1] = (word)complex_descr;
16897         GC_dirty(op + lw - 1);
16898         REACHABLE_AFTER_DIRTY(complex_descr);
16899         if (EXPECT(GC_general_register_disappearing_link(
16900                                                 (void **)(op + lw - 1), op)
16901                   == GC_NO_MEMORY, FALSE))
16902 #endif
16903         {
16904             return (*GC_get_oom_fn())(lb);
16905         }
16906     }
16907     return op;
16908 }
16909 #include <stdio.h>
16910 #include <limits.h>
16911 #include <stdarg.h>
16912 #ifndef MSWINCE
16913 #include <signal.h>
16914 #endif
16915 #ifdef GC_SOLARIS_THREADS
16916 #include <sys/syscall.h>
16917 #endif
16918 #if defined(UNIX_LIKE) || defined(CYGWIN32) || defined(SYMBIAN) \
16919     || (defined(CONSOLE_LOG) && defined(MSWIN32))
16920 #include <fcntl.h>
16921 #include <sys/types.h>
16922 #include <sys/stat.h>
16923 #endif
16924 #if defined(CONSOLE_LOG) && defined(MSWIN32) && defined(_MSC_VER)
16925 #include <io.h>
16926 #endif
16927 #ifdef NONSTOP
16928 #include <floss.h>
16929 #endif
16930 #ifdef THREADS
16931 #ifdef PCR
16932 #include "il/PCR_IL.h"
16933     GC_INNER PCR_Th_ML GC_allocate_ml;
16934 #elif defined(SN_TARGET_PSP2)
16935     GC_INNER WapiMutex GC_allocate_ml_PSP2 = { 0, NULL };
16936 #elif defined(GC_DEFN_ALLOCATE_ML) || defined(SN_TARGET_PS3)
16937 #include <pthread.h>
16938     GC_INNER pthread_mutex_t GC_allocate_ml;
16939 #endif
16940 #endif
16941 #ifdef DYNAMIC_LOADING
16942 #define GC_REGISTER_MAIN_STATIC_DATA() GC_register_main_static_data()
16943 #elif defined(GC_DONT_REGISTER_MAIN_STATIC_DATA)
16944 #define GC_REGISTER_MAIN_STATIC_DATA() FALSE
16945 #else
16946 #define GC_REGISTER_MAIN_STATIC_DATA() TRUE
16947 #endif
16948 #ifdef NEED_CANCEL_DISABLE_COUNT
16949   __thread unsigned char GC_cancel_disable_count = 0;
16950 #endif
16951 GC_FAR struct _GC_arrays GC_arrays ;
16952 GC_INNER unsigned GC_n_mark_procs = GC_RESERVED_MARK_PROCS;
16953 GC_INNER unsigned GC_n_kinds = GC_N_KINDS_INITIAL_VALUE;
16954 GC_INNER GC_bool GC_debugging_started = FALSE;
16955 ptr_t GC_stackbottom = 0;
16956 #ifdef IA64
16957   ptr_t GC_register_stackbottom = 0;
16958 #endif
16959 int GC_dont_gc = FALSE;
16960 int GC_dont_precollect = FALSE;
16961 GC_bool GC_quiet = 0;
16962 #if !defined(NO_CLOCK) || !defined(SMALL_CONFIG)
16963   int GC_print_stats = 0;
16964 #endif
16965 #ifdef GC_PRINT_BACK_HEIGHT
16966   GC_INNER GC_bool GC_print_back_height = TRUE;
16967 #else
16968   GC_INNER GC_bool GC_print_back_height = FALSE;
16969 #endif
16970 #ifndef NO_DEBUGGING
16971 #ifdef GC_DUMP_REGULARLY
16972     GC_INNER GC_bool GC_dump_regularly = TRUE;
16973 #else
16974     GC_INNER GC_bool GC_dump_regularly = FALSE;
16975 #endif
16976 #ifndef NO_CLOCK
16977     STATIC CLOCK_TYPE GC_init_time;
16978 #endif
16979 #endif
16980 #ifdef KEEP_BACK_PTRS
16981   GC_INNER long GC_backtraces = 0;
16982 #endif
16983 #ifdef FIND_LEAK
16984   int GC_find_leak = 1;
16985 #else
16986   int GC_find_leak = 0;
16987 #endif
16988 #ifndef SHORT_DBG_HDRS
16989 #ifdef GC_FINDLEAK_DELAY_FREE
16990     GC_INNER GC_bool GC_findleak_delay_free = TRUE;
16991 #else
16992     GC_INNER GC_bool GC_findleak_delay_free = FALSE;
16993 #endif
16994 #endif
16995 #ifdef ALL_INTERIOR_POINTERS
16996   int GC_all_interior_pointers = 1;
16997 #else
16998   int GC_all_interior_pointers = 0;
16999 #endif
17000 #ifdef FINALIZE_ON_DEMAND
17001   int GC_finalize_on_demand = 1;
17002 #else
17003   int GC_finalize_on_demand = 0;
17004 #endif
17005 #ifdef JAVA_FINALIZATION
17006   int GC_java_finalization = 1;
17007 #else
17008   int GC_java_finalization = 0;
17009 #endif
17010 GC_finalizer_notifier_proc GC_finalizer_notifier =
17011                                         (GC_finalizer_notifier_proc)0;
17012 #ifdef GC_FORCE_UNMAP_ON_GCOLLECT
17013   GC_INNER GC_bool GC_force_unmap_on_gcollect = TRUE;
17014 #else
17015   GC_INNER GC_bool GC_force_unmap_on_gcollect = FALSE;
17016 #endif
17017 #ifndef GC_LARGE_ALLOC_WARN_INTERVAL
17018 #define GC_LARGE_ALLOC_WARN_INTERVAL 5
17019 #endif
17020 GC_INNER long GC_large_alloc_warn_interval = GC_LARGE_ALLOC_WARN_INTERVAL;
17021 STATIC void * GC_CALLBACK GC_default_oom_fn(
17022                                         size_t bytes_requested GC_ATTR_UNUSED)
17023 {
17024     return(0);
17025 }
17026 GC_oom_func GC_oom_fn = GC_default_oom_fn;
17027 #ifdef CAN_HANDLE_FORK
17028 #ifdef HANDLE_FORK
17029     GC_INNER int GC_handle_fork = 1;
17030 #else
17031     GC_INNER int GC_handle_fork = FALSE;
17032 #endif
17033 #elif !defined(HAVE_NO_FORK)
17034   GC_API void GC_CALL GC_atfork_prepare(void)
17035   {
17036 #ifdef THREADS
17037       ABORT("fork() handling unsupported");
17038 #endif
17039   }
17040   GC_API void GC_CALL GC_atfork_parent(void)
17041   {
17042   }
17043   GC_API void GC_CALL GC_atfork_child(void)
17044   {
17045   }
17046 #endif
17047 GC_API void GC_CALL GC_set_handle_fork(int value GC_ATTR_UNUSED)
17048 {
17049 #ifdef CAN_HANDLE_FORK
17050     if (!GC_is_initialized)
17051       GC_handle_fork = value >= -1 ? value : 1;
17052 #elif defined(THREADS) || (defined(DARWIN) && defined(MPROTECT_VDB))
17053     if (!GC_is_initialized && value) {
17054 #ifndef SMALL_CONFIG
17055         GC_init();
17056 #ifndef THREADS
17057           if (GC_manual_vdb)
17058             return;
17059 #endif
17060 #endif
17061       ABORT("fork() handling unsupported");
17062     }
17063 #else
17064 #endif
17065 }
17066 STATIC void GC_init_size_map(void)
17067 {
17068     size_t i;
17069       GC_size_map[0] = 1;
17070     for (i = 1; i <= GRANULES_TO_BYTES(TINY_FREELISTS-1) - EXTRA_BYTES; i++) {
17071         GC_size_map[i] = ROUNDED_UP_GRANULES(i);
17072 #ifndef _MSC_VER
17073           GC_ASSERT(GC_size_map[i] < TINY_FREELISTS);
17074 #endif
17075     }
17076 }
17077 #ifndef SMALL_CLEAR_SIZE
17078 #define SMALL_CLEAR_SIZE 256
17079 #endif
17080 #if defined(ALWAYS_SMALL_CLEAR_STACK) || defined(STACK_NOT_SCANNED)
17081   GC_API void * GC_CALL GC_clear_stack(void *arg)
17082   {
17083 #ifndef STACK_NOT_SCANNED
17084       word volatile dummy[SMALL_CLEAR_SIZE];
17085       BZERO(( void *)dummy, sizeof(dummy));
17086 #endif
17087     return arg;
17088   }
17089 #else
17090 #ifdef THREADS
17091 #define BIG_CLEAR_SIZE 2048
17092 #else
17093     STATIC word GC_stack_last_cleared = 0;
17094     STATIC ptr_t GC_min_sp = NULL;
17095     STATIC ptr_t GC_high_water = NULL;
17096     STATIC word GC_bytes_allocd_at_reset = 0;
17097 #define DEGRADE_RATE 50
17098 #endif
17099 #if defined(ASM_CLEAR_CODE)
17100     void *GC_clear_stack_inner(void *, ptr_t);
17101 #else
17102     void *GC_clear_stack_inner(void *arg,
17103 #if defined(__APPLE_CC__) && !GC_CLANG_PREREQ(6, 0)
17104                                volatile
17105 #endif
17106                                ptr_t limit)
17107     {
17108 #define CLEAR_SIZE 213
17109       volatile word dummy[CLEAR_SIZE];
17110       BZERO(( void *)dummy, sizeof(dummy));
17111       if ((word)GC_approx_sp() COOLER_THAN (word)limit) {
17112         (void)GC_clear_stack_inner(arg, limit);
17113       }
17114 #if defined(CPPCHECK)
17115         GC_noop1(dummy[0]);
17116 #else
17117         GC_noop1(COVERT_DATAFLOW(dummy));
17118 #endif
17119       return(arg);
17120     }
17121 #endif
17122 #ifdef THREADS
17123     GC_ATTR_NO_SANITIZE_THREAD
17124     static unsigned next_random_no(void)
17125     {
17126       static unsigned random_no = 0;
17127       return ++random_no % 13;
17128     }
17129 #endif
17130   GC_API void * GC_CALL GC_clear_stack(void *arg)
17131   {
17132     ptr_t sp = GC_approx_sp();
17133 #ifdef THREADS
17134         word volatile dummy[SMALL_CLEAR_SIZE];
17135 #endif
17136 #define SLOP 400
17137 #define GC_SLOP 4000
17138 #define CLEAR_THRESHOLD 100000
17139 #ifdef THREADS
17140       if (next_random_no() == 0) {
17141         ptr_t limit = sp;
17142         MAKE_HOTTER(limit, BIG_CLEAR_SIZE*sizeof(word));
17143         limit = (ptr_t)((word)limit & ~0xf);
17144         return GC_clear_stack_inner(arg, limit);
17145       }
17146       BZERO((void *)dummy, SMALL_CLEAR_SIZE*sizeof(word));
17147 #else
17148       if (GC_gc_no > GC_stack_last_cleared) {
17149         if (GC_stack_last_cleared == 0)
17150           GC_high_water = (ptr_t)GC_stackbottom;
17151         GC_min_sp = GC_high_water;
17152         GC_stack_last_cleared = GC_gc_no;
17153         GC_bytes_allocd_at_reset = GC_bytes_allocd;
17154       }
17155       MAKE_COOLER(GC_high_water, WORDS_TO_BYTES(DEGRADE_RATE) + GC_SLOP);
17156       if ((word)sp HOTTER_THAN (word)GC_high_water) {
17157           GC_high_water = sp;
17158       }
17159       MAKE_HOTTER(GC_high_water, GC_SLOP);
17160       {
17161         ptr_t limit = GC_min_sp;
17162         MAKE_HOTTER(limit, SLOP);
17163         if ((word)sp COOLER_THAN (word)limit) {
17164           limit = (ptr_t)((word)limit & ~0xf);
17165           GC_min_sp = sp;
17166           return GC_clear_stack_inner(arg, limit);
17167         }
17168       }
17169       if (GC_bytes_allocd - GC_bytes_allocd_at_reset > CLEAR_THRESHOLD) {
17170         GC_min_sp = sp;
17171         MAKE_HOTTER(GC_min_sp, CLEAR_THRESHOLD/4);
17172         if ((word)GC_min_sp HOTTER_THAN (word)GC_high_water)
17173           GC_min_sp = GC_high_water;
17174         GC_bytes_allocd_at_reset = GC_bytes_allocd;
17175       }
17176 #endif
17177     return arg;
17178   }
17179 #endif
17180 GC_API void * GC_CALL GC_base(void * p)
17181 {
17182     ptr_t r;
17183     struct hblk *h;
17184     bottom_index *bi;
17185     hdr *candidate_hdr;
17186     r = (ptr_t)p;
17187     if (!EXPECT(GC_is_initialized, TRUE)) return 0;
17188     h = HBLKPTR(r);
17189     GET_BI(r, bi);
17190     candidate_hdr = HDR_FROM_BI(bi, r);
17191     if (candidate_hdr == 0) return(0);
17192         while (IS_FORWARDING_ADDR_OR_NIL(candidate_hdr)) {
17193            h = FORWARDED_ADDR(h,candidate_hdr);
17194            r = (ptr_t)h;
17195            candidate_hdr = HDR(h);
17196         }
17197     if (HBLK_IS_FREE(candidate_hdr)) return(0);
17198         r = (ptr_t)((word)r & ~(WORDS_TO_BYTES(1) - 1));
17199         {
17200             size_t offset = HBLKDISPL(r);
17201             word sz = candidate_hdr -> hb_sz;
17202             size_t obj_displ = offset % sz;
17203             ptr_t limit;
17204             r -= obj_displ;
17205             limit = r + sz;
17206             if ((word)limit > (word)(h + 1) && sz <= HBLKSIZE) {
17207                 return(0);
17208             }
17209             if ((word)p >= (word)limit) return(0);
17210         }
17211     return((void *)r);
17212 }
17213 GC_API int GC_CALL GC_is_heap_ptr(const void *p)
17214 {
17215     bottom_index *bi;
17216     GC_ASSERT(GC_is_initialized);
17217     GET_BI(p, bi);
17218     return HDR_FROM_BI(bi, p) != 0;
17219 }
17220 GC_API size_t GC_CALL GC_size(const void * p)
17221 {
17222     hdr * hhdr = HDR(p);
17223     return (size_t)hhdr->hb_sz;
17224 }
17225 GC_API size_t GC_CALL GC_get_heap_size(void)
17226 {
17227     return (size_t)(GC_heapsize - GC_unmapped_bytes);
17228 }
17229 GC_API size_t GC_CALL GC_get_obtained_from_os_bytes(void)
17230 {
17231     return (size_t)GC_our_mem_bytes;
17232 }
17233 GC_API size_t GC_CALL GC_get_free_bytes(void)
17234 {
17235     return (size_t)(GC_large_free_bytes - GC_unmapped_bytes);
17236 }
17237 GC_API size_t GC_CALL GC_get_unmapped_bytes(void)
17238 {
17239     return (size_t)GC_unmapped_bytes;
17240 }
17241 GC_API size_t GC_CALL GC_get_bytes_since_gc(void)
17242 {
17243     return (size_t)GC_bytes_allocd;
17244 }
17245 GC_API size_t GC_CALL GC_get_total_bytes(void)
17246 {
17247     return (size_t)(GC_bytes_allocd + GC_bytes_allocd_before_gc);
17248 }
17249 #ifndef GC_GET_HEAP_USAGE_NOT_NEEDED
17250 GC_API size_t GC_CALL GC_get_size_map_at(int i)
17251 {
17252   if ((unsigned)i > MAXOBJBYTES)
17253     return GC_SIZE_MAX;
17254   return GRANULES_TO_BYTES(GC_size_map[i]);
17255 }
17256 GC_API void GC_CALL GC_get_heap_usage_safe(GC_word *pheap_size,
17257                         GC_word *pfree_bytes, GC_word *punmapped_bytes,
17258                         GC_word *pbytes_since_gc, GC_word *ptotal_bytes)
17259 {
17260   DCL_LOCK_STATE;
17261   LOCK();
17262   if (pheap_size != NULL)
17263     *pheap_size = GC_heapsize - GC_unmapped_bytes;
17264   if (pfree_bytes != NULL)
17265     *pfree_bytes = GC_large_free_bytes - GC_unmapped_bytes;
17266   if (punmapped_bytes != NULL)
17267     *punmapped_bytes = GC_unmapped_bytes;
17268   if (pbytes_since_gc != NULL)
17269     *pbytes_since_gc = GC_bytes_allocd;
17270   if (ptotal_bytes != NULL)
17271     *ptotal_bytes = GC_bytes_allocd + GC_bytes_allocd_before_gc;
17272   UNLOCK();
17273 }
17274   GC_INNER word GC_reclaimed_bytes_before_gc = 0;
17275   static void fill_prof_stats(struct GC_prof_stats_s *pstats)
17276   {
17277     pstats->heapsize_full = GC_heapsize;
17278     pstats->free_bytes_full = GC_large_free_bytes;
17279     pstats->unmapped_bytes = GC_unmapped_bytes;
17280     pstats->bytes_allocd_since_gc = GC_bytes_allocd;
17281     pstats->allocd_bytes_before_gc = GC_bytes_allocd_before_gc;
17282     pstats->non_gc_bytes = GC_non_gc_bytes;
17283     pstats->gc_no = GC_gc_no;
17284 #ifdef PARALLEL_MARK
17285       pstats->markers_m1 = (word)((signed_word)GC_markers_m1);
17286 #else
17287       pstats->markers_m1 = 0;
17288 #endif
17289     pstats->bytes_reclaimed_since_gc = GC_bytes_found > 0 ?
17290                                         (word)GC_bytes_found : 0;
17291     pstats->reclaimed_bytes_before_gc = GC_reclaimed_bytes_before_gc;
17292     pstats->expl_freed_bytes_since_gc = GC_bytes_freed;
17293     pstats->obtained_from_os_bytes = GC_our_mem_bytes;
17294   }
17295 #include <string.h>
17296   GC_API size_t GC_CALL GC_get_prof_stats(struct GC_prof_stats_s *pstats,
17297                                           size_t stats_sz)
17298   {
17299     struct GC_prof_stats_s stats;
17300     DCL_LOCK_STATE;
17301     LOCK();
17302     fill_prof_stats(stats_sz >= sizeof(stats) ? pstats : &stats);
17303     UNLOCK();
17304     if (stats_sz == sizeof(stats)) {
17305       return sizeof(stats);
17306     } else if (stats_sz > sizeof(stats)) {
17307       memset((char *)pstats + sizeof(stats), 0xff, stats_sz - sizeof(stats));
17308       return sizeof(stats);
17309     } else {
17310       if (EXPECT(stats_sz > 0, TRUE))
17311         BCOPY(&stats, pstats, stats_sz);
17312       return stats_sz;
17313     }
17314   }
17315 #ifdef THREADS
17316     GC_API size_t GC_CALL GC_get_prof_stats_unsafe(
17317                                             struct GC_prof_stats_s *pstats,
17318                                             size_t stats_sz)
17319     {
17320       struct GC_prof_stats_s stats;
17321       if (stats_sz >= sizeof(stats)) {
17322         fill_prof_stats(pstats);
17323         if (stats_sz > sizeof(stats))
17324           memset((char *)pstats + sizeof(stats), 0xff,
17325                  stats_sz - sizeof(stats));
17326         return sizeof(stats);
17327       } else {
17328         if (EXPECT(stats_sz > 0, TRUE)) {
17329           fill_prof_stats(&stats);
17330           BCOPY(&stats, pstats, stats_sz);
17331         }
17332         return stats_sz;
17333       }
17334     }
17335 #endif
17336 #endif
17337 #if defined(GC_DARWIN_THREADS) || defined(GC_OPENBSD_UTHREADS) \
17338     || defined(GC_WIN32_THREADS) || (defined(NACL) && defined(THREADS))
17339   GC_API void GC_CALL GC_set_suspend_signal(int sig GC_ATTR_UNUSED)
17340   {
17341   }
17342   GC_API void GC_CALL GC_set_thr_restart_signal(int sig GC_ATTR_UNUSED)
17343   {
17344   }
17345   GC_API int GC_CALL GC_get_suspend_signal(void)
17346   {
17347     return -1;
17348   }
17349   GC_API int GC_CALL GC_get_thr_restart_signal(void)
17350   {
17351     return -1;
17352   }
17353 #endif
17354 #if !defined(_MAX_PATH) && (defined(MSWIN32) || defined(MSWINCE) \
17355                             || defined(CYGWIN32))
17356 #define _MAX_PATH MAX_PATH
17357 #endif
17358 #ifdef GC_READ_ENV_FILE
17359   STATIC char *GC_envfile_content = NULL;
17360   STATIC unsigned GC_envfile_length = 0;
17361 #ifndef GC_ENVFILE_MAXLEN
17362 #define GC_ENVFILE_MAXLEN 0x4000
17363 #endif
17364 #define GC_ENV_FILE_EXT ".gc.env"
17365   STATIC void GC_envfile_init(void)
17366   {
17367 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
17368       HANDLE hFile;
17369       char *content;
17370       unsigned ofs;
17371       unsigned len;
17372       DWORD nBytesRead;
17373       TCHAR path[_MAX_PATH + 0x10];
17374       size_t bytes_to_get;
17375       len = (unsigned)GetModuleFileName(NULL , path,
17376                                         _MAX_PATH + 1);
17377       if (len > 4 && path[len - 4] == (TCHAR)'.') {
17378         len -= 4;
17379       }
17380       BCOPY(TEXT(GC_ENV_FILE_EXT), &path[len], sizeof(TEXT(GC_ENV_FILE_EXT)));
17381       hFile = CreateFile(path, GENERIC_READ,
17382                          FILE_SHARE_READ | FILE_SHARE_WRITE,
17383                          NULL , OPEN_EXISTING,
17384                          FILE_ATTRIBUTE_NORMAL, NULL );
17385       if (hFile == INVALID_HANDLE_VALUE)
17386         return;
17387       len = (unsigned)GetFileSize(hFile, NULL);
17388       if (len <= 1 || len >= GC_ENVFILE_MAXLEN) {
17389         CloseHandle(hFile);
17390         return;
17391       }
17392       GC_ASSERT(GC_page_size != 0);
17393       bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP((size_t)len + 1);
17394       content = (char *)GET_MEM(bytes_to_get);
17395       if (content == NULL) {
17396         CloseHandle(hFile);
17397         return;
17398       }
17399       GC_add_to_our_memory(content, bytes_to_get);
17400       ofs = 0;
17401       nBytesRead = (DWORD)-1L;
17402       while (ReadFile(hFile, content + ofs, len - ofs + 1, &nBytesRead,
17403                       NULL ) && nBytesRead != 0) {
17404         if ((ofs += nBytesRead) > len)
17405           break;
17406       }
17407       CloseHandle(hFile);
17408       if (ofs != len || nBytesRead != 0) {
17409         return;
17410       }
17411       content[ofs] = '\0';
17412       while (ofs-- > 0) {
17413        if (content[ofs] == '\r' || content[ofs] == '\n')
17414          content[ofs] = '\0';
17415       }
17416       GC_ASSERT(NULL == GC_envfile_content);
17417       GC_envfile_length = len + 1;
17418       GC_envfile_content = content;
17419 #endif
17420   }
17421   GC_INNER char * GC_envfile_getenv(const char *name)
17422   {
17423     char *p;
17424     char *end_of_content;
17425     unsigned namelen;
17426 #ifndef NO_GETENV
17427       p = getenv(name);
17428       if (p != NULL)
17429         return *p != '\0' ? p : NULL;
17430 #endif
17431     p = GC_envfile_content;
17432     if (p == NULL)
17433       return NULL;
17434     namelen = strlen(name);
17435     if (namelen == 0)
17436       return NULL;
17437     for (end_of_content = p + GC_envfile_length;
17438          p != end_of_content; p += strlen(p) + 1) {
17439       if (strncmp(p, name, namelen) == 0 && *(p += namelen) == '=') {
17440         p++;
17441         return *p != '\0' ? p : NULL;
17442       }
17443     }
17444     return NULL;
17445   }
17446 #endif
17447 GC_INNER GC_bool GC_is_initialized = FALSE;
17448 GC_API int GC_CALL GC_is_init_called(void)
17449 {
17450   return GC_is_initialized;
17451 }
17452 #if defined(GC_WIN32_THREADS) \
17453     && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
17454   GC_INNER CRITICAL_SECTION GC_write_cs;
17455 #endif
17456 #ifndef DONT_USE_ATEXIT
17457 #if !defined(PCR) && !defined(SMALL_CONFIG)
17458     static GC_bool skip_gc_atexit = FALSE;
17459 #else
17460 #define skip_gc_atexit FALSE
17461 #endif
17462   STATIC void GC_exit_check(void)
17463   {
17464     if (GC_find_leak && !skip_gc_atexit) {
17465 #ifdef THREADS
17466         GC_in_thread_creation = TRUE;
17467         GC_gcollect();
17468         GC_in_thread_creation = FALSE;
17469 #else
17470         GC_gcollect();
17471 #endif
17472     }
17473   }
17474 #endif
17475 #if defined(UNIX_LIKE) && !defined(NO_DEBUGGING)
17476   static void looping_handler(int sig)
17477   {
17478     GC_err_printf("Caught signal %d: looping in handler\n", sig);
17479     for (;;) {
17480     }
17481   }
17482   static GC_bool installed_looping_handler = FALSE;
17483   static void maybe_install_looping_handler(void)
17484   {
17485     if (!installed_looping_handler && 0 != GETENV("GC_LOOP_ON_ABORT")) {
17486       GC_set_and_save_fault_handler(looping_handler);
17487       installed_looping_handler = TRUE;
17488     }
17489   }
17490 #else
17491 #define maybe_install_looping_handler()
17492 #endif
17493 #define GC_DEFAULT_STDOUT_FD 1
17494 #define GC_DEFAULT_STDERR_FD 2
17495 #if !defined(OS2) && !defined(MACOS) && !defined(GC_ANDROID_LOG) \
17496     && !defined(NN_PLATFORM_CTR) && !defined(NINTENDO_SWITCH) \
17497     && (!defined(MSWIN32) || defined(CONSOLE_LOG)) && !defined(MSWINCE)
17498   STATIC int GC_stdout = GC_DEFAULT_STDOUT_FD;
17499   STATIC int GC_stderr = GC_DEFAULT_STDERR_FD;
17500   STATIC int GC_log = GC_DEFAULT_STDERR_FD;
17501 #ifndef MSWIN32
17502     GC_API void GC_CALL GC_set_log_fd(int fd)
17503     {
17504       GC_log = fd;
17505     }
17506 #endif
17507 #endif
17508 #ifdef MSGBOX_ON_ERROR
17509   STATIC void GC_win32_MessageBoxA(const char *msg, const char *caption,
17510                                    unsigned flags)
17511   {
17512 #ifndef DONT_USE_USER32_DLL
17513       (void)MessageBoxA(NULL, msg, caption, flags);
17514 #else
17515       HINSTANCE hU32 = LoadLibrary(TEXT("user32.dll"));
17516       if (hU32) {
17517         FARPROC pfn = GetProcAddress(hU32, "MessageBoxA");
17518         if (pfn)
17519           (void)(*(int (WINAPI *)(HWND, LPCSTR, LPCSTR, UINT))(word)pfn)(
17520                               NULL , msg, caption, flags);
17521         (void)FreeLibrary(hU32);
17522       }
17523 #endif
17524   }
17525 #endif
17526 #if defined(THREADS) && defined(UNIX_LIKE) && !defined(NO_GETCONTEXT)
17527   static void callee_saves_pushed_dummy_fn(ptr_t data GC_ATTR_UNUSED,
17528                                            void * context GC_ATTR_UNUSED) {}
17529 #endif
17530 #ifndef SMALL_CONFIG
17531 #ifdef MANUAL_VDB
17532     static GC_bool manual_vdb_allowed = TRUE;
17533 #else
17534     static GC_bool manual_vdb_allowed = FALSE;
17535 #endif
17536   GC_API void GC_CALL GC_set_manual_vdb_allowed(int value)
17537   {
17538     manual_vdb_allowed = (GC_bool)value;
17539   }
17540   GC_API int GC_CALL GC_get_manual_vdb_allowed(void)
17541   {
17542     return (int)manual_vdb_allowed;
17543   }
17544 #endif
17545 STATIC word GC_parse_mem_size_arg(const char *str)
17546 {
17547   word result = 0;
17548   if (*str != '\0') {
17549     char *endptr;
17550     char ch;
17551     result = (word)STRTOULL(str, &endptr, 10);
17552     ch = *endptr;
17553     if (ch != '\0') {
17554       if (*(endptr + 1) != '\0')
17555         return 0;
17556       switch (ch) {
17557       case 'K':
17558       case 'k':
17559         result <<= 10;
17560         break;
17561       case 'M':
17562       case 'm':
17563         result <<= 20;
17564         break;
17565       case 'G':
17566       case 'g':
17567         result <<= 30;
17568         break;
17569       default:
17570         result = 0;
17571       }
17572     }
17573   }
17574   return result;
17575 }
17576 #define GC_LOG_STD_NAME "gc.log"
17577 GC_API void GC_CALL GC_init(void)
17578 {
17579     word initial_heap_sz;
17580     IF_CANCEL(int cancel_state;)
17581 #if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
17582       DCL_LOCK_STATE;
17583 #endif
17584     if (EXPECT(GC_is_initialized, TRUE)) return;
17585 #ifdef REDIRECT_MALLOC
17586       {
17587         static GC_bool init_started = FALSE;
17588         if (init_started)
17589           ABORT("Redirected malloc() called during GC init");
17590         init_started = TRUE;
17591       }
17592 #endif
17593 #if defined(GC_INITIAL_HEAP_SIZE) && !defined(CPPCHECK)
17594       initial_heap_sz = GC_INITIAL_HEAP_SIZE;
17595 #else
17596       initial_heap_sz = MINHINCR * HBLKSIZE;
17597 #endif
17598     DISABLE_CANCEL(cancel_state);
17599 #ifdef THREADS
17600 #ifndef GC_ALWAYS_MULTITHREADED
17601         GC_ASSERT(!GC_need_to_lock);
17602 #endif
17603 #ifdef SN_TARGET_PS3
17604         {
17605           pthread_mutexattr_t mattr;
17606           if (0 != pthread_mutexattr_init(&mattr)) {
17607             ABORT("pthread_mutexattr_init failed");
17608           }
17609           if (0 != pthread_mutex_init(&GC_allocate_ml, &mattr)) {
17610             ABORT("pthread_mutex_init failed");
17611           }
17612           (void)pthread_mutexattr_destroy(&mattr);
17613         }
17614 #endif
17615 #endif
17616 #if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
17617 #ifndef SPIN_COUNT
17618 #define SPIN_COUNT 4000
17619 #endif
17620 #ifdef MSWINRT_FLAVOR
17621         InitializeCriticalSectionAndSpinCount(&GC_allocate_ml, SPIN_COUNT);
17622 #else
17623         {
17624 #ifndef MSWINCE
17625             FARPROC pfn = 0;
17626             HMODULE hK32 = GetModuleHandle(TEXT("kernel32.dll"));
17627             if (hK32)
17628               pfn = GetProcAddress(hK32,
17629                                    "InitializeCriticalSectionAndSpinCount");
17630             if (pfn) {
17631               (*(BOOL (WINAPI *)(LPCRITICAL_SECTION, DWORD))(word)pfn)(
17632                                 &GC_allocate_ml, SPIN_COUNT);
17633             } else
17634 #endif
17635            InitializeCriticalSection(&GC_allocate_ml);
17636         }
17637 #endif
17638 #endif
17639 #if defined(GC_WIN32_THREADS) \
17640        && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
17641       InitializeCriticalSection(&GC_write_cs);
17642 #endif
17643     GC_setpagesize();
17644 #ifdef MSWIN32
17645       GC_init_win32();
17646 #endif
17647 #ifdef GC_READ_ENV_FILE
17648       GC_envfile_init();
17649 #endif
17650 #if !defined(NO_CLOCK) || !defined(SMALL_CONFIG)
17651 #ifdef GC_PRINT_VERBOSE_STATS
17652         GC_print_stats = VERBOSE;
17653 #else
17654         if (0 != GETENV("GC_PRINT_VERBOSE_STATS")) {
17655           GC_print_stats = VERBOSE;
17656         } else if (0 != GETENV("GC_PRINT_STATS")) {
17657           GC_print_stats = 1;
17658         }
17659 #endif
17660 #endif
17661 #if ((defined(UNIX_LIKE) && !defined(GC_ANDROID_LOG)) \
17662         || (defined(CONSOLE_LOG) && defined(MSWIN32)) \
17663         || defined(CYGWIN32) || defined(SYMBIAN)) && !defined(SMALL_CONFIG)
17664         {
17665           char * file_name = TRUSTED_STRING(GETENV("GC_LOG_FILE"));
17666 #ifdef GC_LOG_TO_FILE_ALWAYS
17667             if (NULL == file_name)
17668               file_name = GC_LOG_STD_NAME;
17669 #else
17670             if (0 != file_name)
17671 #endif
17672           {
17673 #if defined(_MSC_VER)
17674               int log_d = _open(file_name, O_CREAT | O_WRONLY | O_APPEND);
17675 #else
17676               int log_d = open(file_name, O_CREAT | O_WRONLY | O_APPEND, 0644);
17677 #endif
17678             if (log_d < 0) {
17679               GC_err_printf("Failed to open %s as log file\n", file_name);
17680             } else {
17681               char *str;
17682               GC_log = log_d;
17683               str = GETENV("GC_ONLY_LOG_TO_FILE");
17684 #ifdef GC_ONLY_LOG_TO_FILE
17685                 if (str != NULL && *str == '0' && *(str + 1) == '\0')
17686 #else
17687                 if (str == NULL || (*str == '0' && *(str + 1) == '\0'))
17688 #endif
17689               {
17690                 GC_stdout = log_d;
17691                 GC_stderr = log_d;
17692               }
17693             }
17694           }
17695         }
17696 #endif
17697 #if !defined(NO_DEBUGGING) && !defined(GC_DUMP_REGULARLY)
17698       if (0 != GETENV("GC_DUMP_REGULARLY")) {
17699         GC_dump_regularly = TRUE;
17700       }
17701 #endif
17702 #ifdef KEEP_BACK_PTRS
17703       {
17704         char * backtraces_string = GETENV("GC_BACKTRACES");
17705         if (0 != backtraces_string) {
17706           GC_backtraces = atol(backtraces_string);
17707           if (backtraces_string[0] == '\0') GC_backtraces = 1;
17708         }
17709       }
17710 #endif
17711     if (0 != GETENV("GC_FIND_LEAK")) {
17712       GC_find_leak = 1;
17713     }
17714 #ifndef SHORT_DBG_HDRS
17715       if (0 != GETENV("GC_FINDLEAK_DELAY_FREE")) {
17716         GC_findleak_delay_free = TRUE;
17717       }
17718 #endif
17719     if (0 != GETENV("GC_ALL_INTERIOR_POINTERS")) {
17720       GC_all_interior_pointers = 1;
17721     }
17722     if (0 != GETENV("GC_DONT_GC")) {
17723 #ifdef LINT2
17724         GC_disable();
17725 #else
17726         GC_dont_gc = 1;
17727 #endif
17728     }
17729     if (0 != GETENV("GC_PRINT_BACK_HEIGHT")) {
17730       GC_print_back_height = TRUE;
17731     }
17732     if (0 != GETENV("GC_NO_BLACKLIST_WARNING")) {
17733       GC_large_alloc_warn_interval = LONG_MAX;
17734     }
17735     {
17736       char * addr_string = GETENV("GC_TRACE");
17737       if (0 != addr_string) {
17738 #ifndef ENABLE_TRACE
17739           WARN("Tracing not enabled: Ignoring GC_TRACE value\n", 0);
17740 #else
17741           word addr = (word)STRTOULL(addr_string, NULL, 16);
17742           if (addr < 0x1000)
17743               WARN("Unlikely trace address: %p\n", (void *)addr);
17744           GC_trace_addr = (ptr_t)addr;
17745 #endif
17746       }
17747     }
17748 #ifdef GC_COLLECT_AT_MALLOC
17749       {
17750         char * string = GETENV("GC_COLLECT_AT_MALLOC");
17751         if (0 != string) {
17752           size_t min_lb = (size_t)STRTOULL(string, NULL, 10);
17753           if (min_lb > 0)
17754             GC_dbg_collect_at_malloc_min_lb = min_lb;
17755         }
17756       }
17757 #endif
17758 #if !defined(GC_DISABLE_INCREMENTAL) && !defined(NO_CLOCK)
17759       {
17760         char * time_limit_string = GETENV("GC_PAUSE_TIME_TARGET");
17761         if (0 != time_limit_string) {
17762           long time_limit = atol(time_limit_string);
17763           if (time_limit > 0) {
17764             GC_time_limit = time_limit;
17765           }
17766         }
17767       }
17768 #endif
17769 #ifndef SMALL_CONFIG
17770       {
17771         char * full_freq_string = GETENV("GC_FULL_FREQUENCY");
17772         if (full_freq_string != NULL) {
17773           int full_freq = atoi(full_freq_string);
17774           if (full_freq > 0)
17775             GC_full_freq = full_freq;
17776         }
17777       }
17778 #endif
17779     {
17780       char * interval_string = GETENV("GC_LARGE_ALLOC_WARN_INTERVAL");
17781       if (0 != interval_string) {
17782         long interval = atol(interval_string);
17783         if (interval <= 0) {
17784           WARN("GC_LARGE_ALLOC_WARN_INTERVAL environment variable has "
17785                "bad value: Ignoring\n", 0);
17786         } else {
17787           GC_large_alloc_warn_interval = interval;
17788         }
17789       }
17790     }
17791     {
17792         char * space_divisor_string = GETENV("GC_FREE_SPACE_DIVISOR");
17793         if (space_divisor_string != NULL) {
17794           int space_divisor = atoi(space_divisor_string);
17795           if (space_divisor > 0)
17796             GC_free_space_divisor = (unsigned)space_divisor;
17797         }
17798     }
17799 #ifdef USE_MUNMAP
17800       {
17801         char * string = GETENV("GC_UNMAP_THRESHOLD");
17802         if (string != NULL) {
17803           if (*string == '0' && *(string + 1) == '\0') {
17804             GC_unmap_threshold = 0;
17805           } else {
17806             int unmap_threshold = atoi(string);
17807             if (unmap_threshold > 0)
17808               GC_unmap_threshold = unmap_threshold;
17809           }
17810         }
17811       }
17812       {
17813         char * string = GETENV("GC_FORCE_UNMAP_ON_GCOLLECT");
17814         if (string != NULL) {
17815           if (*string == '0' && *(string + 1) == '\0') {
17816             GC_force_unmap_on_gcollect = FALSE;
17817           } else {
17818             GC_force_unmap_on_gcollect = TRUE;
17819           }
17820         }
17821       }
17822       {
17823         char * string = GETENV("GC_USE_ENTIRE_HEAP");
17824         if (string != NULL) {
17825           if (*string == '0' && *(string + 1) == '\0') {
17826             GC_use_entire_heap = FALSE;
17827           } else {
17828             GC_use_entire_heap = TRUE;
17829           }
17830         }
17831       }
17832 #endif
17833 #if !defined(NO_DEBUGGING) && !defined(NO_CLOCK)
17834       GET_TIME(GC_init_time);
17835 #endif
17836     maybe_install_looping_handler();
17837 #if ALIGNMENT > GC_DS_TAGS
17838       if (EXTRA_BYTES != 0)
17839         GC_obj_kinds[NORMAL].ok_descriptor = (word)(-ALIGNMENT) | GC_DS_LENGTH;
17840 #endif
17841     GC_exclude_static_roots_inner(beginGC_arrays, endGC_arrays);
17842     GC_exclude_static_roots_inner(beginGC_obj_kinds, endGC_obj_kinds);
17843 #ifdef SEPARATE_GLOBALS
17844       GC_exclude_static_roots_inner(beginGC_objfreelist, endGC_objfreelist);
17845       GC_exclude_static_roots_inner(beginGC_aobjfreelist, endGC_aobjfreelist);
17846 #endif
17847 #if defined(USE_PROC_FOR_LIBRARIES) && defined(GC_LINUX_THREADS)
17848         WARN("USE_PROC_FOR_LIBRARIES + GC_LINUX_THREADS performs poorly.\n", 0);
17849 #endif
17850 #if defined(SEARCH_FOR_DATA_START)
17851         GC_init_linux_data_start();
17852 #endif
17853 #if defined(NETBSD) && defined(__ELF__)
17854         GC_init_netbsd_elf();
17855 #endif
17856 #if !defined(THREADS) || defined(GC_PTHREADS) \
17857         || defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) \
17858         || defined(GC_WIN32_THREADS) || defined(GC_SOLARIS_THREADS)
17859       if (GC_stackbottom == 0) {
17860         GC_stackbottom = GC_get_main_stack_base();
17861 #if (defined(LINUX) || defined(HPUX)) && defined(IA64)
17862           GC_register_stackbottom = GC_get_register_stack_base();
17863 #endif
17864       } else {
17865 #if (defined(LINUX) || defined(HPUX)) && defined(IA64)
17866           if (GC_register_stackbottom == 0) {
17867             WARN("GC_register_stackbottom should be set with GC_stackbottom\n", 0);
17868             GC_register_stackbottom = GC_get_register_stack_base();
17869           }
17870 #endif
17871       }
17872 #endif
17873 #if !defined(CPPCHECK)
17874       GC_STATIC_ASSERT(sizeof(ptr_t) == sizeof(word));
17875       GC_STATIC_ASSERT(sizeof(signed_word) == sizeof(word));
17876 #if !defined(_AUX_SOURCE) || defined(__GNUC__)
17877         GC_STATIC_ASSERT((word)(-1) > (word)0);
17878 #endif
17879       GC_STATIC_ASSERT((signed_word)(-1) < (signed_word)0);
17880 #endif
17881     GC_STATIC_ASSERT(sizeof (struct hblk) == HBLKSIZE);
17882 #ifndef THREADS
17883       GC_ASSERT(!((word)GC_stackbottom HOTTER_THAN (word)GC_approx_sp()));
17884 #endif
17885     GC_init_headers();
17886 #ifndef GC_DISABLE_INCREMENTAL
17887       if (GC_incremental || 0 != GETENV("GC_ENABLE_INCREMENTAL")) {
17888 #if defined(BASE_ATOMIC_OPS_EMULATED) || defined(CHECKSUMS) \
17889            || defined(REDIRECT_MALLOC) || defined(REDIRECT_MALLOC_IN_HEADER) \
17890            || defined(SMALL_CONFIG)
17891 #else
17892           if (manual_vdb_allowed) {
17893               GC_manual_vdb = TRUE;
17894               GC_incremental = TRUE;
17895           } else
17896 #endif
17897          {
17898           GC_incremental = GC_dirty_init();
17899           GC_ASSERT(GC_bytes_allocd == 0);
17900         }
17901       }
17902 #endif
17903       if (GC_REGISTER_MAIN_STATIC_DATA()) GC_register_data_segments();
17904     GC_bl_init();
17905     GC_mark_init();
17906     {
17907         char * sz_str = GETENV("GC_INITIAL_HEAP_SIZE");
17908         if (sz_str != NULL) {
17909           initial_heap_sz = GC_parse_mem_size_arg(sz_str);
17910           if (initial_heap_sz <= MINHINCR * HBLKSIZE) {
17911             WARN("Bad initial heap size %s - ignoring it.\n", sz_str);
17912           }
17913         }
17914     }
17915     {
17916         char * sz_str = GETENV("GC_MAXIMUM_HEAP_SIZE");
17917         if (sz_str != NULL) {
17918           word max_heap_sz = GC_parse_mem_size_arg(sz_str);
17919           if (max_heap_sz < initial_heap_sz) {
17920             WARN("Bad maximum heap size %s - ignoring it.\n", sz_str);
17921           }
17922           if (0 == GC_max_retries) GC_max_retries = 2;
17923           GC_set_max_heap_size(max_heap_sz);
17924         }
17925     }
17926 #if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
17927         LOCK();
17928 #endif
17929     if (!GC_expand_hp_inner(divHBLKSZ(initial_heap_sz))) {
17930         GC_err_printf("Can't start up: not enough memory\n");
17931         EXIT();
17932     } else {
17933         GC_requested_heapsize += initial_heap_sz;
17934     }
17935     if (GC_all_interior_pointers)
17936       GC_initialize_offsets();
17937     GC_register_displacement_inner(0L);
17938 #if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC)
17939       if (!GC_all_interior_pointers) {
17940         GC_register_displacement_inner(sizeof(void *));
17941       }
17942 #endif
17943     GC_init_size_map();
17944 #ifdef PCR
17945       if (PCR_IL_Lock(PCR_Bool_false, PCR_allSigsBlocked, PCR_waitForever)
17946           != PCR_ERes_okay) {
17947           ABORT("Can't lock load state");
17948       } else if (PCR_IL_Unlock() != PCR_ERes_okay) {
17949           ABORT("Can't unlock load state");
17950       }
17951       PCR_IL_Unlock();
17952       GC_pcr_install();
17953 #endif
17954     GC_is_initialized = TRUE;
17955 #if defined(GC_PTHREADS) || defined(GC_WIN32_THREADS)
17956 #if defined(LINT2) \
17957            && !(defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED))
17958           LOCK();
17959           GC_thr_init();
17960           UNLOCK();
17961 #else
17962           GC_thr_init();
17963 #endif
17964 #ifdef PARALLEL_MARK
17965 #if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
17966             UNLOCK();
17967 #endif
17968           GC_start_mark_threads_inner();
17969 #if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
17970             LOCK();
17971 #endif
17972 #endif
17973 #endif
17974     COND_DUMP;
17975     if (!GC_dont_precollect || GC_incremental) {
17976         GC_gcollect_inner();
17977     }
17978 #if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
17979         UNLOCK();
17980 #endif
17981 #if defined(THREADS) && defined(UNIX_LIKE) && !defined(NO_GETCONTEXT)
17982       if (GC_dont_gc || GC_dont_precollect)
17983         GC_with_callee_saves_pushed(callee_saves_pushed_dummy_fn, NULL);
17984 #endif
17985 #ifndef DONT_USE_ATEXIT
17986       if (GC_find_leak) {
17987         atexit(GC_exit_check);
17988       }
17989 #endif
17990 #if defined(PARALLEL_MARK) || defined(THREAD_LOCAL_ALLOC) \
17991        || (defined(GC_ALWAYS_MULTITHREADED) && defined(GC_WIN32_THREADS) \
17992            && !defined(GC_NO_THREADS_DISCOVERY))
17993         GC_init_parallel();
17994 #endif
17995 #if defined(DYNAMIC_LOADING) && defined(DARWIN)
17996         GC_init_dyld();
17997 #endif
17998     RESTORE_CANCEL(cancel_state);
17999 }
18000 GC_API void GC_CALL GC_enable_incremental(void)
18001 {
18002 #if !defined(GC_DISABLE_INCREMENTAL) && !defined(KEEP_BACK_PTRS)
18003     DCL_LOCK_STATE;
18004     if (!GC_find_leak && 0 == GETENV("GC_DISABLE_INCREMENTAL")) {
18005       LOCK();
18006       if (!GC_incremental) {
18007         GC_setpagesize();
18008         maybe_install_looping_handler();
18009         if (!GC_is_initialized) {
18010           UNLOCK();
18011           GC_incremental = TRUE;
18012           GC_init();
18013           LOCK();
18014         } else {
18015 #if !defined(BASE_ATOMIC_OPS_EMULATED) && !defined(CHECKSUMS) \
18016              && !defined(REDIRECT_MALLOC) \
18017              && !defined(REDIRECT_MALLOC_IN_HEADER) && !defined(SMALL_CONFIG)
18018             if (manual_vdb_allowed) {
18019               GC_manual_vdb = TRUE;
18020               GC_incremental = TRUE;
18021             } else
18022 #endif
18023            {
18024             GC_incremental = GC_dirty_init();
18025           }
18026         }
18027         if (GC_incremental && !GC_dont_gc) {
18028           IF_CANCEL(int cancel_state;)
18029           DISABLE_CANCEL(cancel_state);
18030           if (GC_bytes_allocd > 0) {
18031             GC_gcollect_inner();
18032           }
18033           GC_read_dirty(FALSE);
18034           RESTORE_CANCEL(cancel_state);
18035         }
18036       }
18037       UNLOCK();
18038       return;
18039     }
18040 #endif
18041   GC_init();
18042 }
18043 #if defined(THREADS)
18044   GC_API void GC_CALL GC_start_mark_threads(void)
18045   {
18046 #if defined(PARALLEL_MARK) && defined(CAN_HANDLE_FORK) \
18047        && !defined(THREAD_SANITIZER)
18048       IF_CANCEL(int cancel_state;)
18049       DISABLE_CANCEL(cancel_state);
18050       GC_start_mark_threads_inner();
18051       RESTORE_CANCEL(cancel_state);
18052 #else
18053       GC_ASSERT(I_DONT_HOLD_LOCK());
18054 #endif
18055   }
18056 #endif
18057   GC_API void GC_CALL GC_deinit(void)
18058   {
18059     if (GC_is_initialized) {
18060       GC_is_initialized = FALSE;
18061 #if defined(GC_WIN32_THREADS) && (defined(MSWIN32) || defined(MSWINCE))
18062 #if !defined(CONSOLE_LOG) || defined(MSWINCE)
18063           DeleteCriticalSection(&GC_write_cs);
18064 #endif
18065         DeleteCriticalSection(&GC_allocate_ml);
18066 #endif
18067     }
18068   }
18069 #if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
18070 #if defined(_MSC_VER) && defined(_DEBUG) && !defined(MSWINCE)
18071 #include <crtdbg.h>
18072 #endif
18073   STATIC HANDLE GC_log = 0;
18074 #ifdef THREADS
18075 #if defined(PARALLEL_MARK) && !defined(GC_ALWAYS_MULTITHREADED)
18076 #define IF_NEED_TO_LOCK(x) if (GC_parallel || GC_need_to_lock) x
18077 #else
18078 #define IF_NEED_TO_LOCK(x) if (GC_need_to_lock) x
18079 #endif
18080 #else
18081 #define IF_NEED_TO_LOCK(x)
18082 #endif
18083 #ifdef MSWINRT_FLAVOR
18084 #include <windows.storage.h>
18085     DECLSPEC_IMPORT HRESULT WINAPI RoGetActivationFactory(
18086                                         HSTRING activatableClassId,
18087                                         REFIID iid, void** factory);
18088     static GC_bool getWinRTLogPath(wchar_t* buf, size_t bufLen)
18089     {
18090       static const GUID kIID_IApplicationDataStatics = {
18091         0x5612147B, 0xE843, 0x45E3,
18092         0x94, 0xD8, 0x06, 0x16, 0x9E, 0x3C, 0x8E, 0x17
18093       };
18094       static const GUID kIID_IStorageItem = {
18095         0x4207A996, 0xCA2F, 0x42F7,
18096         0xBD, 0xE8, 0x8B, 0x10, 0x45, 0x7A, 0x7F, 0x30
18097       };
18098       GC_bool result = FALSE;
18099       HSTRING_HEADER appDataClassNameHeader;
18100       HSTRING appDataClassName;
18101       __x_ABI_CWindows_CStorage_CIApplicationDataStatics* appDataStatics = 0;
18102       GC_ASSERT(bufLen > 0);
18103       if (SUCCEEDED(WindowsCreateStringReference(
18104                       RuntimeClass_Windows_Storage_ApplicationData,
18105                       (sizeof(RuntimeClass_Windows_Storage_ApplicationData)-1)
18106                         / sizeof(wchar_t),
18107                       &appDataClassNameHeader, &appDataClassName))
18108           && SUCCEEDED(RoGetActivationFactory(appDataClassName,
18109                                               &kIID_IApplicationDataStatics,
18110                                               &appDataStatics))) {
18111         __x_ABI_CWindows_CStorage_CIApplicationData* appData = NULL;
18112         __x_ABI_CWindows_CStorage_CIStorageFolder* tempFolder = NULL;
18113         __x_ABI_CWindows_CStorage_CIStorageItem* tempFolderItem = NULL;
18114         HSTRING tempPath = NULL;
18115         if (SUCCEEDED(appDataStatics->lpVtbl->get_Current(appDataStatics,
18116                                                           &appData))
18117             && SUCCEEDED(appData->lpVtbl->get_TemporaryFolder(appData,
18118                                                               &tempFolder))
18119             && SUCCEEDED(tempFolder->lpVtbl->QueryInterface(tempFolder,
18120                                                         &kIID_IStorageItem,
18121                                                         &tempFolderItem))
18122             && SUCCEEDED(tempFolderItem->lpVtbl->get_Path(tempFolderItem,
18123                                                           &tempPath))) {
18124           UINT32 tempPathLen;
18125           const wchar_t* tempPathBuf =
18126                           WindowsGetStringRawBuffer(tempPath, &tempPathLen);
18127           buf[0] = '\0';
18128           if (wcsncat_s(buf, bufLen, tempPathBuf, tempPathLen) == 0
18129               && wcscat_s(buf, bufLen, L"\\") == 0
18130               && wcscat_s(buf, bufLen, TEXT(GC_LOG_STD_NAME)) == 0)
18131             result = TRUE;
18132           WindowsDeleteString(tempPath);
18133         }
18134         if (tempFolderItem != NULL)
18135           tempFolderItem->lpVtbl->Release(tempFolderItem);
18136         if (tempFolder != NULL)
18137           tempFolder->lpVtbl->Release(tempFolder);
18138         if (appData != NULL)
18139           appData->lpVtbl->Release(appData);
18140         appDataStatics->lpVtbl->Release(appDataStatics);
18141       }
18142       return result;
18143     }
18144 #endif
18145   STATIC HANDLE GC_CreateLogFile(void)
18146   {
18147     HANDLE hFile;
18148 #ifdef MSWINRT_FLAVOR
18149       TCHAR pathBuf[_MAX_PATH + 0x10];
18150       hFile = INVALID_HANDLE_VALUE;
18151       if (getWinRTLogPath(pathBuf, _MAX_PATH + 1)) {
18152         CREATEFILE2_EXTENDED_PARAMETERS extParams;
18153         BZERO(&extParams, sizeof(extParams));
18154         extParams.dwSize = sizeof(extParams);
18155         extParams.dwFileAttributes = FILE_ATTRIBUTE_NORMAL;
18156         extParams.dwFileFlags = GC_print_stats == VERBOSE ? 0
18157                                     : FILE_FLAG_WRITE_THROUGH;
18158         hFile = CreateFile2(pathBuf, GENERIC_WRITE, FILE_SHARE_READ,
18159                             CREATE_ALWAYS, &extParams);
18160       }
18161 #else
18162     TCHAR *logPath;
18163 #if defined(NO_GETENV_WIN32) && defined(CPPCHECK)
18164 #define appendToFile FALSE
18165 #else
18166       BOOL appendToFile = FALSE;
18167 #endif
18168 #if !defined(NO_GETENV_WIN32) || !defined(OLD_WIN32_LOG_FILE)
18169       TCHAR pathBuf[_MAX_PATH + 0x10];
18170       logPath = pathBuf;
18171 #endif
18172 #ifndef NO_GETENV_WIN32
18173       if (GetEnvironmentVariable(TEXT("GC_LOG_FILE"), pathBuf,
18174                                  _MAX_PATH + 1) - 1U < (DWORD)_MAX_PATH) {
18175         appendToFile = TRUE;
18176       } else
18177 #endif
18178      {
18179 #ifdef OLD_WIN32_LOG_FILE
18180         logPath = TEXT(GC_LOG_STD_NAME);
18181 #else
18182         int len = (int)GetModuleFileName(NULL , pathBuf,
18183                                          _MAX_PATH + 1);
18184         if (len > 4 && pathBuf[len - 4] == (TCHAR)'.') {
18185           len -= 4;
18186         }
18187         BCOPY(TEXT(".") TEXT(GC_LOG_STD_NAME), &pathBuf[len],
18188               sizeof(TEXT(".") TEXT(GC_LOG_STD_NAME)));
18189 #endif
18190     }
18191     hFile = CreateFile(logPath, GENERIC_WRITE, FILE_SHARE_READ,
18192                        NULL ,
18193                        appendToFile ? OPEN_ALWAYS : CREATE_ALWAYS,
18194                        GC_print_stats == VERBOSE ? FILE_ATTRIBUTE_NORMAL :
18195                             FILE_ATTRIBUTE_NORMAL | FILE_FLAG_WRITE_THROUGH,
18196                        NULL );
18197 #ifndef NO_GETENV_WIN32
18198       if (appendToFile && hFile != INVALID_HANDLE_VALUE) {
18199         LONG posHigh = 0;
18200         (void)SetFilePointer(hFile, 0, &posHigh, FILE_END);
18201       }
18202 #endif
18203 #undef appendToFile
18204 #endif
18205     return hFile;
18206   }
18207   STATIC int GC_write(const char *buf, size_t len)
18208   {
18209       BOOL res;
18210       DWORD written;
18211 #if defined(THREADS) && defined(GC_ASSERTIONS)
18212         static GC_bool inside_write = FALSE;
18213         if (inside_write)
18214           return -1;
18215 #endif
18216       if (len == 0)
18217           return 0;
18218       IF_NEED_TO_LOCK(EnterCriticalSection(&GC_write_cs));
18219 #if defined(THREADS) && defined(GC_ASSERTIONS)
18220         if (GC_write_disabled) {
18221           inside_write = TRUE;
18222           ABORT("Assertion failure: GC_write called with write_disabled");
18223         }
18224 #endif
18225       if (GC_log == 0) {
18226         GC_log = GC_CreateLogFile();
18227       }
18228       if (GC_log == INVALID_HANDLE_VALUE) {
18229         IF_NEED_TO_LOCK(LeaveCriticalSection(&GC_write_cs));
18230 #ifdef NO_DEBUGGING
18231           return 0;
18232 #else
18233           return -1;
18234 #endif
18235       }
18236       res = WriteFile(GC_log, buf, (DWORD)len, &written, NULL);
18237 #if defined(_MSC_VER) && defined(_DEBUG) && !defined(NO_CRT)
18238 #ifdef MSWINCE
18239               {
18240                   WCHAR wbuf[1024];
18241                   wbuf[MultiByteToWideChar(CP_ACP, 0 ,
18242                                 buf, len, wbuf,
18243                                 sizeof(wbuf) / sizeof(wbuf[0]) - 1)] = 0;
18244                   OutputDebugStringW(wbuf);
18245               }
18246 #else
18247               _CrtDbgReport(_CRT_WARN, NULL, 0, NULL, "%.*s", len, buf);
18248 #endif
18249 #endif
18250       IF_NEED_TO_LOCK(LeaveCriticalSection(&GC_write_cs));
18251       return res ? (int)written : -1;
18252   }
18253 #define WRITE(f, buf, len) GC_write(buf, len)
18254 #elif defined(OS2) || defined(MACOS)
18255   STATIC FILE * GC_stdout = NULL;
18256   STATIC FILE * GC_stderr = NULL;
18257   STATIC FILE * GC_log = NULL;
18258   STATIC void GC_set_files(void)
18259   {
18260     if (GC_stdout == NULL) {
18261       GC_stdout = stdout;
18262     }
18263     if (GC_stderr == NULL) {
18264       GC_stderr = stderr;
18265     }
18266     if (GC_log == NULL) {
18267       GC_log = stderr;
18268     }
18269   }
18270   GC_INLINE int GC_write(FILE *f, const char *buf, size_t len)
18271   {
18272     int res = fwrite(buf, 1, len, f);
18273     fflush(f);
18274     return res;
18275   }
18276 #define WRITE(f, buf, len) (GC_set_files(), GC_write(f, buf, len))
18277 #elif defined(GC_ANDROID_LOG)
18278 #include <android/log.h>
18279 #ifndef GC_ANDROID_LOG_TAG
18280 #define GC_ANDROID_LOG_TAG "BDWGC"
18281 #endif
18282 #define GC_stdout ANDROID_LOG_DEBUG
18283 #define GC_stderr ANDROID_LOG_ERROR
18284 #define GC_log GC_stdout
18285 #define WRITE(level, buf, unused_len) \
18286                 __android_log_write(level, GC_ANDROID_LOG_TAG, buf)
18287 #elif defined(NN_PLATFORM_CTR)
18288   int n3ds_log_write(const char* text, int length);
18289 #define WRITE(level, buf, len) n3ds_log_write(buf, len)
18290 #elif defined(NINTENDO_SWITCH)
18291   int switch_log_write(const char* text, int length);
18292 #define WRITE(level, buf, len) switch_log_write(buf, len)
18293 #else
18294 #if !defined(GC_NO_TYPES) && !defined(SN_TARGET_PSP2)
18295 #if !defined(AMIGA) && !defined(MSWIN32) && !defined(MSWIN_XBOX1) \
18296        && !defined(__CC_ARM)
18297 #include <unistd.h>
18298 #endif
18299 #if !defined(ECOS) && !defined(NOSYS)
18300 #include <errno.h>
18301 #endif
18302 #endif
18303   STATIC int GC_write(int fd, const char *buf, size_t len)
18304   {
18305 #if defined(ECOS) || defined(PLATFORM_WRITE) || defined(SN_TARGET_PSP2) \
18306        || defined(NOSYS)
18307 #ifdef ECOS
18308 #else
18309 #endif
18310       return len;
18311 #else
18312       int bytes_written = 0;
18313       IF_CANCEL(int cancel_state;)
18314       DISABLE_CANCEL(cancel_state);
18315       while ((unsigned)bytes_written < len) {
18316 #ifdef GC_SOLARIS_THREADS
18317              int result = syscall(SYS_write, fd, buf + bytes_written,
18318                                              len - bytes_written);
18319 #elif defined(_MSC_VER)
18320              int result = _write(fd, buf + bytes_written,
18321                                  (unsigned)(len - bytes_written));
18322 #else
18323              int result = write(fd, buf + bytes_written, len - bytes_written);
18324 #endif
18325          if (-1 == result) {
18326              if (EAGAIN == errno)
18327                continue;
18328              RESTORE_CANCEL(cancel_state);
18329              return(result);
18330          }
18331          bytes_written += result;
18332       }
18333       RESTORE_CANCEL(cancel_state);
18334       return(bytes_written);
18335 #endif
18336   }
18337 #define WRITE(f, buf, len) GC_write(f, buf, len)
18338 #endif
18339 #define BUFSZ 1024
18340 #if defined(DJGPP) || defined(__STRICT_ANSI__)
18341 #define GC_VSNPRINTF(buf, bufsz, format, args) vsprintf(buf, format, args)
18342 #elif defined(_MSC_VER)
18343 #ifdef MSWINCE
18344 #define GC_VSNPRINTF StringCchVPrintfA
18345 #else
18346 #define GC_VSNPRINTF _vsnprintf
18347 #endif
18348 #else
18349 #define GC_VSNPRINTF vsnprintf
18350 #endif
18351 #define GC_PRINTF_FILLBUF(buf, format) \
18352         do { \
18353           va_list args; \
18354           va_start(args, format); \
18355           (buf)[sizeof(buf) - 1] = 0x15;  \
18356           (void)GC_VSNPRINTF(buf, sizeof(buf) - 1, format, args); \
18357           va_end(args); \
18358           if ((buf)[sizeof(buf) - 1] != 0x15) \
18359             ABORT("GC_printf clobbered stack"); \
18360         } while (0)
18361 void GC_printf(const char *format, ...)
18362 {
18363     if (!GC_quiet) {
18364       char buf[BUFSZ + 1];
18365       GC_PRINTF_FILLBUF(buf, format);
18366 #ifdef NACL
18367         (void)WRITE(GC_stdout, buf, strlen(buf));
18368 #else
18369         if (WRITE(GC_stdout, buf, strlen(buf)) < 0
18370 #if defined(CYGWIN32) || (defined(CONSOLE_LOG) && defined(MSWIN32))
18371               && GC_stdout != GC_DEFAULT_STDOUT_FD
18372 #endif
18373            ) {
18374           ABORT("write to stdout failed");
18375         }
18376 #endif
18377     }
18378 }
18379 void GC_err_printf(const char *format, ...)
18380 {
18381     char buf[BUFSZ + 1];
18382     GC_PRINTF_FILLBUF(buf, format);
18383     GC_err_puts(buf);
18384 }
18385 void GC_log_printf(const char *format, ...)
18386 {
18387     char buf[BUFSZ + 1];
18388     GC_PRINTF_FILLBUF(buf, format);
18389 #ifdef NACL
18390       (void)WRITE(GC_log, buf, strlen(buf));
18391 #else
18392       if (WRITE(GC_log, buf, strlen(buf)) < 0
18393 #if defined(CYGWIN32) || (defined(CONSOLE_LOG) && defined(MSWIN32))
18394             && GC_log != GC_DEFAULT_STDERR_FD
18395 #endif
18396          ) {
18397         ABORT("write to GC log failed");
18398       }
18399 #endif
18400 }
18401 #ifndef GC_ANDROID_LOG
18402 #define GC_warn_printf GC_err_printf
18403 #else
18404   GC_INNER void GC_info_log_printf(const char *format, ...)
18405   {
18406     char buf[BUFSZ + 1];
18407     GC_PRINTF_FILLBUF(buf, format);
18408     (void)WRITE(ANDROID_LOG_INFO, buf, 0 );
18409   }
18410   GC_INNER void GC_verbose_log_printf(const char *format, ...)
18411   {
18412     char buf[BUFSZ + 1];
18413     GC_PRINTF_FILLBUF(buf, format);
18414     (void)WRITE(ANDROID_LOG_VERBOSE, buf, 0);
18415   }
18416   STATIC void GC_warn_printf(const char *format, ...)
18417   {
18418     char buf[BUFSZ + 1];
18419     GC_PRINTF_FILLBUF(buf, format);
18420     (void)WRITE(ANDROID_LOG_WARN, buf, 0);
18421   }
18422 #endif
18423 void GC_err_puts(const char *s)
18424 {
18425     (void)WRITE(GC_stderr, s, strlen(s));
18426 }
18427 STATIC void GC_CALLBACK GC_default_warn_proc(char *msg, GC_word arg)
18428 {
18429     GC_warn_printf(msg, arg);
18430 }
18431 GC_INNER GC_warn_proc GC_current_warn_proc = GC_default_warn_proc;
18432 GC_API void GC_CALLBACK GC_ignore_warn_proc(char *msg, GC_word arg)
18433 {
18434     if (GC_print_stats) {
18435       GC_default_warn_proc(msg, arg);
18436     }
18437 }
18438 GC_API void GC_CALL GC_set_warn_proc(GC_warn_proc p)
18439 {
18440     DCL_LOCK_STATE;
18441     GC_ASSERT(NONNULL_ARG_NOT_NULL(p));
18442 #ifdef GC_WIN32_THREADS
18443 #ifdef CYGWIN32
18444         GC_ASSERT(GC_is_initialized);
18445 #else
18446         if (!GC_is_initialized) GC_init();
18447 #endif
18448 #endif
18449     LOCK();
18450     GC_current_warn_proc = p;
18451     UNLOCK();
18452 }
18453 GC_API GC_warn_proc GC_CALL GC_get_warn_proc(void)
18454 {
18455     GC_warn_proc result;
18456     DCL_LOCK_STATE;
18457     LOCK();
18458     result = GC_current_warn_proc;
18459     UNLOCK();
18460     return(result);
18461 }
18462 #if !defined(PCR) && !defined(SMALL_CONFIG)
18463   STATIC void GC_CALLBACK GC_default_on_abort(const char *msg)
18464   {
18465 #ifndef DONT_USE_ATEXIT
18466       skip_gc_atexit = TRUE;
18467 #endif
18468     if (msg != NULL) {
18469 #ifdef MSGBOX_ON_ERROR
18470         GC_win32_MessageBoxA(msg, "Fatal error in GC", MB_ICONERROR | MB_OK);
18471 #endif
18472 #ifndef GC_ANDROID_LOG
18473 #if defined(GC_WIN32_THREADS) && defined(GC_ASSERTIONS) \
18474          && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
18475         if (!GC_write_disabled)
18476 #endif
18477       {
18478         if (WRITE(GC_stderr, msg, strlen(msg)) >= 0)
18479           (void)WRITE(GC_stderr, "\n", 1);
18480       }
18481 #else
18482       __android_log_assert("*" , GC_ANDROID_LOG_TAG, "%s\n", msg);
18483 #endif
18484     }
18485 #if !defined(NO_DEBUGGING) && !defined(GC_ANDROID_LOG)
18486       if (GETENV("GC_LOOP_ON_ABORT") != NULL) {
18487             for(;;) {
18488             }
18489       }
18490 #endif
18491   }
18492   GC_abort_func GC_on_abort = GC_default_on_abort;
18493   GC_API void GC_CALL GC_set_abort_func(GC_abort_func fn)
18494   {
18495       DCL_LOCK_STATE;
18496       GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
18497       LOCK();
18498       GC_on_abort = fn;
18499       UNLOCK();
18500   }
18501   GC_API GC_abort_func GC_CALL GC_get_abort_func(void)
18502   {
18503       GC_abort_func fn;
18504       DCL_LOCK_STATE;
18505       LOCK();
18506       fn = GC_on_abort;
18507       UNLOCK();
18508       return fn;
18509   }
18510 #endif
18511 GC_API void GC_CALL GC_enable(void)
18512 {
18513     DCL_LOCK_STATE;
18514     LOCK();
18515     GC_ASSERT(GC_dont_gc != 0);
18516     GC_dont_gc--;
18517     UNLOCK();
18518 }
18519 GC_API void GC_CALL GC_disable(void)
18520 {
18521     DCL_LOCK_STATE;
18522     LOCK();
18523     GC_dont_gc++;
18524     UNLOCK();
18525 }
18526 GC_API int GC_CALL GC_is_disabled(void)
18527 {
18528     return GC_dont_gc != 0;
18529 }
18530 GC_API void ** GC_CALL GC_new_free_list_inner(void)
18531 {
18532     void *result;
18533     GC_ASSERT(I_HOLD_LOCK());
18534     result = GC_INTERNAL_MALLOC((MAXOBJGRANULES+1) * sizeof(ptr_t), PTRFREE);
18535     if (NULL == result) ABORT("Failed to allocate freelist for new kind");
18536     BZERO(result, (MAXOBJGRANULES+1)*sizeof(ptr_t));
18537     return (void **)result;
18538 }
18539 GC_API void ** GC_CALL GC_new_free_list(void)
18540 {
18541     void ** result;
18542     DCL_LOCK_STATE;
18543     LOCK();
18544     result = GC_new_free_list_inner();
18545     UNLOCK();
18546     return result;
18547 }
18548 GC_API unsigned GC_CALL GC_new_kind_inner(void **fl, GC_word descr,
18549                                           int adjust, int clear)
18550 {
18551     unsigned result = GC_n_kinds;
18552     GC_ASSERT(NONNULL_ARG_NOT_NULL(fl));
18553     GC_ASSERT(adjust == FALSE || adjust == TRUE);
18554     GC_ASSERT(clear == TRUE
18555               || (descr == 0 && adjust == FALSE && clear == FALSE));
18556     if (result < MAXOBJKINDS) {
18557       GC_ASSERT(result > 0);
18558       GC_n_kinds++;
18559       GC_obj_kinds[result].ok_freelist = fl;
18560       GC_obj_kinds[result].ok_reclaim_list = 0;
18561       GC_obj_kinds[result].ok_descriptor = descr;
18562       GC_obj_kinds[result].ok_relocate_descr = adjust;
18563       GC_obj_kinds[result].ok_init = (GC_bool)clear;
18564 #ifdef ENABLE_DISCLAIM
18565         GC_obj_kinds[result].ok_mark_unconditionally = FALSE;
18566         GC_obj_kinds[result].ok_disclaim_proc = 0;
18567 #endif
18568     } else {
18569       ABORT("Too many kinds");
18570     }
18571     return result;
18572 }
18573 GC_API unsigned GC_CALL GC_new_kind(void **fl, GC_word descr, int adjust,
18574                                     int clear)
18575 {
18576     unsigned result;
18577     DCL_LOCK_STATE;
18578     LOCK();
18579     result = GC_new_kind_inner(fl, descr, adjust, clear);
18580     UNLOCK();
18581     return result;
18582 }
18583 GC_API unsigned GC_CALL GC_new_proc_inner(GC_mark_proc proc)
18584 {
18585     unsigned result = GC_n_mark_procs;
18586     if (result < MAX_MARK_PROCS) {
18587       GC_n_mark_procs++;
18588       GC_mark_procs[result] = proc;
18589     } else {
18590       ABORT("Too many mark procedures");
18591     }
18592     return result;
18593 }
18594 GC_API unsigned GC_CALL GC_new_proc(GC_mark_proc proc)
18595 {
18596     unsigned result;
18597     DCL_LOCK_STATE;
18598     LOCK();
18599     result = GC_new_proc_inner(proc);
18600     UNLOCK();
18601     return result;
18602 }
18603 GC_API void * GC_CALL GC_call_with_alloc_lock(GC_fn_type fn, void *client_data)
18604 {
18605     void * result;
18606     DCL_LOCK_STATE;
18607 #ifdef THREADS
18608       LOCK();
18609 #endif
18610     result = (*fn)(client_data);
18611 #ifdef THREADS
18612       UNLOCK();
18613 #endif
18614     return(result);
18615 }
18616 GC_API void * GC_CALL GC_call_with_stack_base(GC_stack_base_func fn, void *arg)
18617 {
18618     struct GC_stack_base base;
18619     void *result;
18620     base.mem_base = (void *)&base;
18621 #ifdef IA64
18622       base.reg_base = (void *)GC_save_regs_in_stack();
18623 #endif
18624     result = fn(&base, arg);
18625     GC_noop1(COVERT_DATAFLOW(&base));
18626     return result;
18627 }
18628 #ifndef THREADS
18629 GC_INNER ptr_t GC_blocked_sp = NULL;
18630 #ifdef IA64
18631     STATIC ptr_t GC_blocked_register_sp = NULL;
18632 #endif
18633 GC_INNER struct GC_traced_stack_sect_s *GC_traced_stack_sect = NULL;
18634 GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type fn,
18635                                              void * client_data)
18636 {
18637     struct GC_traced_stack_sect_s stacksect;
18638     GC_ASSERT(GC_is_initialized);
18639     if ((word)GC_stackbottom HOTTER_THAN (word)(&stacksect))
18640       GC_stackbottom = (ptr_t)COVERT_DATAFLOW(&stacksect);
18641     if (GC_blocked_sp == NULL) {
18642       client_data = fn(client_data);
18643       GC_noop1(COVERT_DATAFLOW(&stacksect));
18644       return client_data;
18645     }
18646     stacksect.saved_stack_ptr = GC_blocked_sp;
18647 #ifdef IA64
18648       stacksect.backing_store_end = GC_save_regs_in_stack();
18649       stacksect.saved_backing_store_ptr = GC_blocked_register_sp;
18650 #endif
18651     stacksect.prev = GC_traced_stack_sect;
18652     GC_blocked_sp = NULL;
18653     GC_traced_stack_sect = &stacksect;
18654     client_data = fn(client_data);
18655     GC_ASSERT(GC_blocked_sp == NULL);
18656     GC_ASSERT(GC_traced_stack_sect == &stacksect);
18657 #if defined(CPPCHECK)
18658       GC_noop1((word)GC_traced_stack_sect - (word)GC_blocked_sp);
18659 #endif
18660     GC_traced_stack_sect = stacksect.prev;
18661 #ifdef IA64
18662       GC_blocked_register_sp = stacksect.saved_backing_store_ptr;
18663 #endif
18664     GC_blocked_sp = stacksect.saved_stack_ptr;
18665     return client_data;
18666 }
18667 STATIC void GC_do_blocking_inner(ptr_t data, void * context GC_ATTR_UNUSED)
18668 {
18669     struct blocking_data * d = (struct blocking_data *) data;
18670     GC_ASSERT(GC_is_initialized);
18671     GC_ASSERT(GC_blocked_sp == NULL);
18672 #ifdef SPARC
18673         GC_blocked_sp = GC_save_regs_in_stack();
18674 #else
18675         GC_blocked_sp = (ptr_t) &d;
18676 #endif
18677 #ifdef IA64
18678         GC_blocked_register_sp = GC_save_regs_in_stack();
18679 #endif
18680     d -> client_data = (d -> fn)(d -> client_data);
18681 #ifdef SPARC
18682         GC_ASSERT(GC_blocked_sp != NULL);
18683 #else
18684         GC_ASSERT(GC_blocked_sp == (ptr_t)(&d));
18685 #endif
18686 #if defined(CPPCHECK)
18687       GC_noop1((word)GC_blocked_sp);
18688 #endif
18689     GC_blocked_sp = NULL;
18690 }
18691   GC_API void GC_CALL GC_set_stackbottom(void *gc_thread_handle,
18692                                          const struct GC_stack_base *sb)
18693   {
18694     GC_ASSERT(sb -> mem_base != NULL);
18695     GC_ASSERT(NULL == gc_thread_handle || &GC_stackbottom == gc_thread_handle);
18696     GC_ASSERT(NULL == GC_blocked_sp
18697               && NULL == GC_traced_stack_sect);
18698     (void)gc_thread_handle;
18699     GC_stackbottom = (char *)sb->mem_base;
18700 #ifdef IA64
18701       GC_register_stackbottom = (ptr_t)sb->reg_base;
18702 #endif
18703   }
18704   GC_API void * GC_CALL GC_get_my_stackbottom(struct GC_stack_base *sb)
18705   {
18706     GC_ASSERT(GC_is_initialized);
18707     sb -> mem_base = GC_stackbottom;
18708 #ifdef IA64
18709       sb -> reg_base = GC_register_stackbottom;
18710 #endif
18711     return &GC_stackbottom;
18712   }
18713 #endif
18714 GC_API void * GC_CALL GC_do_blocking(GC_fn_type fn, void * client_data)
18715 {
18716     struct blocking_data my_data;
18717     my_data.fn = fn;
18718     my_data.client_data = client_data;
18719     GC_with_callee_saves_pushed(GC_do_blocking_inner, (ptr_t)(&my_data));
18720     return my_data.client_data;
18721 }
18722 #if !defined(NO_DEBUGGING)
18723   GC_API void GC_CALL GC_dump(void)
18724   {
18725     DCL_LOCK_STATE;
18726     LOCK();
18727     GC_dump_named(NULL);
18728     UNLOCK();
18729   }
18730   GC_API void GC_CALL GC_dump_named(const char *name)
18731   {
18732 #ifndef NO_CLOCK
18733       CLOCK_TYPE current_time;
18734       GET_TIME(current_time);
18735 #endif
18736     if (name != NULL) {
18737       GC_printf("***GC Dump %s\n", name);
18738     } else {
18739       GC_printf("***GC Dump collection #%lu\n", (unsigned long)GC_gc_no);
18740     }
18741 #ifndef NO_CLOCK
18742       GC_printf("Time since GC init: %lu ms\n",
18743                 MS_TIME_DIFF(current_time, GC_init_time));
18744 #endif
18745     GC_printf("\n***Static roots:\n");
18746     GC_print_static_roots();
18747     GC_printf("\n***Heap sections:\n");
18748     GC_print_heap_sects();
18749     GC_printf("\n***Free blocks:\n");
18750     GC_print_hblkfreelist();
18751     GC_printf("\n***Blocks in use:\n");
18752     GC_print_block_list();
18753   }
18754 #endif
18755 static void block_add_size(struct hblk *h, word pbytes)
18756 {
18757   hdr *hhdr = HDR(h);
18758   *(word *)pbytes += (WORDS_TO_BYTES(hhdr->hb_sz) + (HBLKSIZE - 1))
18759                         & ~(word)(HBLKSIZE - 1);
18760 }
18761 GC_API size_t GC_CALL GC_get_memory_use(void)
18762 {
18763   word bytes = 0;
18764   DCL_LOCK_STATE;
18765   LOCK();
18766   GC_apply_to_all_blocks(block_add_size, (word)(&bytes));
18767   UNLOCK();
18768   return (size_t)bytes;
18769 }
18770 GC_API GC_word GC_CALL GC_get_gc_no(void)
18771 {
18772     return GC_gc_no;
18773 }
18774 #ifdef THREADS
18775   GC_API int GC_CALL GC_get_parallel(void)
18776   {
18777     return GC_parallel;
18778   }
18779   GC_API void GC_CALL GC_alloc_lock(void)
18780   {
18781     DCL_LOCK_STATE;
18782     LOCK();
18783   }
18784   GC_API void GC_CALL GC_alloc_unlock(void)
18785   {
18786     UNLOCK();
18787   }
18788   GC_INNER GC_on_thread_event_proc GC_on_thread_event = 0;
18789   GC_API void GC_CALL GC_set_on_thread_event(GC_on_thread_event_proc fn)
18790   {
18791     DCL_LOCK_STATE;
18792     LOCK();
18793     GC_on_thread_event = fn;
18794     UNLOCK();
18795   }
18796   GC_API GC_on_thread_event_proc GC_CALL GC_get_on_thread_event(void)
18797   {
18798     GC_on_thread_event_proc fn;
18799     DCL_LOCK_STATE;
18800     LOCK();
18801     fn = GC_on_thread_event;
18802     UNLOCK();
18803     return fn;
18804   }
18805 #endif
18806 GC_API void GC_CALL GC_set_oom_fn(GC_oom_func fn)
18807 {
18808     GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
18809     DCL_LOCK_STATE;
18810     LOCK();
18811     GC_oom_fn = fn;
18812     UNLOCK();
18813 }
18814 GC_API GC_oom_func GC_CALL GC_get_oom_fn(void)
18815 {
18816     GC_oom_func fn;
18817     DCL_LOCK_STATE;
18818     LOCK();
18819     fn = GC_oom_fn;
18820     UNLOCK();
18821     return fn;
18822 }
18823 GC_API void GC_CALL GC_set_on_heap_resize(GC_on_heap_resize_proc fn)
18824 {
18825     DCL_LOCK_STATE;
18826     LOCK();
18827     GC_on_heap_resize = fn;
18828     UNLOCK();
18829 }
18830 GC_API GC_on_heap_resize_proc GC_CALL GC_get_on_heap_resize(void)
18831 {
18832     GC_on_heap_resize_proc fn;
18833     DCL_LOCK_STATE;
18834     LOCK();
18835     fn = GC_on_heap_resize;
18836     UNLOCK();
18837     return fn;
18838 }
18839 GC_API void GC_CALL GC_set_finalizer_notifier(GC_finalizer_notifier_proc fn)
18840 {
18841     DCL_LOCK_STATE;
18842     LOCK();
18843     GC_finalizer_notifier = fn;
18844     UNLOCK();
18845 }
18846 GC_API GC_finalizer_notifier_proc GC_CALL GC_get_finalizer_notifier(void)
18847 {
18848     GC_finalizer_notifier_proc fn;
18849     DCL_LOCK_STATE;
18850     LOCK();
18851     fn = GC_finalizer_notifier;
18852     UNLOCK();
18853     return fn;
18854 }
18855 GC_API void GC_CALL GC_set_find_leak(int value)
18856 {
18857     GC_find_leak = value;
18858 }
18859 GC_API int GC_CALL GC_get_find_leak(void)
18860 {
18861     return GC_find_leak;
18862 }
18863 GC_API void GC_CALL GC_set_all_interior_pointers(int value)
18864 {
18865     DCL_LOCK_STATE;
18866     GC_all_interior_pointers = value ? 1 : 0;
18867     if (GC_is_initialized) {
18868       LOCK();
18869       GC_initialize_offsets();
18870       if (!GC_all_interior_pointers)
18871         GC_bl_init_no_interiors();
18872       UNLOCK();
18873     }
18874 }
18875 GC_API int GC_CALL GC_get_all_interior_pointers(void)
18876 {
18877     return GC_all_interior_pointers;
18878 }
18879 GC_API void GC_CALL GC_set_finalize_on_demand(int value)
18880 {
18881     GC_ASSERT(value != -1);
18882     GC_finalize_on_demand = value;
18883 }
18884 GC_API int GC_CALL GC_get_finalize_on_demand(void)
18885 {
18886     return GC_finalize_on_demand;
18887 }
18888 GC_API void GC_CALL GC_set_java_finalization(int value)
18889 {
18890     GC_ASSERT(value != -1);
18891     GC_java_finalization = value;
18892 }
18893 GC_API int GC_CALL GC_get_java_finalization(void)
18894 {
18895     return GC_java_finalization;
18896 }
18897 GC_API void GC_CALL GC_set_dont_expand(int value)
18898 {
18899     GC_ASSERT(value != -1);
18900     GC_dont_expand = value;
18901 }
18902 GC_API int GC_CALL GC_get_dont_expand(void)
18903 {
18904     return GC_dont_expand;
18905 }
18906 GC_API void GC_CALL GC_set_no_dls(int value)
18907 {
18908     GC_ASSERT(value != -1);
18909     GC_no_dls = value;
18910 }
18911 GC_API int GC_CALL GC_get_no_dls(void)
18912 {
18913     return GC_no_dls;
18914 }
18915 GC_API void GC_CALL GC_set_non_gc_bytes(GC_word value)
18916 {
18917     GC_non_gc_bytes = value;
18918 }
18919 GC_API GC_word GC_CALL GC_get_non_gc_bytes(void)
18920 {
18921     return GC_non_gc_bytes;
18922 }
18923 GC_API void GC_CALL GC_set_free_space_divisor(GC_word value)
18924 {
18925     GC_ASSERT(value > 0);
18926     GC_free_space_divisor = value;
18927 }
18928 GC_API GC_word GC_CALL GC_get_free_space_divisor(void)
18929 {
18930     return GC_free_space_divisor;
18931 }
18932 GC_API void GC_CALL GC_set_max_retries(GC_word value)
18933 {
18934     GC_ASSERT((GC_signed_word)value != -1);
18935     GC_max_retries = value;
18936 }
18937 GC_API GC_word GC_CALL GC_get_max_retries(void)
18938 {
18939     return GC_max_retries;
18940 }
18941 GC_API void GC_CALL GC_set_dont_precollect(int value)
18942 {
18943     GC_ASSERT(value != -1);
18944     GC_dont_precollect = value;
18945 }
18946 GC_API int GC_CALL GC_get_dont_precollect(void)
18947 {
18948     return GC_dont_precollect;
18949 }
18950 GC_API void GC_CALL GC_set_full_freq(int value)
18951 {
18952     GC_ASSERT(value >= 0);
18953     GC_full_freq = value;
18954 }
18955 GC_API int GC_CALL GC_get_full_freq(void)
18956 {
18957     return GC_full_freq;
18958 }
18959 GC_API void GC_CALL GC_set_time_limit(unsigned long value)
18960 {
18961     GC_ASSERT((long)value != -1L);
18962     GC_time_limit = value;
18963 }
18964 GC_API unsigned long GC_CALL GC_get_time_limit(void)
18965 {
18966     return GC_time_limit;
18967 }
18968 GC_API void GC_CALL GC_set_force_unmap_on_gcollect(int value)
18969 {
18970     GC_force_unmap_on_gcollect = (GC_bool)value;
18971 }
18972 GC_API int GC_CALL GC_get_force_unmap_on_gcollect(void)
18973 {
18974     return (int)GC_force_unmap_on_gcollect;
18975 }
18976 GC_API void GC_CALL GC_abort_on_oom(void)
18977 {
18978     GC_err_printf("Insufficient memory for the allocation\n");
18979     EXIT();
18980 }
18981 #ifdef THREADS
18982   GC_API void GC_CALL GC_stop_world_external(void)
18983   {
18984     GC_ASSERT(GC_is_initialized);
18985     LOCK();
18986 #ifdef THREAD_LOCAL_ALLOC
18987       GC_ASSERT(!GC_world_stopped);
18988 #endif
18989     STOP_WORLD();
18990 #ifdef THREAD_LOCAL_ALLOC
18991       GC_world_stopped = TRUE;
18992 #endif
18993   }
18994   GC_API void GC_CALL GC_start_world_external(void)
18995   {
18996 #ifdef THREAD_LOCAL_ALLOC
18997       GC_ASSERT(GC_world_stopped);
18998       GC_world_stopped = FALSE;
18999 #else
19000       GC_ASSERT(GC_is_initialized);
19001 #endif
19002     START_WORLD();
19003     UNLOCK();
19004   }
19005 #endif
19006 #if !defined(OS2) && !defined(PCR) && !defined(AMIGA) && !defined(MACOS) \
19007     && !defined(MSWINCE) && !defined(SN_TARGET_ORBIS) \
19008     && !defined(SN_TARGET_PSP2) && !defined(__CC_ARM)
19009 #include <sys/types.h>
19010 #if !defined(MSWIN32) && !defined(MSWIN_XBOX1)
19011 #include <unistd.h>
19012 #endif
19013 #endif
19014 #include <stdio.h>
19015 #if defined(MSWINCE) || defined(SN_TARGET_PS3)
19016 #define SIGSEGV 0
19017 #else
19018 #include <signal.h>
19019 #endif
19020 #if defined(UNIX_LIKE) || defined(CYGWIN32) || defined(NACL) \
19021     || defined(SYMBIAN)
19022 #include <fcntl.h>
19023 #endif
19024 #if defined(LINUX) || defined(LINUX_STACKBOTTOM)
19025 #include <ctype.h>
19026 #endif
19027 #ifdef AMIGA
19028 #define GC_AMIGA_DEF
19029 #if !defined(GC_AMIGA_DEF) && !defined(GC_AMIGA_SB) && !defined(GC_AMIGA_DS) && !defined(GC_AMIGA_AM)
19030 #include <stdio.h>
19031 #include <signal.h>
19032 #define GC_AMIGA_DEF
19033 #define GC_AMIGA_SB
19034 #define GC_AMIGA_DS
19035 #define GC_AMIGA_AM
19036 #endif
19037 #ifdef GC_AMIGA_DEF
19038 #ifndef __GNUC__
19039 #include <exec/exec.h>
19040 #endif
19041 #include <proto/exec.h>
19042 #include <proto/dos.h>
19043 #include <dos/dosextens.h>
19044 #include <workbench/startup.h>
19045 #endif
19046 #ifdef GC_AMIGA_SB
19047 ptr_t GC_get_main_stack_base(void)
19048 {
19049     struct Process *proc = (struct Process*)SysBase->ThisTask;
19050     if (proc->pr_Task.tc_Node.ln_Type==NT_PROCESS
19051         && proc->pr_CLI != NULL) {
19052         return (char *)proc->pr_ReturnAddr + sizeof(ULONG);
19053     } else {
19054         return (char *)proc->pr_Task.tc_SPUpper;
19055     }
19056 }
19057 #endif
19058 #ifdef GC_AMIGA_DS
19059    void GC_register_data_segments(void)
19060    {
19061      struct Process     *proc;
19062      struct CommandLineInterface *cli;
19063      BPTR myseglist;
19064      ULONG *data;
19065 #ifdef __GNUC__
19066         ULONG dataSegSize;
19067         GC_bool found_segment = FALSE;
19068         extern char __data_size[];
19069         dataSegSize=__data_size+8;
19070 #endif
19071         proc= (struct Process*)SysBase->ThisTask;
19072         myseglist = proc->pr_SegList;
19073         if (proc->pr_Task.tc_Node.ln_Type==NT_PROCESS) {
19074           if (proc->pr_CLI != NULL) {
19075             cli = BADDR(proc->pr_CLI);
19076             myseglist = cli->cli_Module;
19077           }
19078         } else {
19079           ABORT("Not a Process.");
19080         }
19081         if (myseglist == NULL) {
19082             ABORT("Arrrgh.. can't find segments, aborting");
19083         }
19084         for (data = (ULONG *)BADDR(myseglist); data != NULL;
19085              data = (ULONG *)BADDR(data[0])) {
19086           if ((ULONG)GC_register_data_segments < (ULONG)(&data[1])
19087               || (ULONG)GC_register_data_segments > (ULONG)(&data[1])
19088                                                     + data[-1]) {
19089 #ifdef __GNUC__
19090                 if (dataSegSize == data[-1]) {
19091                   found_segment = TRUE;
19092                 }
19093 #endif
19094               GC_add_roots_inner((char *)&data[1],
19095                                  ((char *)&data[1]) + data[-1], FALSE);
19096           }
19097         }
19098 #ifdef __GNUC__
19099            if (!found_segment) {
19100              ABORT("Can`t find correct Segments.\nSolution: Use an newer version of ixemul.library");
19101            }
19102 #endif
19103    }
19104 #endif
19105 #ifdef GC_AMIGA_AM
19106 #ifndef GC_AMIGA_FASTALLOC
19107 void *GC_amiga_allocwrapper(size_t size,void *(*AllocFunction)(size_t size2)){
19108         return (*AllocFunction)(size);
19109 }
19110 void *(*GC_amiga_allocwrapper_do)(size_t size,void *(*AllocFunction)(size_t size2))
19111         =GC_amiga_allocwrapper;
19112 #else
19113 void *GC_amiga_allocwrapper_firsttime(size_t size,void *(*AllocFunction)(size_t size2));
19114 void *(*GC_amiga_allocwrapper_do)(size_t size,void *(*AllocFunction)(size_t size2))
19115         =GC_amiga_allocwrapper_firsttime;
19116 struct GC_Amiga_AllocedMemoryHeader{
19117         ULONG size;
19118         struct GC_Amiga_AllocedMemoryHeader *next;
19119 };
19120 struct GC_Amiga_AllocedMemoryHeader *GC_AMIGAMEM=(struct GC_Amiga_AllocedMemoryHeader *)(int)~(NULL);
19121 ULONG GC_AMIGA_MEMF = MEMF_FAST | MEMF_CLEAR;
19122 #ifndef GC_AMIGA_ONLYFAST
19123 BOOL GC_amiga_dontalloc=FALSE;
19124 #endif
19125 #ifdef GC_AMIGA_PRINTSTATS
19126 int succ=0,succ2=0;
19127 int nsucc=0,nsucc2=0;
19128 int nullretries=0;
19129 int numcollects=0;
19130 int chipa=0;
19131 int allochip=0;
19132 int allocfast=0;
19133 int cur0=0;
19134 int cur1=0;
19135 int cur10=0;
19136 int cur50=0;
19137 int cur150=0;
19138 int cur151=0;
19139 int ncur0=0;
19140 int ncur1=0;
19141 int ncur10=0;
19142 int ncur50=0;
19143 int ncur150=0;
19144 int ncur151=0;
19145 #endif
19146 void GC_amiga_free_all_mem(void){
19147         struct GC_Amiga_AllocedMemoryHeader *gc_am=(struct GC_Amiga_AllocedMemoryHeader *)(~(int)(GC_AMIGAMEM));
19148 #ifdef GC_AMIGA_PRINTSTATS
19149         printf("\n\n"
19150                 "%d bytes of chip-mem, and %d bytes of fast-mem where allocated from the OS.\n",
19151                 allochip,allocfast
19152         );
19153         printf(
19154                 "%d bytes of chip-mem were returned from the GC_AMIGA_FASTALLOC supported allocating functions.\n",
19155                 chipa
19156         );
19157         printf("\n");
19158         printf("GC_gcollect was called %d times to avoid returning NULL or start allocating with the MEMF_ANY flag.\n",numcollects);
19159         printf("%d of them was a success. (the others had to use allocation from the OS.)\n",nullretries);
19160         printf("\n");
19161         printf("Succeeded forcing %d gc-allocations (%d bytes) of chip-mem to be fast-mem.\n",succ,succ2);
19162         printf("Failed forcing %d gc-allocations (%d bytes) of chip-mem to be fast-mem.\n",nsucc,nsucc2);
19163         printf("\n");
19164         printf(
19165                 "Number of retries before succeeding a chip->fast force:\n"
19166                 "0: %d, 1: %d, 2-9: %d, 10-49: %d, 50-149: %d, >150: %d\n",
19167                 cur0,cur1,cur10,cur50,cur150,cur151
19168         );
19169         printf(
19170                 "Number of retries before giving up a chip->fast force:\n"
19171                 "0: %d, 1: %d, 2-9: %d, 10-49: %d, 50-149: %d, >150: %d\n",
19172                 ncur0,ncur1,ncur10,ncur50,ncur150,ncur151
19173         );
19174 #endif
19175         while(gc_am!=NULL){
19176                 struct GC_Amiga_AllocedMemoryHeader *temp = gc_am->next;
19177                 FreeMem(gc_am,gc_am->size);
19178                 gc_am=(struct GC_Amiga_AllocedMemoryHeader *)(~(int)(temp));
19179         }
19180 }
19181 #ifndef GC_AMIGA_ONLYFAST
19182 char *chipmax;
19183 size_t latestsize;
19184 #endif
19185 #ifdef GC_AMIGA_FASTALLOC
19186 void *GC_amiga_get_mem(size_t size){
19187         struct GC_Amiga_AllocedMemoryHeader *gc_am;
19188 #ifndef GC_AMIGA_ONLYFAST
19189         if(GC_amiga_dontalloc==TRUE){
19190                 return NULL;
19191         }
19192         if(GC_AMIGA_MEMF==(MEMF_ANY|MEMF_CLEAR) && size>100000 && latestsize<50000) return NULL;
19193 #endif
19194         gc_am=AllocMem((ULONG)(size + sizeof(struct GC_Amiga_AllocedMemoryHeader)),GC_AMIGA_MEMF);
19195         if(gc_am==NULL) return NULL;
19196         gc_am->next=GC_AMIGAMEM;
19197         gc_am->size=size + sizeof(struct GC_Amiga_AllocedMemoryHeader);
19198         GC_AMIGAMEM=(struct GC_Amiga_AllocedMemoryHeader *)(~(int)(gc_am));
19199 #ifdef GC_AMIGA_PRINTSTATS
19200         if((char *)gc_am<chipmax){
19201                 allochip+=size;
19202         }else{
19203                 allocfast+=size;
19204         }
19205 #endif
19206         return gc_am+1;
19207 }
19208 #endif
19209 #ifndef GC_AMIGA_ONLYFAST
19210 #ifdef GC_AMIGA_RETRY
19211 void *GC_amiga_rec_alloc(size_t size,void *(*AllocFunction)(size_t size2),const int rec){
19212         void *ret;
19213         ret=(*AllocFunction)(size);
19214 #ifdef GC_AMIGA_PRINTSTATS
19215         if((char *)ret>chipmax || ret==NULL){
19216                 if(ret==NULL){
19217                         nsucc++;
19218                         nsucc2+=size;
19219                         if(rec==0) ncur0++;
19220                         if(rec==1) ncur1++;
19221                         if(rec>1 && rec<10) ncur10++;
19222                         if(rec>=10 && rec<50) ncur50++;
19223                         if(rec>=50 && rec<150) ncur150++;
19224                         if(rec>=150) ncur151++;
19225                 }else{
19226                         succ++;
19227                         succ2+=size;
19228                         if(rec==0) cur0++;
19229                         if(rec==1) cur1++;
19230                         if(rec>1 && rec<10) cur10++;
19231                         if(rec>=10 && rec<50) cur50++;
19232                         if(rec>=50 && rec<150) cur150++;
19233                         if(rec>=150) cur151++;
19234                 }
19235         }
19236 #endif
19237         if (((char *)ret)<=chipmax && ret!=NULL && (rec<(size>500000?9:size/5000))){
19238                 ret=GC_amiga_rec_alloc(size,AllocFunction,rec+1);
19239         }
19240         return ret;
19241 }
19242 #endif
19243 void *GC_amiga_allocwrapper_any(size_t size,void *(*AllocFunction)(size_t size2)){
19244         void *ret;
19245         GC_amiga_dontalloc=TRUE;
19246         latestsize=size;
19247         ret=(*AllocFunction)(size);
19248         if(((char *)ret) <= chipmax){
19249                 if(ret==NULL){
19250 #ifdef GC_AMIGA_GC
19251                         if(!GC_dont_gc){
19252                                 GC_gcollect();
19253 #ifdef GC_AMIGA_PRINTSTATS
19254                                 numcollects++;
19255 #endif
19256                                 ret=(*AllocFunction)(size);
19257                         }
19258                         if(ret==NULL)
19259 #endif
19260                         {
19261                                 GC_amiga_dontalloc=FALSE;
19262                                 ret=(*AllocFunction)(size);
19263                                 if(ret==NULL){
19264                                         WARN("Out of Memory!  Returning NIL!\n", 0);
19265                                 }
19266                         }
19267 #ifdef GC_AMIGA_PRINTSTATS
19268                         else{
19269                                 nullretries++;
19270                         }
19271                         if(ret!=NULL && (char *)ret<=chipmax) chipa+=size;
19272 #endif
19273                 }
19274 #ifdef GC_AMIGA_RETRY
19275                 else{
19276                         void *ret2;
19277                         if(
19278                                 AllocFunction!=GC_malloc_uncollectable
19279 #ifdef GC_ATOMIC_UNCOLLECTABLE
19280                                 && AllocFunction!=GC_malloc_atomic_uncollectable
19281 #endif
19282                         ){
19283                                 ret2=GC_amiga_rec_alloc(size,AllocFunction,0);
19284                         }else{
19285                                 ret2=(*AllocFunction)(size);
19286 #ifdef GC_AMIGA_PRINTSTATS
19287                                 if((char *)ret2<chipmax || ret2==NULL){
19288                                         nsucc++;
19289                                         nsucc2+=size;
19290                                         ncur0++;
19291                                 }else{
19292                                         succ++;
19293                                         succ2+=size;
19294                                         cur0++;
19295                                 }
19296 #endif
19297                         }
19298                         if(((char *)ret2)>chipmax){
19299                                 GC_free(ret);
19300                                 ret=ret2;
19301                         }else{
19302                                 GC_free(ret2);
19303                         }
19304                 }
19305 #endif
19306         }
19307 #if defined(CPPCHECK)
19308       if (GC_amiga_dontalloc)
19309 #endif
19310         GC_amiga_dontalloc=FALSE;
19311         return ret;
19312 }
19313 void (*GC_amiga_toany)(void)=NULL;
19314 void GC_amiga_set_toany(void (*func)(void)){
19315         GC_amiga_toany=func;
19316 }
19317 #endif
19318 void *GC_amiga_allocwrapper_fast(size_t size,void *(*AllocFunction)(size_t size2)){
19319         void *ret;
19320         ret=(*AllocFunction)(size);
19321         if(ret==NULL){
19322 #ifdef GC_AMIGA_GC
19323                 if(!GC_dont_gc){
19324                         GC_gcollect();
19325 #ifdef GC_AMIGA_PRINTSTATS
19326                         numcollects++;
19327 #endif
19328                         ret=(*AllocFunction)(size);
19329                 }
19330                 if(ret==NULL)
19331 #endif
19332                 {
19333 #ifndef GC_AMIGA_ONLYFAST
19334                         GC_AMIGA_MEMF=MEMF_ANY | MEMF_CLEAR;
19335                         if(GC_amiga_toany!=NULL) (*GC_amiga_toany)();
19336                         GC_amiga_allocwrapper_do=GC_amiga_allocwrapper_any;
19337                         return GC_amiga_allocwrapper_any(size,AllocFunction);
19338 #endif
19339                 }
19340 #ifdef GC_AMIGA_PRINTSTATS
19341                 else{
19342                         nullretries++;
19343                 }
19344 #endif
19345         }
19346         return ret;
19347 }
19348 void *GC_amiga_allocwrapper_firsttime(size_t size,void *(*AllocFunction)(size_t size2)){
19349         atexit(&GC_amiga_free_all_mem);
19350         chipmax=(char *)SysBase->MaxLocMem;
19351         GC_amiga_allocwrapper_do=GC_amiga_allocwrapper_fast;
19352         return GC_amiga_allocwrapper_fast(size,AllocFunction);
19353 }
19354 #endif
19355 void *GC_amiga_realloc(void *old_object,size_t new_size_in_bytes){
19356 #ifndef GC_AMIGA_FASTALLOC
19357         return GC_realloc(old_object,new_size_in_bytes);
19358 #else
19359         void *ret;
19360         latestsize=new_size_in_bytes;
19361         ret=GC_realloc(old_object,new_size_in_bytes);
19362         if(ret==NULL && new_size_in_bytes != 0
19363            && GC_AMIGA_MEMF==(MEMF_FAST | MEMF_CLEAR)){
19364 #ifdef GC_AMIGA_GC
19365                 if(!GC_dont_gc){
19366                         GC_gcollect();
19367 #ifdef GC_AMIGA_PRINTSTATS
19368                         numcollects++;
19369 #endif
19370                         ret=GC_realloc(old_object,new_size_in_bytes);
19371                 }
19372                 if(ret==NULL)
19373 #endif
19374                 {
19375 #ifndef GC_AMIGA_ONLYFAST
19376                         GC_AMIGA_MEMF=MEMF_ANY | MEMF_CLEAR;
19377                         if(GC_amiga_toany!=NULL) (*GC_amiga_toany)();
19378                         GC_amiga_allocwrapper_do=GC_amiga_allocwrapper_any;
19379                         ret=GC_realloc(old_object,new_size_in_bytes);
19380 #endif
19381                 }
19382 #ifdef GC_AMIGA_PRINTSTATS
19383                 else{
19384                         nullretries++;
19385                 }
19386 #endif
19387         }
19388         if(ret==NULL && new_size_in_bytes != 0){
19389                 WARN("Out of Memory!  Returning NIL!\n", 0);
19390         }
19391 #ifdef GC_AMIGA_PRINTSTATS
19392         if(((char *)ret)<chipmax && ret!=NULL){
19393                 chipa+=new_size_in_bytes;
19394         }
19395 #endif
19396         return ret;
19397 #endif
19398 }
19399 #endif
19400 #undef GC_AMIGA_DEF
19401 #endif
19402 #ifdef MACOS
19403 #include <Processes.h>
19404 #endif
19405 #ifdef IRIX5
19406 #include <sys/uio.h>
19407 #include <malloc.h>
19408 #endif
19409 #if defined(MMAP_SUPPORTED) || defined(ADD_HEAP_GUARD_PAGES)
19410 #if defined(USE_MUNMAP) && !defined(USE_MMAP) && !defined(CPPCHECK)
19411 #error Invalid config: USE_MUNMAP requires USE_MMAP
19412 #endif
19413 #include <sys/types.h>
19414 #include <sys/mman.h>
19415 #include <sys/stat.h>
19416 #endif
19417 #if defined(ADD_HEAP_GUARD_PAGES) || defined(LINUX_STACKBOTTOM) \
19418     || defined(MMAP_SUPPORTED) || defined(NEED_PROC_MAPS)
19419 #include <errno.h>
19420 #endif
19421 #ifdef DARWIN
19422 #include <mach-o/getsect.h>
19423 #endif
19424 #ifdef DJGPP
19425   typedef long unsigned int caddr_t;
19426 #endif
19427 #ifdef PCR
19428 #include "mm/PCR_MM.h"
19429 #endif
19430 #if defined(GC_DARWIN_THREADS) && defined(MPROTECT_VDB)
19431 #ifndef GC_DARWIN_STOP_WORLD_H
19432 #define GC_DARWIN_STOP_WORLD_H
19433 #if !defined(GC_DARWIN_THREADS)
19434 #error darwin_stop_world.h included without GC_DARWIN_THREADS defined
19435 #endif
19436 #include <mach/mach.h>
19437 #include <mach/thread_act.h>
19438 EXTERN_C_BEGIN
19439 struct thread_stop_info {
19440   mach_port_t mach_thread;
19441   ptr_t stack_ptr;
19442 };
19443 #ifndef DARWIN_DONT_PARSE_STACK
19444   GC_INNER ptr_t GC_FindTopOfStack(unsigned long);
19445 #endif
19446 #ifdef MPROTECT_VDB
19447   GC_INNER void GC_mprotect_stop(void);
19448   GC_INNER void GC_mprotect_resume(void);
19449 #ifndef GC_NO_THREADS_DISCOVERY
19450     GC_INNER void GC_darwin_register_mach_handler_thread(mach_port_t thread);
19451 #endif
19452 #endif
19453 #if defined(PARALLEL_MARK) && !defined(GC_NO_THREADS_DISCOVERY)
19454   GC_INNER GC_bool GC_is_mach_marker(thread_act_t);
19455 #endif
19456 EXTERN_C_END
19457 #endif
19458 #endif
19459 #if !defined(NO_EXECUTE_PERMISSION)
19460   STATIC GC_bool GC_pages_executable = TRUE;
19461 #else
19462   STATIC GC_bool GC_pages_executable = FALSE;
19463 #endif
19464 #define IGNORE_PAGES_EXECUTABLE 1
19465 #if ((defined(LINUX_STACKBOTTOM) || defined(NEED_PROC_MAPS) \
19466       || defined(PROC_VDB) || defined(SOFT_VDB)) && !defined(PROC_READ)) \
19467     || defined(CPPCHECK)
19468 #define PROC_READ read
19469 #endif
19470 #if defined(LINUX_STACKBOTTOM) || defined(NEED_PROC_MAPS)
19471   STATIC ssize_t GC_repeat_read(int fd, char *buf, size_t count)
19472   {
19473     size_t num_read = 0;
19474     ASSERT_CANCEL_DISABLED();
19475     while (num_read < count) {
19476         ssize_t result = PROC_READ(fd, buf + num_read, count - num_read);
19477         if (result < 0) return result;
19478         if (result == 0) break;
19479         num_read += result;
19480     }
19481     return num_read;
19482   }
19483 #endif
19484 #ifdef NEED_PROC_MAPS
19485 #ifdef THREADS
19486   STATIC size_t GC_get_file_len(int f)
19487   {
19488     size_t total = 0;
19489     ssize_t result;
19490 #define GET_FILE_LEN_BUF_SZ 500
19491     char buf[GET_FILE_LEN_BUF_SZ];
19492     do {
19493         result = PROC_READ(f, buf, sizeof(buf));
19494         if (result == -1) return 0;
19495         total += result;
19496     } while (result > 0);
19497     return total;
19498   }
19499   STATIC size_t GC_get_maps_len(void)
19500   {
19501     int f = open("/proc/self/maps", O_RDONLY);
19502     size_t result;
19503     if (f < 0) return 0;
19504     result = GC_get_file_len(f);
19505     close(f);
19506     return result;
19507   }
19508 #endif
19509 GC_INNER const char * GC_get_maps(void)
19510 {
19511     ssize_t result;
19512     static char *maps_buf = NULL;
19513     static size_t maps_buf_sz = 1;
19514     size_t maps_size;
19515 #ifdef THREADS
19516       size_t old_maps_size = 0;
19517 #endif
19518     GC_ASSERT(I_HOLD_LOCK());
19519 #ifdef THREADS
19520         maps_size = GC_get_maps_len();
19521         if (0 == maps_size)
19522           ABORT("Cannot determine length of /proc/self/maps");
19523 #else
19524         maps_size = 4000;
19525 #endif
19526         do {
19527             int f;
19528             while (maps_size >= maps_buf_sz) {
19529 #ifdef LINT2
19530                 GC_noop1((word)maps_buf);
19531 #else
19532                 GC_scratch_recycle_no_gww(maps_buf, maps_buf_sz);
19533 #endif
19534               while (maps_size >= maps_buf_sz) maps_buf_sz *= 2;
19535               maps_buf = GC_scratch_alloc(maps_buf_sz);
19536               if (NULL == maps_buf)
19537                 ABORT_ARG1("Insufficient space for /proc/self/maps buffer",
19538                         ", %lu bytes requested", (unsigned long)maps_buf_sz);
19539 #ifdef THREADS
19540                 maps_size = GC_get_maps_len();
19541                 if (0 == maps_size)
19542                   ABORT("Cannot determine length of /proc/self/maps");
19543 #endif
19544             }
19545             GC_ASSERT(maps_buf_sz >= maps_size + 1);
19546             f = open("/proc/self/maps", O_RDONLY);
19547             if (-1 == f)
19548               ABORT_ARG1("Cannot open /proc/self/maps",
19549                          ": errno= %d", errno);
19550 #ifdef THREADS
19551               old_maps_size = maps_size;
19552 #endif
19553             maps_size = 0;
19554             do {
19555                 result = GC_repeat_read(f, maps_buf, maps_buf_sz-1);
19556                 if (result <= 0) {
19557                   ABORT_ARG1("Failed to read /proc/self/maps",
19558                              ": errno= %d", result < 0 ? errno : 0);
19559                 }
19560                 maps_size += result;
19561             } while ((size_t)result == maps_buf_sz-1);
19562             close(f);
19563 #ifdef THREADS
19564               if (maps_size > old_maps_size) {
19565                 WARN("Unexpected asynchronous /proc/self/maps growth"
19566                      " (to %" WARN_PRIdPTR " bytes)\n", maps_size);
19567               }
19568 #endif
19569         } while (maps_size >= maps_buf_sz
19570 #ifdef THREADS
19571                    || maps_size < old_maps_size
19572 #endif
19573                 );
19574         maps_buf[maps_size] = '\0';
19575         return maps_buf;
19576 }
19577 #if (defined(DYNAMIC_LOADING) && defined(USE_PROC_FOR_LIBRARIES)) \
19578     || defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
19579     || defined(REDIRECT_MALLOC)
19580   GC_INNER const char *GC_parse_map_entry(const char *maps_ptr,
19581                                           ptr_t *start, ptr_t *end,
19582                                           const char **prot, unsigned *maj_dev,
19583                                           const char **mapping_name)
19584   {
19585     const unsigned char *start_start, *end_start, *maj_dev_start;
19586     const unsigned char *p;
19587     if (maps_ptr == NULL || *maps_ptr == '\0') {
19588         return NULL;
19589     }
19590     p = (const unsigned char *)maps_ptr;
19591     while (isspace(*p)) ++p;
19592     start_start = p;
19593     GC_ASSERT(isxdigit(*start_start));
19594     *start = (ptr_t)strtoul((const char *)start_start, (char **)&p, 16);
19595     GC_ASSERT(*p=='-');
19596     ++p;
19597     end_start = p;
19598     GC_ASSERT(isxdigit(*end_start));
19599     *end = (ptr_t)strtoul((const char *)end_start, (char **)&p, 16);
19600     GC_ASSERT(isspace(*p));
19601     while (isspace(*p)) ++p;
19602     GC_ASSERT(*p == 'r' || *p == '-');
19603     *prot = (const char *)p;
19604     while (!isspace(*p)) ++p;
19605     while (isspace(*p)) p++;
19606     GC_ASSERT(isxdigit(*p));
19607     while (!isspace(*p)) ++p;
19608     while (isspace(*p)) p++;
19609     maj_dev_start = p;
19610     GC_ASSERT(isxdigit(*maj_dev_start));
19611     *maj_dev = strtoul((const char *)maj_dev_start, NULL, 16);
19612     if (mapping_name != NULL) {
19613       while (*p && *p != '\n' && *p != '/' && *p != '[') p++;
19614       *mapping_name = (const char *)p;
19615     }
19616     while (*p && *p++ != '\n');
19617     return (const char *)p;
19618   }
19619 #endif
19620 #if defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR)
19621   GC_INNER GC_bool GC_enclosing_mapping(ptr_t addr, ptr_t *startp,
19622                                         ptr_t *endp)
19623   {
19624     const char *prot;
19625     ptr_t my_start, my_end;
19626     unsigned int maj_dev;
19627     const char *maps_ptr = GC_get_maps();
19628     for (;;) {
19629       maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end,
19630                                     &prot, &maj_dev, 0);
19631       if (NULL == maps_ptr) break;
19632       if (prot[1] == 'w' && maj_dev == 0
19633           && (word)my_end > (word)addr && (word)my_start <= (word)addr) {
19634             *startp = my_start;
19635             *endp = my_end;
19636             return TRUE;
19637       }
19638     }
19639     return FALSE;
19640   }
19641 #endif
19642 #if defined(REDIRECT_MALLOC)
19643   GC_INNER GC_bool GC_text_mapping(char *nm, ptr_t *startp, ptr_t *endp)
19644   {
19645     size_t nm_len = strlen(nm);
19646     const char *prot, *map_path;
19647     ptr_t my_start, my_end;
19648     unsigned int maj_dev;
19649     const char *maps_ptr = GC_get_maps();
19650     for (;;) {
19651       maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end,
19652                                     &prot, &maj_dev, &map_path);
19653       if (NULL == maps_ptr) break;
19654       if (prot[0] == 'r' && prot[1] == '-' && prot[2] == 'x') {
19655           const char *p = map_path;
19656             while (*p != '\0' && *p != '\n' && *p != ' ' && *p != '\t') ++p;
19657             while (*p != '/' && (word)p >= (word)map_path) --p;
19658             ++p;
19659           if (strncmp(nm, p, nm_len) == 0) {
19660             *startp = my_start;
19661             *endp = my_end;
19662             return TRUE;
19663           }
19664       }
19665     }
19666     return FALSE;
19667   }
19668 #endif
19669 #ifdef IA64
19670   static ptr_t backing_store_base_from_proc(void)
19671   {
19672     ptr_t my_start, my_end;
19673     if (!GC_enclosing_mapping(GC_save_regs_in_stack(), &my_start, &my_end)) {
19674         GC_COND_LOG_PRINTF("Failed to find backing store base from /proc\n");
19675         return 0;
19676     }
19677     return my_start;
19678   }
19679 #endif
19680 #endif
19681 #if defined(SEARCH_FOR_DATA_START)
19682 #if defined(LINUX) || defined(HURD)
19683     EXTERN_C_BEGIN
19684 #pragma weak __data_start
19685 #pragma weak data_start
19686     extern int __data_start[], data_start[];
19687     EXTERN_C_END
19688 #endif
19689   ptr_t GC_data_start = NULL;
19690   GC_INNER void GC_init_linux_data_start(void)
19691   {
19692     ptr_t data_end = DATAEND;
19693 #if (defined(LINUX) || defined(HURD)) && defined(USE_PROG_DATA_START)
19694       if (COVERT_DATAFLOW(__data_start) != 0) {
19695         GC_data_start = (ptr_t)(__data_start);
19696       } else {
19697         GC_data_start = (ptr_t)(data_start);
19698       }
19699       if (COVERT_DATAFLOW(GC_data_start) != 0) {
19700         if ((word)GC_data_start > (word)data_end)
19701           ABORT_ARG2("Wrong __data_start/_end pair",
19702                      ": %p .. %p", (void *)GC_data_start, (void *)data_end);
19703         return;
19704       }
19705 #ifdef DEBUG_ADD_DEL_ROOTS
19706         GC_log_printf("__data_start not provided\n");
19707 #endif
19708 #endif
19709     if (GC_no_dls) {
19710       GC_data_start = data_end;
19711       return;
19712     }
19713     GC_data_start = (ptr_t)GC_find_limit(data_end, FALSE);
19714   }
19715 #endif
19716 #ifdef ECOS
19717 #ifndef ECOS_GC_MEMORY_SIZE
19718 #define ECOS_GC_MEMORY_SIZE (448 * 1024)
19719 #endif
19720   static char ecos_gc_memory[ECOS_GC_MEMORY_SIZE];
19721   static char *ecos_gc_brk = ecos_gc_memory;
19722   static void *tiny_sbrk(ptrdiff_t increment)
19723   {
19724     void *p = ecos_gc_brk;
19725     ecos_gc_brk += increment;
19726     if ((word)ecos_gc_brk > (word)(ecos_gc_memory + sizeof(ecos_gc_memory))) {
19727       ecos_gc_brk -= increment;
19728       return NULL;
19729     }
19730     return p;
19731   }
19732 #define sbrk tiny_sbrk
19733 #endif
19734 #if defined(NETBSD) && defined(__ELF__)
19735   ptr_t GC_data_start = NULL;
19736   EXTERN_C_BEGIN
19737   extern char **environ;
19738   EXTERN_C_END
19739   GC_INNER void GC_init_netbsd_elf(void)
19740   {
19741     GC_data_start = (ptr_t)GC_find_limit(&environ, FALSE);
19742   }
19743 #endif
19744 #if defined(ADDRESS_SANITIZER) && (defined(UNIX_LIKE) \
19745                     || defined(NEED_FIND_LIMIT) || defined(MPROTECT_VDB)) \
19746     && !defined(CUSTOM_ASAN_DEF_OPTIONS)
19747   GC_API const char *__asan_default_options(void)
19748   {
19749     return "allow_user_segv_handler=1";
19750   }
19751 #endif
19752 #ifdef OPENBSD
19753   static struct sigaction old_segv_act;
19754   STATIC JMP_BUF GC_jmp_buf_openbsd;
19755   STATIC void GC_fault_handler_openbsd(int sig GC_ATTR_UNUSED)
19756   {
19757      LONGJMP(GC_jmp_buf_openbsd, 1);
19758   }
19759 #ifdef GC_OPENBSD_UTHREADS
19760 #include <sys/syscall.h>
19761     EXTERN_C_BEGIN
19762     extern sigset_t __syscall(quad_t, ...);
19763     EXTERN_C_END
19764   STATIC ptr_t GC_find_limit_openbsd(ptr_t p, ptr_t bound)
19765   {
19766     static volatile ptr_t result;
19767     struct sigaction act;
19768     word pgsz = (word)sysconf(_SC_PAGESIZE);
19769     GC_ASSERT((word)bound >= pgsz);
19770     GC_ASSERT(I_HOLD_LOCK());
19771     act.sa_handler = GC_fault_handler_openbsd;
19772     sigemptyset(&act.sa_mask);
19773     act.sa_flags = SA_NODEFER | SA_RESTART;
19774     sigaction(SIGSEGV, &act, &old_segv_act);
19775     if (SETJMP(GC_jmp_buf_openbsd) == 0) {
19776       result = (ptr_t)((word)p & ~(pgsz-1));
19777       for (;;) {
19778         if ((word)result >= (word)bound - pgsz) {
19779           result = bound;
19780           break;
19781         }
19782         result += pgsz;
19783         GC_noop1((word)(*result));
19784       }
19785     }
19786 #ifdef THREADS
19787       __syscall(SYS_sigprocmask, SIG_UNBLOCK, sigmask(SIGPROF));
19788 #endif
19789     sigaction(SIGSEGV, &old_segv_act, 0);
19790     return(result);
19791   }
19792 #endif
19793   static volatile int firstpass;
19794   STATIC ptr_t GC_skip_hole_openbsd(ptr_t p, ptr_t bound)
19795   {
19796     static volatile ptr_t result;
19797     struct sigaction act;
19798     word pgsz = (word)sysconf(_SC_PAGESIZE);
19799     GC_ASSERT((word)bound >= pgsz);
19800     GC_ASSERT(I_HOLD_LOCK());
19801     act.sa_handler = GC_fault_handler_openbsd;
19802     sigemptyset(&act.sa_mask);
19803     act.sa_flags = SA_NODEFER | SA_RESTART;
19804     sigaction(SIGSEGV, &act, &old_segv_act);
19805     firstpass = 1;
19806     result = (ptr_t)((word)p & ~(pgsz-1));
19807     if (SETJMP(GC_jmp_buf_openbsd) != 0 || firstpass) {
19808       firstpass = 0;
19809       if ((word)result >= (word)bound - pgsz) {
19810         result = bound;
19811       } else {
19812         result += pgsz;
19813         GC_noop1((word)(*result));
19814       }
19815     }
19816     sigaction(SIGSEGV, &old_segv_act, 0);
19817     return(result);
19818   }
19819 #endif
19820 #ifdef OS2
19821 #include <stddef.h>
19822 #if !defined(__IBMC__) && !defined(__WATCOMC__)
19823 struct exe_hdr {
19824     unsigned short      magic_number;
19825     unsigned short      padding[29];
19826     long                new_exe_offset;
19827 };
19828 #define E_MAGIC(x)      (x).magic_number
19829 #define EMAGIC          0x5A4D
19830 #define E_LFANEW(x)     (x).new_exe_offset
19831 struct e32_exe {
19832     unsigned char       magic_number[2];
19833     unsigned char       byte_order;
19834     unsigned char       word_order;
19835     unsigned long       exe_format_level;
19836     unsigned short      cpu;
19837     unsigned short      os;
19838     unsigned long       padding1[13];
19839     unsigned long       object_table_offset;
19840     unsigned long       object_count;
19841     unsigned long       padding2[31];
19842 };
19843 #define E32_MAGIC1(x)   (x).magic_number[0]
19844 #define E32MAGIC1       'L'
19845 #define E32_MAGIC2(x)   (x).magic_number[1]
19846 #define E32MAGIC2       'X'
19847 #define E32_BORDER(x)   (x).byte_order
19848 #define E32LEBO         0
19849 #define E32_WORDER(x)   (x).word_order
19850 #define E32LEWO         0
19851 #define E32_CPU(x)      (x).cpu
19852 #define E32CPU286       1
19853 #define E32_OBJTAB(x)   (x).object_table_offset
19854 #define E32_OBJCNT(x)   (x).object_count
19855 struct o32_obj {
19856     unsigned long       size;
19857     unsigned long       base;
19858     unsigned long       flags;
19859     unsigned long       pagemap;
19860     unsigned long       mapsize;
19861     unsigned long       reserved;
19862 };
19863 #define O32_FLAGS(x)    (x).flags
19864 #define OBJREAD         0x0001L
19865 #define OBJWRITE        0x0002L
19866 #define OBJINVALID      0x0080L
19867 #define O32_SIZE(x)     (x).size
19868 #define O32_BASE(x)     (x).base
19869 #else
19870 #ifndef WORD
19871 #define WORD unsigned short
19872 #endif
19873 #ifndef DWORD
19874 #define DWORD unsigned long
19875 #endif
19876 #define EXE386 1
19877 #include <newexe.h>
19878 #include <exe386.h>
19879 #endif
19880 #define INCL_DOSEXCEPTIONS
19881 #define INCL_DOSPROCESS
19882 #define INCL_DOSERRORS
19883 #define INCL_DOSMODULEMGR
19884 #define INCL_DOSMEMMGR
19885 #include <os2.h>
19886 #endif
19887 GC_INNER size_t GC_page_size = 0;
19888 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
19889 #ifndef VER_PLATFORM_WIN32_CE
19890 #define VER_PLATFORM_WIN32_CE 3
19891 #endif
19892 #if defined(MSWINCE) && defined(THREADS)
19893     GC_INNER GC_bool GC_dont_query_stack_min = FALSE;
19894 #endif
19895   GC_INNER SYSTEM_INFO GC_sysinfo;
19896   GC_INNER void GC_setpagesize(void)
19897   {
19898     GetSystemInfo(&GC_sysinfo);
19899 #if defined(CYGWIN32) && (defined(MPROTECT_VDB) || defined(USE_MUNMAP))
19900       GC_page_size = (size_t)GC_sysinfo.dwAllocationGranularity;
19901       GC_ASSERT(GC_page_size >= (size_t)GC_sysinfo.dwPageSize);
19902 #else
19903       GC_page_size = (size_t)GC_sysinfo.dwPageSize;
19904 #endif
19905 #if defined(MSWINCE) && !defined(_WIN32_WCE_EMULATION)
19906       {
19907         OSVERSIONINFO verInfo;
19908         verInfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
19909         if (!GetVersionEx(&verInfo))
19910           ABORT("GetVersionEx failed");
19911         if (verInfo.dwPlatformId == VER_PLATFORM_WIN32_CE &&
19912             verInfo.dwMajorVersion < 6) {
19913           GC_sysinfo.lpMaximumApplicationAddress = (LPVOID)((word)32 << 20);
19914 #ifdef THREADS
19915             if (verInfo.dwMajorVersion < 5)
19916               GC_dont_query_stack_min = TRUE;
19917 #endif
19918         }
19919       }
19920 #endif
19921   }
19922 #ifndef CYGWIN32
19923 #define is_writable(prot) ((prot) == PAGE_READWRITE \
19924                             || (prot) == PAGE_WRITECOPY \
19925                             || (prot) == PAGE_EXECUTE_READWRITE \
19926                             || (prot) == PAGE_EXECUTE_WRITECOPY)
19927     STATIC word GC_get_writable_length(ptr_t p, ptr_t *base)
19928     {
19929       MEMORY_BASIC_INFORMATION buf;
19930       word result;
19931       word protect;
19932       result = VirtualQuery(p, &buf, sizeof(buf));
19933       if (result != sizeof(buf)) ABORT("Weird VirtualQuery result");
19934       if (base != 0) *base = (ptr_t)(buf.AllocationBase);
19935       protect = (buf.Protect & ~(PAGE_GUARD | PAGE_NOCACHE));
19936       if (!is_writable(protect)) {
19937         return(0);
19938       }
19939       if (buf.State != MEM_COMMIT) return(0);
19940       return(buf.RegionSize);
19941     }
19942     GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
19943     {
19944       ptr_t trunc_sp;
19945       word size;
19946       if (!GC_page_size) GC_setpagesize();
19947       trunc_sp = (ptr_t)((word)GC_approx_sp() & ~(GC_page_size - 1));
19948       size = GC_get_writable_length(trunc_sp, 0);
19949       GC_ASSERT(size != 0);
19950       sb -> mem_base = trunc_sp + size;
19951       return GC_SUCCESS;
19952     }
19953 #else
19954     GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
19955     {
19956 #ifdef X86_64
19957         sb -> mem_base = ((NT_TIB*)NtCurrentTeb())->StackBase;
19958 #else
19959         void * _tlsbase;
19960         __asm__ ("movl %%fs:4, %0"
19961                  : "=r" (_tlsbase));
19962         sb -> mem_base = _tlsbase;
19963 #endif
19964       return GC_SUCCESS;
19965     }
19966 #endif
19967 #define HAVE_GET_STACK_BASE
19968 #else
19969   GC_INNER void GC_setpagesize(void)
19970   {
19971 #if defined(MPROTECT_VDB) || defined(PROC_VDB) || defined(SOFT_VDB) \
19972        || defined(USE_MMAP)
19973       GC_page_size = (size_t)GETPAGESIZE();
19974 #if !defined(CPPCHECK)
19975         if (0 == GC_page_size)
19976           ABORT("getpagesize failed");
19977 #endif
19978 #else
19979       GC_page_size = HBLKSIZE;
19980 #endif
19981   }
19982 #endif
19983 #ifdef HAIKU
19984 #include <kernel/OS.h>
19985   GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
19986   {
19987     thread_info th;
19988     get_thread_info(find_thread(NULL),&th);
19989     sb->mem_base = th.stack_end;
19990     return GC_SUCCESS;
19991   }
19992 #define HAVE_GET_STACK_BASE
19993 #endif
19994 #ifdef OS2
19995   GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
19996   {
19997     PTIB ptib;
19998     PPIB ppib;
19999     if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) {
20000       WARN("DosGetInfoBlocks failed\n", 0);
20001       return GC_UNIMPLEMENTED;
20002     }
20003     sb->mem_base = ptib->tib_pstacklimit;
20004     return GC_SUCCESS;
20005   }
20006 #define HAVE_GET_STACK_BASE
20007 #endif
20008 #ifdef AMIGA
20009 #define GC_AMIGA_SB
20010 #undef GC_AMIGA_SB
20011 #define GET_MAIN_STACKBASE_SPECIAL
20012 #endif
20013 #if defined(NEED_FIND_LIMIT) || defined(UNIX_LIKE)
20014     typedef void (*GC_fault_handler_t)(int);
20015 #if defined(SUNOS5SIGS) || defined(IRIX5) || defined(OSF1) \
20016        || defined(HAIKU) || defined(HURD) || defined(FREEBSD) \
20017        || defined(NETBSD)
20018         static struct sigaction old_segv_act;
20019 #if defined(_sigargs)  \
20020            || defined(HURD) || defined(NETBSD) || defined(FREEBSD)
20021             static struct sigaction old_bus_act;
20022 #endif
20023 #else
20024       static GC_fault_handler_t old_segv_handler;
20025 #ifdef HAVE_SIGBUS
20026         static GC_fault_handler_t old_bus_handler;
20027 #endif
20028 #endif
20029     GC_INNER void GC_set_and_save_fault_handler(GC_fault_handler_t h)
20030     {
20031 #if defined(SUNOS5SIGS) || defined(IRIX5) || defined(OSF1) \
20032            || defined(HAIKU) || defined(HURD) || defined(FREEBSD) \
20033            || defined(NETBSD) || defined(OPENBSD)
20034           struct sigaction act;
20035           act.sa_handler = h;
20036 #ifdef SIGACTION_FLAGS_NODEFER_HACK
20037             act.sa_flags = SA_RESTART | SA_NODEFER;
20038 #else
20039             act.sa_flags = SA_RESTART;
20040 #endif
20041           (void) sigemptyset(&act.sa_mask);
20042 #ifdef GC_IRIX_THREADS
20043             (void) sigaction(SIGSEGV, 0, &old_segv_act);
20044             (void) sigaction(SIGSEGV, &act, 0);
20045 #else
20046             (void) sigaction(SIGSEGV, &act, &old_segv_act);
20047 #if defined(IRIX5) && defined(_sigargs)  \
20048                || defined(HURD) || defined(NETBSD) || defined(FREEBSD)
20049               (void) sigaction(SIGBUS, &act, &old_bus_act);
20050 #endif
20051 #endif
20052 #else
20053           old_segv_handler = signal(SIGSEGV, h);
20054 #ifdef HAVE_SIGBUS
20055             old_bus_handler = signal(SIGBUS, h);
20056 #endif
20057 #endif
20058 #if defined(CPPCHECK) && defined(ADDRESS_SANITIZER)
20059           GC_noop1((word)&__asan_default_options);
20060 #endif
20061     }
20062 #endif
20063 #if defined(NEED_FIND_LIMIT) \
20064      || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))
20065 #define MIN_PAGE_SIZE 256
20066     GC_INNER JMP_BUF GC_jmp_buf;
20067     STATIC void GC_fault_handler(int sig GC_ATTR_UNUSED)
20068     {
20069         LONGJMP(GC_jmp_buf, 1);
20070     }
20071     GC_INNER void GC_setup_temporary_fault_handler(void)
20072     {
20073         GC_ASSERT(I_HOLD_LOCK());
20074         GC_set_and_save_fault_handler(GC_fault_handler);
20075     }
20076     GC_INNER void GC_reset_fault_handler(void)
20077     {
20078 #if defined(SUNOS5SIGS) || defined(IRIX5) || defined(OSF1) \
20079            || defined(HAIKU) || defined(HURD) || defined(FREEBSD) \
20080            || defined(NETBSD) || defined(OPENBSD)
20081           (void) sigaction(SIGSEGV, &old_segv_act, 0);
20082 #if defined(IRIX5) && defined(_sigargs)  \
20083              || defined(HURD) || defined(NETBSD)
20084               (void) sigaction(SIGBUS, &old_bus_act, 0);
20085 #endif
20086 #else
20087           (void) signal(SIGSEGV, old_segv_handler);
20088 #ifdef HAVE_SIGBUS
20089             (void) signal(SIGBUS, old_bus_handler);
20090 #endif
20091 #endif
20092     }
20093     GC_ATTR_NO_SANITIZE_ADDR
20094     STATIC ptr_t GC_find_limit_with_bound(ptr_t p, GC_bool up, ptr_t bound)
20095     {
20096         static volatile ptr_t result;
20097         GC_ASSERT(up ? (word)bound >= MIN_PAGE_SIZE
20098                      : (word)bound <= ~(word)MIN_PAGE_SIZE);
20099         GC_ASSERT(I_HOLD_LOCK());
20100         GC_setup_temporary_fault_handler();
20101         if (SETJMP(GC_jmp_buf) == 0) {
20102             result = (ptr_t)(((word)(p))
20103                               & ~(MIN_PAGE_SIZE-1));
20104             for (;;) {
20105                 if (up) {
20106                     if ((word)result >= (word)bound - MIN_PAGE_SIZE) {
20107                       result = bound;
20108                       break;
20109                     }
20110                     result += MIN_PAGE_SIZE;
20111                 } else {
20112                     if ((word)result <= (word)bound + MIN_PAGE_SIZE) {
20113                       result = bound - MIN_PAGE_SIZE;
20114                       break;
20115                     }
20116                     result -= MIN_PAGE_SIZE;
20117                 }
20118                 GC_noop1((word)(*result));
20119             }
20120         }
20121         GC_reset_fault_handler();
20122         if (!up) {
20123             result += MIN_PAGE_SIZE;
20124         }
20125         return(result);
20126     }
20127     void * GC_find_limit(void * p, int up)
20128     {
20129         return GC_find_limit_with_bound((ptr_t)p, (GC_bool)up,
20130                                         up ? (ptr_t)GC_WORD_MAX : 0);
20131     }
20132 #endif
20133 #ifdef HPUX_MAIN_STACKBOTTOM
20134 #include <sys/param.h>
20135 #include <sys/pstat.h>
20136   STATIC ptr_t GC_hpux_main_stack_base(void)
20137   {
20138     struct pst_vm_status vm_status;
20139     int i = 0;
20140     while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) {
20141       if (vm_status.pst_type == PS_STACK)
20142         return (ptr_t)vm_status.pst_vaddr;
20143     }
20144 #ifdef STACK_GROWS_UP
20145       return (ptr_t)GC_find_limit(GC_approx_sp(),  FALSE);
20146 #else
20147       return (ptr_t)GC_find_limit(GC_approx_sp(), TRUE);
20148 #endif
20149   }
20150 #endif
20151 #ifdef HPUX_STACKBOTTOM
20152 #include <sys/param.h>
20153 #include <sys/pstat.h>
20154   GC_INNER ptr_t GC_get_register_stack_base(void)
20155   {
20156     struct pst_vm_status vm_status;
20157     int i = 0;
20158     while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) {
20159       if (vm_status.pst_type == PS_RSESTACK) {
20160         return (ptr_t) vm_status.pst_vaddr;
20161       }
20162     }
20163     return (ptr_t)(((word)GC_stackbottom - BACKING_STORE_DISPLACEMENT - 1)
20164                    & ~(BACKING_STORE_ALIGNMENT - 1));
20165   }
20166 #endif
20167 #ifdef LINUX_STACKBOTTOM
20168 #include <sys/types.h>
20169 #include <sys/stat.h>
20170 #define STAT_SKIP 27
20171 #ifdef USE_LIBC_PRIVATES
20172     EXTERN_C_BEGIN
20173 #pragma weak __libc_stack_end
20174     extern ptr_t __libc_stack_end;
20175 #ifdef IA64
20176 #pragma weak __libc_ia64_register_backing_store_base
20177       extern ptr_t __libc_ia64_register_backing_store_base;
20178 #endif
20179     EXTERN_C_END
20180 #endif
20181 #ifdef IA64
20182     GC_INNER ptr_t GC_get_register_stack_base(void)
20183     {
20184       ptr_t result;
20185 #ifdef USE_LIBC_PRIVATES
20186         if (0 != &__libc_ia64_register_backing_store_base
20187             && 0 != __libc_ia64_register_backing_store_base) {
20188           return __libc_ia64_register_backing_store_base;
20189         }
20190 #endif
20191       result = backing_store_base_from_proc();
20192       if (0 == result) {
20193           result = (ptr_t)GC_find_limit(GC_save_regs_in_stack(), FALSE);
20194       }
20195       return result;
20196     }
20197 #endif
20198   STATIC ptr_t GC_linux_main_stack_base(void)
20199   {
20200 #define STAT_BUF_SIZE 4096
20201     char stat_buf[STAT_BUF_SIZE];
20202     int f;
20203     word result;
20204     ssize_t i, buf_offset = 0, len;
20205 #ifdef USE_LIBC_PRIVATES
20206       if (0 != &__libc_stack_end && 0 != __libc_stack_end ) {
20207 #if defined(IA64)
20208           if (((word)__libc_stack_end & 0xfff) + 0x10 < 0x1000) {
20209             return __libc_stack_end + 0x10;
20210           }
20211 #elif defined(SPARC)
20212           if (__libc_stack_end != (ptr_t) (unsigned long)0x1)
20213             return __libc_stack_end;
20214 #else
20215           return __libc_stack_end;
20216 #endif
20217       }
20218 #endif
20219     f = open("/proc/self/stat", O_RDONLY);
20220     if (-1 == f)
20221       ABORT_ARG1("Could not open /proc/self/stat", ": errno= %d", errno);
20222     len = GC_repeat_read(f, stat_buf, sizeof(stat_buf));
20223     if (len < 0)
20224       ABORT_ARG1("Failed to read /proc/self/stat",
20225                  ": errno= %d", errno);
20226     close(f);
20227     for (i = 0; i < STAT_SKIP; ++i) {
20228       while (buf_offset < len && isspace(stat_buf[buf_offset++])) {
20229       }
20230       while (buf_offset < len && !isspace(stat_buf[buf_offset++])) {
20231       }
20232     }
20233     while (buf_offset < len && isspace(stat_buf[buf_offset])) {
20234       buf_offset++;
20235     }
20236     for (i = 0; buf_offset + i < len; i++) {
20237       if (!isdigit(stat_buf[buf_offset + i])) break;
20238     }
20239     if (buf_offset + i >= len) ABORT("Could not parse /proc/self/stat");
20240     stat_buf[buf_offset + i] = '\0';
20241     result = (word)STRTOULL(&stat_buf[buf_offset], NULL, 10);
20242     if (result < 0x100000 || (result & (sizeof(word) - 1)) != 0)
20243       ABORT_ARG1("Absurd stack bottom value",
20244                  ": 0x%lx", (unsigned long)result);
20245     return (ptr_t)result;
20246   }
20247 #endif
20248 #ifdef FREEBSD_STACKBOTTOM
20249 #include <unistd.h>
20250 #include <sys/types.h>
20251 #include <sys/sysctl.h>
20252   STATIC ptr_t GC_freebsd_main_stack_base(void)
20253   {
20254     int nm[2] = {CTL_KERN, KERN_USRSTACK};
20255     ptr_t base;
20256     size_t len = sizeof(ptr_t);
20257     int r = sysctl(nm, 2, &base, &len, NULL, 0);
20258     if (r) ABORT("Error getting main stack base");
20259     return base;
20260   }
20261 #endif
20262 #if defined(ECOS) || defined(NOSYS)
20263   ptr_t GC_get_main_stack_base(void)
20264   {
20265     return STACKBOTTOM;
20266   }
20267 #define GET_MAIN_STACKBASE_SPECIAL
20268 #elif defined(SYMBIAN)
20269   EXTERN_C_BEGIN
20270   extern int GC_get_main_symbian_stack_base(void);
20271   EXTERN_C_END
20272   ptr_t GC_get_main_stack_base(void)
20273   {
20274     return (ptr_t)GC_get_main_symbian_stack_base();
20275   }
20276 #define GET_MAIN_STACKBASE_SPECIAL
20277 #elif defined(EMSCRIPTEN)
20278 #include <emscripten.h>
20279   static void* emscripten_stack_base;
20280   static void scan_stack_cb(void *begin, void *end)
20281   {
20282     (void)begin;
20283     emscripten_stack_base = end;
20284   }
20285   ptr_t GC_get_main_stack_base(void)
20286   {
20287     emscripten_scan_stack(scan_stack_cb);
20288     return (ptr_t)emscripten_stack_base;
20289   }
20290 #define GET_MAIN_STACKBASE_SPECIAL
20291 #elif !defined(AMIGA) && !defined(HAIKU) && !defined(OS2) \
20292       && !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) \
20293       && !defined(GC_OPENBSD_THREADS) \
20294       && (!defined(GC_SOLARIS_THREADS) || defined(_STRICT_STDC))
20295 #if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
20296      && (defined(THREADS) || defined(USE_GET_STACKBASE_FOR_MAIN))
20297 #include <pthread.h>
20298 #ifdef HAVE_PTHREAD_NP_H
20299 #include <pthread_np.h>
20300 #endif
20301 #elif defined(DARWIN) && !defined(NO_PTHREAD_GET_STACKADDR_NP)
20302 #include <pthread.h>
20303 #undef STACKBOTTOM
20304 #define STACKBOTTOM (ptr_t)pthread_get_stackaddr_np(pthread_self())
20305 #endif
20306   ptr_t GC_get_main_stack_base(void)
20307   {
20308     ptr_t result;
20309 #if (defined(HAVE_PTHREAD_ATTR_GET_NP) \
20310         || defined(HAVE_PTHREAD_GETATTR_NP)) \
20311        && (defined(USE_GET_STACKBASE_FOR_MAIN) \
20312            || (defined(THREADS) && !defined(REDIRECT_MALLOC)))
20313       pthread_attr_t attr;
20314       void *stackaddr;
20315       size_t size;
20316 #ifdef HAVE_PTHREAD_ATTR_GET_NP
20317         if (pthread_attr_init(&attr) == 0
20318             && (pthread_attr_get_np(pthread_self(), &attr) == 0
20319                 ? TRUE : (pthread_attr_destroy(&attr), FALSE)))
20320 #else
20321         if (pthread_getattr_np(pthread_self(), &attr) == 0)
20322 #endif
20323       {
20324         if (pthread_attr_getstack(&attr, &stackaddr, &size) == 0
20325             && stackaddr != NULL) {
20326           (void)pthread_attr_destroy(&attr);
20327 #ifdef STACK_GROWS_DOWN
20328             stackaddr = (char *)stackaddr + size;
20329 #endif
20330           return (ptr_t)stackaddr;
20331         }
20332         (void)pthread_attr_destroy(&attr);
20333       }
20334       WARN("pthread_getattr_np or pthread_attr_getstack failed"
20335            " for main thread\n", 0);
20336 #endif
20337 #ifdef STACKBOTTOM
20338       result = STACKBOTTOM;
20339 #else
20340 #ifdef HEURISTIC1
20341 #define STACKBOTTOM_ALIGNMENT_M1 ((word)STACK_GRAN - 1)
20342 #ifdef STACK_GROWS_DOWN
20343           result = (ptr_t)(((word)GC_approx_sp() + STACKBOTTOM_ALIGNMENT_M1)
20344                            & ~STACKBOTTOM_ALIGNMENT_M1);
20345 #else
20346           result = (ptr_t)((word)GC_approx_sp() & ~STACKBOTTOM_ALIGNMENT_M1);
20347 #endif
20348 #elif defined(HPUX_MAIN_STACKBOTTOM)
20349         result = GC_hpux_main_stack_base();
20350 #elif defined(LINUX_STACKBOTTOM)
20351         result = GC_linux_main_stack_base();
20352 #elif defined(FREEBSD_STACKBOTTOM)
20353         result = GC_freebsd_main_stack_base();
20354 #elif defined(HEURISTIC2)
20355         {
20356           ptr_t sp = GC_approx_sp();
20357 #ifdef STACK_GROWS_DOWN
20358             result = (ptr_t)GC_find_limit(sp, TRUE);
20359 #if defined(HEURISTIC2_LIMIT) && !defined(CPPCHECK)
20360               if ((word)result > (word)HEURISTIC2_LIMIT
20361                   && (word)sp < (word)HEURISTIC2_LIMIT) {
20362                 result = HEURISTIC2_LIMIT;
20363               }
20364 #endif
20365 #else
20366             result = (ptr_t)GC_find_limit(sp, FALSE);
20367 #if defined(HEURISTIC2_LIMIT) && !defined(CPPCHECK)
20368               if ((word)result < (word)HEURISTIC2_LIMIT
20369                   && (word)sp > (word)HEURISTIC2_LIMIT) {
20370                 result = HEURISTIC2_LIMIT;
20371               }
20372 #endif
20373 #endif
20374         }
20375 #elif defined(STACK_NOT_SCANNED) || defined(CPPCHECK)
20376         result = NULL;
20377 #else
20378 #error None of HEURISTIC* and *STACKBOTTOM defined!
20379 #endif
20380 #if defined(STACK_GROWS_DOWN) && !defined(CPPCHECK)
20381         if (result == 0)
20382           result = (ptr_t)(signed_word)(-sizeof(ptr_t));
20383 #endif
20384 #endif
20385 #if !defined(CPPCHECK)
20386       GC_ASSERT((word)GC_approx_sp() HOTTER_THAN (word)result);
20387 #endif
20388     return(result);
20389   }
20390 #define GET_MAIN_STACKBASE_SPECIAL
20391 #endif
20392 #if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
20393     && defined(THREADS) && !defined(HAVE_GET_STACK_BASE)
20394 #include <pthread.h>
20395 #ifdef HAVE_PTHREAD_NP_H
20396 #include <pthread_np.h>
20397 #endif
20398   GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
20399   {
20400     pthread_attr_t attr;
20401     size_t size;
20402 #ifdef IA64
20403       DCL_LOCK_STATE;
20404 #endif
20405 #ifdef HAVE_PTHREAD_ATTR_GET_NP
20406       if (pthread_attr_init(&attr) != 0)
20407         ABORT("pthread_attr_init failed");
20408       if (pthread_attr_get_np(pthread_self(), &attr) != 0) {
20409         WARN("pthread_attr_get_np failed\n", 0);
20410         (void)pthread_attr_destroy(&attr);
20411         return GC_UNIMPLEMENTED;
20412       }
20413 #else
20414       if (pthread_getattr_np(pthread_self(), &attr) != 0) {
20415         WARN("pthread_getattr_np failed\n", 0);
20416         return GC_UNIMPLEMENTED;
20417       }
20418 #endif
20419     if (pthread_attr_getstack(&attr, &(b -> mem_base), &size) != 0) {
20420         ABORT("pthread_attr_getstack failed");
20421     }
20422     (void)pthread_attr_destroy(&attr);
20423 #ifdef STACK_GROWS_DOWN
20424         b -> mem_base = (char *)(b -> mem_base) + size;
20425 #endif
20426 #ifdef IA64
20427       LOCK();
20428       {
20429         IF_CANCEL(int cancel_state;)
20430         ptr_t bsp;
20431         ptr_t next_stack;
20432         DISABLE_CANCEL(cancel_state);
20433         bsp = GC_save_regs_in_stack();
20434         next_stack = GC_greatest_stack_base_below(bsp);
20435         if (0 == next_stack) {
20436           b -> reg_base = GC_find_limit(bsp, FALSE);
20437         } else {
20438           b -> reg_base = GC_find_limit_with_bound(bsp, FALSE, next_stack);
20439         }
20440         RESTORE_CANCEL(cancel_state);
20441       }
20442       UNLOCK();
20443 #endif
20444     return GC_SUCCESS;
20445   }
20446 #define HAVE_GET_STACK_BASE
20447 #endif
20448 #if defined(GC_DARWIN_THREADS) && !defined(NO_PTHREAD_GET_STACKADDR_NP)
20449 #include <pthread.h>
20450   GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
20451   {
20452     b->mem_base = pthread_get_stackaddr_np(pthread_self());
20453     GC_ASSERT((word)GC_approx_sp() HOTTER_THAN (word)b->mem_base);
20454     return GC_SUCCESS;
20455   }
20456 #define HAVE_GET_STACK_BASE
20457 #endif
20458 #ifdef GC_OPENBSD_THREADS
20459 #include <sys/signal.h>
20460 #include <pthread.h>
20461 #include <pthread_np.h>
20462   GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
20463   {
20464     stack_t stack;
20465     if (pthread_stackseg_np(pthread_self(), &stack))
20466       ABORT("pthread_stackseg_np(self) failed");
20467     sb->mem_base = stack.ss_sp;
20468     return GC_SUCCESS;
20469   }
20470 #define HAVE_GET_STACK_BASE
20471 #endif
20472 #if defined(GC_SOLARIS_THREADS) && !defined(_STRICT_STDC)
20473 #include <thread.h>
20474 #include <signal.h>
20475 #include <pthread.h>
20476   static pthread_t stackbase_main_self = 0;
20477   static void *stackbase_main_ss_sp = NULL;
20478   GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
20479   {
20480     stack_t s;
20481     pthread_t self = pthread_self();
20482     if (self == stackbase_main_self)
20483       {
20484         b -> mem_base = stackbase_main_ss_sp;
20485         GC_ASSERT(b -> mem_base != NULL);
20486         return GC_SUCCESS;
20487       }
20488     if (thr_stksegment(&s)) {
20489       ABORT("thr_stksegment failed");
20490     }
20491     GC_ASSERT((word)GC_approx_sp() HOTTER_THAN (word)s.ss_sp);
20492     if (!stackbase_main_self && thr_main() != 0)
20493       {
20494         stackbase_main_ss_sp = s.ss_sp;
20495         stackbase_main_self = self;
20496       }
20497     b -> mem_base = s.ss_sp;
20498     return GC_SUCCESS;
20499   }
20500 #define HAVE_GET_STACK_BASE
20501 #endif
20502 #ifdef GC_RTEMS_PTHREADS
20503   GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
20504   {
20505     sb->mem_base = rtems_get_stack_bottom();
20506     return GC_SUCCESS;
20507   }
20508 #define HAVE_GET_STACK_BASE
20509 #endif
20510 #ifndef HAVE_GET_STACK_BASE
20511 #ifdef NEED_FIND_LIMIT
20512     GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
20513     {
20514       IF_CANCEL(int cancel_state;)
20515       DCL_LOCK_STATE;
20516       LOCK();
20517       DISABLE_CANCEL(cancel_state);
20518 #ifdef STACK_GROWS_DOWN
20519         b -> mem_base = GC_find_limit(GC_approx_sp(), TRUE);
20520 #ifdef IA64
20521           b -> reg_base = GC_find_limit(GC_save_regs_in_stack(), FALSE);
20522 #endif
20523 #else
20524         b -> mem_base = GC_find_limit(GC_approx_sp(), FALSE);
20525 #endif
20526       RESTORE_CANCEL(cancel_state);
20527       UNLOCK();
20528       return GC_SUCCESS;
20529     }
20530 #else
20531     GC_API int GC_CALL GC_get_stack_base(
20532                                 struct GC_stack_base *b GC_ATTR_UNUSED)
20533     {
20534 #if defined(GET_MAIN_STACKBASE_SPECIAL) && !defined(THREADS) \
20535          && !defined(IA64)
20536         b->mem_base = GC_get_main_stack_base();
20537         return GC_SUCCESS;
20538 #else
20539         return GC_UNIMPLEMENTED;
20540 #endif
20541     }
20542 #endif
20543 #endif
20544 #ifndef GET_MAIN_STACKBASE_SPECIAL
20545   ptr_t GC_get_main_stack_base(void)
20546   {
20547     struct GC_stack_base sb;
20548     if (GC_get_stack_base(&sb) != GC_SUCCESS)
20549       ABORT("GC_get_stack_base failed");
20550     GC_ASSERT((word)GC_approx_sp() HOTTER_THAN (word)sb.mem_base);
20551     return (ptr_t)sb.mem_base;
20552   }
20553 #endif
20554 #ifdef OS2
20555 void GC_register_data_segments(void)
20556 {
20557     PTIB ptib;
20558     PPIB ppib;
20559     HMODULE module_handle;
20560 #define PBUFSIZ 512
20561     UCHAR path[PBUFSIZ];
20562     FILE * myexefile;
20563     struct exe_hdr hdrdos;
20564     struct e32_exe hdr386;
20565     struct o32_obj seg;
20566     int nsegs;
20567 #if defined(CPPCHECK)
20568         hdrdos.padding[0] = 0;
20569         hdr386.exe_format_level = 0;
20570         hdr386.os = 0;
20571         hdr386.padding1[0] = 0;
20572         hdr386.padding2[0] = 0;
20573         seg.pagemap = 0;
20574         seg.mapsize = 0;
20575         seg.reserved = 0;
20576 #endif
20577     if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) {
20578         ABORT("DosGetInfoBlocks failed");
20579     }
20580     module_handle = ppib -> pib_hmte;
20581     if (DosQueryModuleName(module_handle, PBUFSIZ, path) != NO_ERROR) {
20582         ABORT("DosQueryModuleName failed");
20583     }
20584     myexefile = fopen(path, "rb");
20585     if (myexefile == 0) {
20586         ABORT_ARG1("Failed to open executable", ": %s", path);
20587     }
20588     if (fread((char *)(&hdrdos), 1, sizeof(hdrdos), myexefile)
20589           < sizeof(hdrdos)) {
20590         ABORT_ARG1("Could not read MSDOS header", " from: %s", path);
20591     }
20592     if (E_MAGIC(hdrdos) != EMAGIC) {
20593         ABORT_ARG1("Bad DOS magic number", " in file: %s", path);
20594     }
20595     if (fseek(myexefile, E_LFANEW(hdrdos), SEEK_SET) != 0) {
20596         ABORT_ARG1("Bad DOS magic number", " in file: %s", path);
20597     }
20598     if (fread((char *)(&hdr386), 1, sizeof(hdr386), myexefile)
20599           < sizeof(hdr386)) {
20600         ABORT_ARG1("Could not read OS/2 header", " from: %s", path);
20601     }
20602     if (E32_MAGIC1(hdr386) != E32MAGIC1 || E32_MAGIC2(hdr386) != E32MAGIC2) {
20603         ABORT_ARG1("Bad OS/2 magic number", " in file: %s", path);
20604     }
20605     if (E32_BORDER(hdr386) != E32LEBO || E32_WORDER(hdr386) != E32LEWO) {
20606         ABORT_ARG1("Bad byte order in executable", " file: %s", path);
20607     }
20608     if (E32_CPU(hdr386) == E32CPU286) {
20609         ABORT_ARG1("GC cannot handle 80286 executables", ": %s", path);
20610     }
20611     if (fseek(myexefile, E_LFANEW(hdrdos) + E32_OBJTAB(hdr386),
20612               SEEK_SET) != 0) {
20613         ABORT_ARG1("Seek to object table failed", " in file: %s", path);
20614     }
20615     for (nsegs = E32_OBJCNT(hdr386); nsegs > 0; nsegs--) {
20616       int flags;
20617       if (fread((char *)(&seg), 1, sizeof(seg), myexefile) < sizeof(seg)) {
20618         ABORT_ARG1("Could not read obj table entry", " from file: %s", path);
20619       }
20620       flags = O32_FLAGS(seg);
20621       if (!(flags & OBJWRITE)) continue;
20622       if (!(flags & OBJREAD)) continue;
20623       if (flags & OBJINVALID) {
20624           GC_err_printf("Object with invalid pages?\n");
20625           continue;
20626       }
20627       GC_add_roots_inner((ptr_t)O32_BASE(seg),
20628                          (ptr_t)(O32_BASE(seg)+O32_SIZE(seg)), FALSE);
20629     }
20630     (void)fclose(myexefile);
20631 }
20632 #else
20633 #if defined(GWW_VDB)
20634 #ifndef MEM_WRITE_WATCH
20635 #define MEM_WRITE_WATCH 0x200000
20636 #endif
20637 #ifndef WRITE_WATCH_FLAG_RESET
20638 #define WRITE_WATCH_FLAG_RESET 1
20639 #endif
20640 #define GC_ULONG_PTR word
20641     typedef UINT (WINAPI * GetWriteWatch_type)(
20642                                 DWORD, PVOID, GC_ULONG_PTR ,
20643                                 PVOID *, GC_ULONG_PTR *, PULONG);
20644     static FARPROC GetWriteWatch_func;
20645     static DWORD GetWriteWatch_alloc_flag;
20646 #define GC_GWW_AVAILABLE() (GetWriteWatch_func != 0)
20647     static void detect_GetWriteWatch(void)
20648     {
20649       static GC_bool done;
20650       HMODULE hK32;
20651       if (done)
20652         return;
20653 #if defined(MPROTECT_VDB)
20654         {
20655           char * str = GETENV("GC_USE_GETWRITEWATCH");
20656 #if defined(GC_PREFER_MPROTECT_VDB)
20657             if (str == NULL || (*str == '0' && *(str + 1) == '\0')) {
20658               done = TRUE;
20659               return;
20660             }
20661 #else
20662             if (str != NULL && *str == '0' && *(str + 1) == '\0') {
20663               done = TRUE;
20664               return;
20665             }
20666 #endif
20667         }
20668 #endif
20669 #ifdef MSWINRT_FLAVOR
20670         {
20671           MEMORY_BASIC_INFORMATION memInfo;
20672           SIZE_T result = VirtualQuery(GetProcAddress,
20673                                        &memInfo, sizeof(memInfo));
20674           if (result != sizeof(memInfo))
20675             ABORT("Weird VirtualQuery result");
20676           hK32 = (HMODULE)memInfo.AllocationBase;
20677         }
20678 #else
20679         hK32 = GetModuleHandle(TEXT("kernel32.dll"));
20680 #endif
20681       if (hK32 != (HMODULE)0 &&
20682           (GetWriteWatch_func = GetProcAddress(hK32, "GetWriteWatch")) != 0) {
20683         void * page;
20684         GC_ASSERT(GC_page_size != 0);
20685         page = VirtualAlloc(NULL, GC_page_size, MEM_WRITE_WATCH | MEM_RESERVE,
20686                             PAGE_READWRITE);
20687         if (page != NULL) {
20688           PVOID pages[16];
20689           GC_ULONG_PTR count = 16;
20690           DWORD page_size;
20691           if ((*(GetWriteWatch_type)(word)GetWriteWatch_func)(
20692                                         WRITE_WATCH_FLAG_RESET, page,
20693                                         GC_page_size, pages, &count,
20694                                         &page_size) != 0) {
20695             GetWriteWatch_func = 0;
20696           } else {
20697             GetWriteWatch_alloc_flag = MEM_WRITE_WATCH;
20698           }
20699           VirtualFree(page, 0 , MEM_RELEASE);
20700         } else {
20701           GetWriteWatch_func = 0;
20702         }
20703       }
20704       done = TRUE;
20705     }
20706 #else
20707 #define GetWriteWatch_alloc_flag 0
20708 #endif
20709 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
20710 #ifdef MSWIN32
20711   GC_INNER GC_bool GC_no_win32_dlls = FALSE;
20712   GC_INNER GC_bool GC_wnt = FALSE;
20713   GC_INNER void GC_init_win32(void)
20714   {
20715 #if defined(_WIN64) || (defined(_MSC_VER) && _MSC_VER >= 1800)
20716       GC_wnt = TRUE;
20717 #else
20718       DWORD v = GetVersion();
20719       GC_wnt = !(v & 0x80000000);
20720       GC_no_win32_dlls |= ((!GC_wnt) && (v & 0xff) <= 3);
20721 #endif
20722 #ifdef USE_MUNMAP
20723       if (GC_no_win32_dlls) {
20724         GC_unmap_threshold = 0;
20725       }
20726 #endif
20727   }
20728   STATIC ptr_t GC_least_described_address(ptr_t start)
20729   {
20730     MEMORY_BASIC_INFORMATION buf;
20731     LPVOID limit = GC_sysinfo.lpMinimumApplicationAddress;
20732     ptr_t p = (ptr_t)((word)start & ~(GC_page_size - 1));
20733     GC_ASSERT(GC_page_size != 0);
20734     for (;;) {
20735         size_t result;
20736         LPVOID q = (LPVOID)(p - GC_page_size);
20737         if ((word)q > (word)p  || (word)q < (word)limit) break;
20738         result = VirtualQuery(q, &buf, sizeof(buf));
20739         if (result != sizeof(buf) || buf.AllocationBase == 0) break;
20740         p = (ptr_t)(buf.AllocationBase);
20741     }
20742     return p;
20743   }
20744 #endif
20745 #if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
20746   STATIC size_t GC_max_root_size = 100000;
20747   STATIC struct GC_malloc_heap_list {
20748     void * allocation_base;
20749     struct GC_malloc_heap_list *next;
20750   } *GC_malloc_heap_l = 0;
20751   STATIC GC_bool GC_is_malloc_heap_base(void *p)
20752   {
20753     struct GC_malloc_heap_list *q = GC_malloc_heap_l;
20754     while (0 != q) {
20755       if (q -> allocation_base == p) return TRUE;
20756       q = q -> next;
20757     }
20758     return FALSE;
20759   }
20760   STATIC void *GC_get_allocation_base(void *p)
20761   {
20762     MEMORY_BASIC_INFORMATION buf;
20763     size_t result = VirtualQuery(p, &buf, sizeof(buf));
20764     if (result != sizeof(buf)) {
20765       ABORT("Weird VirtualQuery result");
20766     }
20767     return buf.AllocationBase;
20768   }
20769   GC_INNER void GC_add_current_malloc_heap(void)
20770   {
20771     struct GC_malloc_heap_list *new_l = (struct GC_malloc_heap_list *)
20772                  malloc(sizeof(struct GC_malloc_heap_list));
20773     void *candidate;
20774     if (NULL == new_l) return;
20775     candidate = GC_get_allocation_base(new_l);
20776     if (GC_is_malloc_heap_base(candidate)) {
20777         size_t req_size = 10000;
20778         do {
20779           void *p = malloc(req_size);
20780           if (0 == p) {
20781             free(new_l);
20782             return;
20783           }
20784           candidate = GC_get_allocation_base(p);
20785           free(p);
20786           req_size *= 2;
20787         } while (GC_is_malloc_heap_base(candidate)
20788                  && req_size < GC_max_root_size/10 && req_size < 500000);
20789         if (GC_is_malloc_heap_base(candidate)) {
20790           free(new_l);
20791           return;
20792         }
20793     }
20794     GC_COND_LOG_PRINTF("Found new system malloc AllocationBase at %p\n",
20795                        candidate);
20796     new_l -> allocation_base = candidate;
20797     new_l -> next = GC_malloc_heap_l;
20798     GC_malloc_heap_l = new_l;
20799   }
20800   STATIC void GC_free_malloc_heap_list(void)
20801   {
20802     struct GC_malloc_heap_list *q = GC_malloc_heap_l;
20803     GC_malloc_heap_l = NULL;
20804     while (q != NULL) {
20805       struct GC_malloc_heap_list *next = q -> next;
20806       free(q);
20807       q = next;
20808     }
20809   }
20810 #endif
20811   GC_INNER GC_bool GC_is_heap_base(void *p)
20812   {
20813      int i;
20814 #if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
20815        if (GC_root_size > GC_max_root_size)
20816          GC_max_root_size = GC_root_size;
20817        if (GC_is_malloc_heap_base(p))
20818          return TRUE;
20819 #endif
20820      for (i = 0; i < (int)GC_n_heap_bases; i++) {
20821          if (GC_heap_bases[i] == p) return TRUE;
20822      }
20823      return FALSE;
20824   }
20825 #ifdef MSWIN32
20826   STATIC void GC_register_root_section(ptr_t static_root)
20827   {
20828       MEMORY_BASIC_INFORMATION buf;
20829       LPVOID p;
20830       char * base;
20831       char * limit;
20832       if (!GC_no_win32_dlls) return;
20833       p = base = limit = GC_least_described_address(static_root);
20834       while ((word)p < (word)GC_sysinfo.lpMaximumApplicationAddress) {
20835         size_t result = VirtualQuery(p, &buf, sizeof(buf));
20836         char * new_limit;
20837         DWORD protect;
20838         if (result != sizeof(buf) || buf.AllocationBase == 0
20839             || GC_is_heap_base(buf.AllocationBase)) break;
20840         new_limit = (char *)p + buf.RegionSize;
20841         protect = buf.Protect;
20842         if (buf.State == MEM_COMMIT
20843             && is_writable(protect)) {
20844             if ((char *)p == limit) {
20845                 limit = new_limit;
20846             } else {
20847                 if (base != limit) GC_add_roots_inner(base, limit, FALSE);
20848                 base = (char *)p;
20849                 limit = new_limit;
20850             }
20851         }
20852         if ((word)p > (word)new_limit ) break;
20853         p = (LPVOID)new_limit;
20854       }
20855       if (base != limit) GC_add_roots_inner(base, limit, FALSE);
20856   }
20857 #endif
20858   void GC_register_data_segments(void)
20859   {
20860 #ifdef MSWIN32
20861       GC_register_root_section((ptr_t)&GC_pages_executable);
20862 #endif
20863   }
20864 #else
20865 #if (defined(SVR4) || defined(AIX) || defined(DGUX) \
20866       || (defined(LINUX) && defined(SPARC))) && !defined(PCR)
20867   ptr_t GC_SysVGetDataStart(size_t max_page_size, ptr_t etext_addr)
20868   {
20869     word text_end = ((word)(etext_addr) + sizeof(word) - 1)
20870                     & ~(word)(sizeof(word) - 1);
20871     word next_page = ((text_end + (word)max_page_size - 1)
20872                       & ~((word)max_page_size - 1));
20873     word page_offset = (text_end & ((word)max_page_size - 1));
20874     volatile ptr_t result = (char *)(next_page + page_offset);
20875     GC_setup_temporary_fault_handler();
20876     if (SETJMP(GC_jmp_buf) == 0) {
20877 #ifdef AO_HAVE_fetch_and_add
20878           volatile AO_t zero = 0;
20879           (void)AO_fetch_and_add((volatile AO_t *)result, zero);
20880 #else
20881           char v = *result;
20882 #if defined(CPPCHECK)
20883             GC_noop1((word)&v);
20884 #endif
20885           *result = v;
20886 #endif
20887         GC_reset_fault_handler();
20888     } else {
20889         GC_reset_fault_handler();
20890         result = (char *)GC_find_limit(DATAEND, FALSE);
20891     }
20892     return ( ptr_t)result;
20893   }
20894 #endif
20895 #ifdef DATASTART_USES_BSDGETDATASTART
20896   GC_INNER ptr_t GC_FreeBSDGetDataStart(size_t max_page_size,
20897                                         ptr_t etext_addr)
20898   {
20899     word text_end = ((word)(etext_addr) + sizeof(word) - 1)
20900                      & ~(word)(sizeof(word) - 1);
20901     volatile word next_page = (text_end + (word)max_page_size - 1)
20902                               & ~((word)max_page_size - 1);
20903     volatile ptr_t result = (ptr_t)text_end;
20904     GC_setup_temporary_fault_handler();
20905     if (SETJMP(GC_jmp_buf) == 0) {
20906         for (; next_page < (word)DATAEND; next_page += (word)max_page_size)
20907             *(volatile char *)next_page;
20908         GC_reset_fault_handler();
20909     } else {
20910         GC_reset_fault_handler();
20911         result = (ptr_t)GC_find_limit(DATAEND, FALSE);
20912     }
20913     return(result);
20914   }
20915 #endif
20916 #ifdef AMIGA
20917 #define GC_AMIGA_DS
20918 #undef GC_AMIGA_DS
20919 #elif defined(OPENBSD)
20920 void GC_register_data_segments(void)
20921 {
20922   ptr_t region_start = DATASTART;
20923   if ((word)region_start - 1U >= (word)DATAEND)
20924     ABORT_ARG2("Wrong DATASTART/END pair",
20925                ": %p .. %p", (void *)region_start, (void *)DATAEND);
20926   for (;;) {
20927 #ifdef GC_OPENBSD_UTHREADS
20928       ptr_t region_end = GC_find_limit_openbsd(region_start, DATAEND);
20929 #else
20930       ptr_t region_end = GC_find_limit_with_bound(region_start, TRUE, DATAEND);
20931 #endif
20932     GC_add_roots_inner(region_start, region_end, FALSE);
20933     if ((word)region_end >= (word)DATAEND)
20934       break;
20935     region_start = GC_skip_hole_openbsd(region_end, DATAEND);
20936   }
20937 }
20938 #else
20939 #if !defined(PCR) && !defined(MACOS) && defined(REDIRECT_MALLOC) \
20940      && defined(GC_SOLARIS_THREADS)
20941     EXTERN_C_BEGIN
20942     extern caddr_t sbrk(int);
20943     EXTERN_C_END
20944 #endif
20945   void GC_register_data_segments(void)
20946   {
20947 #if !defined(PCR) && !defined(MACOS)
20948 #if defined(REDIRECT_MALLOC) && defined(GC_SOLARIS_THREADS)
20949         GC_ASSERT(DATASTART);
20950         {
20951           ptr_t p = (ptr_t)sbrk(0);
20952           if ((word)DATASTART < (word)p)
20953             GC_add_roots_inner(DATASTART, p, FALSE);
20954         }
20955 #else
20956         if ((word)DATASTART - 1U >= (word)DATAEND) {
20957           ABORT_ARG2("Wrong DATASTART/END pair",
20958                      ": %p .. %p", (void *)DATASTART, (void *)DATAEND);
20959         }
20960         GC_add_roots_inner(DATASTART, DATAEND, FALSE);
20961 #ifdef GC_HAVE_DATAREGION2
20962           if ((word)DATASTART2 - 1U >= (word)DATAEND2)
20963             ABORT_ARG2("Wrong DATASTART/END2 pair",
20964                        ": %p .. %p", (void *)DATASTART2, (void *)DATAEND2);
20965           GC_add_roots_inner(DATASTART2, DATAEND2, FALSE);
20966 #endif
20967 #endif
20968 #endif
20969 #if defined(MACOS)
20970     {
20971 #if defined(THINK_C)
20972         extern void* GC_MacGetDataStart(void);
20973         GC_add_roots_inner((ptr_t)GC_MacGetDataStart(),
20974                            (ptr_t)LMGetCurrentA5(), FALSE);
20975 #else
20976 #if defined(__MWERKS__)
20977 #if !__POWERPC__
20978           extern void* GC_MacGetDataStart(void);
20979 #if __option(far_data)
20980           extern void* GC_MacGetDataEnd(void);
20981 #endif
20982           GC_add_roots_inner((ptr_t)GC_MacGetDataStart(),
20983                              (ptr_t)LMGetCurrentA5(), FALSE);
20984 #if __option(far_data)
20985           GC_add_roots_inner((ptr_t)LMGetCurrentA5(),
20986                              (ptr_t)GC_MacGetDataEnd(), FALSE);
20987 #endif
20988 #else
20989           extern char __data_start__[], __data_end__[];
20990           GC_add_roots_inner((ptr_t)&__data_start__,
20991                              (ptr_t)&__data_end__, FALSE);
20992 #endif
20993 #endif
20994 #endif
20995     }
20996 #endif
20997   }
20998 #endif
20999 #endif
21000 #endif
21001 #if !defined(OS2) && !defined(PCR) && !defined(AMIGA) \
21002      && !defined(USE_WINALLOC) && !defined(MACOS) && !defined(DOS4GW) \
21003      && !defined(NINTENDO_SWITCH) && !defined(NONSTOP) \
21004      && !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PS3) \
21005      && !defined(SN_TARGET_PSP2) && !defined(RTEMS) && !defined(__CC_ARM)
21006 #define SBRK_ARG_T ptrdiff_t
21007 #if defined(MMAP_SUPPORTED)
21008 #ifdef USE_MMAP_FIXED
21009 #define GC_MMAP_FLAGS MAP_FIXED | MAP_PRIVATE
21010 #else
21011 #define GC_MMAP_FLAGS MAP_PRIVATE
21012 #endif
21013 #ifdef USE_MMAP_ANON
21014 #define zero_fd -1
21015 #if defined(MAP_ANONYMOUS) && !defined(CPPCHECK)
21016 #define OPT_MAP_ANON MAP_ANONYMOUS
21017 #else
21018 #define OPT_MAP_ANON MAP_ANON
21019 #endif
21020 #else
21021   static int zero_fd = -1;
21022 #define OPT_MAP_ANON 0
21023 #endif
21024 #ifndef MSWIN_XBOX1
21025 #if defined(SYMBIAN) && !defined(USE_MMAP_ANON)
21026       EXTERN_C_BEGIN
21027       extern char *GC_get_private_path_and_zero_file(void);
21028       EXTERN_C_END
21029 #endif
21030   STATIC ptr_t GC_unix_mmap_get_mem(size_t bytes)
21031   {
21032     void *result;
21033     static ptr_t last_addr = HEAP_START;
21034 #ifndef USE_MMAP_ANON
21035       static GC_bool initialized = FALSE;
21036       if (!EXPECT(initialized, TRUE)) {
21037 #ifdef SYMBIAN
21038           char *path = GC_get_private_path_and_zero_file();
21039           if (path != NULL) {
21040             zero_fd = open(path, O_RDWR | O_CREAT, 0644);
21041             free(path);
21042           }
21043 #else
21044           zero_fd = open("/dev/zero", O_RDONLY);
21045 #endif
21046           if (zero_fd == -1)
21047             ABORT("Could not open /dev/zero");
21048           if (fcntl(zero_fd, F_SETFD, FD_CLOEXEC) == -1)
21049             WARN("Could not set FD_CLOEXEC for /dev/zero\n", 0);
21050           initialized = TRUE;
21051       }
21052 #endif
21053     GC_ASSERT(GC_page_size != 0);
21054     if (bytes & (GC_page_size - 1)) ABORT("Bad GET_MEM arg");
21055     result = mmap(last_addr, bytes, (PROT_READ | PROT_WRITE)
21056                                     | (GC_pages_executable ? PROT_EXEC : 0),
21057                   GC_MMAP_FLAGS | OPT_MAP_ANON, zero_fd, 0);
21058 #undef IGNORE_PAGES_EXECUTABLE
21059     if (EXPECT(MAP_FAILED == result, FALSE)) {
21060       if (HEAP_START == last_addr && GC_pages_executable && EACCES == errno)
21061         ABORT("Cannot allocate executable pages");
21062       return NULL;
21063     }
21064     last_addr = (ptr_t)(((word)result + bytes + GC_page_size - 1)
21065                         & ~(GC_page_size - 1));
21066 #if !defined(LINUX)
21067       if (last_addr == 0) {
21068         munmap(result, ~GC_page_size - (size_t)result + 1);
21069         return GC_unix_mmap_get_mem(bytes);
21070       }
21071 #else
21072       GC_ASSERT(last_addr != 0);
21073 #endif
21074     if (((word)result % HBLKSIZE) != 0)
21075       ABORT(
21076        "GC_unix_get_mem: Memory returned by mmap is not aligned to HBLKSIZE.");
21077     return((ptr_t)result);
21078   }
21079 #endif
21080 #endif
21081 #if defined(USE_MMAP)
21082   ptr_t GC_unix_get_mem(size_t bytes)
21083   {
21084     return GC_unix_mmap_get_mem(bytes);
21085   }
21086 #else
21087 STATIC ptr_t GC_unix_sbrk_get_mem(size_t bytes)
21088 {
21089   ptr_t result;
21090 #ifdef IRIX5
21091     __LOCK_MALLOC();
21092 #endif
21093   {
21094     ptr_t cur_brk = (ptr_t)sbrk(0);
21095     SBRK_ARG_T lsbs = (word)cur_brk & (GC_page_size-1);
21096     GC_ASSERT(GC_page_size != 0);
21097     if ((SBRK_ARG_T)bytes < 0) {
21098         result = 0;
21099         goto out;
21100     }
21101     if (lsbs != 0) {
21102         if((ptr_t)sbrk((SBRK_ARG_T)GC_page_size - lsbs) == (ptr_t)(-1)) {
21103             result = 0;
21104             goto out;
21105         }
21106     }
21107 #ifdef ADD_HEAP_GUARD_PAGES
21108       {
21109         ptr_t guard = (ptr_t)sbrk((SBRK_ARG_T)GC_page_size);
21110         if (mprotect(guard, GC_page_size, PROT_NONE) != 0)
21111             ABORT("ADD_HEAP_GUARD_PAGES: mprotect failed");
21112       }
21113 #endif
21114     result = (ptr_t)sbrk((SBRK_ARG_T)bytes);
21115     if (result == (ptr_t)(-1)) result = 0;
21116   }
21117  out:
21118 #ifdef IRIX5
21119     __UNLOCK_MALLOC();
21120 #endif
21121   return(result);
21122 }
21123 ptr_t GC_unix_get_mem(size_t bytes)
21124 {
21125 #if defined(MMAP_SUPPORTED)
21126     static GC_bool sbrk_failed = FALSE;
21127     ptr_t result = 0;
21128     if (GC_pages_executable) {
21129         return GC_unix_mmap_get_mem(bytes);
21130     }
21131     if (!sbrk_failed) result = GC_unix_sbrk_get_mem(bytes);
21132     if (0 == result) {
21133         sbrk_failed = TRUE;
21134         result = GC_unix_mmap_get_mem(bytes);
21135     }
21136     if (0 == result) {
21137         result = GC_unix_sbrk_get_mem(bytes);
21138     }
21139     return result;
21140 #else
21141     return GC_unix_sbrk_get_mem(bytes);
21142 #endif
21143 }
21144 #endif
21145 #endif
21146 #ifdef OS2
21147 void * os2_alloc(size_t bytes)
21148 {
21149     void * result;
21150     if (DosAllocMem(&result, bytes, (PAG_READ | PAG_WRITE | PAG_COMMIT)
21151                                     | (GC_pages_executable ? PAG_EXECUTE : 0))
21152                     != NO_ERROR) {
21153         return(0);
21154     }
21155     if (result == 0) return(os2_alloc(bytes));
21156     return(result);
21157 }
21158 #endif
21159 #ifdef MSWIN_XBOX1
21160     ptr_t GC_durango_get_mem(size_t bytes)
21161     {
21162       if (0 == bytes) return NULL;
21163       return (ptr_t)VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_TOP_DOWN,
21164                                  PAGE_READWRITE);
21165     }
21166 #elif defined(MSWINCE)
21167   ptr_t GC_wince_get_mem(size_t bytes)
21168   {
21169     ptr_t result = 0;
21170     word i;
21171     GC_ASSERT(GC_page_size != 0);
21172     bytes = ROUNDUP_PAGESIZE(bytes);
21173     for (i = 0; i < GC_n_heap_bases; i++) {
21174         if (((word)(-(signed_word)GC_heap_lengths[i])
21175              & (GC_sysinfo.dwAllocationGranularity-1))
21176             >= bytes) {
21177             result = GC_heap_bases[i] + GC_heap_lengths[i];
21178             break;
21179         }
21180     }
21181     if (i == GC_n_heap_bases) {
21182         size_t res_bytes =
21183             SIZET_SAT_ADD(bytes, (size_t)GC_sysinfo.dwAllocationGranularity-1)
21184             & ~((size_t)GC_sysinfo.dwAllocationGranularity-1);
21185         result = (ptr_t) VirtualAlloc(NULL, res_bytes,
21186                                 MEM_RESERVE | MEM_TOP_DOWN,
21187                                 GC_pages_executable ? PAGE_EXECUTE_READWRITE :
21188                                                       PAGE_READWRITE);
21189         if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result");
21190         if (GC_n_heap_bases >= MAX_HEAP_SECTS) ABORT("Too many heap sections");
21191         if (result == NULL) return NULL;
21192         GC_heap_bases[GC_n_heap_bases] = result;
21193         GC_heap_lengths[GC_n_heap_bases] = 0;
21194         GC_n_heap_bases++;
21195     }
21196     result = (ptr_t) VirtualAlloc(result, bytes, MEM_COMMIT,
21197                               GC_pages_executable ? PAGE_EXECUTE_READWRITE :
21198                                                     PAGE_READWRITE);
21199 #undef IGNORE_PAGES_EXECUTABLE
21200     if (result != NULL) {
21201         if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result");
21202         GC_heap_lengths[i] += bytes;
21203     }
21204     return(result);
21205   }
21206 #elif (defined(USE_WINALLOC) && !defined(MSWIN_XBOX1)) || defined(CYGWIN32)
21207 #ifdef USE_GLOBAL_ALLOC
21208 #define GLOBAL_ALLOC_TEST 1
21209 #else
21210 #define GLOBAL_ALLOC_TEST GC_no_win32_dlls
21211 #endif
21212 #if (defined(GC_USE_MEM_TOP_DOWN) && defined(USE_WINALLOC)) \
21213      || defined(CPPCHECK)
21214     DWORD GC_mem_top_down = MEM_TOP_DOWN;
21215 #else
21216 #define GC_mem_top_down 0
21217 #endif
21218   ptr_t GC_win32_get_mem(size_t bytes)
21219   {
21220     ptr_t result;
21221 #ifndef USE_WINALLOC
21222     result = GC_unix_get_mem(bytes);
21223 #else
21224 #if defined(MSWIN32) && !defined(MSWINRT_FLAVOR)
21225       if (GLOBAL_ALLOC_TEST) {
21226         result = (ptr_t)GlobalAlloc(0, SIZET_SAT_ADD(bytes, HBLKSIZE));
21227         result = (ptr_t)(((word)result + HBLKSIZE - 1)
21228                          & ~(word)(HBLKSIZE - 1));
21229       } else
21230 #endif
21231      {
21232 #ifdef MPROTECT_VDB
21233 #ifdef GWW_VDB
21234 #define VIRTUAL_ALLOC_PAD (GC_GWW_AVAILABLE() ? 0 : 1)
21235 #else
21236 #define VIRTUAL_ALLOC_PAD 1
21237 #endif
21238 #else
21239 #define VIRTUAL_ALLOC_PAD 0
21240 #endif
21241         result = (ptr_t) VirtualAlloc(NULL,
21242                             SIZET_SAT_ADD(bytes, VIRTUAL_ALLOC_PAD),
21243                             GetWriteWatch_alloc_flag
21244                                 | (MEM_COMMIT | MEM_RESERVE)
21245                                 | GC_mem_top_down,
21246                             GC_pages_executable ? PAGE_EXECUTE_READWRITE :
21247                                                   PAGE_READWRITE);
21248 #undef IGNORE_PAGES_EXECUTABLE
21249     }
21250 #endif
21251     if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result");
21252     if (GC_n_heap_bases >= MAX_HEAP_SECTS) ABORT("Too many heap sections");
21253     if (0 != result) GC_heap_bases[GC_n_heap_bases++] = result;
21254     return(result);
21255   }
21256 #endif
21257 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) \
21258     || defined(MSWIN_XBOX1)
21259   GC_API void GC_CALL GC_win32_free_heap(void)
21260   {
21261 #if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC) \
21262        && !defined(MSWIN_XBOX1)
21263       GC_free_malloc_heap_list();
21264 #endif
21265 #if (defined(USE_WINALLOC) && !defined(MSWIN_XBOX1) \
21266         && !defined(MSWINCE)) || defined(CYGWIN32)
21267 #ifndef MSWINRT_FLAVOR
21268 #ifndef CYGWIN32
21269           if (GLOBAL_ALLOC_TEST)
21270 #endif
21271         {
21272           while (GC_n_heap_bases-- > 0) {
21273 #ifdef CYGWIN32
21274 #else
21275               GlobalFree(GC_heap_bases[GC_n_heap_bases]);
21276 #endif
21277             GC_heap_bases[GC_n_heap_bases] = 0;
21278           }
21279           return;
21280         }
21281 #endif
21282 #ifndef CYGWIN32
21283         while (GC_n_heap_bases > 0) {
21284           VirtualFree(GC_heap_bases[--GC_n_heap_bases], 0, MEM_RELEASE);
21285           GC_heap_bases[GC_n_heap_bases] = 0;
21286         }
21287 #endif
21288 #endif
21289   }
21290 #endif
21291 #ifdef AMIGA
21292 #define GC_AMIGA_AM
21293 #undef GC_AMIGA_AM
21294 #endif
21295 #if defined(HAIKU)
21296 #include <stdlib.h>
21297   ptr_t GC_haiku_get_mem(size_t bytes)
21298   {
21299     void* mem;
21300     GC_ASSERT(GC_page_size != 0);
21301     if (posix_memalign(&mem, GC_page_size, bytes) == 0)
21302       return mem;
21303     return NULL;
21304   }
21305 #endif
21306 #if (defined(USE_MUNMAP) || defined(MPROTECT_VDB)) && !defined(USE_WINALLOC)
21307 #define ABORT_ON_REMAP_FAIL(C_msg_prefix, start_addr, len) \
21308         ABORT_ARG3(C_msg_prefix " failed", \
21309                    " at %p (length %lu), errno= %d", \
21310                    (void *)(start_addr), (unsigned long)(len), errno)
21311 #endif
21312 #ifdef USE_MUNMAP
21313 #if !defined(NN_PLATFORM_CTR) && !defined(MSWIN32) && !defined(MSWINCE) \
21314     && !defined(MSWIN_XBOX1)
21315 #include <unistd.h>
21316 #ifdef SN_TARGET_PS3
21317 #include <sys/memory.h>
21318 #else
21319 #include <sys/mman.h>
21320 #endif
21321 #include <sys/stat.h>
21322 #include <sys/types.h>
21323 #endif
21324 STATIC ptr_t GC_unmap_start(ptr_t start, size_t bytes)
21325 {
21326     ptr_t result = (ptr_t)(((word)start + GC_page_size - 1)
21327                             & ~(GC_page_size - 1));
21328     GC_ASSERT(GC_page_size != 0);
21329     if ((word)(result + GC_page_size) > (word)(start + bytes)) return 0;
21330     return result;
21331 }
21332 static void block_unmap_inner(ptr_t start_addr, size_t len)
21333 {
21334     if (0 == start_addr) return;
21335 #ifdef USE_WINALLOC
21336       while (len != 0) {
21337           MEMORY_BASIC_INFORMATION mem_info;
21338           word free_len;
21339           if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info))
21340               != sizeof(mem_info))
21341               ABORT("Weird VirtualQuery result");
21342           free_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize;
21343           if (!VirtualFree(start_addr, free_len, MEM_DECOMMIT))
21344               ABORT("VirtualFree failed");
21345           GC_unmapped_bytes += free_len;
21346           start_addr += free_len;
21347           len -= free_len;
21348       }
21349 #elif defined(SN_TARGET_PS3)
21350       ps3_free_mem(start_addr, len);
21351 #else
21352       if (len != 0) {
21353 #if defined(AIX) || defined(CYGWIN32) || defined(HAIKU) \
21354            || (defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE)) \
21355            || defined(HPUX)
21356           if (mprotect(start_addr, len, PROT_NONE))
21357             ABORT_ON_REMAP_FAIL("unmap: mprotect", start_addr, len);
21358 #elif defined(EMSCRIPTEN)
21359 #else
21360           void * result = mmap(start_addr, len, PROT_NONE,
21361                                MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON,
21362                                zero_fd, 0);
21363           if (EXPECT(MAP_FAILED == result, FALSE))
21364             ABORT_ON_REMAP_FAIL("unmap: mmap", start_addr, len);
21365           if (result != (void *)start_addr)
21366             ABORT("unmap: mmap() result differs from start_addr");
21367 #if defined(CPPCHECK) || defined(LINT2)
21368             GC_noop1((word)result);
21369 #endif
21370 #endif
21371         GC_unmapped_bytes += len;
21372       }
21373 #endif
21374 }
21375 GC_INNER void GC_unmap(ptr_t start, size_t bytes)
21376 {
21377     ptr_t start_addr = GC_unmap_start(start, bytes);
21378     ptr_t end_addr = GC_unmap_end(start, bytes);
21379     block_unmap_inner(start_addr, (size_t)(end_addr - start_addr));
21380 }
21381 GC_INNER void GC_remap(ptr_t start, size_t bytes)
21382 {
21383     ptr_t start_addr = GC_unmap_start(start, bytes);
21384     ptr_t end_addr = GC_unmap_end(start, bytes);
21385     word len = end_addr - start_addr;
21386     if (0 == start_addr) return;
21387 #ifdef USE_WINALLOC
21388       while (len != 0) {
21389           MEMORY_BASIC_INFORMATION mem_info;
21390           word alloc_len;
21391           ptr_t result;
21392           if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info))
21393               != sizeof(mem_info))
21394               ABORT("Weird VirtualQuery result");
21395           alloc_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize;
21396           result = (ptr_t)VirtualAlloc(start_addr, alloc_len, MEM_COMMIT,
21397                                        GC_pages_executable
21398                                                 ? PAGE_EXECUTE_READWRITE
21399                                                 : PAGE_READWRITE);
21400           if (result != start_addr) {
21401               if (GetLastError() == ERROR_NOT_ENOUGH_MEMORY ||
21402                   GetLastError() == ERROR_OUTOFMEMORY) {
21403                   ABORT("Not enough memory to process remapping");
21404               } else {
21405                   ABORT("VirtualAlloc remapping failed");
21406               }
21407           }
21408 #ifdef LINT2
21409             GC_noop1((word)result);
21410 #endif
21411           GC_unmapped_bytes -= alloc_len;
21412           start_addr += alloc_len;
21413           len -= alloc_len;
21414       }
21415 #else
21416       {
21417 #if defined(NACL) || defined(NETBSD)
21418           void *result = mmap(start_addr, len, (PROT_READ | PROT_WRITE)
21419                                     | (GC_pages_executable ? PROT_EXEC : 0),
21420                                    MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON,
21421                                    zero_fd, 0 );
21422           if (EXPECT(MAP_FAILED == result, FALSE))
21423             ABORT_ON_REMAP_FAIL("remap: mmap", start_addr, len);
21424           if (result != (void *)start_addr)
21425             ABORT("remap: mmap() result differs from start_addr");
21426 #if defined(CPPCHECK) || defined(LINT2)
21427             GC_noop1((word)result);
21428 #endif
21429 #else
21430           if (mprotect(start_addr, len, (PROT_READ | PROT_WRITE)
21431                             | (GC_pages_executable ? PROT_EXEC : 0)))
21432             ABORT_ON_REMAP_FAIL("remap: mprotect", start_addr, len);
21433 #endif
21434       }
21435 #undef IGNORE_PAGES_EXECUTABLE
21436       GC_unmapped_bytes -= len;
21437 #endif
21438 }
21439 GC_INNER void GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2,
21440                            size_t bytes2)
21441 {
21442     ptr_t start1_addr = GC_unmap_start(start1, bytes1);
21443     ptr_t end1_addr = GC_unmap_end(start1, bytes1);
21444     ptr_t start2_addr = GC_unmap_start(start2, bytes2);
21445     ptr_t start_addr = end1_addr;
21446     ptr_t end_addr = start2_addr;
21447     GC_ASSERT(start1 + bytes1 == start2);
21448     if (0 == start1_addr) start_addr = GC_unmap_start(start1, bytes1 + bytes2);
21449     if (0 == start2_addr) end_addr = GC_unmap_end(start1, bytes1 + bytes2);
21450     block_unmap_inner(start_addr, (size_t)(end_addr - start_addr));
21451 }
21452 #endif
21453 #ifndef THREADS
21454 #ifdef EMSCRIPTEN
21455     static void scan_regs_cb(void *begin, void *end)
21456     {
21457       GC_push_all_stack((ptr_t)begin, (ptr_t)end);
21458     }
21459     STATIC void GC_CALLBACK GC_default_push_other_roots(void)
21460     {
21461       emscripten_scan_registers(scan_regs_cb);
21462     }
21463 #else
21464 #define GC_default_push_other_roots 0
21465 #endif
21466 #else
21467 #ifdef PCR
21468 PCR_ERes GC_push_thread_stack(PCR_Th_T *t, PCR_Any dummy)
21469 {
21470     struct PCR_ThCtl_TInfoRep info;
21471     PCR_ERes result;
21472     info.ti_stkLow = info.ti_stkHi = 0;
21473     result = PCR_ThCtl_GetInfo(t, &info);
21474     GC_push_all_stack((ptr_t)(info.ti_stkLow), (ptr_t)(info.ti_stkHi));
21475     return(result);
21476 }
21477 PCR_ERes GC_push_old_obj(void *p, size_t size, PCR_Any data)
21478 {
21479     GC_push_all_stack((ptr_t)p, (ptr_t)p + size);
21480     return(PCR_ERes_okay);
21481 }
21482 extern struct PCR_MM_ProcsRep * GC_old_allocator;
21483 STATIC void GC_CALLBACK GC_default_push_other_roots(void)
21484 {
21485           if ((*(GC_old_allocator->mmp_enumerate))(PCR_Bool_false,
21486                                                    GC_push_old_obj, 0)
21487               != PCR_ERes_okay) {
21488               ABORT("Old object enumeration failed");
21489           }
21490         if (PCR_ERes_IsErr(
21491                 PCR_ThCtl_ApplyToAllOtherThreads(GC_push_thread_stack,0))
21492             || PCR_ERes_IsErr(GC_push_thread_stack(PCR_Th_CurrThread(), 0))) {
21493           ABORT("Thread stack marking failed");
21494         }
21495 }
21496 #elif defined(SN_TARGET_PS3)
21497     STATIC void GC_CALLBACK GC_default_push_other_roots(void)
21498     {
21499       ABORT("GC_default_push_other_roots is not implemented");
21500     }
21501     void GC_push_thread_structures(void)
21502     {
21503       ABORT("GC_push_thread_structures is not implemented");
21504     }
21505 #else
21506     STATIC void GC_CALLBACK GC_default_push_other_roots(void)
21507     {
21508       GC_push_all_stacks();
21509     }
21510 #endif
21511 #endif
21512 GC_push_other_roots_proc GC_push_other_roots = GC_default_push_other_roots;
21513 GC_API void GC_CALL GC_set_push_other_roots(GC_push_other_roots_proc fn)
21514 {
21515     GC_push_other_roots = fn;
21516 }
21517 GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void)
21518 {
21519     return GC_push_other_roots;
21520 }
21521 #if defined(SOFT_VDB) && !defined(NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK) \
21522     || (defined(GLIBC_2_19_TSX_BUG) && defined(THREADS))
21523   GC_INNER int GC_parse_version(int *pminor, const char *pverstr) {
21524     char *endp;
21525     unsigned long value = strtoul(pverstr, &endp, 10);
21526     int major = (int)value;
21527     if (major < 0 || (char *)pverstr == endp || (unsigned)major != value) {
21528       return -1;
21529     }
21530     if (*endp != '.') {
21531       *pminor = -1;
21532     } else {
21533       value = strtoul(endp + 1, &endp, 10);
21534       *pminor = (int)value;
21535       if (*pminor < 0 || (unsigned)(*pminor) != value) {
21536         return -1;
21537       }
21538     }
21539     return major;
21540   }
21541 #endif
21542 #if (defined(CHECKSUMS) && (defined(GWW_VDB) || defined(SOFT_VDB))) \
21543     || defined(PROC_VDB)
21544     STATIC void GC_or_pages(page_hash_table pht1, page_hash_table pht2)
21545     {
21546       unsigned i;
21547       for (i = 0; i < PHT_SIZE; i++) pht1[i] |= pht2[i];
21548     }
21549 #endif
21550 #ifdef GWW_VDB
21551 #define GC_GWW_BUF_LEN (MAXHINCR * HBLKSIZE / 4096 )
21552   static PVOID gww_buf[GC_GWW_BUF_LEN];
21553 #ifndef MPROTECT_VDB
21554 #define GC_gww_dirty_init GC_dirty_init
21555 #endif
21556     GC_INNER GC_bool GC_gww_dirty_init(void)
21557     {
21558       detect_GetWriteWatch();
21559       return GC_GWW_AVAILABLE();
21560     }
21561   GC_INLINE void GC_gww_read_dirty(GC_bool output_unneeded)
21562   {
21563     word i;
21564     if (!output_unneeded)
21565       BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
21566     for (i = 0; i != GC_n_heap_sects; ++i) {
21567       GC_ULONG_PTR count;
21568       do {
21569         PVOID * pages = gww_buf;
21570         DWORD page_size;
21571         count = GC_GWW_BUF_LEN;
21572         if ((*(GetWriteWatch_type)(word)GetWriteWatch_func)(
21573                                         WRITE_WATCH_FLAG_RESET,
21574                                         GC_heap_sects[i].hs_start,
21575                                         GC_heap_sects[i].hs_bytes,
21576                                         pages, &count, &page_size) != 0) {
21577           static int warn_count = 0;
21578           struct hblk * start = (struct hblk *)GC_heap_sects[i].hs_start;
21579           static struct hblk *last_warned = 0;
21580           size_t nblocks = divHBLKSZ(GC_heap_sects[i].hs_bytes);
21581           if (i != 0 && last_warned != start && warn_count++ < 5) {
21582             last_warned = start;
21583             WARN("GC_gww_read_dirty unexpectedly failed at %p: "
21584                  "Falling back to marking all pages dirty\n", start);
21585           }
21586           if (!output_unneeded) {
21587             unsigned j;
21588             for (j = 0; j < nblocks; ++j) {
21589               word hash = PHT_HASH(start + j);
21590               set_pht_entry_from_index(GC_grungy_pages, hash);
21591             }
21592           }
21593           count = 1;
21594         } else  if (!output_unneeded) {
21595           PVOID * pages_end = pages + count;
21596           while (pages != pages_end) {
21597             struct hblk * h = (struct hblk *) *pages++;
21598             struct hblk * h_end = (struct hblk *) ((char *) h + page_size);
21599             do {
21600               set_pht_entry_from_index(GC_grungy_pages, PHT_HASH(h));
21601             } while ((word)(++h) < (word)h_end);
21602           }
21603         }
21604       } while (count == GC_GWW_BUF_LEN);
21605     }
21606 #ifdef CHECKSUMS
21607       GC_ASSERT(!output_unneeded);
21608       GC_or_pages(GC_written_pages, GC_grungy_pages);
21609 #endif
21610   }
21611 #elif defined(SOFT_VDB)
21612   static int clear_refs_fd = -1;
21613 #define GC_GWW_AVAILABLE() (clear_refs_fd != -1)
21614 #else
21615 #define GC_GWW_AVAILABLE() FALSE
21616 #endif
21617 #ifdef DEFAULT_VDB
21618   GC_INNER GC_bool GC_dirty_init(void)
21619   {
21620     GC_VERBOSE_LOG_PRINTF("Initializing DEFAULT_VDB...\n");
21621     return TRUE;
21622   }
21623 #endif
21624 #ifndef GC_DISABLE_INCREMENTAL
21625 #if !defined(THREADS) || defined(HAVE_LOCKFREE_AO_OR)
21626 #define async_set_pht_entry_from_index(db, index) \
21627                         set_pht_entry_from_index_concurrent(db, index)
21628 #elif defined(AO_HAVE_test_and_set_acquire)
21629     GC_INNER volatile AO_TS_t GC_fault_handler_lock = AO_TS_INITIALIZER;
21630     static void async_set_pht_entry_from_index(volatile page_hash_table db,
21631                                                size_t index)
21632     {
21633       GC_acquire_dirty_lock();
21634       set_pht_entry_from_index(db, index);
21635       GC_release_dirty_lock();
21636     }
21637 #else
21638 #error No test_and_set operation: Introduces a race.
21639 #endif
21640 #endif
21641 #ifdef MPROTECT_VDB
21642 #ifdef DARWIN
21643 #include <mach/vm_map.h>
21644     STATIC mach_port_t GC_task_self = 0;
21645 #define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
21646         if (vm_protect(GC_task_self, (vm_address_t)(addr), (vm_size_t)(len), \
21647                        FALSE, VM_PROT_READ \
21648                               | ((allow_write) ? VM_PROT_WRITE : 0) \
21649                               | (GC_pages_executable ? VM_PROT_EXECUTE : 0)) \
21650                 == KERN_SUCCESS) {} else ABORT(C_msg_prefix \
21651                                                "vm_protect() failed")
21652 #elif !defined(USE_WINALLOC)
21653 #include <sys/mman.h>
21654 #include <signal.h>
21655 #if !defined(CYGWIN32) && !defined(HAIKU)
21656 #include <sys/syscall.h>
21657 #endif
21658 #define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
21659         if (mprotect((caddr_t)(addr), (size_t)(len), \
21660                      PROT_READ | ((allow_write) ? PROT_WRITE : 0) \
21661                      | (GC_pages_executable ? PROT_EXEC : 0)) >= 0) { \
21662         } else if (GC_pages_executable) { \
21663             ABORT_ON_REMAP_FAIL(C_msg_prefix \
21664                                     "mprotect vdb executable pages", \
21665                                 addr, len); \
21666         } else ABORT_ON_REMAP_FAIL(C_msg_prefix "mprotect vdb", addr, len)
21667 #undef IGNORE_PAGES_EXECUTABLE
21668 #else
21669 #ifndef MSWINCE
21670 #include <signal.h>
21671 #endif
21672     static DWORD protect_junk;
21673 #define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
21674         if (VirtualProtect(addr, len, \
21675                            GC_pages_executable ? \
21676                                 ((allow_write) ? PAGE_EXECUTE_READWRITE : \
21677                                                  PAGE_EXECUTE_READ) : \
21678                                  (allow_write) ? PAGE_READWRITE : \
21679                                                  PAGE_READONLY, \
21680                            &protect_junk)) { \
21681         } else ABORT_ARG1(C_msg_prefix "VirtualProtect failed", \
21682                           ": errcode= 0x%X", (unsigned)GetLastError())
21683 #endif
21684 #define PROTECT(addr, len) PROTECT_INNER(addr, len, FALSE, "")
21685 #define UNPROTECT(addr, len) PROTECT_INNER(addr, len, TRUE, "un-")
21686 #if defined(MSWIN32)
21687     typedef LPTOP_LEVEL_EXCEPTION_FILTER SIG_HNDLR_PTR;
21688 #undef SIG_DFL
21689 #define SIG_DFL (LPTOP_LEVEL_EXCEPTION_FILTER)((signed_word)-1)
21690 #elif defined(MSWINCE)
21691     typedef LONG (WINAPI *SIG_HNDLR_PTR)(struct _EXCEPTION_POINTERS *);
21692 #undef SIG_DFL
21693 #define SIG_DFL (SIG_HNDLR_PTR) (-1)
21694 #elif defined(DARWIN)
21695     typedef void (* SIG_HNDLR_PTR)();
21696 #else
21697     typedef void (* SIG_HNDLR_PTR)(int, siginfo_t *, void *);
21698     typedef void (* PLAIN_HNDLR_PTR)(int);
21699 #endif
21700 #if defined(__GLIBC__)
21701 #if __GLIBC__ < 2 || __GLIBC__ == 2 && __GLIBC_MINOR__ < 2
21702 #error glibc too old?
21703 #endif
21704 #endif
21705 #ifndef DARWIN
21706   STATIC SIG_HNDLR_PTR GC_old_segv_handler = 0;
21707 #if defined(FREEBSD) || defined(HPUX) || defined(HURD) || defined(LINUX)
21708     STATIC SIG_HNDLR_PTR GC_old_bus_handler = 0;
21709 #ifndef LINUX
21710       STATIC GC_bool GC_old_bus_handler_used_si = FALSE;
21711 #endif
21712 #endif
21713 #if !defined(MSWIN32) && !defined(MSWINCE)
21714     STATIC GC_bool GC_old_segv_handler_used_si = FALSE;
21715 #endif
21716 #endif
21717 #ifdef THREADS
21718   GC_ATTR_NO_SANITIZE_THREAD
21719   static GC_bool is_header_found_async(void *addr)
21720   {
21721 #ifdef HASH_TL
21722       hdr *result;
21723       GET_HDR((ptr_t)addr, result);
21724       return result != NULL;
21725 #else
21726       return HDR_INNER(addr) != NULL;
21727 #endif
21728   }
21729 #else
21730 #define is_header_found_async(addr) (HDR(addr) != NULL)
21731 #endif
21732 #ifndef DARWIN
21733 #if !defined(MSWIN32) && !defined(MSWINCE)
21734 #include <errno.h>
21735 #if defined(FREEBSD) || defined(HURD) || defined(HPUX)
21736 #define SIG_OK (sig == SIGBUS || sig == SIGSEGV)
21737 #else
21738 #define SIG_OK (sig == SIGSEGV)
21739 #endif
21740 #if defined(FREEBSD)
21741 #ifndef SEGV_ACCERR
21742 #define SEGV_ACCERR 2
21743 #endif
21744 #if defined(AARCH64) || defined(ARM32) || defined(MIPS) \
21745          || __FreeBSD__ >= 7
21746 #define CODE_OK (si -> si_code == SEGV_ACCERR)
21747 #elif defined(POWERPC)
21748 #define AIM
21749 #include <machine/trap.h>
21750 #define CODE_OK (si -> si_code == EXC_DSI \
21751                         || si -> si_code == SEGV_ACCERR)
21752 #else
21753 #define CODE_OK (si -> si_code == BUS_PAGE_FAULT \
21754                         || si -> si_code == SEGV_ACCERR)
21755 #endif
21756 #elif defined(OSF1)
21757 #define CODE_OK (si -> si_code == 2 )
21758 #elif defined(IRIX5)
21759 #define CODE_OK (si -> si_code == EACCES)
21760 #elif defined(CYGWIN32) || defined(HAIKU) || defined(HURD)
21761 #define CODE_OK TRUE
21762 #elif defined(LINUX)
21763 #define CODE_OK TRUE
21764 #elif defined(HPUX)
21765 #define CODE_OK (si -> si_code == SEGV_ACCERR \
21766                       || si -> si_code == BUS_ADRERR \
21767                       || si -> si_code == BUS_UNKNOWN \
21768                       || si -> si_code == SEGV_UNKNOWN \
21769                       || si -> si_code == BUS_OBJERR)
21770 #elif defined(SUNOS5SIGS)
21771 #define CODE_OK (si -> si_code == SEGV_ACCERR)
21772 #endif
21773 #ifndef NO_GETCONTEXT
21774 #include <ucontext.h>
21775 #endif
21776     STATIC void GC_write_fault_handler(int sig, siginfo_t *si, void *raw_sc)
21777 #else
21778 #define SIG_OK (exc_info -> ExceptionRecord -> ExceptionCode \
21779                      == STATUS_ACCESS_VIOLATION)
21780 #define CODE_OK (exc_info -> ExceptionRecord -> ExceptionInformation[0] \
21781                       == 1)
21782     STATIC LONG WINAPI GC_write_fault_handler(
21783                                 struct _EXCEPTION_POINTERS *exc_info)
21784 #endif
21785   {
21786 #if !defined(MSWIN32) && !defined(MSWINCE)
21787         char *addr = (char *)si->si_addr;
21788 #else
21789         char * addr = (char *) (exc_info -> ExceptionRecord
21790                                 -> ExceptionInformation[1]);
21791 #endif
21792     if (SIG_OK && CODE_OK) {
21793         struct hblk * h = (struct hblk *)((word)addr & ~(GC_page_size-1));
21794         GC_bool in_allocd_block;
21795         size_t i;
21796         GC_ASSERT(GC_page_size != 0);
21797 #ifdef CHECKSUMS
21798           GC_record_fault(h);
21799 #endif
21800 #ifdef SUNOS5SIGS
21801             in_allocd_block = FALSE;
21802             for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
21803               if (is_header_found_async(&h[i])) {
21804                 in_allocd_block = TRUE;
21805                 break;
21806               }
21807             }
21808 #else
21809             in_allocd_block = is_header_found_async(addr);
21810 #endif
21811         if (!in_allocd_block) {
21812             SIG_HNDLR_PTR old_handler;
21813 #if defined(MSWIN32) || defined(MSWINCE)
21814                 old_handler = GC_old_segv_handler;
21815 #else
21816                 GC_bool used_si;
21817 #if defined(FREEBSD) || defined(HURD) || defined(HPUX)
21818                 if (sig == SIGBUS) {
21819                    old_handler = GC_old_bus_handler;
21820                    used_si = GC_old_bus_handler_used_si;
21821                 } else
21822 #endif
21823                  {
21824                    old_handler = GC_old_segv_handler;
21825                    used_si = GC_old_segv_handler_used_si;
21826                 }
21827 #endif
21828             if (old_handler == (SIG_HNDLR_PTR)(signed_word)SIG_DFL) {
21829 #if !defined(MSWIN32) && !defined(MSWINCE)
21830                     ABORT_ARG1("Unexpected bus error or segmentation fault",
21831                                " at %p", (void *)addr);
21832 #else
21833                     return(EXCEPTION_CONTINUE_SEARCH);
21834 #endif
21835             } else {
21836 #if defined(MSWIN32) || defined(MSWINCE)
21837                     return((*old_handler)(exc_info));
21838 #else
21839                     if (used_si)
21840                       ((SIG_HNDLR_PTR)old_handler) (sig, si, raw_sc);
21841                     else
21842                       ((PLAIN_HNDLR_PTR)(signed_word)old_handler)(sig);
21843                     return;
21844 #endif
21845             }
21846         }
21847         UNPROTECT(h, GC_page_size);
21848         for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
21849             word index = PHT_HASH(h+i);
21850             async_set_pht_entry_from_index(GC_dirty_pages, index);
21851         }
21852 #if defined(MSWIN32) || defined(MSWINCE)
21853             return(EXCEPTION_CONTINUE_EXECUTION);
21854 #else
21855             return;
21856 #endif
21857     }
21858 #if defined(MSWIN32) || defined(MSWINCE)
21859       return EXCEPTION_CONTINUE_SEARCH;
21860 #else
21861       ABORT_ARG1("Unexpected bus error or segmentation fault",
21862                  " at %p", (void *)addr);
21863 #endif
21864   }
21865 #if defined(GC_WIN32_THREADS) && !defined(CYGWIN32)
21866     GC_INNER void GC_set_write_fault_handler(void)
21867     {
21868       SetUnhandledExceptionFilter(GC_write_fault_handler);
21869     }
21870 #endif
21871 #ifdef SOFT_VDB
21872     static GC_bool soft_dirty_init(void);
21873 #endif
21874   GC_INNER GC_bool GC_dirty_init(void)
21875   {
21876 #if !defined(MSWIN32) && !defined(MSWINCE)
21877       struct sigaction act, oldact;
21878       act.sa_flags = SA_RESTART | SA_SIGINFO;
21879       act.sa_sigaction = GC_write_fault_handler;
21880       (void)sigemptyset(&act.sa_mask);
21881 #if defined(THREADS) && !defined(GC_OPENBSD_UTHREADS) \
21882          && !defined(GC_WIN32_THREADS) && !defined(NACL)
21883         (void)sigaddset(&act.sa_mask, GC_get_suspend_signal());
21884 #endif
21885 #endif
21886     GC_VERBOSE_LOG_PRINTF(
21887                 "Initializing mprotect virtual dirty bit implementation\n");
21888     if (GC_page_size % HBLKSIZE != 0) {
21889         ABORT("Page size not multiple of HBLKSIZE");
21890     }
21891 #ifdef GWW_VDB
21892       if (GC_gww_dirty_init()) {
21893         GC_COND_LOG_PRINTF("Using GetWriteWatch()\n");
21894         return TRUE;
21895       }
21896 #elif defined(SOFT_VDB)
21897       if (soft_dirty_init()) {
21898         GC_COND_LOG_PRINTF("Using soft-dirty bit feature\n");
21899         return TRUE;
21900       }
21901 #endif
21902 #ifdef MSWIN32
21903       GC_old_segv_handler = SetUnhandledExceptionFilter(
21904                                         GC_write_fault_handler);
21905       if (GC_old_segv_handler != NULL) {
21906         GC_COND_LOG_PRINTF("Replaced other UnhandledExceptionFilter\n");
21907       } else {
21908           GC_old_segv_handler = SIG_DFL;
21909       }
21910 #elif defined(MSWINCE)
21911 #else
21912 #if defined(GC_IRIX_THREADS)
21913         sigaction(SIGSEGV, 0, &oldact);
21914         sigaction(SIGSEGV, &act, 0);
21915 #else
21916         {
21917           int res = sigaction(SIGSEGV, &act, &oldact);
21918           if (res != 0) ABORT("Sigaction failed");
21919         }
21920 #endif
21921       if (oldact.sa_flags & SA_SIGINFO) {
21922         GC_old_segv_handler = oldact.sa_sigaction;
21923         GC_old_segv_handler_used_si = TRUE;
21924       } else {
21925         GC_old_segv_handler = (SIG_HNDLR_PTR)(signed_word)oldact.sa_handler;
21926         GC_old_segv_handler_used_si = FALSE;
21927       }
21928       if (GC_old_segv_handler == (SIG_HNDLR_PTR)(signed_word)SIG_IGN) {
21929         WARN("Previously ignored segmentation violation!?\n", 0);
21930         GC_old_segv_handler = (SIG_HNDLR_PTR)(signed_word)SIG_DFL;
21931       }
21932       if (GC_old_segv_handler != (SIG_HNDLR_PTR)(signed_word)SIG_DFL) {
21933         GC_VERBOSE_LOG_PRINTF("Replaced other SIGSEGV handler\n");
21934       }
21935 #if defined(HPUX) || defined(LINUX) || defined(HURD) \
21936        || (defined(FREEBSD) && (defined(__GLIBC__) || defined(SUNOS5SIGS)))
21937       sigaction(SIGBUS, &act, &oldact);
21938       if ((oldact.sa_flags & SA_SIGINFO) != 0) {
21939         GC_old_bus_handler = oldact.sa_sigaction;
21940 #if !defined(LINUX)
21941           GC_old_bus_handler_used_si = TRUE;
21942 #endif
21943       } else {
21944         GC_old_bus_handler = (SIG_HNDLR_PTR)(signed_word)oldact.sa_handler;
21945       }
21946       if (GC_old_bus_handler == (SIG_HNDLR_PTR)(signed_word)SIG_IGN) {
21947         WARN("Previously ignored bus error!?\n", 0);
21948 #if !defined(LINUX)
21949           GC_old_bus_handler = (SIG_HNDLR_PTR)(signed_word)SIG_DFL;
21950 #else
21951 #endif
21952       } else if (GC_old_bus_handler != (SIG_HNDLR_PTR)(signed_word)SIG_DFL) {
21953           GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n");
21954       }
21955 #endif
21956 #endif
21957 #if defined(CPPCHECK) && defined(ADDRESS_SANITIZER)
21958       GC_noop1((word)&__asan_default_options);
21959 #endif
21960     return TRUE;
21961   }
21962 #endif
21963 GC_API int GC_CALL GC_incremental_protection_needs(void)
21964 {
21965     GC_ASSERT(GC_is_initialized);
21966 #if defined(GWW_VDB) || defined(SOFT_VDB)
21967       if (GC_GWW_AVAILABLE())
21968         return GC_PROTECTS_NONE;
21969 #endif
21970     if (GC_page_size == HBLKSIZE) {
21971         return GC_PROTECTS_POINTER_HEAP;
21972     } else {
21973         return GC_PROTECTS_POINTER_HEAP | GC_PROTECTS_PTRFREE_HEAP;
21974     }
21975 }
21976 #define HAVE_INCREMENTAL_PROTECTION_NEEDS
21977 #define IS_PTRFREE(hhdr) ((hhdr)->hb_descr == 0)
21978 #define PAGE_ALIGNED(x) !((word)(x) & (GC_page_size - 1))
21979 STATIC void GC_protect_heap(void)
21980 {
21981     unsigned i;
21982     GC_bool protect_all =
21983         (0 != (GC_incremental_protection_needs() & GC_PROTECTS_PTRFREE_HEAP));
21984     GC_ASSERT(GC_page_size != 0);
21985     for (i = 0; i < GC_n_heap_sects; i++) {
21986         ptr_t start = GC_heap_sects[i].hs_start;
21987         size_t len = GC_heap_sects[i].hs_bytes;
21988         if (protect_all) {
21989           PROTECT(start, len);
21990         } else {
21991           struct hblk * current;
21992           struct hblk * current_start;
21993           struct hblk * limit;
21994           GC_ASSERT(PAGE_ALIGNED(len));
21995           GC_ASSERT(PAGE_ALIGNED(start));
21996           current_start = current = (struct hblk *)start;
21997           limit = (struct hblk *)(start + len);
21998           while ((word)current < (word)limit) {
21999             hdr * hhdr;
22000             word nhblks;
22001             GC_bool is_ptrfree;
22002             GC_ASSERT(PAGE_ALIGNED(current));
22003             GET_HDR(current, hhdr);
22004             if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
22005               GC_ASSERT(current_start == current);
22006               current_start = ++current;
22007               continue;
22008             }
22009             if (HBLK_IS_FREE(hhdr)) {
22010               GC_ASSERT(PAGE_ALIGNED(hhdr -> hb_sz));
22011               nhblks = divHBLKSZ(hhdr -> hb_sz);
22012               is_ptrfree = TRUE;
22013             } else {
22014               nhblks = OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
22015               is_ptrfree = IS_PTRFREE(hhdr);
22016             }
22017             if (is_ptrfree) {
22018               if ((word)current_start < (word)current) {
22019                 PROTECT(current_start, (ptr_t)current - (ptr_t)current_start);
22020               }
22021               current_start = (current += nhblks);
22022             } else {
22023               current += nhblks;
22024             }
22025           }
22026           if ((word)current_start < (word)current) {
22027             PROTECT(current_start, (ptr_t)current - (ptr_t)current_start);
22028           }
22029         }
22030     }
22031 }
22032 #endif
22033 #if !defined(THREADS) && (defined(PROC_VDB) || defined(SOFT_VDB))
22034   static pid_t saved_proc_pid;
22035 #endif
22036 #ifdef PROC_VDB
22037 #include <errno.h>
22038 #include <sys/types.h>
22039 #include <sys/signal.h>
22040 #include <sys/syscall.h>
22041 #include <sys/stat.h>
22042 #ifdef GC_NO_SYS_FAULT_H
22043 #define PG_MODIFIED 1
22044     struct prpageheader {
22045       int dummy[2];
22046       unsigned long pr_nmap;
22047       unsigned long pr_npage;
22048     };
22049     struct prasmap {
22050       char *pr_vaddr;
22051       size_t pr_npage;
22052       char dummy1[64+8];
22053       unsigned pr_mflags;
22054       unsigned pr_pagesize;
22055       int dummy2[2];
22056     };
22057 #else
22058 #include <sys/fault.h>
22059 #include <sys/procfs.h>
22060 #endif
22061 #define INITIAL_BUF_SZ 16384
22062   STATIC size_t GC_proc_buf_size = INITIAL_BUF_SZ;
22063   STATIC char *GC_proc_buf = NULL;
22064   STATIC int GC_proc_fd = -1;
22065   static GC_bool proc_dirty_open_files(void)
22066   {
22067     char buf[40];
22068     pid_t pid = getpid();
22069     (void)snprintf(buf, sizeof(buf), "/proc/%ld/pagedata", (long)pid);
22070     buf[sizeof(buf) - 1] = '\0';
22071     GC_proc_fd = open(buf, O_RDONLY);
22072     if (-1 == GC_proc_fd) {
22073       WARN("/proc open failed; cannot enable GC incremental mode\n", 0);
22074       return FALSE;
22075     }
22076     if (syscall(SYS_fcntl, GC_proc_fd, F_SETFD, FD_CLOEXEC) == -1)
22077       WARN("Could not set FD_CLOEXEC for /proc\n", 0);
22078 #ifndef THREADS
22079       saved_proc_pid = pid;
22080 #endif
22081     return TRUE;
22082   }
22083 #ifdef CAN_HANDLE_FORK
22084     GC_INNER void GC_dirty_update_child(void)
22085     {
22086       if (-1 == GC_proc_fd)
22087         return;
22088       close(GC_proc_fd);
22089       if (!proc_dirty_open_files())
22090         GC_incremental = FALSE;
22091     }
22092 #endif
22093 GC_INNER GC_bool GC_dirty_init(void)
22094 {
22095     if (GC_bytes_allocd != 0 || GC_bytes_allocd_before_gc != 0) {
22096       memset(GC_written_pages, 0xff, sizeof(page_hash_table));
22097       GC_VERBOSE_LOG_PRINTF(
22098                 "Allocated %lu bytes: all pages may have been written\n",
22099                 (unsigned long)(GC_bytes_allocd + GC_bytes_allocd_before_gc));
22100     }
22101     if (!proc_dirty_open_files())
22102       return FALSE;
22103     GC_proc_buf = GC_scratch_alloc(GC_proc_buf_size);
22104     if (GC_proc_buf == NULL)
22105       ABORT("Insufficient space for /proc read");
22106     return TRUE;
22107 }
22108 GC_INLINE void GC_proc_read_dirty(GC_bool output_unneeded)
22109 {
22110     int nmaps;
22111     char * bufp = GC_proc_buf;
22112     int i;
22113 #ifndef THREADS
22114       if (getpid() != saved_proc_pid
22115           && (-1 == GC_proc_fd
22116               || (close(GC_proc_fd), !proc_dirty_open_files()))) {
22117         if (!output_unneeded)
22118           memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
22119         memset(GC_written_pages, 0xff, sizeof(page_hash_table));
22120         return;
22121       }
22122 #endif
22123     BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
22124     if (PROC_READ(GC_proc_fd, bufp, GC_proc_buf_size) <= 0) {
22125         size_t new_size = 2 * GC_proc_buf_size;
22126         char *new_buf;
22127         WARN("/proc read failed: GC_proc_buf_size= %" WARN_PRIdPTR "\n",
22128              (signed_word)GC_proc_buf_size);
22129         new_buf = GC_scratch_alloc(new_size);
22130         if (new_buf != 0) {
22131             GC_scratch_recycle_no_gww(bufp, GC_proc_buf_size);
22132             GC_proc_buf = bufp = new_buf;
22133             GC_proc_buf_size = new_size;
22134         }
22135         if (PROC_READ(GC_proc_fd, bufp, GC_proc_buf_size) <= 0) {
22136             WARN("Insufficient space for /proc read\n", 0);
22137             if (!output_unneeded)
22138               memset(GC_grungy_pages, 0xff, sizeof (page_hash_table));
22139             memset(GC_written_pages, 0xff, sizeof(page_hash_table));
22140             return;
22141         }
22142     }
22143     nmaps = ((struct prpageheader *)bufp) -> pr_nmap;
22144 #ifdef DEBUG_DIRTY_BITS
22145       GC_log_printf("Proc VDB read: pr_nmap= %u, pr_npage= %lu\n",
22146                     nmaps, ((struct prpageheader *)bufp)->pr_npage);
22147 #endif
22148 #if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK)
22149       GC_noop1(((struct prpageheader *)bufp)->dummy[0]);
22150 #endif
22151     bufp += sizeof(struct prpageheader);
22152     for (i = 0; i < nmaps; i++) {
22153         struct prasmap * map = (struct prasmap *)bufp;
22154         ptr_t vaddr = (ptr_t)(map -> pr_vaddr);
22155         unsigned long npages = map -> pr_npage;
22156         unsigned pagesize = map -> pr_pagesize;
22157         ptr_t limit;
22158 #if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK)
22159           GC_noop1(map->dummy1[0] + map->dummy2[0]);
22160 #endif
22161 #ifdef DEBUG_DIRTY_BITS
22162           GC_log_printf(
22163                 "pr_vaddr= %p, npage= %lu, mflags= 0x%x, pagesize= 0x%x\n",
22164                 (void *)vaddr, npages, map->pr_mflags, pagesize);
22165 #endif
22166         bufp += sizeof(struct prasmap);
22167         limit = vaddr + pagesize * npages;
22168         for (; (word)vaddr < (word)limit; vaddr += pagesize) {
22169             if ((*bufp++) & PG_MODIFIED) {
22170                 struct hblk * h;
22171                 ptr_t next_vaddr = vaddr + pagesize;
22172 #ifdef DEBUG_DIRTY_BITS
22173                   GC_log_printf("dirty page at: %p\n", (void *)vaddr);
22174 #endif
22175                 for (h = (struct hblk *)vaddr;
22176                      (word)h < (word)next_vaddr; h++) {
22177                     word index = PHT_HASH(h);
22178                     set_pht_entry_from_index(GC_grungy_pages, index);
22179                 }
22180             }
22181         }
22182         bufp = (char *)(((word)bufp + (sizeof(long)-1))
22183                         & ~(word)(sizeof(long)-1));
22184     }
22185 #ifdef DEBUG_DIRTY_BITS
22186       GC_log_printf("Proc VDB read done\n");
22187 #endif
22188     GC_or_pages(GC_written_pages, GC_grungy_pages);
22189 }
22190 #endif
22191 #ifdef SOFT_VDB
22192 #ifndef VDB_BUF_SZ
22193 #define VDB_BUF_SZ 16384
22194 #endif
22195   static int open_proc_fd(pid_t pid, const char *proc_filename, int mode)
22196   {
22197     int f;
22198     char buf[40];
22199     (void)snprintf(buf, sizeof(buf), "/proc/%ld/%s", (long)pid,
22200                    proc_filename);
22201     buf[sizeof(buf) - 1] = '\0';
22202     f = open(buf, mode);
22203     if (-1 == f) {
22204       WARN("/proc/self/%s open failed; cannot enable GC incremental mode\n",
22205            proc_filename);
22206     } else if (fcntl(f, F_SETFD, FD_CLOEXEC) == -1) {
22207       WARN("Could not set FD_CLOEXEC for /proc\n", 0);
22208     }
22209     return f;
22210   }
22211 #include <stdint.h>
22212   typedef uint64_t pagemap_elem_t;
22213   static pagemap_elem_t *soft_vdb_buf;
22214   static int pagemap_fd;
22215   static GC_bool soft_dirty_open_files(void)
22216   {
22217     pid_t pid = getpid();
22218     clear_refs_fd = open_proc_fd(pid, "clear_refs", O_WRONLY);
22219     if (-1 == clear_refs_fd)
22220       return FALSE;
22221     pagemap_fd = open_proc_fd(pid, "pagemap", O_RDONLY);
22222     if (-1 == pagemap_fd) {
22223       close(clear_refs_fd);
22224       clear_refs_fd = -1;
22225       return FALSE;
22226     }
22227 #ifndef THREADS
22228       saved_proc_pid = pid;
22229 #endif
22230     return TRUE;
22231   }
22232 #ifdef CAN_HANDLE_FORK
22233     GC_INNER void GC_dirty_update_child(void)
22234     {
22235       if (-1 == clear_refs_fd)
22236         return;
22237       close(clear_refs_fd);
22238       close(pagemap_fd);
22239       if (!soft_dirty_open_files())
22240         GC_incremental = FALSE;
22241     }
22242 #endif
22243 #define PM_SOFTDIRTY_MASK ((pagemap_elem_t)1 << 55)
22244   static GC_bool detect_soft_dirty_supported(ptr_t vaddr)
22245   {
22246     off_t fpos;
22247     pagemap_elem_t buf[1];
22248     *vaddr = 1;
22249     GC_ASSERT(GC_page_size != 0);
22250     fpos = (off_t)((word)vaddr / GC_page_size * sizeof(pagemap_elem_t));
22251     if (lseek(pagemap_fd, fpos, SEEK_SET) == (off_t)(-1))
22252       return FALSE;
22253     if (PROC_READ(pagemap_fd, buf, sizeof(buf)) != (int)sizeof(buf))
22254       return FALSE;
22255     return (buf[0] & PM_SOFTDIRTY_MASK) != 0;
22256   }
22257 #ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK
22258 #include <sys/utsname.h>
22259 #include <string.h>
22260     static GC_bool ensure_min_linux_ver(int major, int minor) {
22261       struct utsname info;
22262       int actual_major;
22263       int actual_minor = -1;
22264       if (uname(&info) == -1) {
22265         return FALSE;
22266       }
22267       if (strcmp(info.sysname, "Linux")) {
22268         WARN("Cannot ensure Linux version as running on other OS: %s\n",
22269              info.sysname);
22270         return FALSE;
22271       }
22272       actual_major = GC_parse_version(&actual_minor, info.release);
22273       return actual_major > major
22274              || (actual_major == major && actual_minor >= minor);
22275     }
22276 #endif
22277 #ifdef MPROTECT_VDB
22278     static GC_bool soft_dirty_init(void)
22279 #else
22280     GC_INNER GC_bool GC_dirty_init(void)
22281 #endif
22282   {
22283     GC_ASSERT(NULL == soft_vdb_buf);
22284 #ifdef MPROTECT_VDB
22285       char * str = GETENV("GC_USE_GETWRITEWATCH");
22286 #ifdef GC_PREFER_MPROTECT_VDB
22287         if (str == NULL || (*str == '0' && *(str + 1) == '\0'))
22288           return FALSE;
22289 #else
22290         if (str != NULL && *str == '0' && *(str + 1) == '\0')
22291           return FALSE;
22292 #endif
22293 #endif
22294 #ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK
22295       if (!ensure_min_linux_ver(3, 18)) {
22296         GC_COND_LOG_PRINTF(
22297             "Running on old kernel lacking correct soft-dirty bit support\n");
22298         return FALSE;
22299       }
22300 #endif
22301     if (!soft_dirty_open_files())
22302       return FALSE;
22303     soft_vdb_buf = (pagemap_elem_t *)GC_scratch_alloc(VDB_BUF_SZ);
22304     if (NULL == soft_vdb_buf)
22305       ABORT("Insufficient space for /proc pagemap buffer");
22306     if (!detect_soft_dirty_supported((ptr_t)soft_vdb_buf)) {
22307       GC_COND_LOG_PRINTF("Soft-dirty bit is not supported by kernel\n");
22308       GC_scratch_recycle_no_gww(soft_vdb_buf, VDB_BUF_SZ);
22309       soft_vdb_buf = NULL;
22310       close(clear_refs_fd);
22311       clear_refs_fd = -1;
22312       close(pagemap_fd);
22313       return FALSE;
22314     }
22315     return TRUE;
22316   }
22317   static off_t pagemap_buf_fpos;
22318   static size_t pagemap_buf_len;
22319   static const pagemap_elem_t *pagemap_buffered_read(size_t *pres,
22320                                                      off_t fpos, size_t len,
22321                                                      off_t next_fpos_hint)
22322   {
22323     ssize_t res;
22324     size_t ofs;
22325     GC_ASSERT(len > 0);
22326     if (pagemap_buf_fpos <= fpos
22327         && fpos < pagemap_buf_fpos + (off_t)pagemap_buf_len) {
22328       ofs = (size_t)(fpos - pagemap_buf_fpos);
22329       res = (ssize_t)(pagemap_buf_fpos + pagemap_buf_len - fpos);
22330     } else {
22331       off_t aligned_pos = fpos & ~(GC_page_size < VDB_BUF_SZ
22332                                     ? GC_page_size-1 : VDB_BUF_SZ-1);
22333       for (;;) {
22334         size_t count;
22335         if ((0 == pagemap_buf_len
22336              || pagemap_buf_fpos + (off_t)pagemap_buf_len != aligned_pos)
22337             && lseek(pagemap_fd, aligned_pos, SEEK_SET) == (off_t)(-1))
22338           ABORT_ARG2("Failed to lseek /proc/self/pagemap",
22339                      ": offset= %lu, errno= %d", (unsigned long)fpos, errno);
22340         ofs = (size_t)(fpos - aligned_pos);
22341         GC_ASSERT(ofs < VDB_BUF_SZ);
22342         if (next_fpos_hint > aligned_pos
22343             && next_fpos_hint - aligned_pos < VDB_BUF_SZ) {
22344           count = VDB_BUF_SZ;
22345         } else {
22346           count = len + ofs;
22347           if (count > VDB_BUF_SZ)
22348             count = VDB_BUF_SZ;
22349         }
22350         GC_ASSERT(count % sizeof(pagemap_elem_t) == 0);
22351         res = PROC_READ(pagemap_fd, soft_vdb_buf, count);
22352         if (res > (ssize_t)ofs)
22353           break;
22354         if (res <= 0)
22355           ABORT_ARG1("Failed to read /proc/self/pagemap",
22356                      ": errno= %d", res < 0 ? errno : 0);
22357         aligned_pos = fpos;
22358       }
22359       pagemap_buf_fpos = aligned_pos;
22360       pagemap_buf_len = (size_t)res;
22361       res -= (ssize_t)ofs;
22362     }
22363     GC_ASSERT(ofs % sizeof(pagemap_elem_t) == 0);
22364     *pres = (size_t)res < len ? (size_t)res : len;
22365     return &soft_vdb_buf[ofs / sizeof(pagemap_elem_t)];
22366   }
22367   static void soft_set_grungy_pages(ptr_t vaddr , ptr_t limit,
22368                                     ptr_t next_start_hint)
22369   {
22370     GC_ASSERT(GC_page_size != 0);
22371     while ((word)vaddr < (word)limit) {
22372       size_t res;
22373       word limit_buf;
22374       const pagemap_elem_t *bufp = pagemap_buffered_read(&res,
22375                 (off_t)((word)vaddr / GC_page_size * sizeof(pagemap_elem_t)),
22376                 (size_t)((((word)limit - (word)vaddr + GC_page_size-1)
22377                          / GC_page_size) * sizeof(pagemap_elem_t)),
22378                 (off_t)((word)next_start_hint / GC_page_size
22379                         * sizeof(pagemap_elem_t)));
22380       if (res % sizeof(pagemap_elem_t) != 0) {
22381         memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
22382         WARN("Incomplete read of pagemap, not multiple of entry size\n", 0);
22383         break;
22384       }
22385       limit_buf = ((word)vaddr & ~(GC_page_size-1))
22386                   + (res / sizeof(pagemap_elem_t)) * GC_page_size;
22387       for (; (word)vaddr < limit_buf; vaddr += GC_page_size, bufp++)
22388         if ((*bufp & PM_SOFTDIRTY_MASK) != 0) {
22389           struct hblk * h;
22390           ptr_t next_vaddr = vaddr + GC_page_size;
22391 #ifdef DEBUG_DIRTY_BITS
22392             GC_log_printf("dirty page at: %p\n", (void *)vaddr);
22393 #endif
22394           for (h = (struct hblk *)vaddr; (word)h < (word)next_vaddr; h++) {
22395             word index = PHT_HASH(h);
22396             set_pht_entry_from_index(GC_grungy_pages, index);
22397           }
22398         }
22399     }
22400   }
22401   GC_INLINE void GC_soft_read_dirty(GC_bool output_unneeded)
22402   {
22403     ssize_t res;
22404 #ifndef THREADS
22405       if (getpid() != saved_proc_pid
22406           && (-1 == clear_refs_fd
22407               || (close(clear_refs_fd), close(pagemap_fd),
22408                   !soft_dirty_open_files()))) {
22409         if (!output_unneeded) {
22410           memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
22411 #ifdef CHECKSUMS
22412             memset(GC_written_pages, 0xff, sizeof(page_hash_table));
22413 #endif
22414         }
22415         return;
22416       }
22417 #endif
22418     if (!output_unneeded) {
22419       word i;
22420       BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
22421       pagemap_buf_len = 0;
22422       for (i = 0; i != GC_n_heap_sects; ++i) {
22423         ptr_t vaddr = GC_heap_sects[i].hs_start;
22424         soft_set_grungy_pages(vaddr, vaddr + GC_heap_sects[i].hs_bytes,
22425                               i < GC_n_heap_sects-1 ?
22426                                     GC_heap_sects[i+1].hs_start : NULL);
22427       }
22428 #ifdef CHECKSUMS
22429         GC_or_pages(GC_written_pages, GC_grungy_pages);
22430 #endif
22431 #ifndef NO_VDB_FOR_STATIC_ROOTS
22432         for (i = 0; (int)i < n_root_sets; ++i) {
22433           soft_set_grungy_pages(GC_static_roots[i].r_start,
22434                                 GC_static_roots[i].r_end,
22435                                 (int)i < n_root_sets-1 ?
22436                                     GC_static_roots[i+1].r_start : NULL);
22437         }
22438 #endif
22439     }
22440     res = write(clear_refs_fd, "4\n", 2);
22441     if (res != 2)
22442       ABORT_ARG1("Failed to write to /proc/self/clear_refs",
22443                  ": errno= %d", res < 0 ? errno : 0);
22444   }
22445 #endif
22446 #ifdef PCR_VDB
22447 #include "vd/PCR_VD.h"
22448 #define NPAGES (32*1024)
22449 PCR_VD_DB GC_grungy_bits[NPAGES];
22450 STATIC ptr_t GC_vd_base = NULL;
22451 GC_INNER GC_bool GC_dirty_init(void)
22452 {
22453     GC_vd_base = GC_heap_sects[0].hs_start;
22454     if (GC_vd_base == 0) {
22455         ABORT("Bad initial heap segment");
22456     }
22457     if (PCR_VD_Start(HBLKSIZE, GC_vd_base, NPAGES*HBLKSIZE)
22458         != PCR_ERes_okay) {
22459         ABORT("Dirty bit initialization failed");
22460     }
22461     return TRUE;
22462 }
22463 #endif
22464 #ifndef GC_DISABLE_INCREMENTAL
22465   GC_INNER GC_bool GC_manual_vdb = FALSE;
22466   GC_INNER void GC_dirty_inner(const void *p)
22467   {
22468     word index = PHT_HASH(p);
22469 #if defined(MPROTECT_VDB)
22470       GC_ASSERT(GC_manual_vdb);
22471 #endif
22472     async_set_pht_entry_from_index(GC_dirty_pages, index);
22473   }
22474   GC_INNER void GC_read_dirty(GC_bool output_unneeded)
22475   {
22476     if (GC_manual_vdb
22477 #if defined(MPROTECT_VDB)
22478           || !GC_GWW_AVAILABLE()
22479 #endif
22480         ) {
22481       if (!output_unneeded)
22482         BCOPY(( void *)GC_dirty_pages, GC_grungy_pages,
22483               sizeof(GC_dirty_pages));
22484       BZERO(( void *)GC_dirty_pages,
22485             sizeof(GC_dirty_pages));
22486 #ifdef MPROTECT_VDB
22487         if (!GC_manual_vdb)
22488           GC_protect_heap();
22489 #endif
22490       return;
22491     }
22492 #ifdef GWW_VDB
22493       GC_gww_read_dirty(output_unneeded);
22494 #elif defined(PROC_VDB)
22495       GC_proc_read_dirty(output_unneeded);
22496 #elif defined(SOFT_VDB)
22497       GC_soft_read_dirty(output_unneeded);
22498 #elif defined(PCR_VDB)
22499       {
22500         static int onhs = 0;
22501         int nhs = GC_n_heap_sects;
22502         for (; onhs < nhs; onhs++) {
22503             PCR_VD_WriteProtectEnable(
22504                     GC_heap_sects[onhs].hs_start,
22505                     GC_heap_sects[onhs].hs_bytes);
22506         }
22507       }
22508       if (PCR_VD_Clear(GC_vd_base, NPAGES*HBLKSIZE, GC_grungy_bits)
22509           != PCR_ERes_okay) {
22510         ABORT("Dirty bit read failed");
22511       }
22512 #endif
22513   }
22514 #if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB)
22515     GC_INNER GC_bool GC_is_vdb_for_static_roots(void)
22516     {
22517       if (GC_manual_vdb) return FALSE;
22518 #if defined(MPROTECT_VDB)
22519         return GC_GWW_AVAILABLE();
22520 #else
22521         GC_ASSERT(GC_incremental);
22522         return TRUE;
22523 #endif
22524     }
22525 #endif
22526   GC_INNER GC_bool GC_page_was_dirty(struct hblk *h)
22527   {
22528     word index;
22529 #ifdef PCR_VDB
22530       if (!GC_manual_vdb) {
22531         if ((word)h < (word)GC_vd_base
22532             || (word)h >= (word)(GC_vd_base + NPAGES * HBLKSIZE)) {
22533           return TRUE;
22534         }
22535         return GC_grungy_bits[h-(struct hblk*)GC_vd_base] & PCR_VD_DB_dirtyBit;
22536       }
22537 #elif defined(DEFAULT_VDB)
22538       if (!GC_manual_vdb)
22539         return TRUE;
22540 #elif defined(PROC_VDB)
22541       if (GC_manual_vdb)
22542 #endif
22543       {
22544         if (NULL == HDR(h))
22545           return TRUE;
22546       }
22547     index = PHT_HASH(h);
22548     return get_pht_entry_from_index(GC_grungy_pages, index);
22549   }
22550 #if defined(CHECKSUMS) || defined(PROC_VDB)
22551     GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk *h)
22552     {
22553 #if defined(GWW_VDB) || defined(PROC_VDB) || defined(SOFT_VDB)
22554         word index;
22555 #ifdef MPROTECT_VDB
22556           if (!GC_GWW_AVAILABLE())
22557             return TRUE;
22558 #endif
22559 #if defined(PROC_VDB)
22560           if (GC_manual_vdb)
22561 #endif
22562         {
22563           if (NULL == HDR(h))
22564             return TRUE;
22565         }
22566         index = PHT_HASH(h);
22567         return get_pht_entry_from_index(GC_written_pages, index);
22568 #else
22569         (void)h;
22570         return TRUE;
22571 #endif
22572     }
22573 #endif
22574   GC_INNER void GC_remove_protection(struct hblk *h, word nblocks,
22575                                      GC_bool is_ptrfree)
22576   {
22577 #ifdef PCR_VDB
22578       (void)is_ptrfree;
22579       if (!GC_auto_incremental)
22580         return;
22581       PCR_VD_WriteProtectDisable(h, nblocks*HBLKSIZE);
22582       PCR_VD_WriteProtectEnable(h, nblocks*HBLKSIZE);
22583 #elif defined(MPROTECT_VDB)
22584       struct hblk * h_trunc;
22585       struct hblk * h_end;
22586       struct hblk * current;
22587       if (!GC_auto_incremental || GC_GWW_AVAILABLE())
22588         return;
22589       GC_ASSERT(GC_page_size != 0);
22590       h_trunc = (struct hblk *)((word)h & ~(GC_page_size-1));
22591       h_end = (struct hblk *)(((word)(h + nblocks) + GC_page_size - 1)
22592                               & ~(GC_page_size - 1));
22593       if (h_end == h_trunc + 1 &&
22594         get_pht_entry_from_index(GC_dirty_pages, PHT_HASH(h_trunc))) {
22595         return;
22596       }
22597       for (current = h_trunc; (word)current < (word)h_end; ++current) {
22598         word index = PHT_HASH(current);
22599         if (!is_ptrfree || (word)current < (word)h
22600             || (word)current >= (word)(h + nblocks)) {
22601           async_set_pht_entry_from_index(GC_dirty_pages, index);
22602         }
22603       }
22604       UNPROTECT(h_trunc, (ptr_t)h_end - (ptr_t)h_trunc);
22605 #else
22606       (void)h; (void)nblocks; (void)is_ptrfree;
22607 #endif
22608   }
22609 #endif
22610 #if defined(MPROTECT_VDB) && defined(DARWIN)
22611 #include <mach/mach.h>
22612 #include <mach/mach_error.h>
22613 #include <mach/exception.h>
22614 #include <mach/task.h>
22615 #include <pthread.h>
22616 EXTERN_C_BEGIN
22617 extern boolean_t
22618 exc_server(mach_msg_header_t *, mach_msg_header_t *);
22619 extern kern_return_t
22620 exception_raise(mach_port_t, mach_port_t, mach_port_t, exception_type_t,
22621                 exception_data_t, mach_msg_type_number_t);
22622 extern kern_return_t
22623 exception_raise_state(mach_port_t, mach_port_t, mach_port_t, exception_type_t,
22624                       exception_data_t, mach_msg_type_number_t,
22625                       thread_state_flavor_t*, thread_state_t,
22626                       mach_msg_type_number_t, thread_state_t,
22627                       mach_msg_type_number_t*);
22628 extern kern_return_t
22629 exception_raise_state_identity(mach_port_t, mach_port_t, mach_port_t,
22630                                exception_type_t, exception_data_t,
22631                                mach_msg_type_number_t, thread_state_flavor_t*,
22632                                thread_state_t, mach_msg_type_number_t,
22633                                thread_state_t, mach_msg_type_number_t*);
22634 GC_API_OSCALL kern_return_t
22635 catch_exception_raise(mach_port_t exception_port, mach_port_t thread,
22636                       mach_port_t task, exception_type_t exception,
22637                       exception_data_t code,
22638                       mach_msg_type_number_t code_count);
22639 GC_API_OSCALL kern_return_t
22640 catch_exception_raise_state(mach_port_name_t exception_port,
22641                 int exception, exception_data_t code,
22642                 mach_msg_type_number_t codeCnt, int flavor,
22643                 thread_state_t old_state, int old_stateCnt,
22644                 thread_state_t new_state, int new_stateCnt);
22645 GC_API_OSCALL kern_return_t
22646 catch_exception_raise_state_identity(mach_port_name_t exception_port,
22647                 mach_port_t thread, mach_port_t task, int exception,
22648                 exception_data_t code, mach_msg_type_number_t codeCnt,
22649                 int flavor, thread_state_t old_state, int old_stateCnt,
22650                 thread_state_t new_state, int new_stateCnt);
22651 EXTERN_C_END
22652 GC_API_OSCALL kern_return_t
22653 catch_exception_raise_state(mach_port_name_t exception_port GC_ATTR_UNUSED,
22654     int exception GC_ATTR_UNUSED, exception_data_t code GC_ATTR_UNUSED,
22655     mach_msg_type_number_t codeCnt GC_ATTR_UNUSED, int flavor GC_ATTR_UNUSED,
22656     thread_state_t old_state GC_ATTR_UNUSED, int old_stateCnt GC_ATTR_UNUSED,
22657     thread_state_t new_state GC_ATTR_UNUSED, int new_stateCnt GC_ATTR_UNUSED)
22658 {
22659   ABORT_RET("Unexpected catch_exception_raise_state invocation");
22660   return(KERN_INVALID_ARGUMENT);
22661 }
22662 GC_API_OSCALL kern_return_t
22663 catch_exception_raise_state_identity(
22664     mach_port_name_t exception_port GC_ATTR_UNUSED,
22665     mach_port_t thread GC_ATTR_UNUSED, mach_port_t task GC_ATTR_UNUSED,
22666     int exception GC_ATTR_UNUSED, exception_data_t code GC_ATTR_UNUSED,
22667     mach_msg_type_number_t codeCnt GC_ATTR_UNUSED, int flavor GC_ATTR_UNUSED,
22668     thread_state_t old_state GC_ATTR_UNUSED, int old_stateCnt GC_ATTR_UNUSED,
22669     thread_state_t new_state GC_ATTR_UNUSED, int new_stateCnt GC_ATTR_UNUSED)
22670 {
22671   ABORT_RET("Unexpected catch_exception_raise_state_identity invocation");
22672   return(KERN_INVALID_ARGUMENT);
22673 }
22674 #define MAX_EXCEPTION_PORTS 16
22675 static struct {
22676   mach_msg_type_number_t count;
22677   exception_mask_t      masks[MAX_EXCEPTION_PORTS];
22678   exception_handler_t   ports[MAX_EXCEPTION_PORTS];
22679   exception_behavior_t  behaviors[MAX_EXCEPTION_PORTS];
22680   thread_state_flavor_t flavors[MAX_EXCEPTION_PORTS];
22681 } GC_old_exc_ports;
22682 STATIC struct ports_s {
22683   void (*volatile os_callback[3])(void);
22684   mach_port_t exception;
22685 #if defined(THREADS)
22686     mach_port_t reply;
22687 #endif
22688 } GC_ports = {
22689   {
22690     (void (*)(void))catch_exception_raise,
22691     (void (*)(void))catch_exception_raise_state,
22692     (void (*)(void))catch_exception_raise_state_identity
22693   },
22694 #ifdef THREADS
22695     0,
22696 #endif
22697   0
22698 };
22699 typedef struct {
22700     mach_msg_header_t head;
22701 } GC_msg_t;
22702 typedef enum {
22703     GC_MP_NORMAL,
22704     GC_MP_DISCARDING,
22705     GC_MP_STOPPED
22706 } GC_mprotect_state_t;
22707 #ifdef THREADS
22708 #define ID_STOP 1
22709 #define ID_RESUME 2
22710 #define ID_ACK 3
22711   STATIC GC_mprotect_state_t GC_mprotect_state = GC_MP_NORMAL;
22712   STATIC void GC_mprotect_thread_notify(mach_msg_id_t id)
22713   {
22714     struct buf_s {
22715       GC_msg_t msg;
22716       mach_msg_trailer_t trailer;
22717     } buf;
22718     mach_msg_return_t r;
22719     buf.msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0);
22720     buf.msg.head.msgh_size = sizeof(buf.msg);
22721     buf.msg.head.msgh_remote_port = GC_ports.exception;
22722     buf.msg.head.msgh_local_port = MACH_PORT_NULL;
22723     buf.msg.head.msgh_id = id;
22724     r = mach_msg(&buf.msg.head, MACH_SEND_MSG | MACH_RCV_MSG | MACH_RCV_LARGE,
22725                  sizeof(buf.msg), sizeof(buf), GC_ports.reply,
22726                  MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
22727     if (r != MACH_MSG_SUCCESS)
22728       ABORT("mach_msg failed in GC_mprotect_thread_notify");
22729     if (buf.msg.head.msgh_id != ID_ACK)
22730       ABORT("Invalid ack in GC_mprotect_thread_notify");
22731   }
22732   STATIC void GC_mprotect_thread_reply(void)
22733   {
22734     GC_msg_t msg;
22735     mach_msg_return_t r;
22736     msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0);
22737     msg.head.msgh_size = sizeof(msg);
22738     msg.head.msgh_remote_port = GC_ports.reply;
22739     msg.head.msgh_local_port = MACH_PORT_NULL;
22740     msg.head.msgh_id = ID_ACK;
22741     r = mach_msg(&msg.head, MACH_SEND_MSG, sizeof(msg), 0, MACH_PORT_NULL,
22742                  MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
22743     if (r != MACH_MSG_SUCCESS)
22744       ABORT("mach_msg failed in GC_mprotect_thread_reply");
22745   }
22746   GC_INNER void GC_mprotect_stop(void)
22747   {
22748     GC_mprotect_thread_notify(ID_STOP);
22749   }
22750   GC_INNER void GC_mprotect_resume(void)
22751   {
22752     GC_mprotect_thread_notify(ID_RESUME);
22753   }
22754 #else
22755 #define GC_mprotect_state GC_MP_NORMAL
22756 #endif
22757 struct mp_reply_s {
22758   mach_msg_header_t head;
22759   char data[256];
22760 };
22761 struct mp_msg_s {
22762   mach_msg_header_t head;
22763   mach_msg_body_t msgh_body;
22764   char data[1024];
22765 };
22766 STATIC void *GC_mprotect_thread(void *arg)
22767 {
22768   mach_msg_return_t r;
22769   struct mp_reply_s reply;
22770   struct mp_msg_s msg;
22771   mach_msg_id_t id;
22772   if ((word)arg == GC_WORD_MAX) return 0;
22773 #if defined(CPPCHECK)
22774     reply.data[0] = 0;
22775     msg.data[0] = 0;
22776 #endif
22777 #if defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID)
22778     (void)pthread_setname_np("GC-mprotect");
22779 #endif
22780 #if defined(THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
22781     GC_darwin_register_mach_handler_thread(mach_thread_self());
22782 #endif
22783   for(;;) {
22784     r = mach_msg(&msg.head, MACH_RCV_MSG | MACH_RCV_LARGE |
22785                  (GC_mprotect_state == GC_MP_DISCARDING ? MACH_RCV_TIMEOUT : 0),
22786                  0, sizeof(msg), GC_ports.exception,
22787                  GC_mprotect_state == GC_MP_DISCARDING ? 0
22788                  : MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
22789     id = r == MACH_MSG_SUCCESS ? msg.head.msgh_id : -1;
22790 #if defined(THREADS)
22791       if(GC_mprotect_state == GC_MP_DISCARDING) {
22792         if(r == MACH_RCV_TIMED_OUT) {
22793           GC_mprotect_state = GC_MP_STOPPED;
22794           GC_mprotect_thread_reply();
22795           continue;
22796         }
22797         if(r == MACH_MSG_SUCCESS && (id == ID_STOP || id == ID_RESUME))
22798           ABORT("Out of order mprotect thread request");
22799       }
22800 #endif
22801     if (r != MACH_MSG_SUCCESS) {
22802       ABORT_ARG2("mach_msg failed",
22803                  ": errcode= %d (%s)", (int)r, mach_error_string(r));
22804     }
22805     switch(id) {
22806 #if defined(THREADS)
22807         case ID_STOP:
22808           if(GC_mprotect_state != GC_MP_NORMAL)
22809             ABORT("Called mprotect_stop when state wasn't normal");
22810           GC_mprotect_state = GC_MP_DISCARDING;
22811           break;
22812         case ID_RESUME:
22813           if(GC_mprotect_state != GC_MP_STOPPED)
22814             ABORT("Called mprotect_resume when state wasn't stopped");
22815           GC_mprotect_state = GC_MP_NORMAL;
22816           GC_mprotect_thread_reply();
22817           break;
22818 #endif
22819         default:
22820           if(!exc_server(&msg.head, &reply.head))
22821             ABORT("exc_server failed");
22822           r = mach_msg(&reply.head, MACH_SEND_MSG, reply.head.msgh_size, 0,
22823                        MACH_PORT_NULL, MACH_MSG_TIMEOUT_NONE,
22824                        MACH_PORT_NULL);
22825           if(r != MACH_MSG_SUCCESS) {
22826 #ifdef BROKEN_EXCEPTION_HANDLING
22827               GC_err_printf("mach_msg failed with %d %s while sending "
22828                             "exc reply\n", (int)r, mach_error_string(r));
22829 #else
22830               ABORT("mach_msg failed while sending exception reply");
22831 #endif
22832           }
22833     }
22834   }
22835 }
22836 #ifdef BROKEN_EXCEPTION_HANDLING
22837   STATIC int GC_sigbus_count = 0;
22838   STATIC void GC_darwin_sigbus(int num, siginfo_t *sip, void *context)
22839   {
22840     if (num != SIGBUS)
22841       ABORT("Got a non-sigbus signal in the sigbus handler");
22842     if (GC_sigbus_count >= 8) {
22843       ABORT("Got more than 8 SIGBUSs in a row!");
22844     } else {
22845       GC_sigbus_count++;
22846       WARN("Ignoring SIGBUS\n", 0);
22847     }
22848   }
22849 #endif
22850 GC_INNER GC_bool GC_dirty_init(void)
22851 {
22852   kern_return_t r;
22853   mach_port_t me;
22854   pthread_t thread;
22855   pthread_attr_t attr;
22856   exception_mask_t mask;
22857 #ifdef CAN_HANDLE_FORK
22858     if (GC_handle_fork) {
22859       WARN("Can't turn on GC incremental mode as fork()"
22860            " handling requested\n", 0);
22861       return FALSE;
22862     }
22863 #endif
22864   GC_VERBOSE_LOG_PRINTF("Initializing mach/darwin mprotect"
22865                         " virtual dirty bit implementation\n");
22866 #ifdef BROKEN_EXCEPTION_HANDLING
22867     WARN("Enabling workarounds for various darwin "
22868          "exception handling bugs\n", 0);
22869 #endif
22870   if (GC_page_size % HBLKSIZE != 0) {
22871     ABORT("Page size not multiple of HBLKSIZE");
22872   }
22873   GC_task_self = me = mach_task_self();
22874   r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.exception);
22875   if (r != KERN_SUCCESS)
22876     ABORT("mach_port_allocate failed (exception port)");
22877   r = mach_port_insert_right(me, GC_ports.exception, GC_ports.exception,
22878                              MACH_MSG_TYPE_MAKE_SEND);
22879   if (r != KERN_SUCCESS)
22880     ABORT("mach_port_insert_right failed (exception port)");
22881 #if defined(THREADS)
22882      r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.reply);
22883      if(r != KERN_SUCCESS)
22884        ABORT("mach_port_allocate failed (reply port)");
22885 #endif
22886   mask = EXC_MASK_BAD_ACCESS;
22887   r = task_get_exception_ports(me, mask, GC_old_exc_ports.masks,
22888                                &GC_old_exc_ports.count, GC_old_exc_ports.ports,
22889                                GC_old_exc_ports.behaviors,
22890                                GC_old_exc_ports.flavors);
22891   if (r != KERN_SUCCESS)
22892     ABORT("task_get_exception_ports failed");
22893   r = task_set_exception_ports(me, mask, GC_ports.exception, EXCEPTION_DEFAULT,
22894                                GC_MACH_THREAD_STATE);
22895   if (r != KERN_SUCCESS)
22896     ABORT("task_set_exception_ports failed");
22897   if (pthread_attr_init(&attr) != 0)
22898     ABORT("pthread_attr_init failed");
22899   if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0)
22900     ABORT("pthread_attr_setdetachedstate failed");
22901 #undef pthread_create
22902   if (pthread_create(&thread, &attr, GC_mprotect_thread, NULL) != 0)
22903     ABORT("pthread_create failed");
22904   (void)pthread_attr_destroy(&attr);
22905 #ifdef BROKEN_EXCEPTION_HANDLING
22906     {
22907       struct sigaction sa, oldsa;
22908       sa.sa_handler = (SIG_HNDLR_PTR)GC_darwin_sigbus;
22909       sigemptyset(&sa.sa_mask);
22910       sa.sa_flags = SA_RESTART|SA_SIGINFO;
22911       if (sigaction(SIGBUS, &sa, &oldsa) < 0)
22912         ABORT("sigaction failed");
22913       if (oldsa.sa_handler != (SIG_HNDLR_PTR)(signed_word)SIG_DFL) {
22914         GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n");
22915       }
22916     }
22917 #endif
22918 #if defined(CPPCHECK)
22919     GC_noop1((word)GC_ports.os_callback[0]);
22920 #endif
22921   return TRUE;
22922 }
22923 STATIC kern_return_t GC_forward_exception(mach_port_t thread, mach_port_t task,
22924                                           exception_type_t exception,
22925                                           exception_data_t data,
22926                                           mach_msg_type_number_t data_count)
22927 {
22928   unsigned int i;
22929   kern_return_t r;
22930   mach_port_t port;
22931   exception_behavior_t behavior;
22932   thread_state_flavor_t flavor;
22933   thread_state_data_t thread_state;
22934   mach_msg_type_number_t thread_state_count = THREAD_STATE_MAX;
22935   for (i=0; i < GC_old_exc_ports.count; i++)
22936     if (GC_old_exc_ports.masks[i] & (1 << exception))
22937       break;
22938   if (i == GC_old_exc_ports.count)
22939     ABORT("No handler for exception!");
22940   port = GC_old_exc_ports.ports[i];
22941   behavior = GC_old_exc_ports.behaviors[i];
22942   flavor = GC_old_exc_ports.flavors[i];
22943   if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) {
22944     r = thread_get_state(thread, flavor, thread_state, &thread_state_count);
22945     if(r != KERN_SUCCESS)
22946       ABORT("thread_get_state failed in forward_exception");
22947     }
22948   switch(behavior) {
22949     case EXCEPTION_STATE:
22950       r = exception_raise_state(port, thread, task, exception, data, data_count,
22951                                 &flavor, thread_state, thread_state_count,
22952                                 thread_state, &thread_state_count);
22953       break;
22954     case EXCEPTION_STATE_IDENTITY:
22955       r = exception_raise_state_identity(port, thread, task, exception, data,
22956                                          data_count, &flavor, thread_state,
22957                                          thread_state_count, thread_state,
22958                                          &thread_state_count);
22959       break;
22960     default:
22961       r = exception_raise(port, thread, task, exception, data, data_count);
22962   }
22963   if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) {
22964     r = thread_set_state(thread, flavor, thread_state, thread_state_count);
22965     if (r != KERN_SUCCESS)
22966       ABORT("thread_set_state failed in forward_exception");
22967   }
22968   return r;
22969 }
22970 #define FWD() GC_forward_exception(thread, task, exception, code, code_count)
22971 #ifdef ARM32
22972 #define DARWIN_EXC_STATE         ARM_EXCEPTION_STATE
22973 #define DARWIN_EXC_STATE_COUNT   ARM_EXCEPTION_STATE_COUNT
22974 #define DARWIN_EXC_STATE_T       arm_exception_state_t
22975 #define DARWIN_EXC_STATE_DAR     THREAD_FLD_NAME(far)
22976 #elif defined(AARCH64)
22977 #define DARWIN_EXC_STATE         ARM_EXCEPTION_STATE64
22978 #define DARWIN_EXC_STATE_COUNT   ARM_EXCEPTION_STATE64_COUNT
22979 #define DARWIN_EXC_STATE_T       arm_exception_state64_t
22980 #define DARWIN_EXC_STATE_DAR     THREAD_FLD_NAME(far)
22981 #elif defined(POWERPC)
22982 #if CPP_WORDSZ == 32
22983 #define DARWIN_EXC_STATE       PPC_EXCEPTION_STATE
22984 #define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE_COUNT
22985 #define DARWIN_EXC_STATE_T     ppc_exception_state_t
22986 #else
22987 #define DARWIN_EXC_STATE       PPC_EXCEPTION_STATE64
22988 #define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE64_COUNT
22989 #define DARWIN_EXC_STATE_T     ppc_exception_state64_t
22990 #endif
22991 #define DARWIN_EXC_STATE_DAR     THREAD_FLD_NAME(dar)
22992 #elif defined(I386) || defined(X86_64)
22993 #if CPP_WORDSZ == 32
22994 #if defined(i386_EXCEPTION_STATE_COUNT) \
22995        && !defined(x86_EXCEPTION_STATE32_COUNT)
22996 #define DARWIN_EXC_STATE           i386_EXCEPTION_STATE
22997 #define DARWIN_EXC_STATE_COUNT     i386_EXCEPTION_STATE_COUNT
22998 #define DARWIN_EXC_STATE_T         i386_exception_state_t
22999 #else
23000 #define DARWIN_EXC_STATE           x86_EXCEPTION_STATE32
23001 #define DARWIN_EXC_STATE_COUNT     x86_EXCEPTION_STATE32_COUNT
23002 #define DARWIN_EXC_STATE_T         x86_exception_state32_t
23003 #endif
23004 #else
23005 #define DARWIN_EXC_STATE       x86_EXCEPTION_STATE64
23006 #define DARWIN_EXC_STATE_COUNT x86_EXCEPTION_STATE64_COUNT
23007 #define DARWIN_EXC_STATE_T     x86_exception_state64_t
23008 #endif
23009 #define DARWIN_EXC_STATE_DAR     THREAD_FLD_NAME(faultvaddr)
23010 #elif !defined(CPPCHECK)
23011 #error FIXME for non-arm/ppc/x86 darwin
23012 #endif
23013 GC_API_OSCALL kern_return_t
23014 catch_exception_raise(mach_port_t exception_port GC_ATTR_UNUSED,
23015                       mach_port_t thread, mach_port_t task GC_ATTR_UNUSED,
23016                       exception_type_t exception, exception_data_t code,
23017                       mach_msg_type_number_t code_count GC_ATTR_UNUSED)
23018 {
23019   kern_return_t r;
23020   char *addr;
23021   thread_state_flavor_t flavor = DARWIN_EXC_STATE;
23022   mach_msg_type_number_t exc_state_count = DARWIN_EXC_STATE_COUNT;
23023   DARWIN_EXC_STATE_T exc_state;
23024   if (exception != EXC_BAD_ACCESS || code[0] != KERN_PROTECTION_FAILURE) {
23025 #ifdef DEBUG_EXCEPTION_HANDLING
23026       GC_log_printf("Exception: 0x%x Code: 0x%x 0x%x in catch...\n",
23027                     exception, code_count > 0 ? code[0] : -1,
23028                     code_count > 1 ? code[1] : -1);
23029 #endif
23030     return FWD();
23031   }
23032   r = thread_get_state(thread, flavor, (natural_t*)&exc_state,
23033                        &exc_state_count);
23034   if(r != KERN_SUCCESS) {
23035 #ifdef BROKEN_EXCEPTION_HANDLING
23036       GC_err_printf("thread_get_state failed in catch_exception_raise\n");
23037       return KERN_SUCCESS;
23038 #else
23039       ABORT("thread_get_state failed in catch_exception_raise");
23040 #endif
23041   }
23042   addr = (char*) exc_state.DARWIN_EXC_STATE_DAR;
23043   if (!is_header_found_async(addr)) {
23044 #ifdef BROKEN_EXCEPTION_HANDLING
23045       static char *last_fault;
23046       static int last_fault_count;
23047       if(addr != last_fault) {
23048         last_fault = addr;
23049         last_fault_count = 0;
23050       }
23051       if(++last_fault_count < 32) {
23052         if(last_fault_count == 1)
23053           WARN("Ignoring KERN_PROTECTION_FAILURE at %p\n", addr);
23054         return KERN_SUCCESS;
23055       }
23056       GC_err_printf("Unexpected KERN_PROTECTION_FAILURE at %p; aborting...\n",
23057                     (void *)addr);
23058       EXIT();
23059 #else
23060       return FWD();
23061 #endif
23062   }
23063 #ifdef BROKEN_EXCEPTION_HANDLING
23064     GC_sigbus_count = 0;
23065 #endif
23066   GC_ASSERT(GC_page_size != 0);
23067   if (GC_mprotect_state == GC_MP_NORMAL) {
23068     struct hblk * h = (struct hblk*)((word)addr & ~(GC_page_size-1));
23069     size_t i;
23070     UNPROTECT(h, GC_page_size);
23071     for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
23072       word index = PHT_HASH(h+i);
23073       async_set_pht_entry_from_index(GC_dirty_pages, index);
23074     }
23075   } else if (GC_mprotect_state == GC_MP_DISCARDING) {
23076   } else {
23077     GC_err_printf("KERN_PROTECTION_FAILURE while world is stopped\n");
23078     return FWD();
23079   }
23080   return KERN_SUCCESS;
23081 }
23082 #undef FWD
23083 #ifndef NO_DESC_CATCH_EXCEPTION_RAISE
23084   __asm__(".desc _catch_exception_raise, 0x10");
23085   __asm__(".desc _catch_exception_raise_state, 0x10");
23086   __asm__(".desc _catch_exception_raise_state_identity, 0x10");
23087 #endif
23088 #endif
23089 #ifndef HAVE_INCREMENTAL_PROTECTION_NEEDS
23090   GC_API int GC_CALL GC_incremental_protection_needs(void)
23091   {
23092     GC_ASSERT(GC_is_initialized);
23093     return GC_PROTECTS_NONE;
23094   }
23095 #endif
23096 #ifdef ECOS
23097 #undef sbrk
23098 #endif
23099 GC_API void GC_CALL GC_set_pages_executable(int value)
23100 {
23101   GC_ASSERT(!GC_is_initialized);
23102   GC_pages_executable = (GC_bool)(value != 0);
23103 }
23104 GC_API int GC_CALL GC_get_pages_executable(void)
23105 {
23106 #ifdef IGNORE_PAGES_EXECUTABLE
23107     return 1;
23108 #else
23109     return (int)GC_pages_executable;
23110 #endif
23111 }
23112 #if defined(I386) && defined(LINUX) && defined(SAVE_CALL_CHAIN)
23113 #include <features.h>
23114     struct frame {
23115         struct frame *fr_savfp;
23116         long    fr_savpc;
23117 #if NARGS > 0
23118           long  fr_arg[NARGS];
23119 #endif
23120     };
23121 #endif
23122 #if defined(SPARC)
23123 #if defined(LINUX)
23124 #include <features.h>
23125 #if defined(SAVE_CALL_CHAIN)
23126      struct frame {
23127         long    fr_local[8];
23128         long    fr_arg[6];
23129         struct frame *fr_savfp;
23130         long    fr_savpc;
23131 #ifndef __arch64__
23132           char  *fr_stret;
23133 #endif
23134         long    fr_argd[6];
23135         long    fr_argx[0];
23136      };
23137 #endif
23138 #elif defined (DRSNX)
23139 #include <sys/sparc/frame.h>
23140 #elif defined(OPENBSD)
23141 #include <frame.h>
23142 #elif defined(FREEBSD) || defined(NETBSD)
23143 #include <machine/frame.h>
23144 #else
23145 #include <sys/frame.h>
23146 #endif
23147 #if NARGS > 6
23148 #error We only know how to get the first 6 arguments
23149 #endif
23150 #endif
23151 #ifdef NEED_CALLINFO
23152 #ifdef LINUX
23153 #include <unistd.h>
23154 #endif
23155 #endif
23156 #if defined(GC_HAVE_BUILTIN_BACKTRACE)
23157 #ifdef _MSC_VER
23158 #ifndef GC_MSVC_DBG_H
23159 #define GC_MSVC_DBG_H
23160 #include <stdlib.h>
23161 #ifdef __cplusplus
23162   extern "C" {
23163 #endif
23164 #if !MSVC_DBG_DLL
23165 #define MSVC_DBG_EXPORT
23166 #elif MSVC_DBG_BUILD
23167 #define MSVC_DBG_EXPORT __declspec(dllexport)
23168 #else
23169 #define MSVC_DBG_EXPORT __declspec(dllimport)
23170 #endif
23171 #ifndef MAX_SYM_NAME
23172 #define MAX_SYM_NAME 2000
23173 #endif
23174 typedef void*  HANDLE;
23175 typedef struct _CONTEXT CONTEXT;
23176 MSVC_DBG_EXPORT size_t GetStackFrames(size_t skip, void* frames[], size_t maxFrames);
23177 MSVC_DBG_EXPORT size_t GetStackFramesFromContext(HANDLE hProcess, HANDLE hThread, CONTEXT* context, size_t skip, void* frames[], size_t maxFrames);
23178 MSVC_DBG_EXPORT size_t GetModuleNameFromAddress(void* address, char* moduleName, size_t size);
23179 MSVC_DBG_EXPORT size_t GetModuleNameFromStack(size_t skip, char* moduleName, size_t size);
23180 MSVC_DBG_EXPORT size_t GetSymbolNameFromAddress(void* address, char* symbolName, size_t size, size_t* offsetBytes);
23181 MSVC_DBG_EXPORT size_t GetSymbolNameFromStack(size_t skip, char* symbolName, size_t size, size_t* offsetBytes);
23182 MSVC_DBG_EXPORT size_t GetFileLineFromAddress(void* address, char* fileName, size_t size, size_t* lineNumber, size_t* offsetBytes);
23183 MSVC_DBG_EXPORT size_t GetFileLineFromStack(size_t skip, char* fileName, size_t size, size_t* lineNumber, size_t* offsetBytes);
23184 MSVC_DBG_EXPORT size_t GetDescriptionFromAddress(void* address, const char* format, char* description, size_t size);
23185 MSVC_DBG_EXPORT size_t GetDescriptionFromStack(void*const frames[], size_t count, const char* format, char* description[], size_t size);
23186 MSVC_DBG_EXPORT int    backtrace(void* addresses[], int count);
23187 MSVC_DBG_EXPORT char** backtrace_symbols(void*const addresses[], int count);
23188 #ifdef __cplusplus
23189   }
23190 #endif
23191 #endif
23192 #else
23193 #include <execinfo.h>
23194 #endif
23195 #endif
23196 #ifdef SAVE_CALL_CHAIN
23197 #if NARGS == 0 && NFRAMES % 2 == 0  \
23198     && defined(GC_HAVE_BUILTIN_BACKTRACE)
23199 #ifdef REDIRECT_MALLOC
23200 #ifdef THREADS
23201     __thread
23202 #endif
23203     GC_bool GC_in_save_callers = FALSE;
23204 #endif
23205 GC_INNER void GC_save_callers(struct callinfo info[NFRAMES])
23206 {
23207   void * tmp_info[NFRAMES + 1];
23208   int npcs, i;
23209 #define IGNORE_FRAMES 1
23210 #ifdef REDIRECT_MALLOC
23211     if (GC_in_save_callers) {
23212       info[0].ci_pc = (word)(&GC_save_callers);
23213       for (i = 1; i < NFRAMES; ++i) info[i].ci_pc = 0;
23214       return;
23215     }
23216     GC_in_save_callers = TRUE;
23217 #endif
23218   GC_ASSERT(I_HOLD_LOCK());
23219   GC_STATIC_ASSERT(sizeof(struct callinfo) == sizeof(void *));
23220   npcs = backtrace((void **)tmp_info, NFRAMES + IGNORE_FRAMES);
23221   if (npcs > IGNORE_FRAMES)
23222     BCOPY(&tmp_info[IGNORE_FRAMES], info,
23223           (npcs - IGNORE_FRAMES) * sizeof(void *));
23224   for (i = npcs - IGNORE_FRAMES; i < NFRAMES; ++i) info[i].ci_pc = 0;
23225 #ifdef REDIRECT_MALLOC
23226     GC_in_save_callers = FALSE;
23227 #endif
23228 }
23229 #else
23230 #if (defined(OPENBSD) || defined(NETBSD) || defined(FREEBSD)) && defined(SPARC)
23231 #define FR_SAVFP fr_fp
23232 #define FR_SAVPC fr_pc
23233 #else
23234 #define FR_SAVFP fr_savfp
23235 #define FR_SAVPC fr_savpc
23236 #endif
23237 #if defined(SPARC) && (defined(__arch64__) || defined(__sparcv9))
23238 #define BIAS 2047
23239 #else
23240 #define BIAS 0
23241 #endif
23242 GC_INNER void GC_save_callers(struct callinfo info[NFRAMES])
23243 {
23244   struct frame *frame;
23245   struct frame *fp;
23246   int nframes = 0;
23247 #ifdef I386
23248     asm("movl %%ebp,%0" : "=r"(frame));
23249     fp = frame;
23250 #else
23251     frame = (struct frame *)GC_save_regs_in_stack();
23252     fp = (struct frame *)((long) frame -> FR_SAVFP + BIAS);
23253 #endif
23254    for (; !((word)fp HOTTER_THAN (word)frame)
23255 #ifndef THREADS
23256             && !((word)GC_stackbottom HOTTER_THAN (word)fp)
23257 #elif defined(STACK_GROWS_UP)
23258             && fp != NULL
23259 #endif
23260           && nframes < NFRAMES;
23261         fp = (struct frame *)((long) fp -> FR_SAVFP + BIAS), nframes++) {
23262 #if NARGS > 0
23263         int i;
23264 #endif
23265       info[nframes].ci_pc = fp->FR_SAVPC;
23266 #if NARGS > 0
23267         for (i = 0; i < NARGS; i++) {
23268           info[nframes].ci_arg[i] = ~(fp->fr_arg[i]);
23269         }
23270 #endif
23271   }
23272   if (nframes < NFRAMES) info[nframes].ci_pc = 0;
23273 }
23274 #endif
23275 #endif
23276 #ifdef NEED_CALLINFO
23277 GC_INNER void GC_print_callers(struct callinfo info[NFRAMES])
23278 {
23279     int i;
23280     static int reentry_count = 0;
23281     DCL_LOCK_STATE;
23282     LOCK();
23283       ++reentry_count;
23284     UNLOCK();
23285 #if NFRAMES == 1
23286       GC_err_printf("\tCaller at allocation:\n");
23287 #else
23288       GC_err_printf("\tCall chain at allocation:\n");
23289 #endif
23290     for (i = 0; i < NFRAMES; i++) {
23291 #if defined(LINUX) && !defined(SMALL_CONFIG)
23292           GC_bool stop = FALSE;
23293 #endif
23294         if (0 == info[i].ci_pc)
23295           break;
23296 #if NARGS > 0
23297         {
23298           int j;
23299           GC_err_printf("\t\targs: ");
23300           for (j = 0; j < NARGS; j++) {
23301             if (j != 0) GC_err_printf(", ");
23302             GC_err_printf("%d (0x%X)", ~(info[i].ci_arg[j]),
23303                                         ~(info[i].ci_arg[j]));
23304           }
23305           GC_err_printf("\n");
23306         }
23307 #endif
23308         if (reentry_count > 1) {
23309             GC_err_printf("\t\t##PC##= 0x%lx\n",
23310                           (unsigned long)info[i].ci_pc);
23311             continue;
23312         }
23313         {
23314           char buf[40];
23315           char *name;
23316 #if defined(GC_HAVE_BUILTIN_BACKTRACE) \
23317              && !defined(GC_BACKTRACE_SYMBOLS_BROKEN)
23318             char **sym_name =
23319               backtrace_symbols((void **)(&(info[i].ci_pc)), 1);
23320             if (sym_name != NULL) {
23321               name = sym_name[0];
23322             } else
23323 #endif
23324            {
23325             (void)snprintf(buf, sizeof(buf), "##PC##= 0x%lx",
23326                            (unsigned long)info[i].ci_pc);
23327             buf[sizeof(buf) - 1] = '\0';
23328             name = buf;
23329           }
23330 #if defined(LINUX) && !defined(SMALL_CONFIG)
23331             do {
23332                 FILE *pipe;
23333 #define EXE_SZ 100
23334                 static char exe_name[EXE_SZ];
23335 #define CMD_SZ 200
23336                 char cmd_buf[CMD_SZ];
23337 #define RESULT_SZ 200
23338                 static char result_buf[RESULT_SZ];
23339                 size_t result_len;
23340                 char *old_preload;
23341 #define PRELOAD_SZ 200
23342                 char preload_buf[PRELOAD_SZ];
23343                 static GC_bool found_exe_name = FALSE;
23344                 static GC_bool will_fail = FALSE;
23345                 if (will_fail)
23346                   break;
23347                 if (!found_exe_name) {
23348                   int ret_code = readlink("/proc/self/exe", exe_name, EXE_SZ);
23349                   if (ret_code < 0 || ret_code >= EXE_SZ
23350                       || exe_name[0] != '/') {
23351                     will_fail = TRUE;
23352                     break;
23353                   }
23354                   exe_name[ret_code] = '\0';
23355                   found_exe_name = TRUE;
23356                 }
23357                 (void)snprintf(cmd_buf, sizeof(cmd_buf),
23358                                "/usr/bin/addr2line -f -e %s 0x%lx",
23359                                exe_name, (unsigned long)info[i].ci_pc);
23360                 cmd_buf[sizeof(cmd_buf) - 1] = '\0';
23361                 old_preload = GETENV("LD_PRELOAD");
23362                 if (0 != old_preload) {
23363                   size_t old_len = strlen(old_preload);
23364                   if (old_len >= PRELOAD_SZ) {
23365                     will_fail = TRUE;
23366                     break;
23367                   }
23368                   BCOPY(old_preload, preload_buf, old_len + 1);
23369                   unsetenv ("LD_PRELOAD");
23370                 }
23371                 pipe = popen(cmd_buf, "r");
23372                 if (0 != old_preload
23373                     && 0 != setenv ("LD_PRELOAD", preload_buf, 0)) {
23374                   WARN("Failed to reset LD_PRELOAD\n", 0);
23375                 }
23376                 if (NULL == pipe) {
23377                   will_fail = TRUE;
23378                   break;
23379                 }
23380                 result_len = fread(result_buf, 1, RESULT_SZ - 1, pipe);
23381                 (void)pclose(pipe);
23382                 if (0 == result_len) {
23383                   will_fail = TRUE;
23384                   break;
23385                 }
23386                 if (result_buf[result_len - 1] == '\n') --result_len;
23387                 result_buf[result_len] = 0;
23388                 if (result_buf[0] == '?'
23389                     || (result_buf[result_len-2] == ':'
23390                         && result_buf[result_len-1] == '0'))
23391                   break;
23392                 {
23393                   char * nl = strchr(result_buf, '\n');
23394                   if (nl != NULL
23395                       && (word)nl < (word)(result_buf + result_len)) {
23396                     *nl = ':';
23397                   }
23398                   if (strncmp(result_buf, "main",
23399                               nl != NULL
23400                                 ? (size_t)((word)nl
23401                                            - COVERT_DATAFLOW(result_buf))
23402                                 : result_len) == 0) {
23403                     stop = TRUE;
23404                   }
23405                 }
23406                 if (result_len < RESULT_SZ - 25) {
23407                   (void)snprintf(&result_buf[result_len],
23408                                  sizeof(result_buf) - result_len,
23409                                  " [0x%lx]", (unsigned long)info[i].ci_pc);
23410                   result_buf[sizeof(result_buf) - 1] = '\0';
23411                 }
23412 #if defined(CPPCHECK)
23413                   GC_noop1((unsigned char)name[0]);
23414 #endif
23415                 name = result_buf;
23416             } while (0);
23417 #endif
23418           GC_err_printf("\t\t%s\n", name);
23419 #if defined(GC_HAVE_BUILTIN_BACKTRACE) \
23420              && !defined(GC_BACKTRACE_SYMBOLS_BROKEN)
23421             if (sym_name != NULL)
23422               free(sym_name);
23423 #endif
23424         }
23425 #if defined(LINUX) && !defined(SMALL_CONFIG)
23426           if (stop)
23427             break;
23428 #endif
23429     }
23430     LOCK();
23431       --reentry_count;
23432     UNLOCK();
23433 }
23434 #endif
23435 #if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
23436   void GC_print_address_map(void)
23437   {
23438     const char *maps = GC_get_maps();
23439     GC_err_printf("---------- Begin address map ----------\n");
23440     GC_err_puts(maps);
23441     GC_err_printf("---------- End address map ----------\n");
23442   }
23443 #endif
23444 #if defined(THREAD_LOCAL_ALLOC)
23445 #ifndef THREADS
23446 #error "invalid config - THREAD_LOCAL_ALLOC requires GC_THREADS"
23447 #endif
23448 #ifndef GC_THREAD_LOCAL_ALLOC_H
23449 #define GC_THREAD_LOCAL_ALLOC_H
23450 #ifdef THREAD_LOCAL_ALLOC
23451 #if defined(USE_HPUX_TLS)
23452 #error USE_HPUX_TLS macro was replaced by USE_COMPILER_TLS
23453 #endif
23454 #include <stdlib.h>
23455 EXTERN_C_BEGIN
23456 #if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC) \
23457     && !defined(USE_WIN32_COMPILER_TLS) && !defined(USE_COMPILER_TLS) \
23458     && !defined(USE_CUSTOM_SPECIFIC)
23459 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
23460 #if defined(CYGWIN32) && GC_GNUC_PREREQ(4, 0)
23461 #if defined(__clang__)
23462 #define USE_PTHREAD_SPECIFIC
23463 #else
23464 #define USE_COMPILER_TLS
23465 #endif
23466 #elif defined(__GNUC__) || defined(MSWINCE)
23467 #define USE_WIN32_SPECIFIC
23468 #else
23469 #define USE_WIN32_COMPILER_TLS
23470 #endif
23471 #elif (defined(LINUX) && !defined(ARM32) && !defined(AVR32) \
23472          && GC_GNUC_PREREQ(3, 3) \
23473          && !(defined(__clang__) && defined(HOST_ANDROID))) \
23474        || (defined(FREEBSD) \
23475            || (defined(NETBSD) && __NetBSD_Version__ >= 600000000 ) \
23476             && (GC_GNUC_PREREQ(4, 4) || GC_CLANG_PREREQ(3, 9))) \
23477        || (defined(HOST_ANDROID) && defined(ARM32) \
23478             && (GC_GNUC_PREREQ(4, 6) || GC_CLANG_PREREQ_FULL(3, 8, 256229)))
23479 #define USE_COMPILER_TLS
23480 #elif defined(GC_DGUX386_THREADS) || defined(GC_OSF1_THREADS) \
23481        || defined(GC_AIX_THREADS) || defined(GC_DARWIN_THREADS) \
23482        || defined(GC_FREEBSD_THREADS) || defined(GC_NETBSD_THREADS) \
23483        || defined(GC_LINUX_THREADS) || defined(GC_HAIKU_THREADS) \
23484        || defined(GC_RTEMS_PTHREADS)
23485 #define USE_PTHREAD_SPECIFIC
23486 #elif defined(GC_HPUX_THREADS)
23487 #ifdef __GNUC__
23488 #define USE_PTHREAD_SPECIFIC
23489 #else
23490 #define USE_COMPILER_TLS
23491 #endif
23492 #else
23493 #define USE_CUSTOM_SPECIFIC
23494 #endif
23495 #endif
23496 #ifndef THREAD_FREELISTS_KINDS
23497 #ifdef ENABLE_DISCLAIM
23498 #define THREAD_FREELISTS_KINDS (NORMAL+2)
23499 #else
23500 #define THREAD_FREELISTS_KINDS (NORMAL+1)
23501 #endif
23502 #endif
23503 typedef struct thread_local_freelists {
23504   void * _freelists[THREAD_FREELISTS_KINDS][TINY_FREELISTS];
23505 #define ptrfree_freelists _freelists[PTRFREE]
23506 #define normal_freelists _freelists[NORMAL]
23507 #ifdef GC_GCJ_SUPPORT
23508     void * gcj_freelists[TINY_FREELISTS];
23509 #define ERROR_FL ((void *)GC_WORD_MAX)
23510 #endif
23511 #define DIRECT_GRANULES (HBLKSIZE/GRANULE_BYTES)
23512 } *GC_tlfs;
23513 #if defined(USE_PTHREAD_SPECIFIC)
23514 #define GC_getspecific pthread_getspecific
23515 #define GC_setspecific pthread_setspecific
23516 #define GC_key_create pthread_key_create
23517 #define GC_remove_specific(key) pthread_setspecific(key, NULL)
23518 #define GC_remove_specific_after_fork(key, t) (void)0
23519   typedef pthread_key_t GC_key_t;
23520 #elif defined(USE_COMPILER_TLS) || defined(USE_WIN32_COMPILER_TLS)
23521 #define GC_getspecific(x) (x)
23522 #define GC_setspecific(key, v) ((key) = (v), 0)
23523 #define GC_key_create(key, d) 0
23524 #define GC_remove_specific(key)
23525 #define GC_remove_specific_after_fork(key, t) (void)0
23526   typedef void * GC_key_t;
23527 #elif defined(USE_WIN32_SPECIFIC)
23528 #define GC_getspecific TlsGetValue
23529 #define GC_setspecific(key, v) !TlsSetValue(key, v)
23530 #ifndef TLS_OUT_OF_INDEXES
23531 #define TLS_OUT_OF_INDEXES (DWORD)0xFFFFFFFF
23532 #endif
23533 #define GC_key_create(key, d) \
23534         ((d) != 0 || (*(key) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? -1 : 0)
23535 #define GC_remove_specific(key)
23536 #define GC_remove_specific_after_fork(key, t) (void)0
23537   typedef DWORD GC_key_t;
23538 #elif defined(USE_CUSTOM_SPECIFIC)
23539   EXTERN_C_END
23540 #ifndef GC_SPECIFIC_H
23541 #define GC_SPECIFIC_H
23542 #include <errno.h>
23543 EXTERN_C_BEGIN
23544 #define MALLOC_CLEAR(n) GC_INTERNAL_MALLOC(n, NORMAL)
23545 #define TS_CACHE_SIZE 1024
23546 #define CACHE_HASH(n) ((((n) >> 8) ^ (n)) & (TS_CACHE_SIZE - 1))
23547 #define TS_HASH_SIZE 1024
23548 #define HASH(p) \
23549           ((unsigned)((((word)(p)) >> 8) ^ (word)(p)) & (TS_HASH_SIZE - 1))
23550 #ifdef GC_ASSERTIONS
23551   typedef GC_hidden_pointer ts_entry_value_t;
23552 #define TS_HIDE_VALUE(p) GC_HIDE_POINTER(p)
23553 #define TS_REVEAL_PTR(p) GC_REVEAL_POINTER(p)
23554 #else
23555   typedef void * ts_entry_value_t;
23556 #define TS_HIDE_VALUE(p) (p)
23557 #define TS_REVEAL_PTR(p) (p)
23558 #endif
23559 typedef struct thread_specific_entry {
23560         volatile AO_t qtid;
23561         ts_entry_value_t value;
23562         struct thread_specific_entry *next;
23563         pthread_t thread;
23564 } tse;
23565 #define quick_thread_id() (((word)GC_approx_sp()) >> 12)
23566 #define INVALID_QTID ((word)0)
23567 #define INVALID_THREADID ((pthread_t)0)
23568 union ptse_ao_u {
23569   tse *p;
23570   volatile AO_t ao;
23571 };
23572 typedef struct thread_specific_data {
23573     tse * volatile cache[TS_CACHE_SIZE];
23574     union ptse_ao_u hash[TS_HASH_SIZE];
23575     pthread_mutex_t lock;
23576 } tsd;
23577 typedef tsd * GC_key_t;
23578 #define GC_key_create(key, d) GC_key_create_inner(key)
23579 GC_INNER int GC_key_create_inner(tsd ** key_ptr);
23580 GC_INNER int GC_setspecific(tsd * key, void * value);
23581 #define GC_remove_specific(key) \
23582                         GC_remove_specific_after_fork(key, pthread_self())
23583 GC_INNER void GC_remove_specific_after_fork(tsd * key, pthread_t t);
23584 GC_INNER void * GC_slow_getspecific(tsd * key, word qtid,
23585                                     tse * volatile * cache_entry);
23586 GC_INLINE void * GC_getspecific(tsd * key)
23587 {
23588     word qtid = quick_thread_id();
23589     tse * volatile * entry_ptr = &key->cache[CACHE_HASH(qtid)];
23590     tse * entry = *entry_ptr;
23591     GC_ASSERT(qtid != INVALID_QTID);
23592     if (EXPECT(entry -> qtid == qtid, TRUE)) {
23593       GC_ASSERT(entry -> thread == pthread_self());
23594       return TS_REVEAL_PTR(entry -> value);
23595     }
23596     return GC_slow_getspecific(key, qtid, entry_ptr);
23597 }
23598 EXTERN_C_END
23599 #endif
23600   EXTERN_C_BEGIN
23601 #else
23602 #error implement me
23603 #endif
23604 GC_INNER void GC_init_thread_local(GC_tlfs p);
23605 GC_INNER void GC_destroy_thread_local(GC_tlfs p);
23606 GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p);
23607 #ifdef GC_ASSERTIONS
23608   GC_bool GC_is_thread_tsd_valid(void *tsd);
23609   void GC_check_tls_for(GC_tlfs p);
23610 #if defined(USE_CUSTOM_SPECIFIC)
23611     void GC_check_tsd_marks(tsd *key);
23612 #endif
23613 #endif
23614 #ifndef GC_ATTR_TLS_FAST
23615 #define GC_ATTR_TLS_FAST
23616 #endif
23617 extern
23618 #if defined(USE_COMPILER_TLS)
23619   __thread GC_ATTR_TLS_FAST
23620 #elif defined(USE_WIN32_COMPILER_TLS)
23621   __declspec(thread) GC_ATTR_TLS_FAST
23622 #endif
23623   GC_key_t GC_thread_key;
23624 EXTERN_C_END
23625 #endif
23626 #endif
23627 #include <stdlib.h>
23628 #if defined(USE_COMPILER_TLS)
23629   __thread GC_ATTR_TLS_FAST
23630 #elif defined(USE_WIN32_COMPILER_TLS)
23631   __declspec(thread) GC_ATTR_TLS_FAST
23632 #endif
23633 GC_key_t GC_thread_key;
23634 static GC_bool keys_initialized;
23635 static void return_single_freelist(void *fl, void **gfl)
23636 {
23637     if (*gfl == 0) {
23638       *gfl = fl;
23639     } else {
23640       void *q, **qptr;
23641       GC_ASSERT(GC_size(fl) == GC_size(*gfl));
23642         qptr = &(obj_link(fl));
23643         while ((word)(q = *qptr) >= HBLKSIZE)
23644           qptr = &(obj_link(q));
23645         GC_ASSERT(0 == q);
23646         *qptr = *gfl;
23647         *gfl = fl;
23648     }
23649 }
23650 static void return_freelists(void **fl, void **gfl)
23651 {
23652     int i;
23653     for (i = 1; i < TINY_FREELISTS; ++i) {
23654         if ((word)(fl[i]) >= HBLKSIZE) {
23655           return_single_freelist(fl[i], &gfl[i]);
23656         }
23657         fl[i] = (ptr_t)HBLKSIZE;
23658     }
23659 #ifdef GC_GCJ_SUPPORT
23660       if (fl[0] == ERROR_FL) return;
23661 #endif
23662     if ((word)(fl[0]) >= HBLKSIZE) {
23663         return_single_freelist(fl[0], &gfl[1]);
23664     }
23665 }
23666 #ifdef USE_PTHREAD_SPECIFIC
23667   static void reset_thread_key(void* v) {
23668     pthread_setspecific(GC_thread_key, v);
23669   }
23670 #else
23671 #define reset_thread_key 0
23672 #endif
23673 GC_INNER void GC_init_thread_local(GC_tlfs p)
23674 {
23675     int i, j, res;
23676     GC_ASSERT(I_HOLD_LOCK());
23677     if (!EXPECT(keys_initialized, TRUE)) {
23678         GC_ASSERT((word)&GC_thread_key % sizeof(word) == 0);
23679         res = GC_key_create(&GC_thread_key, reset_thread_key);
23680         if (COVERT_DATAFLOW(res) != 0) {
23681             ABORT("Failed to create key for local allocator");
23682         }
23683         keys_initialized = TRUE;
23684     }
23685     res = GC_setspecific(GC_thread_key, p);
23686     if (COVERT_DATAFLOW(res) != 0) {
23687         ABORT("Failed to set thread specific allocation pointers");
23688     }
23689     for (j = 0; j < TINY_FREELISTS; ++j) {
23690         for (i = 0; i < THREAD_FREELISTS_KINDS; ++i) {
23691             p -> _freelists[i][j] = (void *)(word)1;
23692         }
23693 #ifdef GC_GCJ_SUPPORT
23694             p -> gcj_freelists[j] = (void *)(word)1;
23695 #endif
23696     }
23697 #ifdef GC_GCJ_SUPPORT
23698         p -> gcj_freelists[0] = ERROR_FL;
23699 #endif
23700 }
23701 GC_INNER void GC_destroy_thread_local(GC_tlfs p)
23702 {
23703     int k;
23704     GC_STATIC_ASSERT(THREAD_FREELISTS_KINDS <= MAXOBJKINDS);
23705     for (k = 0; k < THREAD_FREELISTS_KINDS; ++k) {
23706         if (k == (int)GC_n_kinds)
23707             break;
23708         return_freelists(p -> _freelists[k], GC_obj_kinds[k].ok_freelist);
23709     }
23710 #ifdef GC_GCJ_SUPPORT
23711         return_freelists(p -> gcj_freelists, (void **)GC_gcjobjfreelist);
23712 #endif
23713 }
23714 GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind(size_t bytes, int kind)
23715 {
23716     size_t granules;
23717     void *tsd;
23718     void *result;
23719 #if MAXOBJKINDS > THREAD_FREELISTS_KINDS
23720       if (EXPECT(kind >= THREAD_FREELISTS_KINDS, FALSE)) {
23721         return GC_malloc_kind_global(bytes, kind);
23722       }
23723 #endif
23724 #if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
23725     {
23726       GC_key_t k = GC_thread_key;
23727       if (EXPECT(0 == k, FALSE)) {
23728         return GC_malloc_kind_global(bytes, kind);
23729       }
23730       tsd = GC_getspecific(k);
23731     }
23732 #else
23733       if (!EXPECT(keys_initialized, TRUE))
23734         return GC_malloc_kind_global(bytes, kind);
23735       tsd = GC_getspecific(GC_thread_key);
23736 #endif
23737 #if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
23738       if (EXPECT(0 == tsd, FALSE)) {
23739         return GC_malloc_kind_global(bytes, kind);
23740       }
23741 #endif
23742     GC_ASSERT(GC_is_initialized);
23743     GC_ASSERT(GC_is_thread_tsd_valid(tsd));
23744     granules = ROUNDED_UP_GRANULES(bytes);
23745 #if defined(CPPCHECK)
23746 #define MALLOC_KIND_PTRFREE_INIT (void*)1
23747 #else
23748 #define MALLOC_KIND_PTRFREE_INIT NULL
23749 #endif
23750     GC_FAST_MALLOC_GRANS(result, granules,
23751                          ((GC_tlfs)tsd) -> _freelists[kind], DIRECT_GRANULES,
23752                          kind, GC_malloc_kind_global(bytes, kind),
23753                          (void)(kind == PTRFREE ? MALLOC_KIND_PTRFREE_INIT
23754                                                : (obj_link(result) = 0)));
23755 #ifdef LOG_ALLOCS
23756       GC_log_printf("GC_malloc_kind(%lu, %d) returned %p, recent GC #%lu\n",
23757                     (unsigned long)bytes, kind, result,
23758                     (unsigned long)GC_gc_no);
23759 #endif
23760     return result;
23761 }
23762 #ifdef GC_GCJ_SUPPORT
23763 GC_API GC_ATTR_MALLOC void * GC_CALL GC_gcj_malloc(size_t bytes,
23764                                     void * ptr_to_struct_containing_descr)
23765 {
23766   if (EXPECT(GC_incremental, FALSE)) {
23767     return GC_core_gcj_malloc(bytes, ptr_to_struct_containing_descr);
23768   } else {
23769     size_t granules = ROUNDED_UP_GRANULES(bytes);
23770     void *result;
23771     void **tiny_fl;
23772     GC_ASSERT(GC_gcjobjfreelist != NULL);
23773     tiny_fl = ((GC_tlfs)GC_getspecific(GC_thread_key))->gcj_freelists;
23774     GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
23775                          GC_gcj_kind,
23776                          GC_core_gcj_malloc(bytes,
23777                                             ptr_to_struct_containing_descr),
23778                          {AO_compiler_barrier();
23779                           *(void **)result = ptr_to_struct_containing_descr;});
23780     return result;
23781   }
23782 }
23783 #endif
23784 GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p)
23785 {
23786     ptr_t q;
23787     int i, j;
23788     for (j = 0; j < TINY_FREELISTS; ++j) {
23789       for (i = 0; i < THREAD_FREELISTS_KINDS; ++i) {
23790         q = (ptr_t)AO_load((volatile AO_t *)&p->_freelists[i][j]);
23791         if ((word)q > HBLKSIZE)
23792           GC_set_fl_marks(q);
23793       }
23794 #ifdef GC_GCJ_SUPPORT
23795         if (EXPECT(j > 0, TRUE)) {
23796           q = (ptr_t)AO_load((volatile AO_t *)&p->gcj_freelists[j]);
23797           if ((word)q > HBLKSIZE)
23798             GC_set_fl_marks(q);
23799         }
23800 #endif
23801     }
23802 }
23803 #if defined(GC_ASSERTIONS)
23804     void GC_check_tls_for(GC_tlfs p)
23805     {
23806         int i, j;
23807         for (j = 1; j < TINY_FREELISTS; ++j) {
23808           for (i = 0; i < THREAD_FREELISTS_KINDS; ++i) {
23809             GC_check_fl_marks(&p->_freelists[i][j]);
23810           }
23811 #ifdef GC_GCJ_SUPPORT
23812             GC_check_fl_marks(&p->gcj_freelists[j]);
23813 #endif
23814         }
23815     }
23816 #endif
23817 #endif
23818 #ifndef GC_PTHREAD_SUPPORT_H
23819 #define GC_PTHREAD_SUPPORT_H
23820 #if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)
23821 #if defined(GC_DARWIN_THREADS)
23822 #else
23823 #ifndef GC_PTHREAD_STOP_WORLD_H
23824 #define GC_PTHREAD_STOP_WORLD_H
23825 EXTERN_C_BEGIN
23826 struct thread_stop_info {
23827 #if !defined(GC_OPENBSD_UTHREADS) && !defined(NACL) \
23828        && !defined(PLATFORM_STOP_WORLD) && !defined(SN_TARGET_PSP2)
23829       volatile AO_t last_stop_count;
23830 #endif
23831     ptr_t stack_ptr;
23832 #ifdef NACL
23833 #ifdef ARM32
23834 #define NACL_GC_REG_STORAGE_SIZE 9
23835 #else
23836 #define NACL_GC_REG_STORAGE_SIZE 20
23837 #endif
23838       ptr_t reg_storage[NACL_GC_REG_STORAGE_SIZE];
23839 #elif defined(PLATFORM_HAVE_GC_REG_STORAGE_SIZE)
23840       word registers[PLATFORM_GC_REG_STORAGE_SIZE];
23841 #endif
23842 };
23843 GC_INNER void GC_stop_init(void);
23844 EXTERN_C_END
23845 #endif
23846 #endif
23847 #ifdef THREAD_LOCAL_ALLOC
23848 #endif
23849 #ifdef THREAD_SANITIZER
23850 #endif
23851 EXTERN_C_BEGIN
23852 typedef struct GC_Thread_Rep {
23853 #ifdef THREAD_SANITIZER
23854       char dummy[sizeof(oh)];
23855 #endif
23856     struct GC_Thread_Rep * next;
23857     pthread_t id;
23858 #ifdef USE_TKILL_ON_ANDROID
23859       pid_t kernel_id;
23860 #endif
23861     struct thread_stop_info stop_info;
23862 #if defined(GC_ENABLE_SUSPEND_THREAD) && !defined(GC_DARWIN_THREADS) \
23863         && !defined(GC_OPENBSD_UTHREADS) && !defined(NACL)
23864       volatile AO_t suspended_ext;
23865 #endif
23866     unsigned char flags;
23867 #define FINISHED 1
23868 #define DETACHED 2
23869 #define MAIN_THREAD 4
23870 #define DISABLED_GC 0x10
23871     unsigned char thread_blocked;
23872     unsigned short finalizer_skipped;
23873     unsigned char finalizer_nested;
23874     ptr_t stack_end;
23875     ptr_t altstack;
23876     word altstack_size;
23877     ptr_t stack;
23878     word stack_size;
23879 #if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
23880       ptr_t topOfStack;
23881 #endif
23882 #ifdef IA64
23883         ptr_t backing_store_end;
23884         ptr_t backing_store_ptr;
23885 #endif
23886     struct GC_traced_stack_sect_s *traced_stack_sect;
23887     void * status;
23888 #ifdef THREAD_LOCAL_ALLOC
23889         struct thread_local_freelists tlfs GC_ATTR_WORD_ALIGNED;
23890 #endif
23891 } * GC_thread;
23892 #ifndef THREAD_TABLE_SZ
23893 #define THREAD_TABLE_SZ 256
23894 #endif
23895 #if CPP_WORDSZ == 64
23896 #define THREAD_TABLE_INDEX(id) \
23897     (int)(((((NUMERIC_THREAD_ID(id) >> 8) ^ NUMERIC_THREAD_ID(id)) >> 16) \
23898           ^ ((NUMERIC_THREAD_ID(id) >> 8) ^ NUMERIC_THREAD_ID(id))) \
23899          % THREAD_TABLE_SZ)
23900 #else
23901 #define THREAD_TABLE_INDEX(id) \
23902                 (int)(((NUMERIC_THREAD_ID(id) >> 16) \
23903                        ^ (NUMERIC_THREAD_ID(id) >> 8) \
23904                        ^ NUMERIC_THREAD_ID(id)) % THREAD_TABLE_SZ)
23905 #endif
23906 GC_EXTERN volatile GC_thread GC_threads[THREAD_TABLE_SZ];
23907 GC_EXTERN GC_bool GC_thr_initialized;
23908 GC_INNER GC_thread GC_lookup_thread(pthread_t id);
23909 #ifdef NACL
23910   GC_EXTERN __thread GC_thread GC_nacl_gc_thread_self;
23911   GC_INNER void GC_nacl_initialize_gc_thread(void);
23912   GC_INNER void GC_nacl_shutdown_gc_thread(void);
23913 #endif
23914 #ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
23915   GC_INNER void GC_unblock_gc_signals(void);
23916 #endif
23917 #ifdef GC_PTHREAD_START_STANDALONE
23918 #define GC_INNER_PTHRSTART
23919 #else
23920 #define GC_INNER_PTHRSTART GC_INNER
23921 #endif
23922 GC_INNER_PTHRSTART void * GC_CALLBACK GC_inner_start_routine(
23923                                         struct GC_stack_base *sb, void *arg);
23924 GC_INNER_PTHRSTART GC_thread GC_start_rtn_prepare_thread(
23925                                         void *(**pstart)(void *),
23926                                         void **pstart_arg,
23927                                         struct GC_stack_base *sb, void *arg);
23928 GC_INNER_PTHRSTART void GC_thread_exit_proc(void *);
23929 EXTERN_C_END
23930 #endif
23931 #endif
23932 #if defined(GC_DARWIN_THREADS)
23933 #include <sys/sysctl.h>
23934 #include <mach/machine.h>
23935 #include <CoreFoundation/CoreFoundation.h>
23936 #ifdef POWERPC
23937 #if CPP_WORDSZ == 32
23938 #define PPC_RED_ZONE_SIZE 224
23939 #elif CPP_WORDSZ == 64
23940 #define PPC_RED_ZONE_SIZE 320
23941 #endif
23942 #endif
23943 #ifndef DARWIN_DONT_PARSE_STACK
23944 typedef struct StackFrame {
23945   unsigned long savedSP;
23946   unsigned long savedCR;
23947   unsigned long savedLR;
23948   unsigned long reserved[2];
23949   unsigned long savedRTOC;
23950 } StackFrame;
23951 GC_INNER ptr_t GC_FindTopOfStack(unsigned long stack_start)
23952 {
23953   StackFrame *frame = (StackFrame *)stack_start;
23954   if (stack_start == 0) {
23955 #ifdef POWERPC
23956 #if CPP_WORDSZ == 32
23957         __asm__ __volatile__ ("lwz %0,0(r1)" : "=r" (frame));
23958 #else
23959         __asm__ __volatile__ ("ld %0,0(r1)" : "=r" (frame));
23960 #endif
23961 #elif defined(ARM32)
23962         volatile ptr_t sp_reg;
23963         __asm__ __volatile__ ("mov %0, r7\n" : "=r" (sp_reg));
23964         frame = (StackFrame *)sp_reg;
23965 #elif defined(AARCH64)
23966         volatile ptr_t sp_reg;
23967         __asm__ __volatile__ ("mov %0, x29\n" : "=r" (sp_reg));
23968         frame = (StackFrame *)sp_reg;
23969 #else
23970 #if defined(CPPCHECK)
23971         GC_noop1((word)&frame);
23972 #endif
23973       ABORT("GC_FindTopOfStack(0) is not implemented");
23974 #endif
23975   }
23976 #ifdef DEBUG_THREADS_EXTRA
23977     GC_log_printf("FindTopOfStack start at sp= %p\n", (void *)frame);
23978 #endif
23979   while (frame->savedSP != 0) {
23980     frame = (StackFrame*)frame->savedSP;
23981     if ((frame->savedLR & ~0x3) == 0 || (frame->savedLR & ~0x3) == ~0x3UL)
23982       break;
23983   }
23984 #ifdef DEBUG_THREADS_EXTRA
23985     GC_log_printf("FindTopOfStack finish at sp= %p\n", (void *)frame);
23986 #endif
23987   return (ptr_t)frame;
23988 }
23989 #endif
23990 #ifdef GC_NO_THREADS_DISCOVERY
23991 #define GC_query_task_threads FALSE
23992 #elif defined(GC_DISCOVER_TASK_THREADS)
23993 #define GC_query_task_threads TRUE
23994 #else
23995   STATIC GC_bool GC_query_task_threads = FALSE;
23996 #endif
23997 GC_API void GC_CALL GC_use_threads_discovery(void)
23998 {
23999 #if defined(GC_NO_THREADS_DISCOVERY) || defined(DARWIN_DONT_PARSE_STACK)
24000     ABORT("Darwin task-threads-based stop and push unsupported");
24001 #else
24002 #ifndef GC_ALWAYS_MULTITHREADED
24003       GC_ASSERT(!GC_need_to_lock);
24004 #endif
24005 #ifndef GC_DISCOVER_TASK_THREADS
24006       GC_query_task_threads = TRUE;
24007 #endif
24008     GC_init_parallel();
24009 #endif
24010 }
24011 #ifndef kCFCoreFoundationVersionNumber_iOS_8_0
24012 #define kCFCoreFoundationVersionNumber_iOS_8_0 1140.1
24013 #endif
24014 STATIC ptr_t GC_stack_range_for(ptr_t *phi, thread_act_t thread, GC_thread p,
24015                                 GC_bool thread_blocked, mach_port_t my_thread,
24016                                 ptr_t *paltstack_lo,
24017                                 ptr_t *paltstack_hi GC_ATTR_UNUSED)
24018 {
24019   ptr_t lo;
24020   if (thread == my_thread) {
24021     GC_ASSERT(!thread_blocked);
24022     lo = GC_approx_sp();
24023 #ifndef DARWIN_DONT_PARSE_STACK
24024       *phi = GC_FindTopOfStack(0);
24025 #endif
24026   } else if (thread_blocked) {
24027 #if defined(CPPCHECK)
24028       if (NULL == p) ABORT("Invalid GC_thread passed to GC_stack_range_for");
24029 #endif
24030     lo = p->stop_info.stack_ptr;
24031 #ifndef DARWIN_DONT_PARSE_STACK
24032       *phi = p->topOfStack;
24033 #endif
24034   } else {
24035     kern_return_t kern_result;
24036     GC_THREAD_STATE_T state;
24037 #if defined(ARM32) && defined(ARM_THREAD_STATE32)
24038       size_t size;
24039       static cpu_type_t cputype = 0;
24040       if (cputype == 0) {
24041         sysctlbyname("hw.cputype", &cputype, &size, NULL, 0);
24042       }
24043       if (cputype == CPU_TYPE_ARM64
24044           || kCFCoreFoundationVersionNumber
24045              >= kCFCoreFoundationVersionNumber_iOS_8_0) {
24046         arm_unified_thread_state_t unified_state;
24047         mach_msg_type_number_t unified_thread_state_count
24048                                         = ARM_UNIFIED_THREAD_STATE_COUNT;
24049 #if defined(CPPCHECK)
24050 #define GC_ARM_UNIFIED_THREAD_STATE 1
24051 #else
24052 #define GC_ARM_UNIFIED_THREAD_STATE ARM_UNIFIED_THREAD_STATE
24053 #endif
24054         kern_result = thread_get_state(thread, GC_ARM_UNIFIED_THREAD_STATE,
24055                                        (natural_t *)&unified_state,
24056                                        &unified_thread_state_count);
24057 #if !defined(CPPCHECK)
24058           if (unified_state.ash.flavor != ARM_THREAD_STATE32) {
24059             ABORT("unified_state flavor should be ARM_THREAD_STATE32");
24060           }
24061 #endif
24062         state = unified_state;
24063       } else
24064 #endif
24065      {
24066       mach_msg_type_number_t thread_state_count = GC_MACH_THREAD_STATE_COUNT;
24067       do {
24068         kern_result = thread_get_state(thread, GC_MACH_THREAD_STATE,
24069                                        (natural_t *)&state,
24070                                        &thread_state_count);
24071       } while (kern_result == KERN_ABORTED);
24072     }
24073 #ifdef DEBUG_THREADS
24074       GC_log_printf("thread_get_state returns %d\n", kern_result);
24075 #endif
24076     if (kern_result != KERN_SUCCESS)
24077       ABORT("thread_get_state failed");
24078 #if defined(I386)
24079       lo = (ptr_t)state.THREAD_FLD(esp);
24080 #ifndef DARWIN_DONT_PARSE_STACK
24081         *phi = GC_FindTopOfStack(state.THREAD_FLD(esp));
24082 #endif
24083       GC_push_one(state.THREAD_FLD(eax));
24084       GC_push_one(state.THREAD_FLD(ebx));
24085       GC_push_one(state.THREAD_FLD(ecx));
24086       GC_push_one(state.THREAD_FLD(edx));
24087       GC_push_one(state.THREAD_FLD(edi));
24088       GC_push_one(state.THREAD_FLD(esi));
24089       GC_push_one(state.THREAD_FLD(ebp));
24090 #elif defined(X86_64)
24091       lo = (ptr_t)state.THREAD_FLD(rsp);
24092 #ifndef DARWIN_DONT_PARSE_STACK
24093         *phi = GC_FindTopOfStack(state.THREAD_FLD(rsp));
24094 #endif
24095       GC_push_one(state.THREAD_FLD(rax));
24096       GC_push_one(state.THREAD_FLD(rbx));
24097       GC_push_one(state.THREAD_FLD(rcx));
24098       GC_push_one(state.THREAD_FLD(rdx));
24099       GC_push_one(state.THREAD_FLD(rdi));
24100       GC_push_one(state.THREAD_FLD(rsi));
24101       GC_push_one(state.THREAD_FLD(rbp));
24102       GC_push_one(state.THREAD_FLD(r8));
24103       GC_push_one(state.THREAD_FLD(r9));
24104       GC_push_one(state.THREAD_FLD(r10));
24105       GC_push_one(state.THREAD_FLD(r11));
24106       GC_push_one(state.THREAD_FLD(r12));
24107       GC_push_one(state.THREAD_FLD(r13));
24108       GC_push_one(state.THREAD_FLD(r14));
24109       GC_push_one(state.THREAD_FLD(r15));
24110 #elif defined(POWERPC)
24111       lo = (ptr_t)(state.THREAD_FLD(r1) - PPC_RED_ZONE_SIZE);
24112 #ifndef DARWIN_DONT_PARSE_STACK
24113         *phi = GC_FindTopOfStack(state.THREAD_FLD(r1));
24114 #endif
24115       GC_push_one(state.THREAD_FLD(r0));
24116       GC_push_one(state.THREAD_FLD(r2));
24117       GC_push_one(state.THREAD_FLD(r3));
24118       GC_push_one(state.THREAD_FLD(r4));
24119       GC_push_one(state.THREAD_FLD(r5));
24120       GC_push_one(state.THREAD_FLD(r6));
24121       GC_push_one(state.THREAD_FLD(r7));
24122       GC_push_one(state.THREAD_FLD(r8));
24123       GC_push_one(state.THREAD_FLD(r9));
24124       GC_push_one(state.THREAD_FLD(r10));
24125       GC_push_one(state.THREAD_FLD(r11));
24126       GC_push_one(state.THREAD_FLD(r12));
24127       GC_push_one(state.THREAD_FLD(r13));
24128       GC_push_one(state.THREAD_FLD(r14));
24129       GC_push_one(state.THREAD_FLD(r15));
24130       GC_push_one(state.THREAD_FLD(r16));
24131       GC_push_one(state.THREAD_FLD(r17));
24132       GC_push_one(state.THREAD_FLD(r18));
24133       GC_push_one(state.THREAD_FLD(r19));
24134       GC_push_one(state.THREAD_FLD(r20));
24135       GC_push_one(state.THREAD_FLD(r21));
24136       GC_push_one(state.THREAD_FLD(r22));
24137       GC_push_one(state.THREAD_FLD(r23));
24138       GC_push_one(state.THREAD_FLD(r24));
24139       GC_push_one(state.THREAD_FLD(r25));
24140       GC_push_one(state.THREAD_FLD(r26));
24141       GC_push_one(state.THREAD_FLD(r27));
24142       GC_push_one(state.THREAD_FLD(r28));
24143       GC_push_one(state.THREAD_FLD(r29));
24144       GC_push_one(state.THREAD_FLD(r30));
24145       GC_push_one(state.THREAD_FLD(r31));
24146 #elif defined(ARM32)
24147       lo = (ptr_t)state.THREAD_FLD(sp);
24148 #ifndef DARWIN_DONT_PARSE_STACK
24149         *phi = GC_FindTopOfStack(state.THREAD_FLD(r[7]));
24150 #endif
24151       {
24152         int j;
24153         for (j = 0; j < 7; j++)
24154           GC_push_one(state.THREAD_FLD(r[j]));
24155         j++;
24156         for (; j <= 12; j++)
24157           GC_push_one(state.THREAD_FLD(r[j]));
24158       }
24159       GC_push_one(state.THREAD_FLD(lr));
24160 #elif defined(AARCH64)
24161       lo = (ptr_t)state.THREAD_FLD(sp);
24162 #ifndef DARWIN_DONT_PARSE_STACK
24163         *phi = GC_FindTopOfStack(state.THREAD_FLD(fp));
24164 #endif
24165       {
24166         int j;
24167         for (j = 0; j <= 28; j++) {
24168           GC_push_one(state.THREAD_FLD(x[j]));
24169         }
24170       }
24171       GC_push_one(state.THREAD_FLD(lr));
24172 #elif defined(CPPCHECK)
24173       lo = NULL;
24174 #else
24175 #error FIXME for non-arm/ppc/x86 architectures
24176 #endif
24177   }
24178 #ifdef DARWIN_DONT_PARSE_STACK
24179     *phi = (p->flags & MAIN_THREAD) != 0 ? GC_stackbottom : p->stack_end;
24180 #endif
24181 #ifdef DARWIN_DONT_PARSE_STACK
24182   if (p->altstack != NULL && (word)p->altstack <= (word)lo
24183       && (word)lo <= (word)p->altstack + p->altstack_size) {
24184     *paltstack_lo = lo;
24185     *paltstack_hi = p->altstack + p->altstack_size;
24186     lo = p->stack;
24187     *phi = p->stack + p->stack_size;
24188   } else
24189 #endif
24190    {
24191     *paltstack_lo = NULL;
24192   }
24193 #ifdef DEBUG_THREADS
24194     GC_log_printf("Darwin: Stack for thread %p is [%p,%p)\n",
24195                   (void *)(word)thread, (void *)lo, (void *)(*phi));
24196 #endif
24197   return lo;
24198 }
24199 GC_INNER void GC_push_all_stacks(void)
24200 {
24201   ptr_t hi, altstack_lo, altstack_hi;
24202   task_t my_task = current_task();
24203   mach_port_t my_thread = mach_thread_self();
24204   GC_bool found_me = FALSE;
24205   int nthreads = 0;
24206   word total_size = 0;
24207   mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ;
24208   if (!EXPECT(GC_thr_initialized, TRUE))
24209     GC_thr_init();
24210 #ifndef DARWIN_DONT_PARSE_STACK
24211     if (GC_query_task_threads) {
24212       int i;
24213       kern_return_t kern_result;
24214       thread_act_array_t act_list = 0;
24215       kern_result = task_threads(my_task, &act_list, &listcount);
24216       if (kern_result != KERN_SUCCESS)
24217         ABORT("task_threads failed");
24218       for (i = 0; i < (int)listcount; i++) {
24219         thread_act_t thread = act_list[i];
24220         ptr_t lo = GC_stack_range_for(&hi, thread, NULL, FALSE, my_thread,
24221                                       &altstack_lo, &altstack_hi);
24222         if (lo) {
24223           GC_ASSERT((word)lo <= (word)hi);
24224           total_size += hi - lo;
24225           GC_push_all_stack(lo, hi);
24226         }
24227         nthreads++;
24228         if (thread == my_thread)
24229           found_me = TRUE;
24230         mach_port_deallocate(my_task, thread);
24231       }
24232       vm_deallocate(my_task, (vm_address_t)act_list,
24233                     sizeof(thread_t) * listcount);
24234     } else
24235 #endif
24236    {
24237     int i;
24238     for (i = 0; i < (int)listcount; i++) {
24239       GC_thread p;
24240       for (p = GC_threads[i]; p != NULL; p = p->next)
24241         if ((p->flags & FINISHED) == 0) {
24242           thread_act_t thread = (thread_act_t)p->stop_info.mach_thread;
24243           ptr_t lo = GC_stack_range_for(&hi, thread, p,
24244                                         (GC_bool)p->thread_blocked,
24245                                         my_thread, &altstack_lo,
24246                                         &altstack_hi);
24247           if (lo) {
24248             GC_ASSERT((word)lo <= (word)hi);
24249             total_size += hi - lo;
24250             GC_push_all_stack_sections(lo, hi, p->traced_stack_sect);
24251           }
24252           if (altstack_lo) {
24253             total_size += altstack_hi - altstack_lo;
24254             GC_push_all_stack(altstack_lo, altstack_hi);
24255           }
24256           nthreads++;
24257           if (thread == my_thread)
24258             found_me = TRUE;
24259         }
24260     }
24261   }
24262   mach_port_deallocate(my_task, my_thread);
24263   GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks\n", nthreads);
24264   if (!found_me && !GC_in_thread_creation)
24265     ABORT("Collecting from unknown thread");
24266   GC_total_stacksize = total_size;
24267 }
24268 #ifndef GC_NO_THREADS_DISCOVERY
24269 #ifdef MPROTECT_VDB
24270     STATIC mach_port_t GC_mach_handler_thread = 0;
24271     STATIC GC_bool GC_use_mach_handler_thread = FALSE;
24272     GC_INNER void GC_darwin_register_mach_handler_thread(mach_port_t thread)
24273     {
24274       GC_mach_handler_thread = thread;
24275       GC_use_mach_handler_thread = TRUE;
24276     }
24277 #endif
24278 #ifndef GC_MAX_MACH_THREADS
24279 #define GC_MAX_MACH_THREADS THREAD_TABLE_SZ
24280 #endif
24281   struct GC_mach_thread {
24282     thread_act_t thread;
24283     GC_bool suspended;
24284   };
24285   struct GC_mach_thread GC_mach_threads[GC_MAX_MACH_THREADS];
24286   STATIC int GC_mach_threads_count = 0;
24287 STATIC GC_bool GC_suspend_thread_list(thread_act_array_t act_list, int count,
24288                                       thread_act_array_t old_list,
24289                                       int old_count, task_t my_task,
24290                                       mach_port_t my_thread)
24291 {
24292   int i;
24293   int j = -1;
24294   GC_bool changed = FALSE;
24295   for (i = 0; i < count; i++) {
24296     thread_act_t thread = act_list[i];
24297     GC_bool found;
24298     kern_return_t kern_result;
24299     if (thread == my_thread
24300 #ifdef MPROTECT_VDB
24301           || (GC_mach_handler_thread == thread && GC_use_mach_handler_thread)
24302 #endif
24303 #ifdef PARALLEL_MARK
24304           || GC_is_mach_marker(thread)
24305 #endif
24306         ) {
24307       mach_port_deallocate(my_task, thread);
24308       continue;
24309     }
24310     found = FALSE;
24311     {
24312       int last_found = j;
24313       while (++j < old_count)
24314         if (old_list[j] == thread) {
24315           found = TRUE;
24316           break;
24317         }
24318       if (!found) {
24319         for (j = 0; j < last_found; j++)
24320           if (old_list[j] == thread) {
24321             found = TRUE;
24322             break;
24323           }
24324       }
24325     }
24326     if (found) {
24327       mach_port_deallocate(my_task, thread);
24328       continue;
24329     }
24330     if (GC_mach_threads_count == GC_MAX_MACH_THREADS)
24331       ABORT("Too many threads");
24332     GC_mach_threads[GC_mach_threads_count].thread = thread;
24333     GC_mach_threads[GC_mach_threads_count].suspended = FALSE;
24334     changed = TRUE;
24335 #ifdef DEBUG_THREADS
24336       GC_log_printf("Suspending %p\n", (void *)(word)thread);
24337 #endif
24338     GC_acquire_dirty_lock();
24339     do {
24340       kern_result = thread_suspend(thread);
24341     } while (kern_result == KERN_ABORTED);
24342     GC_release_dirty_lock();
24343     if (kern_result != KERN_SUCCESS) {
24344       GC_mach_threads[GC_mach_threads_count].suspended = FALSE;
24345     } else {
24346       GC_mach_threads[GC_mach_threads_count].suspended = TRUE;
24347       if (GC_on_thread_event)
24348         GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, (void *)(word)thread);
24349     }
24350     GC_mach_threads_count++;
24351   }
24352   return changed;
24353 }
24354 #endif
24355 GC_INNER void GC_stop_world(void)
24356 {
24357   task_t my_task = current_task();
24358   mach_port_t my_thread = mach_thread_self();
24359   kern_return_t kern_result;
24360 #ifdef DEBUG_THREADS
24361     GC_log_printf("Stopping the world from thread %p\n",
24362                   (void *)(word)my_thread);
24363 #endif
24364 #ifdef PARALLEL_MARK
24365     if (GC_parallel) {
24366       GC_acquire_mark_lock();
24367       GC_ASSERT(GC_fl_builder_count == 0);
24368     }
24369 #endif
24370   if (GC_query_task_threads) {
24371 #ifndef GC_NO_THREADS_DISCOVERY
24372       GC_bool changed;
24373       thread_act_array_t act_list, prev_list;
24374       mach_msg_type_number_t listcount, prevcount;
24375       GC_mach_threads_count = 0;
24376       changed = TRUE;
24377       prev_list = NULL;
24378       prevcount = 0;
24379       do {
24380         kern_result = task_threads(my_task, &act_list, &listcount);
24381         if (kern_result == KERN_SUCCESS) {
24382           changed = GC_suspend_thread_list(act_list, listcount, prev_list,
24383                                            prevcount, my_task, my_thread);
24384           if (prev_list != NULL) {
24385             vm_deallocate(my_task, (vm_address_t)prev_list,
24386                           sizeof(thread_t) * prevcount);
24387           }
24388           prev_list = act_list;
24389           prevcount = listcount;
24390         }
24391       } while (changed);
24392       GC_ASSERT(prev_list != 0);
24393       vm_deallocate(my_task, (vm_address_t)act_list,
24394                     sizeof(thread_t) * listcount);
24395 #endif
24396   } else {
24397     unsigned i;
24398     for (i = 0; i < THREAD_TABLE_SZ; i++) {
24399       GC_thread p;
24400       for (p = GC_threads[i]; p != NULL; p = p->next) {
24401         if ((p->flags & FINISHED) == 0 && !p->thread_blocked &&
24402              p->stop_info.mach_thread != my_thread) {
24403           GC_acquire_dirty_lock();
24404           do {
24405             kern_result = thread_suspend(p->stop_info.mach_thread);
24406           } while (kern_result == KERN_ABORTED);
24407           GC_release_dirty_lock();
24408           if (kern_result != KERN_SUCCESS)
24409             ABORT("thread_suspend failed");
24410           if (GC_on_thread_event)
24411             GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,
24412                                (void *)(word)p->stop_info.mach_thread);
24413         }
24414       }
24415     }
24416   }
24417 #ifdef MPROTECT_VDB
24418     if (GC_auto_incremental) {
24419       GC_mprotect_stop();
24420     }
24421 #endif
24422 #ifdef PARALLEL_MARK
24423     if (GC_parallel)
24424       GC_release_mark_lock();
24425 #endif
24426 #ifdef DEBUG_THREADS
24427     GC_log_printf("World stopped from %p\n", (void *)(word)my_thread);
24428 #endif
24429   mach_port_deallocate(my_task, my_thread);
24430 }
24431 GC_INLINE void GC_thread_resume(thread_act_t thread)
24432 {
24433   kern_return_t kern_result;
24434 #if defined(DEBUG_THREADS) || defined(GC_ASSERTIONS)
24435     struct thread_basic_info info;
24436     mach_msg_type_number_t outCount = THREAD_BASIC_INFO_COUNT;
24437 #if defined(CPPCHECK) && defined(DEBUG_THREADS)
24438       info.run_state = 0;
24439 #endif
24440     kern_result = thread_info(thread, THREAD_BASIC_INFO,
24441                               (thread_info_t)&info, &outCount);
24442     if (kern_result != KERN_SUCCESS)
24443       ABORT("thread_info failed");
24444 #endif
24445 #ifdef DEBUG_THREADS
24446     GC_log_printf("Resuming thread %p with state %d\n", (void *)(word)thread,
24447                   info.run_state);
24448 #endif
24449   kern_result = thread_resume(thread);
24450   if (kern_result != KERN_SUCCESS) {
24451     WARN("thread_resume(%p) failed: mach port invalid\n", thread);
24452   } else if (GC_on_thread_event) {
24453     GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, (void *)(word)thread);
24454   }
24455 }
24456 GC_INNER void GC_start_world(void)
24457 {
24458   task_t my_task = current_task();
24459 #ifdef DEBUG_THREADS
24460     GC_log_printf("World starting\n");
24461 #endif
24462 #ifdef MPROTECT_VDB
24463     if (GC_auto_incremental) {
24464       GC_mprotect_resume();
24465     }
24466 #endif
24467   if (GC_query_task_threads) {
24468 #ifndef GC_NO_THREADS_DISCOVERY
24469       int i, j;
24470       kern_return_t kern_result;
24471       thread_act_array_t act_list;
24472       mach_msg_type_number_t listcount;
24473       kern_result = task_threads(my_task, &act_list, &listcount);
24474       if (kern_result != KERN_SUCCESS)
24475         ABORT("task_threads failed");
24476       j = (int)listcount;
24477       for (i = 0; i < GC_mach_threads_count; i++) {
24478         thread_act_t thread = GC_mach_threads[i].thread;
24479         if (GC_mach_threads[i].suspended) {
24480           int last_found = j;
24481           while (++j < (int)listcount) {
24482             if (act_list[j] == thread)
24483               break;
24484           }
24485           if (j >= (int)listcount) {
24486             for (j = 0; j < last_found; j++) {
24487               if (act_list[j] == thread)
24488                 break;
24489             }
24490           }
24491           if (j != last_found) {
24492             GC_thread_resume(thread);
24493           }
24494         } else {
24495 #ifdef DEBUG_THREADS
24496             GC_log_printf("Not resuming thread %p as it is not suspended\n",
24497                           (void *)(word)thread);
24498 #endif
24499         }
24500         mach_port_deallocate(my_task, thread);
24501       }
24502       for (i = 0; i < (int)listcount; i++)
24503         mach_port_deallocate(my_task, act_list[i]);
24504       vm_deallocate(my_task, (vm_address_t)act_list,
24505                     sizeof(thread_t) * listcount);
24506 #endif
24507   } else {
24508     int i;
24509     mach_port_t my_thread = mach_thread_self();
24510     for (i = 0; i < THREAD_TABLE_SZ; i++) {
24511       GC_thread p;
24512       for (p = GC_threads[i]; p != NULL; p = p->next) {
24513         if ((p->flags & FINISHED) == 0 && !p->thread_blocked &&
24514              p->stop_info.mach_thread != my_thread)
24515           GC_thread_resume(p->stop_info.mach_thread);
24516       }
24517     }
24518     mach_port_deallocate(my_task, my_thread);
24519   }
24520 #ifdef DEBUG_THREADS
24521     GC_log_printf("World started\n");
24522 #endif
24523 }
24524 #endif
24525 #if !defined(MACOS) && !defined(GC_NO_TYPES) && !defined(SN_TARGET_PSP2) \
24526     && !defined(_WIN32_WCE) && !defined(__CC_ARM)
24527 #include <sys/types.h>
24528 #endif
24529 #undef GC_MUST_RESTORE_REDEFINED_DLOPEN
24530 #if defined(GC_PTHREADS) && !defined(GC_NO_DLOPEN) \
24531     && !defined(GC_NO_THREAD_REDIRECTS) && !defined(GC_USE_LD_WRAP)
24532 #undef dlopen
24533 #define GC_MUST_RESTORE_REDEFINED_DLOPEN
24534 #endif
24535 STATIC GC_has_static_roots_func GC_has_static_roots = 0;
24536 #if (defined(DYNAMIC_LOADING) || defined(MSWIN32) || defined(MSWINCE) \
24537     || defined(CYGWIN32)) && !defined(PCR)
24538 #if !defined(DARWIN) && !defined(SCO_ELF) && !defined(SOLARISDL) \
24539     && !defined(AIX) && !defined(DGUX) && !defined(IRIX5) && !defined(HPUX) \
24540     && !defined(CYGWIN32) && !defined(MSWIN32) && !defined(MSWINCE) \
24541     && !(defined(ALPHA) && defined(OSF1)) \
24542     && !(defined(FREEBSD) && defined(__ELF__)) \
24543     && !(defined(LINUX) && defined(__ELF__)) \
24544     && !(defined(NETBSD) && defined(__ELF__)) \
24545     && !(defined(OPENBSD) && (defined(__ELF__) || defined(M68K))) \
24546     && !defined(HAIKU) && !defined(HURD) && !defined(NACL) \
24547     && !defined(CPPCHECK)
24548 #error We only know how to find data segments of dynamic libraries for above.
24549 #error Additional SVR4 variants might not be too hard to add.
24550 #endif
24551 #include <stdio.h>
24552 #ifdef SOLARISDL
24553 #include <sys/elf.h>
24554 #include <dlfcn.h>
24555 #include <link.h>
24556 #endif
24557 #if defined(NETBSD)
24558 #include <sys/param.h>
24559 #include <dlfcn.h>
24560 #include <machine/elf_machdep.h>
24561 #define ELFSIZE ARCH_ELFSIZE
24562 #endif
24563 #if defined(OPENBSD)
24564 #include <sys/param.h>
24565 #if (OpenBSD >= 200519) && !defined(HAVE_DL_ITERATE_PHDR)
24566 #define HAVE_DL_ITERATE_PHDR
24567 #endif
24568 #endif
24569 #if defined(SCO_ELF) || defined(DGUX) || defined(HURD) || defined(NACL) \
24570     || (defined(__ELF__) && (defined(LINUX) || defined(FREEBSD) \
24571                              || defined(NETBSD) || defined(OPENBSD)))
24572 #include <stddef.h>
24573 #if !defined(OPENBSD) && !defined(HOST_ANDROID)
24574 #include <elf.h>
24575 #endif
24576 #ifdef HOST_ANDROID
24577 #ifdef BIONIC_ELFDATA_REDEF_BUG
24578 #include <asm/elf.h>
24579 #include <linux/elf-em.h>
24580 #undef ELF_DATA
24581 #undef EM_ALPHA
24582 #endif
24583 #include <link.h>
24584 #if !defined(GC_DONT_DEFINE_LINK_MAP) && !(__ANDROID_API__ >= 21)
24585       struct link_map {
24586         uintptr_t l_addr;
24587         char* l_name;
24588         uintptr_t l_ld;
24589         struct link_map* l_next;
24590         struct link_map* l_prev;
24591       };
24592       struct r_debug {
24593         int32_t r_version;
24594         struct link_map* r_map;
24595         void (*r_brk)(void);
24596         int32_t r_state;
24597         uintptr_t r_ldbase;
24598       };
24599 #endif
24600 #else
24601     EXTERN_C_BEGIN
24602 #include <link.h>
24603     EXTERN_C_END
24604 #endif
24605 #endif
24606 #ifndef ElfW
24607 #if defined(FREEBSD)
24608 #if __ELF_WORD_SIZE == 32
24609 #define ElfW(type) Elf32_##type
24610 #else
24611 #define ElfW(type) Elf64_##type
24612 #endif
24613 #elif defined(NETBSD) || defined(OPENBSD)
24614 #if ELFSIZE == 32
24615 #define ElfW(type) Elf32_##type
24616 #elif ELFSIZE == 64
24617 #define ElfW(type) Elf64_##type
24618 #else
24619 #error Missing ELFSIZE define
24620 #endif
24621 #else
24622 #if !defined(ELF_CLASS) || ELF_CLASS == ELFCLASS32
24623 #define ElfW(type) Elf32_##type
24624 #else
24625 #define ElfW(type) Elf64_##type
24626 #endif
24627 #endif
24628 #endif
24629 #if defined(SOLARISDL) && !defined(USE_PROC_FOR_LIBRARIES)
24630   EXTERN_C_BEGIN
24631   extern ElfW(Dyn) _DYNAMIC;
24632   EXTERN_C_END
24633   STATIC struct link_map *
24634   GC_FirstDLOpenedLinkMap(void)
24635   {
24636     ElfW(Dyn) *dp;
24637     static struct link_map * cachedResult = 0;
24638     static ElfW(Dyn) *dynStructureAddr = 0;
24639 #ifdef SUNOS53_SHARED_LIB
24640         if( dynStructureAddr == 0 ) {
24641           void* startupSyms = dlopen(0, RTLD_LAZY);
24642           dynStructureAddr = (ElfW(Dyn)*)(word)dlsym(startupSyms, "_DYNAMIC");
24643         }
24644 #else
24645         dynStructureAddr = &_DYNAMIC;
24646 #endif
24647     if (0 == COVERT_DATAFLOW(dynStructureAddr)) {
24648         return(0);
24649     }
24650     if (cachedResult == 0) {
24651         int tag;
24652         for( dp = ((ElfW(Dyn) *)(&_DYNAMIC)); (tag = dp->d_tag) != 0; dp++ ) {
24653             if (tag == DT_DEBUG) {
24654                 struct r_debug *rd = (struct r_debug *)dp->d_un.d_ptr;
24655                 if (rd != NULL) {
24656                     struct link_map *lm = rd->r_map;
24657                     if (lm != NULL)
24658                         cachedResult = lm->l_next;
24659                 }
24660                 break;
24661             }
24662         }
24663     }
24664     return cachedResult;
24665   }
24666 #endif
24667 #ifdef GC_MUST_RESTORE_REDEFINED_DLOPEN
24668 #define dlopen GC_dlopen
24669 #endif
24670 #if defined(SOLARISDL)
24671 #if !defined(PCR) && !defined(GC_SOLARIS_THREADS) && defined(THREADS) \
24672      && !defined(CPPCHECK)
24673 #error Fix mutual exclusion with dlopen
24674 #endif
24675 #ifndef USE_PROC_FOR_LIBRARIES
24676 GC_INNER void GC_register_dynamic_libraries(void)
24677 {
24678   struct link_map *lm;
24679   for (lm = GC_FirstDLOpenedLinkMap(); lm != 0; lm = lm->l_next) {
24680         ElfW(Ehdr) * e;
24681         ElfW(Phdr) * p;
24682         unsigned long offset;
24683         char * start;
24684         int i;
24685         e = (ElfW(Ehdr) *) lm->l_addr;
24686         p = ((ElfW(Phdr) *)(((char *)(e)) + e->e_phoff));
24687         offset = ((unsigned long)(lm->l_addr));
24688         for( i = 0; i < (int)e->e_phnum; i++, p++ ) {
24689           switch( p->p_type ) {
24690             case PT_LOAD:
24691               {
24692                 if( !(p->p_flags & PF_W) ) break;
24693                 start = ((char *)(p->p_vaddr)) + offset;
24694                 GC_add_roots_inner(start, start + p->p_memsz, TRUE);
24695               }
24696               break;
24697             default:
24698               break;
24699           }
24700         }
24701     }
24702 }
24703 #endif
24704 #endif
24705 #if defined(SCO_ELF) || defined(DGUX) || defined(HURD) || defined(NACL) \
24706     || (defined(__ELF__) && (defined(LINUX) || defined(FREEBSD) \
24707                              || defined(NETBSD) || defined(OPENBSD)))
24708 #ifdef USE_PROC_FOR_LIBRARIES
24709 #include <string.h>
24710 #include <sys/stat.h>
24711 #include <fcntl.h>
24712 #include <unistd.h>
24713 #define MAPS_BUF_SIZE (32*1024)
24714 static void sort_heap_sects(struct HeapSect *base, size_t number_of_elements)
24715 {
24716     signed_word n = (signed_word)number_of_elements;
24717     signed_word nsorted = 1;
24718     while (nsorted < n) {
24719       signed_word i;
24720       while (nsorted < n &&
24721              (word)base[nsorted-1].hs_start < (word)base[nsorted].hs_start)
24722           ++nsorted;
24723       if (nsorted == n) break;
24724       GC_ASSERT((word)base[nsorted-1].hs_start > (word)base[nsorted].hs_start);
24725       i = nsorted - 1;
24726       while (i >= 0 && (word)base[i].hs_start > (word)base[i+1].hs_start) {
24727         struct HeapSect tmp = base[i];
24728         base[i] = base[i+1];
24729         base[i+1] = tmp;
24730         --i;
24731       }
24732       GC_ASSERT((word)base[nsorted-1].hs_start < (word)base[nsorted].hs_start);
24733       ++nsorted;
24734     }
24735 }
24736 STATIC void GC_register_map_entries(const char *maps)
24737 {
24738     const char *prot;
24739     ptr_t start, end;
24740     unsigned int maj_dev;
24741     ptr_t least_ha, greatest_ha;
24742     unsigned i;
24743     GC_ASSERT(I_HOLD_LOCK());
24744     sort_heap_sects(GC_our_memory, GC_n_memory);
24745     least_ha = GC_our_memory[0].hs_start;
24746     greatest_ha = GC_our_memory[GC_n_memory-1].hs_start
24747                   + GC_our_memory[GC_n_memory-1].hs_bytes;
24748     for (;;) {
24749         maps = GC_parse_map_entry(maps, &start, &end, &prot, &maj_dev, 0);
24750         if (NULL == maps) break;
24751         if (prot[1] == 'w') {
24752             if ((word)start <= (word)GC_stackbottom
24753                 && (word)end >= (word)GC_stackbottom) {
24754                 continue;
24755             }
24756 #ifdef THREADS
24757               if (GC_segment_is_thread_stack(start, end)) continue;
24758 #endif
24759             if ((word)end <= (word)least_ha
24760                 || (word)start >= (word)greatest_ha) {
24761               GC_add_roots_inner(start, end, TRUE);
24762               continue;
24763             }
24764               i = 0;
24765               while ((word)(GC_our_memory[i].hs_start
24766                                 + GC_our_memory[i].hs_bytes) < (word)start)
24767                   ++i;
24768               GC_ASSERT(i < GC_n_memory);
24769               if ((word)GC_our_memory[i].hs_start <= (word)start) {
24770                   start = GC_our_memory[i].hs_start
24771                           + GC_our_memory[i].hs_bytes;
24772                   ++i;
24773               }
24774               while (i < GC_n_memory
24775                      && (word)GC_our_memory[i].hs_start < (word)end
24776                      && (word)start < (word)end) {
24777                   if ((word)start < (word)GC_our_memory[i].hs_start)
24778                     GC_add_roots_inner(start,
24779                                        GC_our_memory[i].hs_start, TRUE);
24780                   start = GC_our_memory[i].hs_start
24781                           + GC_our_memory[i].hs_bytes;
24782                   ++i;
24783               }
24784               if ((word)start < (word)end)
24785                   GC_add_roots_inner(start, end, TRUE);
24786         } else if (prot[0] == '-' && prot[1] == '-' && prot[2] == '-') {
24787             GC_remove_roots_subregion(start, end);
24788         }
24789     }
24790 }
24791 GC_INNER void GC_register_dynamic_libraries(void)
24792 {
24793     GC_register_map_entries(GC_get_maps());
24794 }
24795 GC_INNER GC_bool GC_register_main_static_data(void)
24796 {
24797     return FALSE;
24798 }
24799 #define HAVE_REGISTER_MAIN_STATIC_DATA
24800 #else
24801 #if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 2) \
24802     || (__GLIBC__ == 2 && __GLIBC_MINOR__ == 2 && defined(DT_CONFIG)) \
24803     || defined(HOST_ANDROID)
24804 #ifndef HAVE_DL_ITERATE_PHDR
24805 #define HAVE_DL_ITERATE_PHDR
24806 #endif
24807 #ifdef HOST_ANDROID
24808     EXTERN_C_BEGIN
24809     extern int dl_iterate_phdr(int (*cb)(struct dl_phdr_info *,
24810                                          size_t, void *),
24811                                void *data);
24812     EXTERN_C_END
24813 #endif
24814 #endif
24815 #if defined(__DragonFly__) || defined(__FreeBSD_kernel__) \
24816     || (defined(FREEBSD) && __FreeBSD__ >= 7)
24817 #ifndef HAVE_DL_ITERATE_PHDR
24818 #define HAVE_DL_ITERATE_PHDR
24819 #endif
24820 #define DL_ITERATE_PHDR_STRONG
24821 #elif defined(HAVE_DL_ITERATE_PHDR)
24822   EXTERN_C_BEGIN
24823 #pragma weak dl_iterate_phdr
24824   EXTERN_C_END
24825 #endif
24826 #if defined(HAVE_DL_ITERATE_PHDR)
24827 #ifdef PT_GNU_RELRO
24828 #define MAX_LOAD_SEGS MAX_ROOT_SETS
24829     static struct load_segment {
24830       ptr_t start;
24831       ptr_t end;
24832       ptr_t start2;
24833       ptr_t end2;
24834     } load_segs[MAX_LOAD_SEGS];
24835     static int n_load_segs;
24836     static GC_bool load_segs_overflow;
24837 #endif
24838 STATIC int GC_register_dynlib_callback(struct dl_phdr_info * info,
24839                                        size_t size, void * ptr)
24840 {
24841   const ElfW(Phdr) * p;
24842   ptr_t start, end;
24843   int i;
24844   if (size < offsetof (struct dl_phdr_info, dlpi_phnum)
24845       + sizeof (info->dlpi_phnum))
24846     return -1;
24847   p = info->dlpi_phdr;
24848   for (i = 0; i < (int)info->dlpi_phnum; i++, p++) {
24849     if (p->p_type == PT_LOAD) {
24850       GC_has_static_roots_func callback = GC_has_static_roots;
24851       if ((p->p_flags & PF_W) == 0) continue;
24852       start = (ptr_t)p->p_vaddr + info->dlpi_addr;
24853       end = start + p->p_memsz;
24854       if (callback != 0 && !callback(info->dlpi_name, start, p->p_memsz))
24855         continue;
24856 #ifdef PT_GNU_RELRO
24857 #if CPP_WORDSZ == 64
24858           start = (ptr_t)((word)start & ~(word)(sizeof(word) - 1));
24859 #endif
24860         if (n_load_segs >= MAX_LOAD_SEGS) {
24861           if (!load_segs_overflow) {
24862             WARN("Too many PT_LOAD segments;"
24863                  " registering as roots directly...\n", 0);
24864             load_segs_overflow = TRUE;
24865           }
24866           GC_add_roots_inner(start, end, TRUE);
24867         } else {
24868           load_segs[n_load_segs].start = start;
24869           load_segs[n_load_segs].end = end;
24870           load_segs[n_load_segs].start2 = 0;
24871           load_segs[n_load_segs].end2 = 0;
24872           ++n_load_segs;
24873         }
24874 #else
24875         GC_add_roots_inner(start, end, TRUE);
24876 #endif
24877     }
24878   }
24879 #ifdef PT_GNU_RELRO
24880     p = info->dlpi_phdr;
24881     for (i = 0; i < (int)info->dlpi_phnum; i++, p++) {
24882       if (p->p_type == PT_GNU_RELRO) {
24883         int j;
24884         start = (ptr_t)p->p_vaddr + info->dlpi_addr;
24885         end = start + p->p_memsz;
24886         for (j = n_load_segs; --j >= 0; ) {
24887           if ((word)start >= (word)load_segs[j].start
24888               && (word)start < (word)load_segs[j].end) {
24889             if (load_segs[j].start2 != 0) {
24890               WARN("More than one GNU_RELRO segment per load one\n",0);
24891             } else {
24892               GC_ASSERT((word)end <=
24893                             (((word)load_segs[j].end + GC_page_size - 1) &
24894                              ~(GC_page_size - 1)));
24895               load_segs[j].end2 = load_segs[j].end;
24896               load_segs[j].end = start;
24897               load_segs[j].start2 = end;
24898             }
24899             break;
24900           }
24901           if (0 == j && 0 == GC_has_static_roots)
24902             WARN("Failed to find PT_GNU_RELRO segment"
24903                  " inside PT_LOAD region\n", 0);
24904         }
24905       }
24906     }
24907 #endif
24908   *(int *)ptr = 1;
24909   return 0;
24910 }
24911 GC_INNER GC_bool GC_register_main_static_data(void)
24912 {
24913 #ifdef DL_ITERATE_PHDR_STRONG
24914     return FALSE;
24915 #else
24916     return 0 == COVERT_DATAFLOW(dl_iterate_phdr);
24917 #endif
24918 }
24919 STATIC GC_bool GC_register_dynamic_libraries_dl_iterate_phdr(void)
24920 {
24921   int did_something;
24922   if (GC_register_main_static_data())
24923     return FALSE;
24924 #ifdef PT_GNU_RELRO
24925     {
24926       static GC_bool excluded_segs = FALSE;
24927       n_load_segs = 0;
24928       load_segs_overflow = FALSE;
24929       if (!EXPECT(excluded_segs, TRUE)) {
24930         GC_exclude_static_roots_inner((ptr_t)load_segs,
24931                                       (ptr_t)load_segs + sizeof(load_segs));
24932         excluded_segs = TRUE;
24933       }
24934     }
24935 #endif
24936   did_something = 0;
24937   dl_iterate_phdr(GC_register_dynlib_callback, &did_something);
24938   if (did_something) {
24939 #ifdef PT_GNU_RELRO
24940       int i;
24941       for (i = 0; i < n_load_segs; ++i) {
24942         if ((word)load_segs[i].end > (word)load_segs[i].start) {
24943           GC_add_roots_inner(load_segs[i].start, load_segs[i].end, TRUE);
24944         }
24945         if ((word)load_segs[i].end2 > (word)load_segs[i].start2) {
24946           GC_add_roots_inner(load_segs[i].start2, load_segs[i].end2, TRUE);
24947         }
24948       }
24949 #endif
24950   } else {
24951       ptr_t datastart, dataend;
24952 #ifdef DATASTART_IS_FUNC
24953         static ptr_t datastart_cached = (ptr_t)GC_WORD_MAX;
24954         if (datastart_cached == (ptr_t)GC_WORD_MAX) {
24955           datastart_cached = DATASTART;
24956         }
24957         datastart = datastart_cached;
24958 #else
24959         datastart = DATASTART;
24960 #endif
24961 #ifdef DATAEND_IS_FUNC
24962         {
24963           static ptr_t dataend_cached = 0;
24964           if (dataend_cached == 0) {
24965             dataend_cached = DATAEND;
24966           }
24967           dataend = dataend_cached;
24968         }
24969 #else
24970         dataend = DATAEND;
24971 #endif
24972       if (NULL == *(char * volatile *)&datastart
24973           || (word)datastart > (word)dataend)
24974         ABORT_ARG2("Wrong DATASTART/END pair",
24975                    ": %p .. %p", (void *)datastart, (void *)dataend);
24976       GC_add_roots_inner(datastart, dataend, TRUE);
24977 #ifdef GC_HAVE_DATAREGION2
24978         if ((word)DATASTART2 - 1U >= (word)DATAEND2) {
24979           ABORT_ARG2("Wrong DATASTART/END2 pair",
24980                      ": %p .. %p", (void *)DATASTART2, (void *)DATAEND2);
24981         }
24982         GC_add_roots_inner(DATASTART2, DATAEND2, TRUE);
24983 #endif
24984   }
24985   return TRUE;
24986 }
24987 #define HAVE_REGISTER_MAIN_STATIC_DATA
24988 #else
24989 #if defined(NETBSD) || defined(OPENBSD)
24990 #include <sys/exec_elf.h>
24991 #ifndef DT_DEBUG
24992 #define DT_DEBUG   21
24993 #endif
24994 #ifndef PT_LOAD
24995 #define PT_LOAD    1
24996 #endif
24997 #ifndef PF_W
24998 #define PF_W       2
24999 #endif
25000 #elif !defined(HOST_ANDROID)
25001 #include <elf.h>
25002 #endif
25003 #ifndef HOST_ANDROID
25004 #include <link.h>
25005 #endif
25006 #endif
25007 EXTERN_C_BEGIN
25008 #ifdef __GNUC__
25009 #pragma weak _DYNAMIC
25010 #endif
25011 extern ElfW(Dyn) _DYNAMIC[];
25012 EXTERN_C_END
25013 STATIC struct link_map *
25014 GC_FirstDLOpenedLinkMap(void)
25015 {
25016     static struct link_map *cachedResult = 0;
25017     if (0 == COVERT_DATAFLOW(_DYNAMIC)) {
25018         return(0);
25019     }
25020     if( cachedResult == 0 ) {
25021 #if defined(NETBSD) && defined(RTLD_DI_LINKMAP)
25022 #if defined(CPPCHECK)
25023 #define GC_RTLD_DI_LINKMAP 2
25024 #else
25025 #define GC_RTLD_DI_LINKMAP RTLD_DI_LINKMAP
25026 #endif
25027         struct link_map *lm = NULL;
25028         if (!dlinfo(RTLD_SELF, GC_RTLD_DI_LINKMAP, &lm) && lm != NULL) {
25029             while (lm->l_prev != NULL) {
25030                 lm = lm->l_prev;
25031             }
25032             cachedResult = lm->l_next;
25033         }
25034 #else
25035         ElfW(Dyn) *dp;
25036         int tag;
25037         for( dp = _DYNAMIC; (tag = dp->d_tag) != 0; dp++ ) {
25038             if (tag == DT_DEBUG) {
25039                 struct r_debug *rd = (struct r_debug *)dp->d_un.d_ptr;
25040                 if (rd != NULL) {
25041                     struct link_map *lm = rd->r_map;
25042                     if (lm != NULL)
25043                         cachedResult = lm->l_next;
25044                 }
25045                 break;
25046             }
25047         }
25048 #endif
25049     }
25050     return cachedResult;
25051 }
25052 GC_INNER void GC_register_dynamic_libraries(void)
25053 {
25054   struct link_map *lm;
25055 #ifdef HAVE_DL_ITERATE_PHDR
25056     if (GC_register_dynamic_libraries_dl_iterate_phdr()) {
25057         return;
25058     }
25059 #endif
25060   for (lm = GC_FirstDLOpenedLinkMap(); lm != 0; lm = lm->l_next)
25061     {
25062         ElfW(Ehdr) * e;
25063         ElfW(Phdr) * p;
25064         unsigned long offset;
25065         char * start;
25066         int i;
25067         e = (ElfW(Ehdr) *) lm->l_addr;
25068 #ifdef HOST_ANDROID
25069           if (e == NULL)
25070             continue;
25071 #endif
25072         p = ((ElfW(Phdr) *)(((char *)(e)) + e->e_phoff));
25073         offset = ((unsigned long)(lm->l_addr));
25074         for( i = 0; i < (int)e->e_phnum; i++, p++ ) {
25075           switch( p->p_type ) {
25076             case PT_LOAD:
25077               {
25078                 if( !(p->p_flags & PF_W) ) break;
25079                 start = ((char *)(p->p_vaddr)) + offset;
25080                 GC_add_roots_inner(start, start + p->p_memsz, TRUE);
25081               }
25082               break;
25083             default:
25084               break;
25085           }
25086         }
25087     }
25088 }
25089 #endif
25090 #endif
25091 #if defined(IRIX5) || (defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX))
25092 #include <sys/procfs.h>
25093 #include <sys/stat.h>
25094 #include <fcntl.h>
25095 #include <elf.h>
25096 #include <errno.h>
25097 #include <signal.h>
25098 #ifndef _sigargs
25099 #define IRIX6
25100 #endif
25101 GC_INNER void GC_register_dynamic_libraries(void)
25102 {
25103     static int fd = -1;
25104     char buf[30];
25105     static prmap_t * addr_map = 0;
25106     static int current_sz = 0;
25107     int needed_sz = 0;
25108     int i;
25109     long flags;
25110     ptr_t start;
25111     ptr_t limit;
25112     ptr_t heap_start = HEAP_START;
25113     ptr_t heap_end = heap_start;
25114 #ifdef SOLARISDL
25115 #define MA_PHYS 0
25116 #endif
25117     if (fd < 0) {
25118       (void)snprintf(buf, sizeof(buf), "/proc/%ld", (long)getpid());
25119       buf[sizeof(buf) - 1] = '\0';
25120       fd = open(buf, O_RDONLY);
25121       if (fd < 0) {
25122         ABORT("/proc open failed");
25123       }
25124     }
25125     if (ioctl(fd, PIOCNMAP, &needed_sz) < 0) {
25126         ABORT_ARG2("/proc PIOCNMAP ioctl failed",
25127                    ": fd= %d, errno= %d", fd, errno);
25128     }
25129     if (needed_sz >= current_sz) {
25130         GC_scratch_recycle_no_gww(addr_map,
25131                                   (size_t)current_sz * sizeof(prmap_t));
25132         current_sz = needed_sz * 2 + 1;
25133         addr_map = (prmap_t *)GC_scratch_alloc(
25134                                 (size_t)current_sz * sizeof(prmap_t));
25135         if (addr_map == NULL)
25136           ABORT("Insufficient memory for address map");
25137     }
25138     if (ioctl(fd, PIOCMAP, addr_map) < 0) {
25139         ABORT_ARG3("/proc PIOCMAP ioctl failed",
25140                    ": errcode= %d, needed_sz= %d, addr_map= %p",
25141                    errno, needed_sz, (void *)addr_map);
25142     }
25143     if (GC_n_heap_sects > 0) {
25144         heap_end = GC_heap_sects[GC_n_heap_sects-1].hs_start
25145                         + GC_heap_sects[GC_n_heap_sects-1].hs_bytes;
25146         if ((word)heap_end < (word)GC_scratch_last_end_ptr)
25147           heap_end = GC_scratch_last_end_ptr;
25148     }
25149     for (i = 0; i < needed_sz; i++) {
25150         flags = addr_map[i].pr_mflags;
25151         if ((flags & (MA_BREAK | MA_STACK | MA_PHYS
25152                       | MA_FETCHOP | MA_NOTCACHED)) != 0) goto irrelevant;
25153         if ((flags & (MA_READ | MA_WRITE)) != (MA_READ | MA_WRITE))
25154             goto irrelevant;
25155         start = (ptr_t)(addr_map[i].pr_vaddr);
25156         if (GC_roots_present(start)) goto irrelevant;
25157         if ((word)start < (word)heap_end && (word)start >= (word)heap_start)
25158                 goto irrelevant;
25159         limit = start + addr_map[i].pr_size;
25160 #ifndef IRIX6
25161           if (addr_map[i].pr_off == 0 && strncmp(start, ELFMAG, 4) == 0) {
25162             caddr_t arg;
25163             int obj;
25164 #define MAP_IRR_SZ 10
25165             static ptr_t map_irr[MAP_IRR_SZ];
25166             static int n_irr = 0;
25167             struct stat buf;
25168             int j;
25169             for (j = 0; j < n_irr; j++) {
25170                 if (map_irr[j] == start) goto irrelevant;
25171             }
25172             arg = (caddr_t)start;
25173             obj = ioctl(fd, PIOCOPENM, &arg);
25174             if (obj >= 0) {
25175                 fstat(obj, &buf);
25176                 close(obj);
25177                 if ((buf.st_mode & 0111) != 0) {
25178                     if (n_irr < MAP_IRR_SZ) {
25179                         map_irr[n_irr++] = start;
25180                     }
25181                     goto irrelevant;
25182                 }
25183             }
25184           }
25185 #endif
25186         GC_add_roots_inner(start, limit, TRUE);
25187       irrelevant: ;
25188     }
25189         if (close(fd) < 0) ABORT("Couldn't close /proc file");
25190         fd = -1;
25191 }
25192 #endif
25193 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
25194 #include <stdlib.h>
25195   STATIC void GC_cond_add_roots(char *base, char * limit)
25196   {
25197 #ifdef GC_WIN32_THREADS
25198       char * curr_base = base;
25199       char * next_stack_lo;
25200       char * next_stack_hi;
25201       if (base == limit) return;
25202       for(;;) {
25203           GC_get_next_stack(curr_base, limit, &next_stack_lo, &next_stack_hi);
25204           if ((word)next_stack_lo >= (word)limit) break;
25205           if ((word)next_stack_lo > (word)curr_base)
25206             GC_add_roots_inner(curr_base, next_stack_lo, TRUE);
25207           curr_base = next_stack_hi;
25208       }
25209       if ((word)curr_base < (word)limit)
25210         GC_add_roots_inner(curr_base, limit, TRUE);
25211 #else
25212       char * stack_top
25213          = (char *)((word)GC_approx_sp() &
25214                     ~(word)(GC_sysinfo.dwAllocationGranularity - 1));
25215       if (base == limit) return;
25216       if ((word)limit > (word)stack_top
25217           && (word)base < (word)GC_stackbottom) {
25218           return;
25219       }
25220       GC_add_roots_inner(base, limit, TRUE);
25221 #endif
25222   }
25223 #ifdef DYNAMIC_LOADING
25224   GC_INNER GC_bool GC_register_main_static_data(void)
25225   {
25226 #if defined(MSWINCE) || defined(CYGWIN32)
25227       return FALSE;
25228 #else
25229       return GC_no_win32_dlls;
25230 #endif
25231   }
25232 #define HAVE_REGISTER_MAIN_STATIC_DATA
25233 #endif
25234 #ifdef DEBUG_VIRTUALQUERY
25235   void GC_dump_meminfo(MEMORY_BASIC_INFORMATION *buf)
25236   {
25237     GC_printf("BaseAddress= 0x%lx, AllocationBase= 0x%lx,"
25238               " RegionSize= 0x%lx(%lu)\n",
25239               buf -> BaseAddress, buf -> AllocationBase,
25240               buf -> RegionSize, buf -> RegionSize);
25241     GC_printf("\tAllocationProtect= 0x%lx, State= 0x%lx, Protect= 0x%lx, "
25242               "Type= 0x%lx\n", buf -> AllocationProtect, buf -> State,
25243               buf -> Protect, buf -> Type);
25244   }
25245 #endif
25246 #if defined(MSWINCE) || defined(CYGWIN32)
25247 #define GC_wnt TRUE
25248 #endif
25249   GC_INNER void GC_register_dynamic_libraries(void)
25250   {
25251     MEMORY_BASIC_INFORMATION buf;
25252     DWORD protect;
25253     LPVOID p;
25254     char * base;
25255     char * limit, * new_limit;
25256 #ifdef MSWIN32
25257       if (GC_no_win32_dlls) return;
25258 #endif
25259     p = GC_sysinfo.lpMinimumApplicationAddress;
25260     base = limit = (char *)p;
25261     while ((word)p < (word)GC_sysinfo.lpMaximumApplicationAddress) {
25262         size_t result = VirtualQuery(p, &buf, sizeof(buf));
25263 #ifdef MSWINCE
25264           if (result == 0) {
25265             new_limit = (char *)
25266                 (((DWORD) p + GC_sysinfo.dwAllocationGranularity)
25267                  & ~(GC_sysinfo.dwAllocationGranularity-1));
25268           } else
25269 #endif
25270          {
25271             if (result != sizeof(buf)) {
25272                 ABORT("Weird VirtualQuery result");
25273             }
25274             new_limit = (char *)p + buf.RegionSize;
25275             protect = buf.Protect;
25276             if (buf.State == MEM_COMMIT
25277                 && (protect == PAGE_EXECUTE_READWRITE
25278                     || protect == PAGE_EXECUTE_WRITECOPY
25279                     || protect == PAGE_READWRITE
25280                     || protect == PAGE_WRITECOPY)
25281                 && (buf.Type == MEM_IMAGE
25282 #ifdef GC_REGISTER_MEM_PRIVATE
25283                       || (protect == PAGE_READWRITE && buf.Type == MEM_PRIVATE)
25284 #else
25285                       || (!GC_wnt && buf.Type == MEM_PRIVATE)
25286 #endif
25287                    )
25288                 && !GC_is_heap_base(buf.AllocationBase)) {
25289 #ifdef DEBUG_VIRTUALQUERY
25290                   GC_dump_meminfo(&buf);
25291 #endif
25292                 if ((char *)p != limit) {
25293                     GC_cond_add_roots(base, limit);
25294                     base = (char *)p;
25295                 }
25296                 limit = new_limit;
25297             }
25298         }
25299         if ((word)p > (word)new_limit ) break;
25300         p = (LPVOID)new_limit;
25301     }
25302     GC_cond_add_roots(base, limit);
25303   }
25304 #endif
25305 #if defined(ALPHA) && defined(OSF1)
25306 #include <loader.h>
25307 EXTERN_C_BEGIN
25308 extern char *sys_errlist[];
25309 extern int sys_nerr;
25310 extern int errno;
25311 EXTERN_C_END
25312 GC_INNER void GC_register_dynamic_libraries(void)
25313 {
25314   ldr_module_t moduleid = LDR_NULL_MODULE;
25315   ldr_process_t mypid = ldr_my_process();
25316     while (TRUE) {
25317       ldr_module_info_t moduleinfo;
25318       size_t modulereturnsize;
25319       ldr_region_t region;
25320       ldr_region_info_t regioninfo;
25321       size_t regionreturnsize;
25322       int status = ldr_next_module(mypid, &moduleid);
25323         if (moduleid == LDR_NULL_MODULE)
25324             break;
25325         if (status != 0) {
25326           ABORT_ARG3("ldr_next_module failed",
25327                      ": status= %d, errcode= %d (%s)", status, errno,
25328                      errno < sys_nerr ? sys_errlist[errno] : "");
25329         }
25330         status = ldr_inq_module(mypid, moduleid, &moduleinfo,
25331                                 sizeof(moduleinfo), &modulereturnsize);
25332         if (status != 0 )
25333             ABORT("ldr_inq_module failed");
25334           if (moduleinfo.lmi_flags & LDR_MAIN)
25335               continue;
25336 #ifdef DL_VERBOSE
25337         GC_log_printf("---Module---\n");
25338         GC_log_printf("Module ID: %ld\n", moduleinfo.lmi_modid);
25339         GC_log_printf("Count of regions: %d\n", moduleinfo.lmi_nregion);
25340         GC_log_printf("Flags for module: %016lx\n", moduleinfo.lmi_flags);
25341         GC_log_printf("Module pathname: \"%s\"\n", moduleinfo.lmi_name);
25342 #endif
25343         for (region = 0; region < moduleinfo.lmi_nregion; region++) {
25344             status = ldr_inq_region(mypid, moduleid, region, ®ioninfo,
25345                                     sizeof(regioninfo), ®ionreturnsize);
25346             if (status != 0 )
25347                 ABORT("ldr_inq_region failed");
25348             if (! (regioninfo.lri_prot & LDR_W))
25349                 continue;
25350 #ifdef DL_VERBOSE
25351             GC_log_printf("--- Region ---\n");
25352             GC_log_printf("Region number: %ld\n", regioninfo.lri_region_no);
25353             GC_log_printf("Protection flags: %016x\n", regioninfo.lri_prot);
25354             GC_log_printf("Virtual address: %p\n", regioninfo.lri_vaddr);
25355             GC_log_printf("Mapped address: %p\n", regioninfo.lri_mapaddr);
25356             GC_log_printf("Region size: %ld\n", regioninfo.lri_size);
25357             GC_log_printf("Region name: \"%s\"\n", regioninfo.lri_name);
25358 #endif
25359           GC_add_roots_inner((char *)regioninfo.lri_mapaddr,
25360                         (char *)regioninfo.lri_mapaddr + regioninfo.lri_size,
25361                         TRUE);
25362         }
25363     }
25364 }
25365 #endif
25366 #if defined(HPUX)
25367 #include <errno.h>
25368 #include <dl.h>
25369 EXTERN_C_BEGIN
25370 extern char *sys_errlist[];
25371 extern int sys_nerr;
25372 EXTERN_C_END
25373 GC_INNER void GC_register_dynamic_libraries(void)
25374 {
25375   int index = 1;
25376     while (TRUE) {
25377       struct shl_descriptor *shl_desc;
25378       int status = shl_get(index, &shl_desc);
25379         if (status != 0) {
25380 #ifdef GC_HPUX_THREADS
25381            break;
25382 #else
25383           if (errno == EINVAL) {
25384             break;
25385           } else {
25386             ABORT_ARG3("shl_get failed",
25387                        ": status= %d, errcode= %d (%s)", status, errno,
25388                        errno < sys_nerr ? sys_errlist[errno] : "");
25389           }
25390 #endif
25391         }
25392 #ifdef DL_VERBOSE
25393         GC_log_printf("---Shared library---\n");
25394         GC_log_printf("filename= \"%s\"\n", shl_desc->filename);
25395         GC_log_printf("index= %d\n", index);
25396         GC_log_printf("handle= %08x\n", (unsigned long) shl_desc->handle);
25397         GC_log_printf("text seg.start= %08x\n", shl_desc->tstart);
25398         GC_log_printf("text seg.end= %08x\n", shl_desc->tend);
25399         GC_log_printf("data seg.start= %08x\n", shl_desc->dstart);
25400         GC_log_printf("data seg.end= %08x\n", shl_desc->dend);
25401         GC_log_printf("ref.count= %lu\n", shl_desc->ref_count);
25402 #endif
25403         GC_add_roots_inner((char *) shl_desc->dstart,
25404                            (char *) shl_desc->dend, TRUE);
25405         index++;
25406     }
25407 }
25408 #endif
25409 #ifdef AIX
25410 #include <alloca.h>
25411 #include <sys/ldr.h>
25412 #include <sys/errno.h>
25413   GC_INNER void GC_register_dynamic_libraries(void)
25414   {
25415       int ldibuflen = 8192;
25416       for (;;) {
25417         int len;
25418         struct ld_info *ldi;
25419 #if defined(CPPCHECK)
25420           char ldibuf[ldibuflen];
25421 #else
25422           char *ldibuf = alloca(ldibuflen);
25423 #endif
25424         len = loadquery(L_GETINFO, ldibuf, ldibuflen);
25425         if (len < 0) {
25426                 if (errno != ENOMEM) {
25427                         ABORT("loadquery failed");
25428                 }
25429                 ldibuflen *= 2;
25430                 continue;
25431         }
25432         ldi = (struct ld_info *)ldibuf;
25433         while (ldi) {
25434                 len = ldi->ldinfo_next;
25435                 GC_add_roots_inner(
25436                                 ldi->ldinfo_dataorg,
25437                                 (ptr_t)(unsigned long)ldi->ldinfo_dataorg
25438                                 + ldi->ldinfo_datasize,
25439                                 TRUE);
25440                 ldi = len ? (struct ld_info *)((char *)ldi + len) : 0;
25441         }
25442         break;
25443       }
25444   }
25445 #endif
25446 #ifdef DARWIN
25447 #ifndef __private_extern__
25448 #define __private_extern__ extern
25449 #include <mach-o/dyld.h>
25450 #undef __private_extern__
25451 #else
25452 #include <mach-o/dyld.h>
25453 #endif
25454 #include <mach-o/getsect.h>
25455 STATIC const struct dyld_sections_s {
25456     const char *seg;
25457     const char *sect;
25458 } GC_dyld_sections[] = {
25459     { SEG_DATA, SECT_DATA },
25460     { SEG_DATA, "__static_data" },
25461     { SEG_DATA, SECT_BSS },
25462     { SEG_DATA, SECT_COMMON },
25463     { SEG_DATA, "__zobj_data" },
25464     { SEG_DATA, "__zobj_bss" }
25465 };
25466 STATIC const char * const GC_dyld_add_sect_fmts[] = {
25467   "__bss%u",
25468   "__pu_bss%u",
25469   "__zo_bss%u",
25470   "__zo_pu_bss%u"
25471 };
25472 #ifndef L2_MAX_OFILE_ALIGNMENT
25473 #define L2_MAX_OFILE_ALIGNMENT 15
25474 #endif
25475 STATIC const char *GC_dyld_name_for_hdr(const struct GC_MACH_HEADER *hdr)
25476 {
25477     unsigned long i, c;
25478     c = _dyld_image_count();
25479     for (i = 0; i < c; i++)
25480       if ((const struct GC_MACH_HEADER *)_dyld_get_image_header(i) == hdr)
25481         return _dyld_get_image_name(i);
25482     return NULL;
25483 }
25484 STATIC void GC_dyld_image_add(const struct GC_MACH_HEADER *hdr,
25485                               intptr_t slide)
25486 {
25487   unsigned long start, end;
25488   unsigned i, j;
25489   const struct GC_MACH_SECTION *sec;
25490   const char *name;
25491   GC_has_static_roots_func callback = GC_has_static_roots;
25492   DCL_LOCK_STATE;
25493   if (GC_no_dls) return;
25494 #ifdef DARWIN_DEBUG
25495     name = GC_dyld_name_for_hdr(hdr);
25496 #else
25497     name = callback != 0 ? GC_dyld_name_for_hdr(hdr) : NULL;
25498 #endif
25499   for (i = 0; i < sizeof(GC_dyld_sections)/sizeof(GC_dyld_sections[0]); i++) {
25500     sec = GC_GETSECTBYNAME(hdr, GC_dyld_sections[i].seg,
25501                            GC_dyld_sections[i].sect);
25502     if (sec == NULL || sec->size < sizeof(word))
25503       continue;
25504     start = slide + sec->addr;
25505     end = start + sec->size;
25506     LOCK();
25507     if (callback == 0 || callback(name, (void*)start, (size_t)sec->size)) {
25508 #ifdef DARWIN_DEBUG
25509         GC_log_printf(
25510               "Adding section __DATA,%s at %p-%p (%lu bytes) from image %s\n",
25511                GC_dyld_sections[i].sect, (void*)start, (void*)end,
25512                (unsigned long)sec->size, name);
25513 #endif
25514       GC_add_roots_inner((ptr_t)start, (ptr_t)end, FALSE);
25515     }
25516     UNLOCK();
25517   }
25518   for (j = 0; j < sizeof(GC_dyld_add_sect_fmts) / sizeof(char *); j++) {
25519     const char *fmt = GC_dyld_add_sect_fmts[j];
25520     for (i = 0; i <= L2_MAX_OFILE_ALIGNMENT; i++) {
25521       char secnam[16];
25522       (void)snprintf(secnam, sizeof(secnam), fmt, (unsigned)i);
25523       secnam[sizeof(secnam) - 1] = '\0';
25524       sec = GC_GETSECTBYNAME(hdr, SEG_DATA, secnam);
25525       if (sec == NULL || sec->size == 0)
25526         continue;
25527       start = slide + sec->addr;
25528       end = start + sec->size;
25529 #ifdef DARWIN_DEBUG
25530         GC_log_printf("Adding on-demand section __DATA,%s at"
25531                       " %p-%p (%lu bytes) from image %s\n",
25532                       secnam, (void*)start, (void*)end,
25533                       (unsigned long)sec->size, name);
25534 #endif
25535       GC_add_roots((char*)start, (char*)end);
25536     }
25537   }
25538 #if defined(DARWIN_DEBUG) && !defined(NO_DEBUGGING)
25539     LOCK();
25540     GC_print_static_roots();
25541     UNLOCK();
25542 #endif
25543 }
25544 STATIC void GC_dyld_image_remove(const struct GC_MACH_HEADER *hdr,
25545                                  intptr_t slide)
25546 {
25547   unsigned long start, end;
25548   unsigned i, j;
25549   const struct GC_MACH_SECTION *sec;
25550 #if defined(DARWIN_DEBUG) && !defined(NO_DEBUGGING)
25551     DCL_LOCK_STATE;
25552 #endif
25553   for (i = 0; i < sizeof(GC_dyld_sections)/sizeof(GC_dyld_sections[0]); i++) {
25554     sec = GC_GETSECTBYNAME(hdr, GC_dyld_sections[i].seg,
25555                            GC_dyld_sections[i].sect);
25556     if (sec == NULL || sec->size == 0)
25557       continue;
25558     start = slide + sec->addr;
25559     end = start + sec->size;
25560 #ifdef DARWIN_DEBUG
25561       GC_log_printf(
25562             "Removing section __DATA,%s at %p-%p (%lu bytes) from image %s\n",
25563             GC_dyld_sections[i].sect, (void*)start, (void*)end,
25564             (unsigned long)sec->size, GC_dyld_name_for_hdr(hdr));
25565 #endif
25566     GC_remove_roots((char*)start, (char*)end);
25567   }
25568   for (j = 0; j < sizeof(GC_dyld_add_sect_fmts) / sizeof(char *); j++) {
25569     const char *fmt = GC_dyld_add_sect_fmts[j];
25570     for (i = 0; i <= L2_MAX_OFILE_ALIGNMENT; i++) {
25571       char secnam[16];
25572       (void)snprintf(secnam, sizeof(secnam), fmt, (unsigned)i);
25573       secnam[sizeof(secnam) - 1] = '\0';
25574       sec = GC_GETSECTBYNAME(hdr, SEG_DATA, secnam);
25575       if (sec == NULL || sec->size == 0)
25576         continue;
25577       start = slide + sec->addr;
25578       end = start + sec->size;
25579 #ifdef DARWIN_DEBUG
25580         GC_log_printf("Removing on-demand section __DATA,%s at"
25581                       " %p-%p (%lu bytes) from image %s\n", secnam,
25582                       (void*)start, (void*)end, (unsigned long)sec->size,
25583                       GC_dyld_name_for_hdr(hdr));
25584 #endif
25585       GC_remove_roots((char*)start, (char*)end);
25586     }
25587   }
25588 #if defined(DARWIN_DEBUG) && !defined(NO_DEBUGGING)
25589     LOCK();
25590     GC_print_static_roots();
25591     UNLOCK();
25592 #endif
25593 }
25594 GC_INNER void GC_register_dynamic_libraries(void)
25595 {
25596 }
25597 GC_INNER void GC_init_dyld(void)
25598 {
25599   static GC_bool initialized = FALSE;
25600   if (initialized) return;
25601 #ifdef DARWIN_DEBUG
25602     GC_log_printf("Registering dyld callbacks...\n");
25603 #endif
25604   _dyld_register_func_for_add_image(
25605         (void (*)(const struct mach_header*, intptr_t))GC_dyld_image_add);
25606   _dyld_register_func_for_remove_image(
25607         (void (*)(const struct mach_header*, intptr_t))GC_dyld_image_remove);
25608   initialized = TRUE;
25609 #ifdef NO_DYLD_BIND_FULLY_IMAGE
25610 #else
25611     if (GC_no_dls) return;
25612     if (GETENV("DYLD_BIND_AT_LAUNCH") == 0) {
25613 #ifdef DARWIN_DEBUG
25614         GC_log_printf("Forcing full bind of GC code...\n");
25615 #endif
25616       if (!_dyld_bind_fully_image_containing_address(
25617                                                   (unsigned long *)GC_malloc))
25618         ABORT("_dyld_bind_fully_image_containing_address failed");
25619     }
25620 #endif
25621 }
25622 #define HAVE_REGISTER_MAIN_STATIC_DATA
25623 GC_INNER GC_bool GC_register_main_static_data(void)
25624 {
25625   return FALSE;
25626 }
25627 #endif
25628 #if defined(HAIKU)
25629 #include <kernel/image.h>
25630   GC_INNER void GC_register_dynamic_libraries(void)
25631   {
25632     image_info info;
25633     int32 cookie = 0;
25634     while (get_next_image_info(0, &cookie, &info) == B_OK) {
25635       ptr_t data = (ptr_t)info.data;
25636       GC_add_roots_inner(data, data + info.data_size, TRUE);
25637     }
25638   }
25639 #endif
25640 #elif defined(PCR)
25641   GC_INNER void GC_register_dynamic_libraries(void)
25642   {
25643     PCR_IL_LoadedFile * p = PCR_IL_GetLastLoadedFile();
25644     PCR_IL_LoadedSegment * q;
25645     while (p != NIL && !(p -> lf_commitPoint)) {
25646         p = p -> lf_prev;
25647     }
25648     for (; p != NIL; p = p -> lf_prev) {
25649       for (q = p -> lf_ls; q != NIL; q = q -> ls_next) {
25650         if ((q -> ls_flags & PCR_IL_SegFlags_Traced_MASK)
25651             == PCR_IL_SegFlags_Traced_on) {
25652           GC_add_roots_inner((ptr_t)q->ls_addr,
25653                              (ptr_t)q->ls_addr + q->ls_bytes, TRUE);
25654         }
25655       }
25656     }
25657   }
25658 #endif
25659 #if !defined(HAVE_REGISTER_MAIN_STATIC_DATA) && defined(DYNAMIC_LOADING)
25660   GC_INNER GC_bool GC_register_main_static_data(void)
25661   {
25662     return TRUE;
25663   }
25664 #endif
25665 GC_API void GC_CALL GC_register_has_static_roots_callback(
25666                                         GC_has_static_roots_func callback)
25667 {
25668     GC_has_static_roots = callback;
25669 }
25670 #if defined(GC_PTHREADS) && !defined(GC_NO_DLOPEN)
25671 #undef GC_MUST_RESTORE_REDEFINED_DLOPEN
25672 #if defined(dlopen) && !defined(GC_USE_LD_WRAP)
25673 #undef dlopen
25674 #define GC_MUST_RESTORE_REDEFINED_DLOPEN
25675 #endif
25676 #ifndef USE_PROC_FOR_LIBRARIES
25677   static void disable_gc_for_dlopen(void)
25678   {
25679     DCL_LOCK_STATE;
25680     LOCK();
25681     while (GC_incremental && GC_collection_in_progress()) {
25682       ENTER_GC();
25683       GC_collect_a_little_inner(1000);
25684       EXIT_GC();
25685     }
25686     ++GC_dont_gc;
25687     UNLOCK();
25688   }
25689 #endif
25690 #ifdef GC_USE_LD_WRAP
25691 #define WRAP_DLFUNC(f) __wrap_##f
25692 #define REAL_DLFUNC(f) __real_##f
25693   void * REAL_DLFUNC(dlopen)(const char *, int);
25694 #else
25695 #define WRAP_DLFUNC(f) GC_##f
25696 #define REAL_DLFUNC(f) f
25697 #endif
25698 GC_API void * WRAP_DLFUNC(dlopen)(const char *path, int mode)
25699 {
25700   void * result;
25701 #ifndef USE_PROC_FOR_LIBRARIES
25702     disable_gc_for_dlopen();
25703 #endif
25704   result = REAL_DLFUNC(dlopen)(path, mode);
25705 #ifndef USE_PROC_FOR_LIBRARIES
25706     GC_enable();
25707 #endif
25708   return(result);
25709 }
25710 #ifdef GC_USE_LD_WRAP
25711   GC_API void *GC_dlopen(const char *path, int mode)
25712   {
25713     return dlopen(path, mode);
25714   }
25715 #endif
25716 #ifdef GC_MUST_RESTORE_REDEFINED_DLOPEN
25717 #define dlopen GC_dlopen
25718 #endif
25719 #endif
25720 #if !defined(PLATFORM_MACH_DEP)
25721 #if !defined(PLATFORM_MACH_DEP) && !defined(SN_TARGET_PSP2)
25722 #include <stdio.h>
25723 #ifdef AMIGA
25724 #ifndef __GNUC__
25725 #include <dos.h>
25726 #else
25727 #include <machine/reg.h>
25728 #endif
25729 #endif
25730 #if defined(MACOS) && defined(__MWERKS__)
25731 #if defined(POWERPC)
25732 #define NONVOLATILE_GPR_COUNT 19
25733   struct ppc_registers {
25734         unsigned long gprs[NONVOLATILE_GPR_COUNT];
25735   };
25736   typedef struct ppc_registers ppc_registers;
25737 #if defined(CPPCHECK)
25738     void getRegisters(ppc_registers* regs);
25739 #else
25740     asm static void getRegisters(register ppc_registers* regs)
25741     {
25742         stmw    r13,regs->gprs
25743         blr
25744     }
25745 #endif
25746   static void PushMacRegisters(void)
25747   {
25748         ppc_registers regs;
25749         int i;
25750         getRegisters(®s);
25751         for (i = 0; i < NONVOLATILE_GPR_COUNT; i++)
25752                 GC_push_one(regs.gprs[i]);
25753   }
25754 #else
25755   asm static void PushMacRegisters(void)
25756   {
25757     sub.w   #4,sp
25758     move.l  a2,(sp)
25759     jsr         GC_push_one
25760     move.l  a3,(sp)
25761     jsr         GC_push_one
25762     move.l  a4,(sp)
25763     jsr         GC_push_one
25764 #if !__option(a6frames)
25765         move.l  a6,(sp)
25766         jsr             GC_push_one
25767 #endif
25768     move.l  d2,(sp)
25769     jsr         GC_push_one
25770     move.l  d3,(sp)
25771     jsr         GC_push_one
25772     move.l  d4,(sp)
25773     jsr         GC_push_one
25774     move.l  d5,(sp)
25775     jsr         GC_push_one
25776     move.l  d6,(sp)
25777     jsr         GC_push_one
25778     move.l  d7,(sp)
25779     jsr         GC_push_one
25780     add.w   #4,sp
25781     rts
25782   }
25783 #endif
25784 #endif
25785 #if defined(SPARC) || defined(IA64)
25786     GC_INNER ptr_t GC_save_regs_ret_val = NULL;
25787 #endif
25788 #undef HAVE_PUSH_REGS
25789 #if defined(USE_ASM_PUSH_REGS)
25790 #define HAVE_PUSH_REGS
25791 #else
25792 #ifdef STACK_NOT_SCANNED
25793     void GC_push_regs(void)
25794     {
25795     }
25796 #define HAVE_PUSH_REGS
25797 #elif defined(M68K) && defined(AMIGA)
25798     void GC_push_regs(void)
25799     {
25800 #ifdef __GNUC__
25801           asm("subq.w &0x4,%sp");
25802           asm("mov.l %a2,(%sp)"); asm("jsr _GC_push_one");
25803           asm("mov.l %a3,(%sp)"); asm("jsr _GC_push_one");
25804           asm("mov.l %a4,(%sp)"); asm("jsr _GC_push_one");
25805           asm("mov.l %a5,(%sp)"); asm("jsr _GC_push_one");
25806           asm("mov.l %a6,(%sp)"); asm("jsr _GC_push_one");
25807           asm("mov.l %d2,(%sp)"); asm("jsr _GC_push_one");
25808           asm("mov.l %d3,(%sp)"); asm("jsr _GC_push_one");
25809           asm("mov.l %d4,(%sp)"); asm("jsr _GC_push_one");
25810           asm("mov.l %d5,(%sp)"); asm("jsr _GC_push_one");
25811           asm("mov.l %d6,(%sp)"); asm("jsr _GC_push_one");
25812           asm("mov.l %d7,(%sp)"); asm("jsr _GC_push_one");
25813           asm("addq.w &0x4,%sp");
25814 #else
25815           GC_push_one(getreg(REG_A2));
25816           GC_push_one(getreg(REG_A3));
25817 #ifndef __SASC
25818             GC_push_one(getreg(REG_A4));
25819 #endif
25820           GC_push_one(getreg(REG_A5));
25821           GC_push_one(getreg(REG_A6));
25822           GC_push_one(getreg(REG_D2));
25823           GC_push_one(getreg(REG_D3));
25824           GC_push_one(getreg(REG_D4));
25825           GC_push_one(getreg(REG_D5));
25826           GC_push_one(getreg(REG_D6));
25827           GC_push_one(getreg(REG_D7));
25828 #endif
25829     }
25830 #define HAVE_PUSH_REGS
25831 #elif defined(MACOS)
25832 #if defined(M68K) && defined(THINK_C) && !defined(CPPCHECK)
25833 #define PushMacReg(reg) \
25834               move.l  reg,(sp) \
25835               jsr             GC_push_one
25836       void GC_push_regs(void)
25837       {
25838           asm {
25839               sub.w   #4,sp          ; reserve space for one parameter.
25840               PushMacReg(a2);
25841               PushMacReg(a3);
25842               PushMacReg(a4);
25843               ; skip a5 (globals), a6 (frame pointer), and a7 (stack pointer)
25844               PushMacReg(d2);
25845               PushMacReg(d3);
25846               PushMacReg(d4);
25847               PushMacReg(d5);
25848               PushMacReg(d6);
25849               PushMacReg(d7);
25850               add.w   #4,sp          ; fix stack.
25851           }
25852       }
25853 #define HAVE_PUSH_REGS
25854 #undef PushMacReg
25855 #elif defined(__MWERKS__)
25856       void GC_push_regs(void)
25857       {
25858           PushMacRegisters();
25859       }
25860 #define HAVE_PUSH_REGS
25861 #endif
25862 #endif
25863 #endif
25864 #if defined(HAVE_PUSH_REGS) && defined(THREADS)
25865 #error GC_push_regs cannot be used with threads
25866 #undef HAVE_PUSH_REGS
25867 #endif
25868 #if !defined(HAVE_PUSH_REGS) && defined(UNIX_LIKE)
25869 #include <signal.h>
25870 #ifndef NO_GETCONTEXT
25871 #if defined(DARWIN) \
25872        && (MAC_OS_X_VERSION_MAX_ALLOWED >= 1060 )
25873 #include <sys/ucontext.h>
25874 #else
25875 #include <ucontext.h>
25876 #endif
25877 #ifdef GETCONTEXT_FPU_EXCMASK_BUG
25878 #include <fenv.h>
25879 #endif
25880 #endif
25881 #endif
25882 GC_ATTR_NO_SANITIZE_ADDR
25883 GC_INNER void GC_with_callee_saves_pushed(void (*fn)(ptr_t, void *),
25884                                           volatile ptr_t arg)
25885 {
25886   volatile int dummy;
25887   volatile ptr_t context = 0;
25888 #if defined(HAVE_PUSH_REGS)
25889     GC_push_regs();
25890 #elif defined(EMSCRIPTEN)
25891 #else
25892 #if defined(UNIX_LIKE) && !defined(NO_GETCONTEXT)
25893       static signed char getcontext_works = 0;
25894       ucontext_t ctxt;
25895 #ifdef GETCONTEXT_FPU_EXCMASK_BUG
25896 #ifdef X86_64
25897           unsigned short old_fcw;
25898 #if defined(CPPCHECK)
25899             GC_noop1((word)&old_fcw);
25900 #endif
25901           __asm__ __volatile__ ("fstcw %0" : "=m" (*&old_fcw));
25902 #else
25903           int except_mask = fegetexcept();
25904 #endif
25905 #endif
25906       if (getcontext_works >= 0) {
25907         if (getcontext(&ctxt) < 0) {
25908           WARN("getcontext failed:"
25909                " using another register retrieval method...\n", 0);
25910         } else {
25911           context = (ptr_t)&ctxt;
25912         }
25913         if (EXPECT(0 == getcontext_works, FALSE))
25914           getcontext_works = context != NULL ? 1 : -1;
25915       }
25916 #ifdef GETCONTEXT_FPU_EXCMASK_BUG
25917 #ifdef X86_64
25918           __asm__ __volatile__ ("fldcw %0" : : "m" (*&old_fcw));
25919           {
25920             unsigned mxcsr;
25921             __asm__ __volatile__ ("stmxcsr %0" : "=m" (*&mxcsr));
25922             mxcsr = (mxcsr & ~(FE_ALL_EXCEPT << 7)) |
25923                         ((old_fcw & FE_ALL_EXCEPT) << 7);
25924             __asm__ __volatile__ ("ldmxcsr %0" : : "m" (*&mxcsr));
25925           }
25926 #else
25927           if (feenableexcept(except_mask) < 0)
25928             ABORT("feenableexcept failed");
25929 #endif
25930 #endif
25931 #if defined(SPARC) || defined(IA64)
25932         GC_save_regs_ret_val = GC_save_regs_in_stack();
25933 #endif
25934       if (NULL == context)
25935 #endif
25936     {
25937 #if defined(HAVE_BUILTIN_UNWIND_INIT)
25938         __builtin_unwind_init();
25939 #elif defined(NO_CRT) && defined(MSWIN32)
25940         CONTEXT ctx;
25941         RtlCaptureContext(&ctx);
25942 #else
25943         jmp_buf regs;
25944         word * i = (word *)®s;
25945         ptr_t lim = (ptr_t)(®s) + sizeof(regs);
25946         for (; (word)i < (word)lim; i++) {
25947             *i = 0;
25948         }
25949 #if defined(MSWIN32) || defined(MSWINCE) || defined(UTS4) \
25950            || defined(OS2) || defined(CX_UX) || defined(__CC_ARM) \
25951            || defined(LINUX) || defined(EWS4800) || defined(RTEMS)
25952           (void) setjmp(regs);
25953 #else
25954           (void) _setjmp(regs);
25955 #endif
25956 #endif
25957     }
25958 #endif
25959   fn(arg, ( void *)context);
25960   GC_noop1(COVERT_DATAFLOW(&dummy));
25961 }
25962 #endif
25963 #endif
25964 #if !defined(PLATFORM_STOP_WORLD)
25965 #if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS) && \
25966     !defined(GC_DARWIN_THREADS) && !defined(PLATFORM_STOP_WORLD) \
25967     && !defined(SN_TARGET_PSP2)
25968 #ifdef NACL
25969 #include <unistd.h>
25970 #include <sys/time.h>
25971   STATIC int GC_nacl_num_gc_threads = 0;
25972   STATIC __thread int GC_nacl_thread_idx = -1;
25973   STATIC volatile int GC_nacl_park_threads_now = 0;
25974   STATIC volatile pthread_t GC_nacl_thread_parker = -1;
25975   GC_INNER __thread GC_thread GC_nacl_gc_thread_self = NULL;
25976   volatile int GC_nacl_thread_parked[MAX_NACL_GC_THREADS];
25977   int GC_nacl_thread_used[MAX_NACL_GC_THREADS];
25978 #elif defined(GC_OPENBSD_UTHREADS)
25979 #include <pthread_np.h>
25980 #else
25981 #include <signal.h>
25982 #include <semaphore.h>
25983 #include <errno.h>
25984 #include <time.h>
25985 #include <unistd.h>
25986 #if (!defined(AO_HAVE_load_acquire) || !defined(AO_HAVE_store_release)) \
25987     && !defined(CPPCHECK)
25988 #error AO_load_acquire and/or AO_store_release are missing;
25989 #error please define AO_REQUIRE_CAS manually
25990 #endif
25991 #undef pthread_sigmask
25992 #ifdef GC_ENABLE_SUSPEND_THREAD
25993   static void *GC_CALLBACK suspend_self_inner(void *client_data);
25994 #endif
25995 #ifdef DEBUG_THREADS
25996 #ifndef NSIG
25997 #if defined(MAXSIG)
25998 #define NSIG (MAXSIG+1)
25999 #elif defined(_NSIG)
26000 #define NSIG _NSIG
26001 #elif defined(__SIGRTMAX)
26002 #define NSIG (__SIGRTMAX+1)
26003 #else
26004 #error define NSIG
26005 #endif
26006 #endif
26007   void GC_print_sig_mask(void)
26008   {
26009     sigset_t blocked;
26010     int i;
26011     if (pthread_sigmask(SIG_BLOCK, NULL, &blocked) != 0)
26012       ABORT("pthread_sigmask failed");
26013     for (i = 1; i < NSIG; i++) {
26014       if (sigismember(&blocked, i))
26015         GC_printf("Signal blocked: %d\n", i);
26016     }
26017   }
26018 #endif
26019 STATIC void GC_remove_allowed_signals(sigset_t *set)
26020 {
26021     if (sigdelset(set, SIGINT) != 0
26022           || sigdelset(set, SIGQUIT) != 0
26023           || sigdelset(set, SIGABRT) != 0
26024           || sigdelset(set, SIGTERM) != 0) {
26025         ABORT("sigdelset failed");
26026     }
26027 #ifdef MPROTECT_VDB
26028       if (sigdelset(set, SIGSEGV) != 0
26029 #ifdef HAVE_SIGBUS
26030             || sigdelset(set, SIGBUS) != 0
26031 #endif
26032           ) {
26033         ABORT("sigdelset failed");
26034       }
26035 #endif
26036 }
26037 static sigset_t suspend_handler_mask;
26038 #define THREAD_RESTARTED 0x1
26039 STATIC volatile AO_t GC_stop_count = 0;
26040 STATIC volatile AO_t GC_world_is_stopped = FALSE;
26041 #ifndef NO_RETRY_SIGNALS
26042   STATIC GC_bool GC_retry_signals = TRUE;
26043 #else
26044   STATIC GC_bool GC_retry_signals = FALSE;
26045 #endif
26046 #ifndef SIG_THR_RESTART
26047 #if defined(GC_HPUX_THREADS) || defined(GC_OSF1_THREADS) \
26048      || defined(GC_NETBSD_THREADS) || defined(GC_USESIGRT_SIGNALS)
26049 #if defined(_SIGRTMIN) && !defined(CPPCHECK)
26050 #define SIG_THR_RESTART _SIGRTMIN + 5
26051 #else
26052 #define SIG_THR_RESTART SIGRTMIN + 5
26053 #endif
26054 #else
26055 #define SIG_THR_RESTART SIGXCPU
26056 #endif
26057 #endif
26058 #define SIGNAL_UNSET (-1)
26059 STATIC int GC_sig_suspend = SIGNAL_UNSET;
26060 STATIC int GC_sig_thr_restart = SIGNAL_UNSET;
26061 GC_API void GC_CALL GC_set_suspend_signal(int sig)
26062 {
26063   if (GC_is_initialized) return;
26064   GC_sig_suspend = sig;
26065 }
26066 GC_API void GC_CALL GC_set_thr_restart_signal(int sig)
26067 {
26068   if (GC_is_initialized) return;
26069   GC_sig_thr_restart = sig;
26070 }
26071 GC_API int GC_CALL GC_get_suspend_signal(void)
26072 {
26073   return GC_sig_suspend != SIGNAL_UNSET ? GC_sig_suspend : SIG_SUSPEND;
26074 }
26075 GC_API int GC_CALL GC_get_thr_restart_signal(void)
26076 {
26077   return GC_sig_thr_restart != SIGNAL_UNSET
26078             ? GC_sig_thr_restart : SIG_THR_RESTART;
26079 }
26080 #if defined(GC_EXPLICIT_SIGNALS_UNBLOCK) \
26081     || !defined(NO_SIGNALS_UNBLOCK_IN_MAIN)
26082   GC_INNER void GC_unblock_gc_signals(void)
26083   {
26084     sigset_t set;
26085     sigemptyset(&set);
26086     GC_ASSERT(GC_sig_suspend != SIGNAL_UNSET);
26087     GC_ASSERT(GC_sig_thr_restart != SIGNAL_UNSET);
26088     sigaddset(&set, GC_sig_suspend);
26089     sigaddset(&set, GC_sig_thr_restart);
26090     if (pthread_sigmask(SIG_UNBLOCK, &set, NULL) != 0)
26091       ABORT("pthread_sigmask failed");
26092   }
26093 #endif
26094 STATIC sem_t GC_suspend_ack_sem;
26095 STATIC void GC_suspend_handler_inner(ptr_t dummy, void *context);
26096 #ifndef NO_SA_SIGACTION
26097   STATIC void GC_suspend_handler(int sig, siginfo_t * info GC_ATTR_UNUSED,
26098                                  void * context GC_ATTR_UNUSED)
26099 #else
26100   STATIC void GC_suspend_handler(int sig)
26101 #endif
26102 {
26103   int old_errno = errno;
26104   if (sig != GC_sig_suspend) {
26105 #if defined(GC_FREEBSD_THREADS)
26106       if (0 == sig) return;
26107 #endif
26108     ABORT("Bad signal in suspend_handler");
26109   }
26110 #if defined(IA64) || defined(HP_PA) || defined(M68K)
26111     GC_with_callee_saves_pushed(GC_suspend_handler_inner, NULL);
26112 #else
26113     {
26114 #ifdef NO_SA_SIGACTION
26115         void *context = 0;
26116 #endif
26117       GC_suspend_handler_inner(NULL, context);
26118     }
26119 #endif
26120   errno = old_errno;
26121 }
26122 #ifdef BASE_ATOMIC_OPS_EMULATED
26123 #define ao_load_acquire_async(p) (*(p))
26124 #define ao_load_async(p) ao_load_acquire_async(p)
26125 #define ao_store_release_async(p, v) (void)(*(p) = (v))
26126 #define ao_store_async(p, v) ao_store_release_async(p, v)
26127 #else
26128 #define ao_load_acquire_async(p) AO_load_acquire(p)
26129 #define ao_load_async(p) AO_load(p)
26130 #define ao_store_release_async(p, v) AO_store_release(p, v)
26131 #define ao_store_async(p, v) AO_store(p, v)
26132 #endif
26133 #ifdef THREAD_SANITIZER
26134   GC_ATTR_NO_SANITIZE_THREAD
26135   static GC_thread GC_lookup_thread_async(pthread_t id)
26136   {
26137     GC_thread p = GC_threads[THREAD_TABLE_INDEX(id)];
26138     while (p != NULL && !THREAD_EQUAL(p->id, id))
26139       p = p->next;
26140     return p;
26141   }
26142 #else
26143 #define GC_lookup_thread_async GC_lookup_thread
26144 #endif
26145 GC_INLINE void GC_store_stack_ptr(GC_thread me)
26146 {
26147 #ifdef SPARC
26148     ao_store_async((volatile AO_t *)&me->stop_info.stack_ptr,
26149              (AO_t)GC_save_regs_in_stack());
26150 #else
26151 #ifdef IA64
26152       me -> backing_store_ptr = GC_save_regs_in_stack();
26153 #endif
26154     ao_store_async((volatile AO_t *)&me->stop_info.stack_ptr,
26155                    (AO_t)GC_approx_sp());
26156 #endif
26157 }
26158 STATIC void GC_suspend_handler_inner(ptr_t dummy GC_ATTR_UNUSED,
26159                                      void * context GC_ATTR_UNUSED)
26160 {
26161   pthread_t self = pthread_self();
26162   GC_thread me;
26163   IF_CANCEL(int cancel_state;)
26164   AO_t my_stop_count = ao_load_acquire_async(&GC_stop_count);
26165   DISABLE_CANCEL(cancel_state);
26166 #ifdef DEBUG_THREADS
26167     GC_log_printf("Suspending %p\n", (void *)self);
26168 #endif
26169   GC_ASSERT(((word)my_stop_count & THREAD_RESTARTED) == 0);
26170   me = GC_lookup_thread_async(self);
26171 #ifdef GC_ENABLE_SUSPEND_THREAD
26172     if (ao_load_async(&me->suspended_ext)) {
26173       GC_store_stack_ptr(me);
26174       sem_post(&GC_suspend_ack_sem);
26175       suspend_self_inner(me);
26176 #ifdef DEBUG_THREADS
26177         GC_log_printf("Continuing %p on GC_resume_thread\n", (void *)self);
26178 #endif
26179       RESTORE_CANCEL(cancel_state);
26180       return;
26181     }
26182 #endif
26183   if (((word)me->stop_info.last_stop_count & ~(word)THREAD_RESTARTED)
26184         == (word)my_stop_count) {
26185       if (!GC_retry_signals) {
26186           WARN("Duplicate suspend signal in thread %p\n", self);
26187       }
26188       RESTORE_CANCEL(cancel_state);
26189       return;
26190   }
26191   GC_store_stack_ptr(me);
26192 #ifdef THREAD_SANITIZER
26193     {
26194       sigset_t set;
26195       sigemptyset(&set);
26196       GC_ASSERT(GC_sig_suspend != SIGNAL_UNSET);
26197       GC_ASSERT(GC_sig_thr_restart != SIGNAL_UNSET);
26198       sigaddset(&set, GC_sig_suspend);
26199       sigaddset(&set, GC_sig_thr_restart);
26200       if (pthread_sigmask(SIG_UNBLOCK, &set, NULL) != 0)
26201         ABORT("pthread_sigmask failed in suspend handler");
26202     }
26203 #endif
26204   sem_post(&GC_suspend_ack_sem);
26205   ao_store_release_async(&me->stop_info.last_stop_count, my_stop_count);
26206   do {
26207       sigsuspend (&suspend_handler_mask);
26208   } while (ao_load_acquire_async(&GC_world_is_stopped)
26209            && ao_load_async(&GC_stop_count) == my_stop_count);
26210 #ifdef DEBUG_THREADS
26211     GC_log_printf("Continuing %p\n", (void *)self);
26212 #endif
26213 #ifndef GC_NETBSD_THREADS_WORKAROUND
26214     if (GC_retry_signals)
26215 #endif
26216   {
26217     sem_post(&GC_suspend_ack_sem);
26218 #ifdef GC_NETBSD_THREADS_WORKAROUND
26219       if (GC_retry_signals)
26220 #endif
26221     {
26222       ao_store_release_async(&me->stop_info.last_stop_count,
26223                              (AO_t)((word)my_stop_count | THREAD_RESTARTED));
26224     }
26225   }
26226   RESTORE_CANCEL(cancel_state);
26227 }
26228 static void suspend_restart_barrier(int n_live_threads)
26229 {
26230     int i;
26231     for (i = 0; i < n_live_threads; i++) {
26232       while (0 != sem_wait(&GC_suspend_ack_sem)) {
26233         if (errno != EINTR)
26234           ABORT("sem_wait failed");
26235       }
26236     }
26237 #ifdef GC_ASSERTIONS
26238       sem_getvalue(&GC_suspend_ack_sem, &i);
26239       GC_ASSERT(0 == i);
26240 #endif
26241 }
26242 static int resend_lost_signals(int n_live_threads,
26243                                int (*suspend_restart_all)(void))
26244 {
26245 #define WAIT_UNIT 3000
26246 #define RETRY_INTERVAL 100000
26247     if (n_live_threads > 0) {
26248       unsigned long wait_usecs = 0;
26249       for (;;) {
26250         int ack_count;
26251         sem_getvalue(&GC_suspend_ack_sem, &ack_count);
26252         if (ack_count == n_live_threads)
26253           break;
26254         if (wait_usecs > RETRY_INTERVAL) {
26255           int newly_sent = suspend_restart_all();
26256           GC_COND_LOG_PRINTF("Resent %d signals after timeout\n", newly_sent);
26257           sem_getvalue(&GC_suspend_ack_sem, &ack_count);
26258           if (newly_sent < n_live_threads - ack_count) {
26259             WARN("Lost some threads while stopping or starting world?!\n", 0);
26260             n_live_threads = ack_count + newly_sent;
26261           }
26262           wait_usecs = 0;
26263         }
26264 #ifdef LINT2
26265 #undef WAIT_UNIT
26266 #define WAIT_UNIT 1
26267           sched_yield();
26268 #elif defined(CPPCHECK)
26269           {
26270             struct timespec ts;
26271             ts.tv_sec = 0;
26272             ts.tv_nsec = WAIT_UNIT * 1000;
26273             (void)nanosleep(&ts, NULL);
26274           }
26275 #else
26276           usleep(WAIT_UNIT);
26277 #endif
26278         wait_usecs += WAIT_UNIT;
26279       }
26280     }
26281     return n_live_threads;
26282 }
26283 #ifdef HAVE_CLOCK_GETTIME
26284 #define TS_NSEC_ADD(ts, ns) \
26285                 (ts.tv_nsec += (ns), \
26286                  (void)(ts.tv_nsec >= 1000000L*1000 ? \
26287                        (ts.tv_nsec -= 1000000L*1000, ts.tv_sec++, 0) : 0))
26288 #endif
26289 static void resend_lost_signals_retry(int n_live_threads,
26290                                       int (*suspend_restart_all)(void))
26291 {
26292 #if defined(HAVE_CLOCK_GETTIME) && !defined(DONT_TIMEDWAIT_ACK_SEM)
26293 #define TIMEOUT_BEFORE_RESEND 10000
26294     int i;
26295     struct timespec ts;
26296     if (n_live_threads > 0 && clock_gettime(CLOCK_REALTIME, &ts) == 0) {
26297       TS_NSEC_ADD(ts, TIMEOUT_BEFORE_RESEND * 1000);
26298       for (i = 0; i < n_live_threads; i++) {
26299         if (0 != sem_timedwait(&GC_suspend_ack_sem, &ts))
26300           break;
26301       }
26302       n_live_threads -= i;
26303     }
26304 #endif
26305   n_live_threads = resend_lost_signals(n_live_threads, suspend_restart_all);
26306   suspend_restart_barrier(n_live_threads);
26307 }
26308 STATIC void GC_restart_handler(int sig)
26309 {
26310 #if defined(DEBUG_THREADS)
26311     int old_errno = errno;
26312 #endif
26313   if (sig != GC_sig_thr_restart)
26314     ABORT("Bad signal in restart handler");
26315 #ifdef DEBUG_THREADS
26316     GC_log_printf("In GC_restart_handler for %p\n", (void *)pthread_self());
26317     errno = old_errno;
26318 #endif
26319 }
26320 #ifdef USE_TKILL_ON_ANDROID
26321     EXTERN_C_BEGIN
26322     extern int tkill(pid_t tid, int sig);
26323     EXTERN_C_END
26324     static int android_thread_kill(pid_t tid, int sig)
26325     {
26326       int ret;
26327       int old_errno = errno;
26328       ret = tkill(tid, sig);
26329       if (ret < 0) {
26330           ret = errno;
26331           errno = old_errno;
26332       }
26333       return ret;
26334     }
26335 #define THREAD_SYSTEM_ID(t) (t)->kernel_id
26336 #define RAISE_SIGNAL(t, sig) android_thread_kill(THREAD_SYSTEM_ID(t), sig)
26337 #else
26338 #define THREAD_SYSTEM_ID(t) (t)->id
26339 #define RAISE_SIGNAL(t, sig) pthread_kill(THREAD_SYSTEM_ID(t), sig)
26340 #endif
26341 #ifdef GC_ENABLE_SUSPEND_THREAD
26342 #include <sys/time.h>
26343     STATIC void GC_brief_async_signal_safe_sleep(void)
26344     {
26345       struct timeval tv;
26346       tv.tv_sec = 0;
26347 #if defined(GC_TIME_LIMIT) && !defined(CPPCHECK)
26348         tv.tv_usec = 1000 * GC_TIME_LIMIT / 2;
26349 #else
26350         tv.tv_usec = 1000 * 50 / 2;
26351 #endif
26352       (void)select(0, 0, 0, 0, &tv);
26353     }
26354     static void *GC_CALLBACK suspend_self_inner(void *client_data) {
26355       GC_thread me = (GC_thread)client_data;
26356       while (ao_load_acquire_async(&me->suspended_ext)) {
26357         GC_brief_async_signal_safe_sleep();
26358       }
26359       return NULL;
26360     }
26361     GC_API void GC_CALL GC_suspend_thread(GC_SUSPEND_THREAD_ID thread) {
26362       GC_thread t;
26363       IF_CANCEL(int cancel_state;)
26364       DCL_LOCK_STATE;
26365       LOCK();
26366       t = GC_lookup_thread((pthread_t)thread);
26367       if (t == NULL || t -> suspended_ext) {
26368         UNLOCK();
26369         return;
26370       }
26371       AO_store_release(&t->suspended_ext, TRUE);
26372       if (THREAD_EQUAL((pthread_t)thread, pthread_self())) {
26373         UNLOCK();
26374         (void)GC_do_blocking(suspend_self_inner, t);
26375         return;
26376       }
26377       if ((t -> flags & FINISHED) != 0) {
26378         UNLOCK();
26379         return;
26380       }
26381       DISABLE_CANCEL(cancel_state);
26382 #ifdef PARALLEL_MARK
26383         if (GC_parallel)
26384           GC_wait_for_reclaim();
26385 #endif
26386       if (GC_manual_vdb) {
26387         GC_acquire_dirty_lock();
26388       }
26389       switch (RAISE_SIGNAL(t, GC_sig_suspend)) {
26390       case 0:
26391         break;
26392       default:
26393         ABORT("pthread_kill failed");
26394       }
26395       GC_ASSERT(GC_thr_initialized);
26396       while (sem_wait(&GC_suspend_ack_sem) != 0) {
26397         if (errno != EINTR)
26398           ABORT("sem_wait for handler failed (suspend_self)");
26399       }
26400       if (GC_manual_vdb)
26401         GC_release_dirty_lock();
26402       RESTORE_CANCEL(cancel_state);
26403       UNLOCK();
26404     }
26405     GC_API void GC_CALL GC_resume_thread(GC_SUSPEND_THREAD_ID thread) {
26406       GC_thread t;
26407       DCL_LOCK_STATE;
26408       LOCK();
26409       t = GC_lookup_thread((pthread_t)thread);
26410       if (t != NULL)
26411         AO_store(&t->suspended_ext, FALSE);
26412       UNLOCK();
26413     }
26414     GC_API int GC_CALL GC_is_thread_suspended(GC_SUSPEND_THREAD_ID thread) {
26415       GC_thread t;
26416       int is_suspended = 0;
26417       DCL_LOCK_STATE;
26418       LOCK();
26419       t = GC_lookup_thread((pthread_t)thread);
26420       if (t != NULL && t -> suspended_ext)
26421         is_suspended = (int)TRUE;
26422       UNLOCK();
26423       return is_suspended;
26424     }
26425 #endif
26426 #undef ao_load_acquire_async
26427 #undef ao_load_async
26428 #undef ao_store_async
26429 #undef ao_store_release_async
26430 #endif
26431 #ifdef IA64
26432 #define IF_IA64(x) x
26433 #else
26434 #define IF_IA64(x)
26435 #endif
26436 GC_INNER void GC_push_all_stacks(void)
26437 {
26438     GC_bool found_me = FALSE;
26439     size_t nthreads = 0;
26440     int i;
26441     GC_thread p;
26442     ptr_t lo, hi;
26443     IF_IA64(ptr_t bs_lo; ptr_t bs_hi;)
26444     struct GC_traced_stack_sect_s *traced_stack_sect;
26445     pthread_t self = pthread_self();
26446     word total_size = 0;
26447     if (!EXPECT(GC_thr_initialized, TRUE))
26448       GC_thr_init();
26449 #ifdef DEBUG_THREADS
26450       GC_log_printf("Pushing stacks from thread %p\n", (void *)self);
26451 #endif
26452     for (i = 0; i < THREAD_TABLE_SZ; i++) {
26453       for (p = GC_threads[i]; p != 0; p = p -> next) {
26454         if (p -> flags & FINISHED) continue;
26455         ++nthreads;
26456         traced_stack_sect = p -> traced_stack_sect;
26457         if (THREAD_EQUAL(p -> id, self)) {
26458             GC_ASSERT(!p->thread_blocked);
26459 #ifdef SPARC
26460                 lo = (ptr_t)GC_save_regs_in_stack();
26461 #else
26462                 lo = GC_approx_sp();
26463 #endif
26464             found_me = TRUE;
26465             IF_IA64(bs_hi = (ptr_t)GC_save_regs_in_stack();)
26466         } else {
26467             lo = (ptr_t)AO_load((volatile AO_t *)&p->stop_info.stack_ptr);
26468             IF_IA64(bs_hi = p -> backing_store_ptr;)
26469             if (traced_stack_sect != NULL
26470                     && traced_stack_sect->saved_stack_ptr == lo) {
26471               traced_stack_sect = traced_stack_sect->prev;
26472             }
26473         }
26474         if ((p -> flags & MAIN_THREAD) == 0) {
26475             hi = p -> stack_end;
26476             IF_IA64(bs_lo = p -> backing_store_end);
26477         } else {
26478             hi = GC_stackbottom;
26479             IF_IA64(bs_lo = BACKING_STORE_BASE;)
26480         }
26481 #ifdef DEBUG_THREADS
26482           GC_log_printf("Stack for thread %p is [%p,%p)\n",
26483                         (void *)p->id, (void *)lo, (void *)hi);
26484 #endif
26485         if (0 == lo) ABORT("GC_push_all_stacks: sp not set!");
26486         if (p->altstack != NULL && (word)p->altstack <= (word)lo
26487             && (word)lo <= (word)p->altstack + p->altstack_size) {
26488           hi = p->altstack + p->altstack_size;
26489         }
26490         GC_push_all_stack_sections(lo, hi, traced_stack_sect);
26491 #ifdef STACK_GROWS_UP
26492           total_size += lo - hi;
26493 #else
26494           total_size += hi - lo;
26495 #endif
26496 #ifdef NACL
26497           GC_push_all_stack((ptr_t)p -> stop_info.reg_storage,
26498               (ptr_t)(p -> stop_info.reg_storage + NACL_GC_REG_STORAGE_SIZE));
26499           total_size += NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t);
26500 #endif
26501 #ifdef IA64
26502 #ifdef DEBUG_THREADS
26503             GC_log_printf("Reg stack for thread %p is [%p,%p)\n",
26504                           (void *)p->id, (void *)bs_lo, (void *)bs_hi);
26505 #endif
26506           GC_push_all_register_sections(bs_lo, bs_hi,
26507                                         THREAD_EQUAL(p -> id, self),
26508                                         traced_stack_sect);
26509           total_size += bs_hi - bs_lo;
26510 #endif
26511       }
26512     }
26513     GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks\n", (int)nthreads);
26514     if (!found_me && !GC_in_thread_creation)
26515       ABORT("Collecting from unknown thread");
26516     GC_total_stacksize = total_size;
26517 }
26518 #ifdef DEBUG_THREADS
26519   pthread_t GC_stopping_thread;
26520   int GC_stopping_pid = 0;
26521 #endif
26522 STATIC int GC_suspend_all(void)
26523 {
26524   int n_live_threads = 0;
26525   int i;
26526 #ifndef NACL
26527     GC_thread p;
26528 #ifndef GC_OPENBSD_UTHREADS
26529       int result;
26530 #endif
26531     pthread_t self = pthread_self();
26532     for (i = 0; i < THREAD_TABLE_SZ; i++) {
26533       for (p = GC_threads[i]; p != 0; p = p -> next) {
26534         if (!THREAD_EQUAL(p -> id, self)) {
26535             if ((p -> flags & FINISHED) != 0) continue;
26536             if (p -> thread_blocked)  continue;
26537 #ifndef GC_OPENBSD_UTHREADS
26538 #ifdef GC_ENABLE_SUSPEND_THREAD
26539                 if (p -> suspended_ext) continue;
26540 #endif
26541               if (AO_load(&p->stop_info.last_stop_count) == GC_stop_count)
26542                 continue;
26543               n_live_threads++;
26544 #endif
26545 #ifdef DEBUG_THREADS
26546               GC_log_printf("Sending suspend signal to %p\n", (void *)p->id);
26547 #endif
26548 #ifdef GC_OPENBSD_UTHREADS
26549               {
26550                 stack_t stack;
26551                 GC_acquire_dirty_lock();
26552                 if (pthread_suspend_np(p -> id) != 0)
26553                   ABORT("pthread_suspend_np failed");
26554                 GC_release_dirty_lock();
26555                 if (pthread_stackseg_np(p->id, &stack))
26556                   ABORT("pthread_stackseg_np failed");
26557                 p -> stop_info.stack_ptr = (ptr_t)stack.ss_sp - stack.ss_size;
26558                 if (GC_on_thread_event)
26559                   GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,
26560                                      (void *)p->id);
26561               }
26562 #else
26563               result = RAISE_SIGNAL(p, GC_sig_suspend);
26564               switch(result) {
26565                 case ESRCH:
26566                     n_live_threads--;
26567                     break;
26568                 case 0:
26569                     if (GC_on_thread_event)
26570                       GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,
26571                                          (void *)(word)THREAD_SYSTEM_ID(p));
26572                     break;
26573                 default:
26574                     ABORT_ARG1("pthread_kill failed at suspend",
26575                                ": errcode= %d", result);
26576               }
26577 #endif
26578         }
26579       }
26580     }
26581 #else
26582 #ifndef NACL_PARK_WAIT_NANOSECONDS
26583 #define NACL_PARK_WAIT_NANOSECONDS (100 * 1000)
26584 #endif
26585 #define NANOS_PER_SECOND (1000UL * 1000 * 1000)
26586     unsigned long num_sleeps = 0;
26587 #ifdef DEBUG_THREADS
26588       GC_log_printf("pthread_stop_world: number of threads: %d\n",
26589                     GC_nacl_num_gc_threads - 1);
26590 #endif
26591     GC_nacl_thread_parker = pthread_self();
26592     GC_nacl_park_threads_now = 1;
26593     if (GC_manual_vdb)
26594       GC_acquire_dirty_lock();
26595     while (1) {
26596       int num_threads_parked = 0;
26597       struct timespec ts;
26598       int num_used = 0;
26599       for (i = 0; i < MAX_NACL_GC_THREADS
26600                   && num_used < GC_nacl_num_gc_threads; i++) {
26601         if (GC_nacl_thread_used[i] == 1) {
26602           num_used++;
26603           if (GC_nacl_thread_parked[i] == 1) {
26604             num_threads_parked++;
26605             if (GC_on_thread_event)
26606               GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, (void *)(word)i);
26607           }
26608         }
26609       }
26610       if (num_threads_parked >= GC_nacl_num_gc_threads - 1)
26611         break;
26612       ts.tv_sec = 0;
26613       ts.tv_nsec = NACL_PARK_WAIT_NANOSECONDS;
26614 #ifdef DEBUG_THREADS
26615         GC_log_printf("Sleep waiting for %d threads to park...\n",
26616                       GC_nacl_num_gc_threads - num_threads_parked - 1);
26617 #endif
26618       nanosleep(&ts, 0);
26619       if (++num_sleeps > NANOS_PER_SECOND / NACL_PARK_WAIT_NANOSECONDS) {
26620         WARN("GC appears stalled waiting for %" WARN_PRIdPTR
26621              " threads to park...\n",
26622              GC_nacl_num_gc_threads - num_threads_parked - 1);
26623         num_sleeps = 0;
26624       }
26625     }
26626     if (GC_manual_vdb)
26627       GC_release_dirty_lock();
26628 #endif
26629   return n_live_threads;
26630 }
26631 GC_INNER void GC_stop_world(void)
26632 {
26633 #if !defined(GC_OPENBSD_UTHREADS) && !defined(NACL)
26634     int n_live_threads;
26635 #endif
26636   GC_ASSERT(I_HOLD_LOCK());
26637 #ifdef DEBUG_THREADS
26638     GC_stopping_thread = pthread_self();
26639     GC_stopping_pid = getpid();
26640     GC_log_printf("Stopping the world from %p\n", (void *)GC_stopping_thread);
26641 #endif
26642 #ifdef PARALLEL_MARK
26643     if (GC_parallel) {
26644       GC_acquire_mark_lock();
26645       GC_ASSERT(GC_fl_builder_count == 0);
26646     }
26647 #endif
26648 #if defined(GC_OPENBSD_UTHREADS) || defined(NACL)
26649     (void)GC_suspend_all();
26650 #else
26651     AO_store(&GC_stop_count,
26652              (AO_t)((word)GC_stop_count + (THREAD_RESTARTED+1)));
26653     if (GC_manual_vdb) {
26654       GC_acquire_dirty_lock();
26655     }
26656     AO_store_release(&GC_world_is_stopped, TRUE);
26657     n_live_threads = GC_suspend_all();
26658     if (GC_retry_signals) {
26659       resend_lost_signals_retry(n_live_threads, GC_suspend_all);
26660     } else {
26661       suspend_restart_barrier(n_live_threads);
26662     }
26663     if (GC_manual_vdb)
26664       GC_release_dirty_lock();
26665 #endif
26666 #ifdef PARALLEL_MARK
26667     if (GC_parallel)
26668       GC_release_mark_lock();
26669 #endif
26670 #ifdef DEBUG_THREADS
26671     GC_log_printf("World stopped from %p\n", (void *)pthread_self());
26672     GC_stopping_thread = 0;
26673 #endif
26674 }
26675 #ifdef NACL
26676 #if defined(__x86_64__)
26677 #define NACL_STORE_REGS() \
26678         do { \
26679           __asm__ __volatile__ ("push %rbx"); \
26680           __asm__ __volatile__ ("push %rbp"); \
26681           __asm__ __volatile__ ("push %r12"); \
26682           __asm__ __volatile__ ("push %r13"); \
26683           __asm__ __volatile__ ("push %r14"); \
26684           __asm__ __volatile__ ("push %r15"); \
26685           __asm__ __volatile__ ("mov %%esp, %0" \
26686                     : "=m" (GC_nacl_gc_thread_self->stop_info.stack_ptr)); \
26687           BCOPY(GC_nacl_gc_thread_self->stop_info.stack_ptr, \
26688                 GC_nacl_gc_thread_self->stop_info.reg_storage, \
26689                 NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t)); \
26690           __asm__ __volatile__ ("naclasp $48, %r15"); \
26691         } while (0)
26692 #elif defined(__i386__)
26693 #define NACL_STORE_REGS() \
26694         do { \
26695           __asm__ __volatile__ ("push %ebx"); \
26696           __asm__ __volatile__ ("push %ebp"); \
26697           __asm__ __volatile__ ("push %esi"); \
26698           __asm__ __volatile__ ("push %edi"); \
26699           __asm__ __volatile__ ("mov %%esp, %0" \
26700                     : "=m" (GC_nacl_gc_thread_self->stop_info.stack_ptr)); \
26701           BCOPY(GC_nacl_gc_thread_self->stop_info.stack_ptr, \
26702                 GC_nacl_gc_thread_self->stop_info.reg_storage, \
26703                 NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t));\
26704           __asm__ __volatile__ ("add $16, %esp"); \
26705         } while (0)
26706 #elif defined(__arm__)
26707 #define NACL_STORE_REGS() \
26708         do { \
26709           __asm__ __volatile__ ("push {r4-r8,r10-r12,lr}"); \
26710           __asm__ __volatile__ ("mov r0, %0" \
26711                 : : "r" (&GC_nacl_gc_thread_self->stop_info.stack_ptr)); \
26712           __asm__ __volatile__ ("bic r0, r0, #0xc0000000"); \
26713           __asm__ __volatile__ ("str sp, [r0]"); \
26714           BCOPY(GC_nacl_gc_thread_self->stop_info.stack_ptr, \
26715                 GC_nacl_gc_thread_self->stop_info.reg_storage, \
26716                 NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t)); \
26717           __asm__ __volatile__ ("add sp, sp, #40"); \
26718           __asm__ __volatile__ ("bic sp, sp, #0xc0000000"); \
26719         } while (0)
26720 #else
26721 #error TODO Please port NACL_STORE_REGS
26722 #endif
26723   GC_API_OSCALL void nacl_pre_syscall_hook(void)
26724   {
26725     if (GC_nacl_thread_idx != -1) {
26726       NACL_STORE_REGS();
26727       GC_nacl_gc_thread_self->stop_info.stack_ptr = GC_approx_sp();
26728       GC_nacl_thread_parked[GC_nacl_thread_idx] = 1;
26729     }
26730   }
26731   GC_API_OSCALL void __nacl_suspend_thread_if_needed(void)
26732   {
26733     if (GC_nacl_park_threads_now) {
26734       pthread_t self = pthread_self();
26735       if (GC_nacl_thread_parker == self)
26736         return;
26737       if (GC_nacl_thread_idx < 0)
26738         return;
26739       if (!GC_nacl_thread_parked[GC_nacl_thread_idx]) {
26740         NACL_STORE_REGS();
26741         GC_nacl_gc_thread_self->stop_info.stack_ptr = GC_approx_sp();
26742       }
26743       GC_nacl_thread_parked[GC_nacl_thread_idx] = 1;
26744       while (GC_nacl_park_threads_now) {
26745       }
26746       GC_nacl_thread_parked[GC_nacl_thread_idx] = 0;
26747       BZERO(GC_nacl_gc_thread_self->stop_info.reg_storage,
26748             NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t));
26749     }
26750   }
26751   GC_API_OSCALL void nacl_post_syscall_hook(void)
26752   {
26753     __nacl_suspend_thread_if_needed();
26754     if (GC_nacl_thread_idx != -1) {
26755       GC_nacl_thread_parked[GC_nacl_thread_idx] = 0;
26756     }
26757   }
26758   STATIC GC_bool GC_nacl_thread_parking_inited = FALSE;
26759   STATIC pthread_mutex_t GC_nacl_thread_alloc_lock = PTHREAD_MUTEX_INITIALIZER;
26760   struct nacl_irt_blockhook {
26761     int (*register_block_hooks)(void (*pre)(void), void (*post)(void));
26762   };
26763   EXTERN_C_BEGIN
26764   extern size_t nacl_interface_query(const char *interface_ident,
26765                                      void *table, size_t tablesize);
26766   EXTERN_C_END
26767   GC_INNER void GC_nacl_initialize_gc_thread(void)
26768   {
26769     int i;
26770     static struct nacl_irt_blockhook gc_hook;
26771     pthread_mutex_lock(&GC_nacl_thread_alloc_lock);
26772     if (!EXPECT(GC_nacl_thread_parking_inited, TRUE)) {
26773       BZERO(GC_nacl_thread_parked, sizeof(GC_nacl_thread_parked));
26774       BZERO(GC_nacl_thread_used, sizeof(GC_nacl_thread_used));
26775       nacl_interface_query("nacl-irt-blockhook-0.1",
26776                            &gc_hook, sizeof(gc_hook));
26777       gc_hook.register_block_hooks(nacl_pre_syscall_hook,
26778                                    nacl_post_syscall_hook);
26779       GC_nacl_thread_parking_inited = TRUE;
26780     }
26781     GC_ASSERT(GC_nacl_num_gc_threads <= MAX_NACL_GC_THREADS);
26782     for (i = 0; i < MAX_NACL_GC_THREADS; i++) {
26783       if (GC_nacl_thread_used[i] == 0) {
26784         GC_nacl_thread_used[i] = 1;
26785         GC_nacl_thread_idx = i;
26786         GC_nacl_num_gc_threads++;
26787         break;
26788       }
26789     }
26790     pthread_mutex_unlock(&GC_nacl_thread_alloc_lock);
26791   }
26792   GC_INNER void GC_nacl_shutdown_gc_thread(void)
26793   {
26794     pthread_mutex_lock(&GC_nacl_thread_alloc_lock);
26795     GC_ASSERT(GC_nacl_thread_idx >= 0);
26796     GC_ASSERT(GC_nacl_thread_idx < MAX_NACL_GC_THREADS);
26797     GC_ASSERT(GC_nacl_thread_used[GC_nacl_thread_idx] != 0);
26798     GC_nacl_thread_used[GC_nacl_thread_idx] = 0;
26799     GC_nacl_thread_idx = -1;
26800     GC_nacl_num_gc_threads--;
26801     pthread_mutex_unlock(&GC_nacl_thread_alloc_lock);
26802   }
26803 #else
26804   STATIC int GC_restart_all(void)
26805   {
26806     int n_live_threads = 0;
26807     int i;
26808     pthread_t self = pthread_self();
26809     GC_thread p;
26810 #ifndef GC_OPENBSD_UTHREADS
26811       int result;
26812 #endif
26813     for (i = 0; i < THREAD_TABLE_SZ; i++) {
26814       for (p = GC_threads[i]; p != NULL; p = p -> next) {
26815         if (!THREAD_EQUAL(p -> id, self)) {
26816           if ((p -> flags & FINISHED) != 0) continue;
26817           if (p -> thread_blocked) continue;
26818 #ifndef GC_OPENBSD_UTHREADS
26819 #ifdef GC_ENABLE_SUSPEND_THREAD
26820               if (p -> suspended_ext) continue;
26821 #endif
26822             if (GC_retry_signals
26823                 && AO_load(&p->stop_info.last_stop_count)
26824                     == (AO_t)((word)GC_stop_count | THREAD_RESTARTED))
26825               continue;
26826             n_live_threads++;
26827 #endif
26828 #ifdef DEBUG_THREADS
26829             GC_log_printf("Sending restart signal to %p\n", (void *)p->id);
26830 #endif
26831 #ifdef GC_OPENBSD_UTHREADS
26832             if (pthread_resume_np(p -> id) != 0)
26833               ABORT("pthread_resume_np failed");
26834             if (GC_on_thread_event)
26835               GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, (void *)p->id);
26836 #else
26837             result = RAISE_SIGNAL(p, GC_sig_thr_restart);
26838             switch(result) {
26839             case ESRCH:
26840               n_live_threads--;
26841               break;
26842             case 0:
26843               if (GC_on_thread_event)
26844                 GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED,
26845                                    (void *)(word)THREAD_SYSTEM_ID(p));
26846               break;
26847             default:
26848               ABORT_ARG1("pthread_kill failed at resume",
26849                          ": errcode= %d", result);
26850             }
26851 #endif
26852         }
26853       }
26854     }
26855     return n_live_threads;
26856   }
26857 #endif
26858 GC_INNER void GC_start_world(void)
26859 {
26860 #ifndef NACL
26861     int n_live_threads;
26862     GC_ASSERT(I_HOLD_LOCK());
26863 #ifdef DEBUG_THREADS
26864       GC_log_printf("World starting\n");
26865 #endif
26866 #ifndef GC_OPENBSD_UTHREADS
26867       AO_store_release(&GC_world_is_stopped, FALSE);
26868 #endif
26869     n_live_threads = GC_restart_all();
26870 #ifdef GC_OPENBSD_UTHREADS
26871       (void)n_live_threads;
26872 #else
26873       if (GC_retry_signals) {
26874         resend_lost_signals_retry(n_live_threads, GC_restart_all);
26875       }
26876 #ifdef GC_NETBSD_THREADS_WORKAROUND
26877         else {
26878           suspend_restart_barrier(n_live_threads);
26879         }
26880 #endif
26881 #endif
26882 #ifdef DEBUG_THREADS
26883       GC_log_printf("World started\n");
26884 #endif
26885 #else
26886 #ifdef DEBUG_THREADS
26887       GC_log_printf("World starting...\n");
26888 #endif
26889     GC_nacl_park_threads_now = 0;
26890     if (GC_on_thread_event)
26891       GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, NULL);
26892 #endif
26893 }
26894 GC_INNER void GC_stop_init(void)
26895 {
26896 #if !defined(GC_OPENBSD_UTHREADS) && !defined(NACL)
26897     struct sigaction act;
26898     char *str;
26899     if (SIGNAL_UNSET == GC_sig_suspend)
26900         GC_sig_suspend = SIG_SUSPEND;
26901     if (SIGNAL_UNSET == GC_sig_thr_restart)
26902         GC_sig_thr_restart = SIG_THR_RESTART;
26903     if (GC_sig_suspend == GC_sig_thr_restart)
26904         ABORT("Cannot use same signal for thread suspend and resume");
26905     if (sem_init(&GC_suspend_ack_sem, GC_SEM_INIT_PSHARED, 0) != 0)
26906         ABORT("sem_init failed");
26907 #ifdef SA_RESTART
26908       act.sa_flags = SA_RESTART
26909 #else
26910       act.sa_flags = 0
26911 #endif
26912 #ifndef NO_SA_SIGACTION
26913                      | SA_SIGINFO
26914 #endif
26915         ;
26916     if (sigfillset(&act.sa_mask) != 0) {
26917         ABORT("sigfillset failed");
26918     }
26919 #ifdef GC_RTEMS_PTHREADS
26920       if(sigprocmask(SIG_UNBLOCK, &act.sa_mask, NULL) != 0) {
26921         ABORT("sigprocmask failed");
26922       }
26923 #endif
26924     GC_remove_allowed_signals(&act.sa_mask);
26925 #ifndef NO_SA_SIGACTION
26926       act.sa_sigaction = GC_suspend_handler;
26927 #else
26928       act.sa_handler = GC_suspend_handler;
26929 #endif
26930     if (sigaction(GC_sig_suspend, &act, NULL) != 0) {
26931         ABORT("Cannot set SIG_SUSPEND handler");
26932     }
26933 #ifndef NO_SA_SIGACTION
26934       act.sa_flags &= ~SA_SIGINFO;
26935 #endif
26936     act.sa_handler = GC_restart_handler;
26937     if (sigaction(GC_sig_thr_restart, &act, NULL) != 0) {
26938         ABORT("Cannot set SIG_THR_RESTART handler");
26939     }
26940     if (sigfillset(&suspend_handler_mask) != 0) ABORT("sigfillset failed");
26941     GC_remove_allowed_signals(&suspend_handler_mask);
26942     if (sigdelset(&suspend_handler_mask, GC_sig_thr_restart) != 0)
26943         ABORT("sigdelset failed");
26944     str = GETENV("GC_RETRY_SIGNALS");
26945     if (str != NULL) {
26946         if (*str == '0' && *(str + 1) == '\0') {
26947             GC_retry_signals = FALSE;
26948         } else {
26949             GC_retry_signals = TRUE;
26950         }
26951     }
26952     if (GC_retry_signals) {
26953       GC_COND_LOG_PRINTF(
26954                 "Will retry suspend and restart signals if necessary\n");
26955     }
26956 #ifndef NO_SIGNALS_UNBLOCK_IN_MAIN
26957       GC_unblock_gc_signals();
26958 #endif
26959 #endif
26960 }
26961 #endif
26962 #endif
26963 #if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)
26964 #include <stdlib.h>
26965 #include <pthread.h>
26966 #include <sched.h>
26967 #include <time.h>
26968 #include <errno.h>
26969 #include <unistd.h>
26970 #if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
26971 #if !defined(GC_RTEMS_PTHREADS)
26972 #include <sys/mman.h>
26973 #endif
26974 #include <sys/time.h>
26975 #include <sys/types.h>
26976 #include <sys/stat.h>
26977 #include <fcntl.h>
26978 #endif
26979 #include <signal.h>
26980 #if defined(GC_DARWIN_THREADS)
26981 #ifndef GC_DARWIN_SEMAPHORE_H
26982 #define GC_DARWIN_SEMAPHORE_H
26983 #if !defined(GC_DARWIN_THREADS)
26984 #error darwin_semaphore.h included with GC_DARWIN_THREADS not defined
26985 #endif
26986 #ifdef __cplusplus
26987   extern "C" {
26988 #endif
26989 typedef struct {
26990     pthread_mutex_t mutex;
26991     pthread_cond_t cond;
26992     int value;
26993 } sem_t;
26994 GC_INLINE int sem_init(sem_t *sem, int pshared, int value) {
26995     if (pshared != 0) {
26996         errno = EPERM;
26997         return -1;
26998     }
26999     sem->value = value;
27000     if (pthread_mutex_init(&sem->mutex, NULL) != 0)
27001       return -1;
27002     if (pthread_cond_init(&sem->cond, NULL) != 0) {
27003       (void)pthread_mutex_destroy(&sem->mutex);
27004       return -1;
27005     }
27006     return 0;
27007 }
27008 GC_INLINE int sem_post(sem_t *sem) {
27009     if (pthread_mutex_lock(&sem->mutex) != 0)
27010       return -1;
27011     sem->value++;
27012     if (pthread_cond_signal(&sem->cond) != 0) {
27013       (void)pthread_mutex_unlock(&sem->mutex);
27014       return -1;
27015     }
27016     return pthread_mutex_unlock(&sem->mutex) != 0 ? -1 : 0;
27017 }
27018 GC_INLINE int sem_wait(sem_t *sem) {
27019     if (pthread_mutex_lock(&sem->mutex) != 0)
27020       return -1;
27021     while (sem->value == 0) {
27022         if (pthread_cond_wait(&sem->cond, &sem->mutex) != 0) {
27023             (void)pthread_mutex_unlock(&sem->mutex);
27024             return -1;
27025         }
27026     }
27027     sem->value--;
27028     return pthread_mutex_unlock(&sem->mutex) != 0 ? -1 : 0;
27029 }
27030 GC_INLINE int sem_destroy(sem_t *sem) {
27031     return pthread_cond_destroy(&sem->cond) != 0
27032            || pthread_mutex_destroy(&sem->mutex) != 0 ? -1 : 0;
27033 }
27034 #ifdef __cplusplus
27035   }
27036 #endif
27037 #endif
27038 #else
27039 #include <semaphore.h>
27040 #endif
27041 #if defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS)
27042 #include <sys/sysctl.h>
27043 #endif
27044 #if defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS)
27045 #include <sys/param.h>
27046 #include <sys/sysctl.h>
27047 #endif
27048 #if !defined(USE_SPIN_LOCK)
27049   GC_INNER pthread_mutex_t GC_allocate_ml = PTHREAD_MUTEX_INITIALIZER;
27050 #endif
27051 #ifdef GC_ASSERTIONS
27052   GC_INNER unsigned long GC_lock_holder = NO_THREAD;
27053 #endif
27054 #if defined(GC_DGUX386_THREADS)
27055 #include <sys/dg_sys_info.h>
27056 #include <sys/_int_psem.h>
27057   typedef unsigned int sem_t;
27058 #endif
27059 #undef pthread_create
27060 #ifndef GC_NO_PTHREAD_SIGMASK
27061 #undef pthread_sigmask
27062 #endif
27063 #ifndef GC_NO_PTHREAD_CANCEL
27064 #undef pthread_cancel
27065 #endif
27066 #ifdef GC_HAVE_PTHREAD_EXIT
27067 #undef pthread_exit
27068 #endif
27069 #undef pthread_join
27070 #undef pthread_detach
27071 #if defined(GC_OSF1_THREADS) && defined(_PTHREAD_USE_MANGLED_NAMES_) \
27072      && !defined(_PTHREAD_USE_PTDNAM_)
27073 #define pthread_create __pthread_create
27074 #define pthread_join __pthread_join
27075 #define pthread_detach __pthread_detach
27076 #ifndef GC_NO_PTHREAD_CANCEL
27077 #define pthread_cancel __pthread_cancel
27078 #endif
27079 #ifdef GC_HAVE_PTHREAD_EXIT
27080 #define pthread_exit __pthread_exit
27081 #endif
27082 #endif
27083 #ifdef GC_USE_LD_WRAP
27084 #define WRAP_FUNC(f) __wrap_##f
27085 #define REAL_FUNC(f) __real_##f
27086     int REAL_FUNC(pthread_create)(pthread_t *,
27087                                   GC_PTHREAD_CREATE_CONST pthread_attr_t *,
27088                                   void *(*start_routine)(void *), void *);
27089     int REAL_FUNC(pthread_join)(pthread_t, void **);
27090     int REAL_FUNC(pthread_detach)(pthread_t);
27091 #ifndef GC_NO_PTHREAD_SIGMASK
27092       int REAL_FUNC(pthread_sigmask)(int, const sigset_t *, sigset_t *);
27093 #endif
27094 #ifndef GC_NO_PTHREAD_CANCEL
27095       int REAL_FUNC(pthread_cancel)(pthread_t);
27096 #endif
27097 #ifdef GC_HAVE_PTHREAD_EXIT
27098       void REAL_FUNC(pthread_exit)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
27099 #endif
27100 #else
27101 #ifdef GC_USE_DLOPEN_WRAP
27102 #include <dlfcn.h>
27103 #define WRAP_FUNC(f) f
27104 #define REAL_FUNC(f) GC_real_##f
27105       typedef int (* GC_pthread_create_t)(pthread_t *,
27106                                     GC_PTHREAD_CREATE_CONST pthread_attr_t *,
27107                                     void * (*)(void *), void *);
27108       static GC_pthread_create_t REAL_FUNC(pthread_create);
27109 #ifndef GC_NO_PTHREAD_SIGMASK
27110         typedef int (* GC_pthread_sigmask_t)(int, const sigset_t *,
27111                                              sigset_t *);
27112         static GC_pthread_sigmask_t REAL_FUNC(pthread_sigmask);
27113 #endif
27114       typedef int (* GC_pthread_join_t)(pthread_t, void **);
27115       static GC_pthread_join_t REAL_FUNC(pthread_join);
27116       typedef int (* GC_pthread_detach_t)(pthread_t);
27117       static GC_pthread_detach_t REAL_FUNC(pthread_detach);
27118 #ifndef GC_NO_PTHREAD_CANCEL
27119         typedef int (* GC_pthread_cancel_t)(pthread_t);
27120         static GC_pthread_cancel_t REAL_FUNC(pthread_cancel);
27121 #endif
27122 #ifdef GC_HAVE_PTHREAD_EXIT
27123         typedef void (* GC_pthread_exit_t)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
27124         static GC_pthread_exit_t REAL_FUNC(pthread_exit);
27125 #endif
27126 #else
27127 #define WRAP_FUNC(f) GC_##f
27128 #if !defined(GC_DGUX386_THREADS)
27129 #define REAL_FUNC(f) f
27130 #else
27131 #define REAL_FUNC(f) __d10_##f
27132 #endif
27133 #endif
27134 #endif
27135 #if defined(GC_USE_LD_WRAP) || defined(GC_USE_DLOPEN_WRAP)
27136   GC_API int GC_pthread_create(pthread_t * t,
27137                                GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
27138                                void * (* fn)(void *), void * arg)
27139   {
27140     return pthread_create(t, a, fn, arg);
27141   }
27142 #ifndef GC_NO_PTHREAD_SIGMASK
27143     GC_API int GC_pthread_sigmask(int how, const sigset_t *mask,
27144                                   sigset_t *old)
27145     {
27146       return pthread_sigmask(how, mask, old);
27147     }
27148 #endif
27149   GC_API int GC_pthread_join(pthread_t t, void **res)
27150   {
27151     return pthread_join(t, res);
27152   }
27153   GC_API int GC_pthread_detach(pthread_t t)
27154   {
27155     return pthread_detach(t);
27156   }
27157 #ifndef GC_NO_PTHREAD_CANCEL
27158     GC_API int GC_pthread_cancel(pthread_t t)
27159     {
27160       return pthread_cancel(t);
27161     }
27162 #endif
27163 #ifdef GC_HAVE_PTHREAD_EXIT
27164     GC_API GC_PTHREAD_EXIT_ATTRIBUTE void GC_pthread_exit(void *retval)
27165     {
27166       pthread_exit(retval);
27167     }
27168 #endif
27169 #endif
27170 #ifdef GC_USE_DLOPEN_WRAP
27171   STATIC GC_bool GC_syms_initialized = FALSE;
27172   STATIC void GC_init_real_syms(void)
27173   {
27174     void *dl_handle;
27175     if (GC_syms_initialized) return;
27176 #ifdef RTLD_NEXT
27177       dl_handle = RTLD_NEXT;
27178 #else
27179       dl_handle = dlopen("libpthread.so.0", RTLD_LAZY);
27180       if (NULL == dl_handle) {
27181         dl_handle = dlopen("libpthread.so", RTLD_LAZY);
27182       }
27183       if (NULL == dl_handle) ABORT("Couldn't open libpthread");
27184 #endif
27185     REAL_FUNC(pthread_create) = (GC_pthread_create_t)(word)
27186                                 dlsym(dl_handle, "pthread_create");
27187 #ifdef RTLD_NEXT
27188       if (REAL_FUNC(pthread_create) == 0)
27189         ABORT("pthread_create not found"
27190               " (probably -lgc is specified after -lpthread)");
27191 #endif
27192 #ifndef GC_NO_PTHREAD_SIGMASK
27193       REAL_FUNC(pthread_sigmask) = (GC_pthread_sigmask_t)(word)
27194                                 dlsym(dl_handle, "pthread_sigmask");
27195 #endif
27196     REAL_FUNC(pthread_join) = (GC_pthread_join_t)(word)
27197                                 dlsym(dl_handle, "pthread_join");
27198     REAL_FUNC(pthread_detach) = (GC_pthread_detach_t)(word)
27199                                   dlsym(dl_handle, "pthread_detach");
27200 #ifndef GC_NO_PTHREAD_CANCEL
27201       REAL_FUNC(pthread_cancel) = (GC_pthread_cancel_t)(word)
27202                                     dlsym(dl_handle, "pthread_cancel");
27203 #endif
27204 #ifdef GC_HAVE_PTHREAD_EXIT
27205       REAL_FUNC(pthread_exit) = (GC_pthread_exit_t)(word)
27206                                   dlsym(dl_handle, "pthread_exit");
27207 #endif
27208     GC_syms_initialized = TRUE;
27209   }
27210 #define INIT_REAL_SYMS() if (EXPECT(GC_syms_initialized, TRUE)) {} \
27211                             else GC_init_real_syms()
27212 #define ASSERT_SYMS_INITIALIZED() GC_ASSERT(GC_syms_initialized)
27213 #else
27214 #define INIT_REAL_SYMS() (void)0
27215 #define ASSERT_SYMS_INITIALIZED() GC_ASSERT(parallel_initialized)
27216 #endif
27217 static GC_bool parallel_initialized = FALSE;
27218 #ifndef GC_ALWAYS_MULTITHREADED
27219   GC_INNER GC_bool GC_need_to_lock = FALSE;
27220 #endif
27221 STATIC int GC_nprocs = 1;
27222 #ifdef THREAD_LOCAL_ALLOC
27223   GC_INNER void GC_mark_thread_local_free_lists(void)
27224   {
27225     int i;
27226     GC_thread p;
27227     for (i = 0; i < THREAD_TABLE_SZ; ++i) {
27228       for (p = GC_threads[i]; 0 != p; p = p -> next) {
27229         if (!(p -> flags & FINISHED))
27230           GC_mark_thread_local_fls_for(&(p->tlfs));
27231       }
27232     }
27233   }
27234 #if defined(GC_ASSERTIONS)
27235     void GC_check_tls(void)
27236     {
27237         int i;
27238         GC_thread p;
27239         for (i = 0; i < THREAD_TABLE_SZ; ++i) {
27240           for (p = GC_threads[i]; 0 != p; p = p -> next) {
27241             if (!(p -> flags & FINISHED))
27242               GC_check_tls_for(&(p->tlfs));
27243           }
27244         }
27245 #if defined(USE_CUSTOM_SPECIFIC)
27246           if (GC_thread_key != 0)
27247             GC_check_tsd_marks(GC_thread_key);
27248 #endif
27249     }
27250 #endif
27251 #endif
27252 #ifndef MAX_MARKERS
27253 #define MAX_MARKERS 16
27254 #endif
27255 #ifdef PARALLEL_MARK
27256 static ptr_t marker_sp[MAX_MARKERS - 1] = {0};
27257 #ifdef IA64
27258   static ptr_t marker_bsp[MAX_MARKERS - 1] = {0};
27259 #endif
27260 #if defined(GC_DARWIN_THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
27261   static mach_port_t marker_mach_threads[MAX_MARKERS - 1] = {0};
27262   GC_INNER GC_bool GC_is_mach_marker(thread_act_t thread)
27263   {
27264     int i;
27265     for (i = 0; i < GC_markers_m1; i++) {
27266       if (marker_mach_threads[i] == thread)
27267         return TRUE;
27268     }
27269     return FALSE;
27270   }
27271 #endif
27272 #ifdef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG
27273   static void set_marker_thread_name(unsigned id)
27274   {
27275     int err = pthread_setname_np(pthread_self(), "GC-marker-%zu",
27276                                  (void*)(size_t)id);
27277     if (err != 0)
27278       WARN("pthread_setname_np failed, errno= %" WARN_PRIdPTR "\n", err);
27279   }
27280 #elif defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) \
27281       || defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID)
27282   static void set_marker_thread_name(unsigned id)
27283   {
27284     char name_buf[16];
27285     int len = sizeof("GC-marker-") - 1;
27286     BCOPY("GC-marker-", name_buf, len);
27287     if (id >= 10)
27288       name_buf[len++] = (char)('0' + (id / 10) % 10);
27289     name_buf[len] = (char)('0' + id % 10);
27290     name_buf[len + 1] = '\0';
27291 #ifdef HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID
27292       (void)pthread_setname_np(name_buf);
27293 #else
27294       if (pthread_setname_np(pthread_self(), name_buf) != 0)
27295         WARN("pthread_setname_np failed\n", 0);
27296 #endif
27297   }
27298 #else
27299 #define set_marker_thread_name(id) (void)(id)
27300 #endif
27301 STATIC void * GC_mark_thread(void * id)
27302 {
27303   word my_mark_no = 0;
27304   IF_CANCEL(int cancel_state;)
27305   if ((word)id == GC_WORD_MAX) return 0;
27306   DISABLE_CANCEL(cancel_state);
27307   set_marker_thread_name((unsigned)(word)id);
27308   marker_sp[(word)id] = GC_approx_sp();
27309 #ifdef IA64
27310     marker_bsp[(word)id] = GC_save_regs_in_stack();
27311 #endif
27312 #if defined(GC_DARWIN_THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
27313     marker_mach_threads[(word)id] = mach_thread_self();
27314 #endif
27315   GC_acquire_mark_lock();
27316   if (0 == --GC_fl_builder_count)
27317     GC_notify_all_builder();
27318   for (;; ++my_mark_no) {
27319     if (my_mark_no < GC_mark_no || my_mark_no > GC_mark_no + 2) {
27320         my_mark_no = GC_mark_no;
27321     }
27322 #ifdef DEBUG_THREADS
27323       GC_log_printf("Starting mark helper for mark number %lu\n",
27324                     (unsigned long)my_mark_no);
27325 #endif
27326     GC_help_marker(my_mark_no);
27327   }
27328 }
27329 STATIC pthread_t GC_mark_threads[MAX_MARKERS];
27330 #ifdef CAN_HANDLE_FORK
27331   static int available_markers_m1 = 0;
27332   static pthread_cond_t mark_cv;
27333 #else
27334 #define available_markers_m1 GC_markers_m1
27335   static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;
27336 #endif
27337 GC_INNER void GC_start_mark_threads_inner(void)
27338 {
27339     int i;
27340     pthread_attr_t attr;
27341 #ifndef NO_MARKER_SPECIAL_SIGMASK
27342       sigset_t set, oldset;
27343 #endif
27344     GC_ASSERT(I_DONT_HOLD_LOCK());
27345     if (available_markers_m1 <= 0) return;
27346 #ifdef CAN_HANDLE_FORK
27347       if (GC_parallel) return;
27348       {
27349         pthread_cond_t mark_cv_local = PTHREAD_COND_INITIALIZER;
27350         BCOPY(&mark_cv_local, &mark_cv, sizeof(mark_cv));
27351       }
27352 #endif
27353     GC_ASSERT(GC_fl_builder_count == 0);
27354     INIT_REAL_SYMS();
27355     if (0 != pthread_attr_init(&attr)) ABORT("pthread_attr_init failed");
27356     if (0 != pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
27357         ABORT("pthread_attr_setdetachstate failed");
27358 #ifdef DEFAULT_STACK_MAYBE_SMALL
27359       {
27360         size_t old_size;
27361         if (pthread_attr_getstacksize(&attr, &old_size) != 0)
27362           ABORT("pthread_attr_getstacksize failed");
27363         if (old_size < MIN_STACK_SIZE
27364             && old_size != 0 ) {
27365           if (pthread_attr_setstacksize(&attr, MIN_STACK_SIZE) != 0)
27366             ABORT("pthread_attr_setstacksize failed");
27367         }
27368       }
27369 #endif
27370 #ifndef NO_MARKER_SPECIAL_SIGMASK
27371       if (sigfillset(&set) != 0)
27372         ABORT("sigfillset failed");
27373 #if !defined(GC_DARWIN_THREADS) && !defined(GC_OPENBSD_UTHREADS) \
27374          && !defined(NACL)
27375         if (sigdelset(&set, GC_get_suspend_signal()) != 0
27376             || sigdelset(&set, GC_get_thr_restart_signal()) != 0)
27377           ABORT("sigdelset failed");
27378 #endif
27379       if (REAL_FUNC(pthread_sigmask)(SIG_BLOCK, &set, &oldset) < 0) {
27380         WARN("pthread_sigmask set failed, no markers started,"
27381              " errno= %" WARN_PRIdPTR "\n", errno);
27382         GC_markers_m1 = 0;
27383         (void)pthread_attr_destroy(&attr);
27384         return;
27385       }
27386 #endif
27387 #ifdef CAN_HANDLE_FORK
27388       GC_markers_m1 = available_markers_m1;
27389 #endif
27390     for (i = 0; i < available_markers_m1; ++i) {
27391       if (0 != REAL_FUNC(pthread_create)(GC_mark_threads + i, &attr,
27392                               GC_mark_thread, (void *)(word)i)) {
27393         WARN("Marker thread creation failed, errno= %" WARN_PRIdPTR "\n",
27394              errno);
27395         GC_markers_m1 = i;
27396         break;
27397       }
27398     }
27399 #ifndef NO_MARKER_SPECIAL_SIGMASK
27400       if (REAL_FUNC(pthread_sigmask)(SIG_SETMASK, &oldset, NULL) < 0) {
27401         WARN("pthread_sigmask restore failed, errno= %" WARN_PRIdPTR "\n",
27402              errno);
27403       }
27404 #endif
27405     (void)pthread_attr_destroy(&attr);
27406     GC_wait_for_markers_init();
27407     GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
27408 }
27409 #endif
27410 GC_INNER GC_bool GC_thr_initialized = FALSE;
27411 GC_INNER volatile GC_thread GC_threads[THREAD_TABLE_SZ] = {0};
27412 void GC_push_thread_structures(void)
27413 {
27414     GC_ASSERT(I_HOLD_LOCK());
27415     GC_PUSH_ALL_SYM(GC_threads);
27416 #if defined(THREAD_LOCAL_ALLOC)
27417       GC_PUSH_ALL_SYM(GC_thread_key);
27418 #endif
27419 }
27420 #ifdef DEBUG_THREADS
27421   STATIC int GC_count_threads(void)
27422   {
27423     int i;
27424     int count = 0;
27425     GC_ASSERT(I_HOLD_LOCK());
27426     for (i = 0; i < THREAD_TABLE_SZ; ++i) {
27427         GC_thread th = GC_threads[i];
27428         while (th) {
27429             if (!(th->flags & FINISHED))
27430                 ++count;
27431             th = th->next;
27432         }
27433     }
27434     return count;
27435   }
27436 #endif
27437 static struct GC_Thread_Rep first_thread;
27438 STATIC GC_thread GC_new_thread(pthread_t id)
27439 {
27440     int hv = THREAD_TABLE_INDEX(id);
27441     GC_thread result;
27442     static GC_bool first_thread_used = FALSE;
27443 #ifdef DEBUG_THREADS
27444         GC_log_printf("Creating thread %p\n", (void *)id);
27445         for (result = GC_threads[hv]; result != NULL; result = result->next)
27446           if (!THREAD_EQUAL(result->id, id)) {
27447             GC_log_printf("Hash collision at GC_threads[%d]\n", hv);
27448             break;
27449           }
27450 #endif
27451     GC_ASSERT(I_HOLD_LOCK());
27452     if (!EXPECT(first_thread_used, TRUE)) {
27453         result = &first_thread;
27454         first_thread_used = TRUE;
27455         GC_ASSERT(NULL == GC_threads[hv]);
27456 #if defined(THREAD_SANITIZER) && defined(CPPCHECK)
27457           GC_noop1(result->dummy[0]);
27458 #endif
27459     } else {
27460         result = (struct GC_Thread_Rep *)
27461                  GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep), NORMAL);
27462         if (result == 0) return(0);
27463     }
27464     result -> id = id;
27465 #ifdef USE_TKILL_ON_ANDROID
27466       result -> kernel_id = gettid();
27467 #endif
27468     result -> next = GC_threads[hv];
27469     GC_threads[hv] = result;
27470 #ifdef NACL
27471       GC_nacl_gc_thread_self = result;
27472       GC_nacl_initialize_gc_thread();
27473 #endif
27474     GC_ASSERT(result -> flags == 0 && result -> thread_blocked == 0);
27475     if (EXPECT(result != &first_thread, TRUE))
27476       GC_dirty(result);
27477     return(result);
27478 }
27479 STATIC void GC_delete_thread(pthread_t id)
27480 {
27481     int hv = THREAD_TABLE_INDEX(id);
27482     GC_thread p = GC_threads[hv];
27483     GC_thread prev = NULL;
27484 #ifdef DEBUG_THREADS
27485       GC_log_printf("Deleting thread %p, n_threads= %d\n",
27486                     (void *)id, GC_count_threads());
27487 #endif
27488 #ifdef NACL
27489       GC_nacl_shutdown_gc_thread();
27490       GC_nacl_gc_thread_self = NULL;
27491 #endif
27492     GC_ASSERT(I_HOLD_LOCK());
27493     while (!THREAD_EQUAL(p -> id, id)) {
27494         prev = p;
27495         p = p -> next;
27496     }
27497     if (prev == 0) {
27498         GC_threads[hv] = p -> next;
27499     } else {
27500         GC_ASSERT(prev != &first_thread);
27501         prev -> next = p -> next;
27502         GC_dirty(prev);
27503     }
27504     if (p != &first_thread) {
27505 #ifdef GC_DARWIN_THREADS
27506         mach_port_deallocate(mach_task_self(), p->stop_info.mach_thread);
27507 #endif
27508       GC_INTERNAL_FREE(p);
27509     }
27510 }
27511 STATIC void GC_delete_gc_thread(GC_thread t)
27512 {
27513     pthread_t id = t -> id;
27514     int hv = THREAD_TABLE_INDEX(id);
27515     GC_thread p = GC_threads[hv];
27516     GC_thread prev = NULL;
27517     GC_ASSERT(I_HOLD_LOCK());
27518     while (p != t) {
27519         prev = p;
27520         p = p -> next;
27521     }
27522     if (prev == 0) {
27523         GC_threads[hv] = p -> next;
27524     } else {
27525         GC_ASSERT(prev != &first_thread);
27526         prev -> next = p -> next;
27527         GC_dirty(prev);
27528     }
27529 #ifdef GC_DARWIN_THREADS
27530         mach_port_deallocate(mach_task_self(), p->stop_info.mach_thread);
27531 #endif
27532     GC_INTERNAL_FREE(p);
27533 #ifdef DEBUG_THREADS
27534       GC_log_printf("Deleted thread %p, n_threads= %d\n",
27535                     (void *)id, GC_count_threads());
27536 #endif
27537 }
27538 GC_INNER GC_thread GC_lookup_thread(pthread_t id)
27539 {
27540     GC_thread p = GC_threads[THREAD_TABLE_INDEX(id)];
27541     while (p != 0 && !THREAD_EQUAL(p -> id, id)) p = p -> next;
27542     return(p);
27543 }
27544 GC_INNER void GC_reset_finalizer_nested(void)
27545 {
27546   GC_thread me = GC_lookup_thread(pthread_self());
27547   me->finalizer_nested = 0;
27548 }
27549 GC_INNER unsigned char *GC_check_finalizer_nested(void)
27550 {
27551   GC_thread me = GC_lookup_thread(pthread_self());
27552   unsigned nesting_level = me->finalizer_nested;
27553   if (nesting_level) {
27554     if (++me->finalizer_skipped < (1U << nesting_level)) return NULL;
27555     me->finalizer_skipped = 0;
27556   }
27557   me->finalizer_nested = (unsigned char)(nesting_level + 1);
27558   return &me->finalizer_nested;
27559 }
27560 #if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
27561   GC_bool GC_is_thread_tsd_valid(void *tsd)
27562   {
27563     GC_thread me;
27564     DCL_LOCK_STATE;
27565     LOCK();
27566     me = GC_lookup_thread(pthread_self());
27567     UNLOCK();
27568     return (word)tsd >= (word)(&me->tlfs)
27569             && (word)tsd < (word)(&me->tlfs) + sizeof(me->tlfs);
27570   }
27571 #endif
27572 GC_API int GC_CALL GC_thread_is_registered(void)
27573 {
27574     pthread_t self = pthread_self();
27575     GC_thread me;
27576     DCL_LOCK_STATE;
27577     LOCK();
27578     me = GC_lookup_thread(self);
27579     UNLOCK();
27580     return me != NULL;
27581 }
27582 static pthread_t main_pthread_id;
27583 static void *main_stack, *main_altstack;
27584 static word main_stack_size, main_altstack_size;
27585 GC_API void GC_CALL GC_register_altstack(void *stack, GC_word stack_size,
27586                                          void *altstack,
27587                                          GC_word altstack_size)
27588 {
27589   GC_thread me;
27590   pthread_t self = pthread_self();
27591   DCL_LOCK_STATE;
27592   LOCK();
27593   me = GC_lookup_thread(self);
27594   if (me != NULL) {
27595     me->stack = (ptr_t)stack;
27596     me->stack_size = stack_size;
27597     me->altstack = (ptr_t)altstack;
27598     me->altstack_size = altstack_size;
27599   } else {
27600     main_pthread_id = self;
27601     main_stack = stack;
27602     main_stack_size = stack_size;
27603     main_altstack = altstack;
27604     main_altstack_size = altstack_size;
27605   }
27606   UNLOCK();
27607 }
27608 #ifdef CAN_HANDLE_FORK
27609 #ifdef CAN_CALL_ATFORK
27610     GC_ATTR_NO_SANITIZE_THREAD
27611 #endif
27612   static void store_to_threads_table(int hv, GC_thread me)
27613   {
27614     GC_threads[hv] = me;
27615   }
27616 STATIC void GC_remove_all_threads_but_me(void)
27617 {
27618     pthread_t self = pthread_self();
27619     int hv;
27620     for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
27621       GC_thread p, next;
27622       GC_thread me = NULL;
27623       for (p = GC_threads[hv]; 0 != p; p = next) {
27624         next = p -> next;
27625         if (THREAD_EQUAL(p -> id, self)
27626             && me == NULL) {
27627           me = p;
27628           p -> next = 0;
27629 #ifdef GC_DARWIN_THREADS
27630             me -> stop_info.mach_thread = mach_thread_self();
27631 #endif
27632 #ifdef USE_TKILL_ON_ANDROID
27633             me -> kernel_id = gettid();
27634 #endif
27635 #if defined(THREAD_LOCAL_ALLOC) && !defined(USE_CUSTOM_SPECIFIC)
27636           {
27637             int res;
27638             res = GC_setspecific(GC_thread_key, &me->tlfs);
27639             if (COVERT_DATAFLOW(res) != 0)
27640               ABORT("GC_setspecific failed (in child)");
27641           }
27642 #endif
27643         } else {
27644 #ifdef THREAD_LOCAL_ALLOC
27645             if (!(p -> flags & FINISHED)) {
27646               GC_remove_specific_after_fork(GC_thread_key, p -> id);
27647             }
27648 #endif
27649 #if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
27650             if (p != &first_thread) GC_INTERNAL_FREE(p);
27651 #endif
27652         }
27653       }
27654       store_to_threads_table(hv, me);
27655     }
27656 }
27657 #endif
27658 #ifdef USE_PROC_FOR_LIBRARIES
27659   GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo, ptr_t hi)
27660   {
27661     int i;
27662     GC_thread p;
27663     GC_ASSERT(I_HOLD_LOCK());
27664 #ifdef PARALLEL_MARK
27665       for (i = 0; i < GC_markers_m1; ++i) {
27666         if ((word)marker_sp[i] > (word)lo && (word)marker_sp[i] < (word)hi)
27667           return TRUE;
27668 #ifdef IA64
27669           if ((word)marker_bsp[i] > (word)lo
27670               && (word)marker_bsp[i] < (word)hi)
27671             return TRUE;
27672 #endif
27673       }
27674 #endif
27675     for (i = 0; i < THREAD_TABLE_SZ; i++) {
27676       for (p = GC_threads[i]; p != 0; p = p -> next) {
27677         if (0 != p -> stack_end) {
27678 #ifdef STACK_GROWS_UP
27679             if ((word)p->stack_end >= (word)lo
27680                 && (word)p->stack_end < (word)hi)
27681               return TRUE;
27682 #else
27683             if ((word)p->stack_end > (word)lo
27684                 && (word)p->stack_end <= (word)hi)
27685               return TRUE;
27686 #endif
27687         }
27688       }
27689     }
27690     return FALSE;
27691   }
27692 #endif
27693 #ifdef IA64
27694   GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound)
27695   {
27696     int i;
27697     GC_thread p;
27698     ptr_t result = 0;
27699     GC_ASSERT(I_HOLD_LOCK());
27700 #ifdef PARALLEL_MARK
27701       for (i = 0; i < GC_markers_m1; ++i) {
27702         if ((word)marker_sp[i] > (word)result
27703             && (word)marker_sp[i] < (word)bound)
27704           result = marker_sp[i];
27705       }
27706 #endif
27707     for (i = 0; i < THREAD_TABLE_SZ; i++) {
27708       for (p = GC_threads[i]; p != 0; p = p -> next) {
27709         if ((word)p->stack_end > (word)result
27710             && (word)p->stack_end < (word)bound) {
27711           result = p -> stack_end;
27712         }
27713       }
27714     }
27715     return result;
27716   }
27717 #endif
27718 #ifndef STAT_READ
27719 #define STAT_READ read
27720 #endif
27721 #ifdef GC_HPUX_THREADS
27722 #define GC_get_nprocs() pthread_num_processors_np()
27723 #elif defined(GC_OSF1_THREADS) || defined(GC_AIX_THREADS) \
27724       || defined(GC_HAIKU_THREADS) || defined(GC_SOLARIS_THREADS) \
27725       || defined(HURD) || defined(HOST_ANDROID) || defined(NACL)
27726   GC_INLINE int GC_get_nprocs(void)
27727   {
27728     int nprocs = (int)sysconf(_SC_NPROCESSORS_ONLN);
27729     return nprocs > 0 ? nprocs : 1;
27730   }
27731 #elif defined(GC_IRIX_THREADS)
27732   GC_INLINE int GC_get_nprocs(void)
27733   {
27734     int nprocs = (int)sysconf(_SC_NPROC_ONLN);
27735     return nprocs > 0 ? nprocs : 1;
27736   }
27737 #elif defined(GC_LINUX_THREADS)
27738   STATIC int GC_get_nprocs(void)
27739   {
27740 #define PROC_STAT_BUF_SZ ((1 + MAX_MARKERS) * 100)
27741     char stat_buf[PROC_STAT_BUF_SZ+1];
27742     int f;
27743     int result, i, len;
27744     f = open("/proc/stat", O_RDONLY);
27745     if (f < 0) {
27746       WARN("Could not open /proc/stat\n", 0);
27747       return 1;
27748     }
27749     len = STAT_READ(f, stat_buf, sizeof(stat_buf)-1);
27750     if (len < 0) {
27751       WARN("Failed to read /proc/stat, errno= %" WARN_PRIdPTR "\n", errno);
27752       close(f);
27753       return 1;
27754     }
27755     stat_buf[len] = '\0';
27756     close(f);
27757     result = 1;
27758     for (i = 0; i < len - 4; ++i) {
27759       if (stat_buf[i] == '\n' && stat_buf[i+1] == 'c'
27760           && stat_buf[i+2] == 'p' && stat_buf[i+3] == 'u') {
27761         int cpu_no = atoi(&stat_buf[i + 4]);
27762         if (cpu_no >= result)
27763           result = cpu_no + 1;
27764       }
27765     }
27766     return result;
27767   }
27768 #elif defined(GC_DGUX386_THREADS)
27769   STATIC int GC_get_nprocs(void)
27770   {
27771     int numCpus;
27772     struct dg_sys_info_pm_info pm_sysinfo;
27773     int status = 0;
27774     status = dg_sys_info((long int *) &pm_sysinfo,
27775         DG_SYS_INFO_PM_INFO_TYPE, DG_SYS_INFO_PM_CURRENT_VERSION);
27776     if (status < 0)
27777        numCpus = -1;
27778     else
27779       numCpus = pm_sysinfo.idle_vp_count;
27780     return(numCpus);
27781   }
27782 #elif defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS) \
27783       || defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS)
27784   STATIC int GC_get_nprocs(void)
27785   {
27786     int mib[] = {CTL_HW,HW_NCPU};
27787     int res;
27788     size_t len = sizeof(res);
27789     sysctl(mib, sizeof(mib)/sizeof(int), &res, &len, NULL, 0);
27790     return res;
27791   }
27792 #else
27793 #define GC_get_nprocs() 1
27794 #endif
27795 #if defined(ARM32) && defined(GC_LINUX_THREADS) && !defined(NACL)
27796   STATIC int GC_get_nprocs_present(void)
27797   {
27798     char stat_buf[16];
27799     int f;
27800     int len;
27801     f = open("/sys/devices/system/cpu/present", O_RDONLY);
27802     if (f < 0)
27803       return -1;
27804     len = STAT_READ(f, stat_buf, sizeof(stat_buf));
27805     close(f);
27806     if (len < 2 || stat_buf[0] != '0' || stat_buf[len - 1] != '\n') {
27807       return 0;
27808     } else if (len == 2) {
27809       return 1;
27810     } else if (stat_buf[1] != '-') {
27811       return 0;
27812     }
27813     stat_buf[len - 1] = '\0';
27814     return atoi(&stat_buf[2]) + 1;
27815   }
27816 #endif
27817 STATIC void GC_wait_for_gc_completion(GC_bool wait_for_all)
27818 {
27819     DCL_LOCK_STATE;
27820 #if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
27821       GC_ASSERT(I_HOLD_LOCK());
27822 #endif
27823     ASSERT_CANCEL_DISABLED();
27824     if (GC_incremental && GC_collection_in_progress()) {
27825         word old_gc_no = GC_gc_no;
27826         while (GC_incremental && GC_collection_in_progress()
27827                && (wait_for_all || old_gc_no == GC_gc_no)) {
27828             ENTER_GC();
27829             GC_in_thread_creation = TRUE;
27830             GC_collect_a_little_inner(1);
27831             GC_in_thread_creation = FALSE;
27832             EXIT_GC();
27833             UNLOCK();
27834             sched_yield();
27835             LOCK();
27836         }
27837     }
27838 }
27839 #ifdef CAN_HANDLE_FORK
27840 IF_CANCEL(static int fork_cancel_state;)
27841 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
27842   GC_ATTR_NO_SANITIZE_THREAD
27843 #endif
27844 static void fork_prepare_proc(void)
27845 {
27846       LOCK();
27847       DISABLE_CANCEL(fork_cancel_state);
27848 #if defined(PARALLEL_MARK)
27849         if (GC_parallel)
27850           GC_wait_for_reclaim();
27851 #endif
27852       GC_wait_for_gc_completion(TRUE);
27853 #if defined(PARALLEL_MARK)
27854         if (GC_parallel)
27855           GC_acquire_mark_lock();
27856 #endif
27857       GC_acquire_dirty_lock();
27858 }
27859 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
27860   GC_ATTR_NO_SANITIZE_THREAD
27861 #endif
27862 static void fork_parent_proc(void)
27863 {
27864     GC_release_dirty_lock();
27865 #if defined(PARALLEL_MARK)
27866       if (GC_parallel)
27867         GC_release_mark_lock();
27868 #endif
27869     RESTORE_CANCEL(fork_cancel_state);
27870     UNLOCK();
27871 }
27872 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
27873   GC_ATTR_NO_SANITIZE_THREAD
27874 #endif
27875 static void fork_child_proc(void)
27876 {
27877     GC_release_dirty_lock();
27878 #if defined(PARALLEL_MARK)
27879       if (GC_parallel)
27880         GC_release_mark_lock();
27881 #endif
27882     GC_remove_all_threads_but_me();
27883 #ifdef PARALLEL_MARK
27884         GC_parallel = FALSE;
27885 #endif
27886 #ifndef GC_DISABLE_INCREMENTAL
27887       GC_dirty_update_child();
27888 #endif
27889     RESTORE_CANCEL(fork_cancel_state);
27890     UNLOCK();
27891 #ifdef USE_PTHREAD_LOCKS
27892       GC_ASSERT(I_DONT_HOLD_LOCK());
27893       (void)pthread_mutex_destroy(&GC_allocate_ml);
27894       if (0 != pthread_mutex_init(&GC_allocate_ml, NULL))
27895         ABORT("pthread_mutex_init failed (in child)");
27896 #endif
27897 }
27898   GC_API void GC_CALL GC_atfork_prepare(void)
27899   {
27900     if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
27901 #if defined(GC_DARWIN_THREADS) && defined(MPROTECT_VDB)
27902       if (GC_auto_incremental) {
27903         GC_ASSERT(0 == GC_handle_fork);
27904         ABORT("Unable to fork while mprotect_thread is running");
27905       }
27906 #endif
27907     if (GC_handle_fork <= 0)
27908       fork_prepare_proc();
27909   }
27910   GC_API void GC_CALL GC_atfork_parent(void)
27911   {
27912     if (GC_handle_fork <= 0)
27913       fork_parent_proc();
27914   }
27915   GC_API void GC_CALL GC_atfork_child(void)
27916   {
27917     if (GC_handle_fork <= 0)
27918       fork_child_proc();
27919   }
27920 #endif
27921 #ifdef INCLUDE_LINUX_THREAD_DESCR
27922   __thread int GC_dummy_thread_local;
27923 #endif
27924 #ifdef PARALLEL_MARK
27925   static void setup_mark_lock(void);
27926   static unsigned required_markers_cnt = 0;
27927 #endif
27928 GC_API void GC_CALL GC_set_markers_count(unsigned markers GC_ATTR_UNUSED)
27929 {
27930 #ifdef PARALLEL_MARK
27931     required_markers_cnt = markers < MAX_MARKERS ? markers : MAX_MARKERS;
27932 #endif
27933 }
27934 GC_INNER void GC_thr_init(void)
27935 {
27936   GC_ASSERT(I_HOLD_LOCK());
27937   if (GC_thr_initialized) return;
27938   GC_thr_initialized = TRUE;
27939   GC_ASSERT((word)&GC_threads % sizeof(word) == 0);
27940 #ifdef CAN_HANDLE_FORK
27941     if (GC_handle_fork) {
27942 #ifdef CAN_CALL_ATFORK
27943         if (pthread_atfork(fork_prepare_proc, fork_parent_proc,
27944                            fork_child_proc) == 0) {
27945           GC_handle_fork = 1;
27946         } else
27947 #endif
27948        if (GC_handle_fork != -1)
27949         ABORT("pthread_atfork failed");
27950     }
27951 #endif
27952 #ifdef INCLUDE_LINUX_THREAD_DESCR
27953     {
27954       ptr_t thread_local_addr = (ptr_t)(&GC_dummy_thread_local);
27955       ptr_t main_thread_start, main_thread_end;
27956       if (!GC_enclosing_mapping(thread_local_addr, &main_thread_start,
27957                                 &main_thread_end)) {
27958         ABORT("Failed to find mapping for main thread thread locals");
27959       } else {
27960         GC_add_roots_inner(main_thread_start, main_thread_end, FALSE);
27961       }
27962     }
27963 #endif
27964   {
27965     pthread_t self = pthread_self();
27966     GC_thread t = GC_new_thread(self);
27967     if (t == NULL)
27968       ABORT("Failed to allocate memory for the initial thread");
27969 #ifdef GC_DARWIN_THREADS
27970       t -> stop_info.mach_thread = mach_thread_self();
27971 #else
27972       t -> stop_info.stack_ptr = GC_approx_sp();
27973 #endif
27974     t -> flags = DETACHED | MAIN_THREAD;
27975     if (THREAD_EQUAL(self, main_pthread_id)) {
27976       t -> stack = (ptr_t)main_stack;
27977       t -> stack_size = main_stack_size;
27978       t -> altstack = (ptr_t)main_altstack;
27979       t -> altstack_size = main_altstack_size;
27980     }
27981   }
27982   {
27983     char * nprocs_string = GETENV("GC_NPROCS");
27984     GC_nprocs = -1;
27985     if (nprocs_string != NULL) GC_nprocs = atoi(nprocs_string);
27986   }
27987   if (GC_nprocs <= 0
27988 #if defined(ARM32) && defined(GC_LINUX_THREADS) && !defined(NACL)
27989         && (GC_nprocs = GC_get_nprocs_present()) <= 1
27990 #endif
27991       )
27992   {
27993     GC_nprocs = GC_get_nprocs();
27994   }
27995   if (GC_nprocs <= 0) {
27996     WARN("GC_get_nprocs() returned %" WARN_PRIdPTR "\n", GC_nprocs);
27997     GC_nprocs = 2;
27998 #ifdef PARALLEL_MARK
27999       available_markers_m1 = 0;
28000 #endif
28001   } else {
28002 #ifdef PARALLEL_MARK
28003       {
28004         char * markers_string = GETENV("GC_MARKERS");
28005         int markers = required_markers_cnt;
28006         if (markers_string != NULL) {
28007           markers = atoi(markers_string);
28008           if (markers <= 0 || markers > MAX_MARKERS) {
28009             WARN("Too big or invalid number of mark threads: %" WARN_PRIdPTR
28010                  "; using maximum threads\n", (signed_word)markers);
28011             markers = MAX_MARKERS;
28012           }
28013         } else if (0 == markers) {
28014           markers = GC_nprocs;
28015 #if defined(GC_MIN_MARKERS) && !defined(CPPCHECK)
28016             if (markers < GC_MIN_MARKERS)
28017               markers = GC_MIN_MARKERS;
28018 #endif
28019           if (markers > MAX_MARKERS)
28020             markers = MAX_MARKERS;
28021         }
28022         available_markers_m1 = markers - 1;
28023       }
28024 #endif
28025   }
28026   GC_COND_LOG_PRINTF("Number of processors: %d\n", GC_nprocs);
28027 #if defined(BASE_ATOMIC_OPS_EMULATED) && !defined(GC_DARWIN_THREADS) \
28028      && !defined(GC_OPENBSD_UTHREADS) && !defined(NACL) \
28029      && !defined(PLATFORM_STOP_WORLD) && !defined(SN_TARGET_PSP2)
28030     {
28031       cpu_set_t mask;
28032       int cpu_set_cnt = 0;
28033       int cpu_lowest_set = 0;
28034       int i = GC_nprocs > 1 ? GC_nprocs : 2;
28035       if (sched_getaffinity(0 ,
28036                             sizeof(mask), &mask) == -1)
28037         ABORT_ARG1("sched_getaffinity failed", ": errno= %d", errno);
28038       while (i-- > 0)
28039         if (CPU_ISSET(i, &mask)) {
28040           cpu_lowest_set = i;
28041           cpu_set_cnt++;
28042         }
28043       if (0 == cpu_set_cnt)
28044         ABORT("sched_getaffinity returned empty mask");
28045       if (cpu_set_cnt > 1) {
28046         CPU_ZERO(&mask);
28047         CPU_SET(cpu_lowest_set, &mask);
28048         if (sched_setaffinity(0, sizeof(mask), &mask) == -1)
28049           ABORT_ARG1("sched_setaffinity failed", ": errno= %d", errno);
28050         WARN("CPU affinity mask is set to %p\n", (word)1 << cpu_lowest_set);
28051       }
28052     }
28053 #endif
28054 #ifndef GC_DARWIN_THREADS
28055     GC_stop_init();
28056 #endif
28057 #ifdef PARALLEL_MARK
28058     if (available_markers_m1 <= 0) {
28059       GC_parallel = FALSE;
28060       GC_COND_LOG_PRINTF(
28061                 "Single marker thread, turning off parallel marking\n");
28062     } else {
28063       setup_mark_lock();
28064     }
28065 #endif
28066 }
28067 GC_INNER void GC_init_parallel(void)
28068 {
28069 #if defined(THREAD_LOCAL_ALLOC)
28070       DCL_LOCK_STATE;
28071 #endif
28072     if (parallel_initialized) return;
28073     parallel_initialized = TRUE;
28074     if (!GC_is_initialized) GC_init();
28075 #if defined(THREAD_LOCAL_ALLOC)
28076       LOCK();
28077       GC_init_thread_local(&(GC_lookup_thread(pthread_self())->tlfs));
28078       UNLOCK();
28079 #endif
28080 }
28081 #ifndef GC_NO_PTHREAD_SIGMASK
28082   GC_API int WRAP_FUNC(pthread_sigmask)(int how, const sigset_t *set,
28083                                         sigset_t *oset)
28084   {
28085     sigset_t fudged_set;
28086     INIT_REAL_SYMS();
28087     if (set != NULL && (how == SIG_BLOCK || how == SIG_SETMASK)) {
28088         int sig_suspend = GC_get_suspend_signal();
28089         fudged_set = *set;
28090         GC_ASSERT(sig_suspend >= 0);
28091         if (sigdelset(&fudged_set, sig_suspend) != 0)
28092             ABORT("sigdelset failed");
28093         set = &fudged_set;
28094     }
28095     return(REAL_FUNC(pthread_sigmask)(how, set, oset));
28096   }
28097 #endif
28098 GC_INNER void GC_do_blocking_inner(ptr_t data, void * context GC_ATTR_UNUSED)
28099 {
28100     struct blocking_data * d = (struct blocking_data *) data;
28101     pthread_t self = pthread_self();
28102     GC_thread me;
28103 #if defined(SPARC) || defined(IA64)
28104         ptr_t stack_ptr = GC_save_regs_in_stack();
28105 #endif
28106 #if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
28107         GC_bool topOfStackUnset = FALSE;
28108 #endif
28109     DCL_LOCK_STATE;
28110     LOCK();
28111     me = GC_lookup_thread(self);
28112     GC_ASSERT(!(me -> thread_blocked));
28113 #ifdef SPARC
28114         me -> stop_info.stack_ptr = stack_ptr;
28115 #else
28116         me -> stop_info.stack_ptr = GC_approx_sp();
28117 #endif
28118 #if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
28119         if (me -> topOfStack == NULL) {
28120             topOfStackUnset = TRUE;
28121             me -> topOfStack = GC_FindTopOfStack(0);
28122         }
28123 #endif
28124 #ifdef IA64
28125         me -> backing_store_ptr = stack_ptr;
28126 #endif
28127     me -> thread_blocked = (unsigned char)TRUE;
28128     UNLOCK();
28129     d -> client_data = (d -> fn)(d -> client_data);
28130     LOCK();
28131 #if defined(CPPCHECK)
28132       GC_noop1((word)&me->thread_blocked);
28133 #endif
28134     me -> thread_blocked = FALSE;
28135 #if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
28136         if (topOfStackUnset)
28137             me -> topOfStack = NULL;
28138 #endif
28139     UNLOCK();
28140 }
28141 GC_API void GC_CALL GC_set_stackbottom(void *gc_thread_handle,
28142                                        const struct GC_stack_base *sb)
28143 {
28144     GC_thread t = (GC_thread)gc_thread_handle;
28145     GC_ASSERT(sb -> mem_base != NULL);
28146     if (!EXPECT(GC_is_initialized, TRUE)) {
28147         GC_ASSERT(NULL == t);
28148     } else {
28149         GC_ASSERT(I_HOLD_LOCK());
28150         if (NULL == t)
28151             t = GC_lookup_thread(pthread_self());
28152         GC_ASSERT((t -> flags & FINISHED) == 0);
28153         GC_ASSERT(!(t -> thread_blocked)
28154                   && NULL == t -> traced_stack_sect);
28155         if ((t -> flags & MAIN_THREAD) == 0) {
28156             t -> stack_end = (ptr_t)sb->mem_base;
28157 #ifdef IA64
28158                 t -> backing_store_end = (ptr_t)sb->reg_base;
28159 #endif
28160             return;
28161         }
28162     }
28163     GC_stackbottom = (char*)sb->mem_base;
28164 #ifdef IA64
28165         GC_register_stackbottom = (ptr_t)sb->reg_base;
28166 #endif
28167 }
28168 GC_API void * GC_CALL GC_get_my_stackbottom(struct GC_stack_base *sb)
28169 {
28170     pthread_t self = pthread_self();
28171     GC_thread me;
28172     DCL_LOCK_STATE;
28173     LOCK();
28174     me = GC_lookup_thread(self);
28175     if ((me -> flags & MAIN_THREAD) == 0) {
28176         sb -> mem_base = me -> stack_end;
28177 #ifdef IA64
28178             sb -> reg_base = me -> backing_store_end;
28179 #endif
28180     } else {
28181         sb -> mem_base = GC_stackbottom;
28182 #ifdef IA64
28183             sb -> reg_base = GC_register_stackbottom;
28184 #endif
28185     }
28186     UNLOCK();
28187     return (void *)me;
28188 }
28189 GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type fn,
28190                                              void * client_data)
28191 {
28192     struct GC_traced_stack_sect_s stacksect;
28193     pthread_t self = pthread_self();
28194     GC_thread me;
28195     DCL_LOCK_STATE;
28196     LOCK();
28197     me = GC_lookup_thread(self);
28198     if ((me -> flags & MAIN_THREAD) == 0) {
28199       GC_ASSERT(me -> stack_end != NULL);
28200       if ((word)me->stack_end HOTTER_THAN (word)(&stacksect))
28201         me -> stack_end = (ptr_t)(&stacksect);
28202     } else {
28203       if ((word)GC_stackbottom HOTTER_THAN (word)(&stacksect))
28204         GC_stackbottom = (ptr_t)COVERT_DATAFLOW(&stacksect);
28205     }
28206     if (!me->thread_blocked) {
28207       UNLOCK();
28208       client_data = fn(client_data);
28209       GC_noop1(COVERT_DATAFLOW(&stacksect));
28210       return client_data;
28211     }
28212     stacksect.saved_stack_ptr = me -> stop_info.stack_ptr;
28213 #ifdef IA64
28214       stacksect.backing_store_end = GC_save_regs_in_stack();
28215       stacksect.saved_backing_store_ptr = me -> backing_store_ptr;
28216 #endif
28217     stacksect.prev = me -> traced_stack_sect;
28218     me -> thread_blocked = FALSE;
28219     me -> traced_stack_sect = &stacksect;
28220     UNLOCK();
28221     client_data = fn(client_data);
28222     GC_ASSERT(me -> thread_blocked == FALSE);
28223     GC_ASSERT(me -> traced_stack_sect == &stacksect);
28224 #if defined(CPPCHECK)
28225       GC_noop1((word)me->traced_stack_sect);
28226 #endif
28227     LOCK();
28228     me -> traced_stack_sect = stacksect.prev;
28229 #ifdef IA64
28230       me -> backing_store_ptr = stacksect.saved_backing_store_ptr;
28231 #endif
28232     me -> thread_blocked = (unsigned char)TRUE;
28233     me -> stop_info.stack_ptr = stacksect.saved_stack_ptr;
28234     UNLOCK();
28235     return client_data;
28236 }
28237 STATIC void GC_unregister_my_thread_inner(GC_thread me)
28238 {
28239     GC_ASSERT(I_HOLD_LOCK());
28240 #ifdef DEBUG_THREADS
28241       GC_log_printf(
28242                 "Unregistering thread %p, gc_thread= %p, n_threads= %d\n",
28243                 (void *)me->id, (void *)me, GC_count_threads());
28244 #endif
28245     GC_ASSERT(!(me -> flags & FINISHED));
28246 #if defined(THREAD_LOCAL_ALLOC)
28247       GC_ASSERT(GC_getspecific(GC_thread_key) == &me->tlfs);
28248       GC_destroy_thread_local(&(me->tlfs));
28249 #endif
28250 #if defined(GC_HAVE_PTHREAD_EXIT) || !defined(GC_NO_PTHREAD_CANCEL)
28251       if ((me -> flags & DISABLED_GC) != 0) {
28252         GC_dont_gc--;
28253       }
28254 #endif
28255     if (me -> flags & DETACHED) {
28256         GC_delete_thread(pthread_self());
28257     } else {
28258         me -> flags |= FINISHED;
28259     }
28260 #if defined(THREAD_LOCAL_ALLOC)
28261       GC_remove_specific(GC_thread_key);
28262 #endif
28263 }
28264 GC_API int GC_CALL GC_unregister_my_thread(void)
28265 {
28266     pthread_t self = pthread_self();
28267     GC_thread me;
28268     IF_CANCEL(int cancel_state;)
28269     DCL_LOCK_STATE;
28270     LOCK();
28271     DISABLE_CANCEL(cancel_state);
28272     GC_wait_for_gc_completion(FALSE);
28273     me = GC_lookup_thread(self);
28274 #ifdef DEBUG_THREADS
28275         GC_log_printf(
28276                 "Called GC_unregister_my_thread on %p, gc_thread= %p\n",
28277                 (void *)self, (void *)me);
28278 #endif
28279     GC_ASSERT(THREAD_EQUAL(me->id, self));
28280     GC_unregister_my_thread_inner(me);
28281     RESTORE_CANCEL(cancel_state);
28282     UNLOCK();
28283     return GC_SUCCESS;
28284 }
28285 GC_INNER_PTHRSTART void GC_thread_exit_proc(void *arg)
28286 {
28287     IF_CANCEL(int cancel_state;)
28288     DCL_LOCK_STATE;
28289 #ifdef DEBUG_THREADS
28290         GC_log_printf("Called GC_thread_exit_proc on %p, gc_thread= %p\n",
28291                       (void *)((GC_thread)arg)->id, arg);
28292 #endif
28293     LOCK();
28294     DISABLE_CANCEL(cancel_state);
28295     GC_wait_for_gc_completion(FALSE);
28296     GC_unregister_my_thread_inner((GC_thread)arg);
28297     RESTORE_CANCEL(cancel_state);
28298     UNLOCK();
28299 }
28300 #if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
28301   GC_API int WRAP_FUNC(pthread_join)(pthread_t thread, void **retval)
28302   {
28303     int result;
28304     GC_thread t;
28305     DCL_LOCK_STATE;
28306     ASSERT_SYMS_INITIALIZED();
28307     LOCK();
28308     t = (GC_thread)COVERT_DATAFLOW(GC_lookup_thread(thread));
28309     UNLOCK();
28310     result = REAL_FUNC(pthread_join)(thread, retval);
28311 #if defined(GC_FREEBSD_THREADS)
28312     if (result == EINTR) result = 0;
28313 #endif
28314     if (result == 0) {
28315         LOCK();
28316         if ((t -> flags & FINISHED) != 0)
28317           GC_delete_gc_thread(t);
28318         UNLOCK();
28319     }
28320     return result;
28321   }
28322   GC_API int WRAP_FUNC(pthread_detach)(pthread_t thread)
28323   {
28324     int result;
28325     GC_thread t;
28326     DCL_LOCK_STATE;
28327     ASSERT_SYMS_INITIALIZED();
28328     LOCK();
28329     t = (GC_thread)COVERT_DATAFLOW(GC_lookup_thread(thread));
28330     UNLOCK();
28331     result = REAL_FUNC(pthread_detach)(thread);
28332     if (result == 0) {
28333       LOCK();
28334       t -> flags |= DETACHED;
28335       if ((t -> flags & FINISHED) != 0) {
28336         GC_delete_gc_thread(t);
28337       }
28338       UNLOCK();
28339     }
28340     return result;
28341   }
28342 #endif
28343 #ifndef GC_NO_PTHREAD_CANCEL
28344   GC_API int WRAP_FUNC(pthread_cancel)(pthread_t thread)
28345   {
28346 #ifdef CANCEL_SAFE
28347       GC_thread t;
28348       DCL_LOCK_STATE;
28349 #endif
28350     INIT_REAL_SYMS();
28351 #ifdef CANCEL_SAFE
28352       LOCK();
28353       t = GC_lookup_thread(thread);
28354       if (t != NULL && (t -> flags & DISABLED_GC) == 0) {
28355         t -> flags |= DISABLED_GC;
28356         GC_dont_gc++;
28357       }
28358       UNLOCK();
28359 #endif
28360     return REAL_FUNC(pthread_cancel)(thread);
28361   }
28362 #endif
28363 #ifdef GC_HAVE_PTHREAD_EXIT
28364   GC_API GC_PTHREAD_EXIT_ATTRIBUTE void WRAP_FUNC(pthread_exit)(void *retval)
28365   {
28366     pthread_t self = pthread_self();
28367     GC_thread me;
28368     DCL_LOCK_STATE;
28369     INIT_REAL_SYMS();
28370     LOCK();
28371     me = GC_lookup_thread(self);
28372     if (me != 0 && (me -> flags & DISABLED_GC) == 0) {
28373       me -> flags |= DISABLED_GC;
28374       GC_dont_gc++;
28375     }
28376     UNLOCK();
28377     REAL_FUNC(pthread_exit)(retval);
28378   }
28379 #endif
28380 GC_INNER GC_bool GC_in_thread_creation = FALSE;
28381 GC_INLINE void GC_record_stack_base(GC_thread me,
28382                                     const struct GC_stack_base *sb)
28383 {
28384 #ifndef GC_DARWIN_THREADS
28385       me -> stop_info.stack_ptr = (ptr_t)sb->mem_base;
28386 #endif
28387     me -> stack_end = (ptr_t)sb->mem_base;
28388     if (me -> stack_end == NULL)
28389       ABORT("Bad stack base in GC_register_my_thread");
28390 #ifdef IA64
28391       me -> backing_store_end = (ptr_t)sb->reg_base;
28392 #endif
28393 }
28394 STATIC GC_thread GC_register_my_thread_inner(const struct GC_stack_base *sb,
28395                                              pthread_t my_pthread)
28396 {
28397     GC_thread me;
28398     GC_in_thread_creation = TRUE;
28399     me = GC_new_thread(my_pthread);
28400     GC_in_thread_creation = FALSE;
28401     if (me == 0)
28402       ABORT("Failed to allocate memory for thread registering");
28403 #ifdef GC_DARWIN_THREADS
28404       me -> stop_info.mach_thread = mach_thread_self();
28405 #endif
28406     GC_record_stack_base(me, sb);
28407 #ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
28408       GC_unblock_gc_signals();
28409 #endif
28410     return me;
28411 }
28412 GC_API void GC_CALL GC_allow_register_threads(void)
28413 {
28414     GC_ASSERT(GC_lookup_thread(pthread_self()) != 0);
28415     set_need_to_lock();
28416 }
28417 GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base *sb)
28418 {
28419     pthread_t self = pthread_self();
28420     GC_thread me;
28421     DCL_LOCK_STATE;
28422     if (GC_need_to_lock == FALSE)
28423         ABORT("Threads explicit registering is not previously enabled");
28424     LOCK();
28425     me = GC_lookup_thread(self);
28426     if (0 == me) {
28427         me = GC_register_my_thread_inner(sb, self);
28428 #if defined(CPPCHECK)
28429           GC_noop1(me->flags);
28430 #endif
28431         me -> flags |= DETACHED;
28432 #if defined(THREAD_LOCAL_ALLOC)
28433           GC_init_thread_local(&(me->tlfs));
28434 #endif
28435         UNLOCK();
28436         return GC_SUCCESS;
28437     } else if ((me -> flags & FINISHED) != 0) {
28438 #ifdef GC_DARWIN_THREADS
28439           me -> stop_info.mach_thread = mach_thread_self();
28440 #endif
28441         GC_record_stack_base(me, sb);
28442         me -> flags &= ~FINISHED;
28443 #ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
28444           GC_unblock_gc_signals();
28445 #endif
28446 #if defined(THREAD_LOCAL_ALLOC)
28447           GC_init_thread_local(&(me->tlfs));
28448 #endif
28449         UNLOCK();
28450         return GC_SUCCESS;
28451     } else {
28452         UNLOCK();
28453         return GC_DUPLICATE;
28454     }
28455 }
28456 struct start_info {
28457     void *(*start_routine)(void *);
28458     void *arg;
28459     word flags;
28460     sem_t registered;
28461 };
28462 GC_INNER_PTHRSTART GC_thread GC_start_rtn_prepare_thread(
28463                                         void *(**pstart)(void *),
28464                                         void **pstart_arg,
28465                                         struct GC_stack_base *sb, void *arg)
28466 {
28467     struct start_info * si = (struct start_info *)arg;
28468     pthread_t self = pthread_self();
28469     GC_thread me;
28470     DCL_LOCK_STATE;
28471 #ifdef DEBUG_THREADS
28472       GC_log_printf("Starting thread %p, pid= %ld, sp= %p\n",
28473                     (void *)self, (long)getpid(), (void *)&arg);
28474 #endif
28475     LOCK();
28476     me = GC_register_my_thread_inner(sb, self);
28477     me -> flags = si -> flags;
28478 #if defined(THREAD_LOCAL_ALLOC)
28479       GC_init_thread_local(&(me->tlfs));
28480 #endif
28481     UNLOCK();
28482     *pstart = si -> start_routine;
28483 #ifdef DEBUG_THREADS
28484       GC_log_printf("start_routine= %p\n", (void *)(signed_word)(*pstart));
28485 #endif
28486     *pstart_arg = si -> arg;
28487     sem_post(&(si -> registered));
28488     return me;
28489 }
28490 #if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
28491   STATIC void * GC_start_routine(void * arg)
28492   {
28493 #ifdef INCLUDE_LINUX_THREAD_DESCR
28494       struct GC_stack_base sb;
28495 #ifdef REDIRECT_MALLOC
28496         GC_disable();
28497 #endif
28498       if (GC_get_stack_base(&sb) != GC_SUCCESS)
28499         ABORT("Failed to get thread stack base");
28500 #ifdef REDIRECT_MALLOC
28501         GC_enable();
28502 #endif
28503       return GC_inner_start_routine(&sb, arg);
28504 #else
28505       return GC_call_with_stack_base(GC_inner_start_routine, arg);
28506 #endif
28507   }
28508   GC_API int WRAP_FUNC(pthread_create)(pthread_t *new_thread,
28509                        GC_PTHREAD_CREATE_CONST pthread_attr_t *attr,
28510                        void *(*start_routine)(void *), void *arg)
28511   {
28512     int result;
28513     int detachstate;
28514     word my_flags = 0;
28515     struct start_info si;
28516     DCL_LOCK_STATE;
28517     INIT_REAL_SYMS();
28518     if (!EXPECT(parallel_initialized, TRUE))
28519       GC_init_parallel();
28520     if (sem_init(&si.registered, GC_SEM_INIT_PSHARED, 0) != 0)
28521       ABORT("sem_init failed");
28522     si.start_routine = start_routine;
28523     si.arg = arg;
28524     LOCK();
28525     if (!EXPECT(GC_thr_initialized, TRUE))
28526       GC_thr_init();
28527 #ifdef GC_ASSERTIONS
28528       {
28529         size_t stack_size = 0;
28530         if (NULL != attr) {
28531           if (pthread_attr_getstacksize(attr, &stack_size) != 0)
28532             ABORT("pthread_attr_getstacksize failed");
28533         }
28534         if (0 == stack_size) {
28535            pthread_attr_t my_attr;
28536            if (pthread_attr_init(&my_attr) != 0)
28537              ABORT("pthread_attr_init failed");
28538            if (pthread_attr_getstacksize(&my_attr, &stack_size) != 0)
28539              ABORT("pthread_attr_getstacksize failed");
28540            (void)pthread_attr_destroy(&my_attr);
28541         }
28542         if (0 == stack_size) {
28543 #ifndef SOLARIS
28544               WARN("Failed to get stack size for assertion checking\n", 0);
28545 #endif
28546             stack_size = 1000000;
28547         }
28548         GC_ASSERT(stack_size >= 65536);
28549       }
28550 #endif
28551     if (NULL == attr) {
28552         detachstate = PTHREAD_CREATE_JOINABLE;
28553     } else {
28554         pthread_attr_getdetachstate(attr, &detachstate);
28555     }
28556     if (PTHREAD_CREATE_DETACHED == detachstate) my_flags |= DETACHED;
28557     si.flags = my_flags;
28558     UNLOCK();
28559 #ifdef DEBUG_THREADS
28560       GC_log_printf("About to start new thread from thread %p\n",
28561                     (void *)pthread_self());
28562 #endif
28563     set_need_to_lock();
28564     result = REAL_FUNC(pthread_create)(new_thread, attr, GC_start_routine,
28565                                        &si);
28566     if (0 == result) {
28567         IF_CANCEL(int cancel_state;)
28568 #ifdef DEBUG_THREADS
28569             GC_log_printf("Started thread %p\n", (void *)(*new_thread));
28570 #endif
28571         DISABLE_CANCEL(cancel_state);
28572         while (0 != sem_wait(&si.registered)) {
28573 #if defined(GC_HAIKU_THREADS)
28574               if (EACCES == errno) continue;
28575 #endif
28576             if (EINTR != errno) ABORT("sem_wait failed");
28577         }
28578         RESTORE_CANCEL(cancel_state);
28579     }
28580     sem_destroy(&si.registered);
28581     return(result);
28582   }
28583 #endif
28584 #if defined(USE_SPIN_LOCK) || !defined(NO_PTHREAD_TRYLOCK)
28585 #define GC_PAUSE_SPIN_CYCLES 10
28586 STATIC void GC_pause(void)
28587 {
28588     int i;
28589     for (i = 0; i < GC_PAUSE_SPIN_CYCLES; ++i) {
28590 #if defined(AO_HAVE_compiler_barrier) \
28591          && !defined(BASE_ATOMIC_OPS_EMULATED)
28592         AO_compiler_barrier();
28593 #else
28594         GC_noop1(i);
28595 #endif
28596     }
28597 }
28598 #endif
28599 #ifndef SPIN_MAX
28600 #define SPIN_MAX 128
28601 #endif
28602 GC_INNER volatile GC_bool GC_collecting = FALSE;
28603 #if (!defined(USE_SPIN_LOCK) && !defined(NO_PTHREAD_TRYLOCK)) \
28604         || defined(PARALLEL_MARK)
28605 #ifdef LOCK_STATS
28606   volatile AO_t GC_spin_count = 0;
28607   volatile AO_t GC_block_count = 0;
28608   volatile AO_t GC_unlocked_count = 0;
28609 #endif
28610 STATIC void GC_generic_lock(pthread_mutex_t * lock)
28611 {
28612 #ifndef NO_PTHREAD_TRYLOCK
28613     unsigned pause_length = 1;
28614     unsigned i;
28615     if (0 == pthread_mutex_trylock(lock)) {
28616 #ifdef LOCK_STATS
28617             (void)AO_fetch_and_add1(&GC_unlocked_count);
28618 #endif
28619         return;
28620     }
28621     for (; pause_length <= SPIN_MAX; pause_length <<= 1) {
28622         for (i = 0; i < pause_length; ++i) {
28623             GC_pause();
28624         }
28625         switch(pthread_mutex_trylock(lock)) {
28626             case 0:
28627 #ifdef LOCK_STATS
28628                     (void)AO_fetch_and_add1(&GC_spin_count);
28629 #endif
28630                 return;
28631             case EBUSY:
28632                 break;
28633             default:
28634                 ABORT("Unexpected error from pthread_mutex_trylock");
28635         }
28636     }
28637 #endif
28638 #ifdef LOCK_STATS
28639         (void)AO_fetch_and_add1(&GC_block_count);
28640 #endif
28641     pthread_mutex_lock(lock);
28642 }
28643 #endif
28644 #if defined(AO_HAVE_char_load) && !defined(BASE_ATOMIC_OPS_EMULATED)
28645 #define is_collecting() \
28646                 ((GC_bool)AO_char_load((unsigned char *)&GC_collecting))
28647 #else
28648 #define is_collecting() GC_collecting
28649 #endif
28650 #if defined(USE_SPIN_LOCK)
28651 GC_INNER volatile AO_TS_t GC_allocate_lock = AO_TS_INITIALIZER;
28652 #define low_spin_max 30
28653 #define high_spin_max SPIN_MAX
28654   static volatile AO_t spin_max = low_spin_max;
28655   static volatile AO_t last_spins = 0;
28656 GC_INNER void GC_lock(void)
28657 {
28658     unsigned my_spin_max;
28659     unsigned my_last_spins;
28660     unsigned i;
28661     if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
28662         return;
28663     }
28664     my_spin_max = (unsigned)AO_load(&spin_max);
28665     my_last_spins = (unsigned)AO_load(&last_spins);
28666     for (i = 0; i < my_spin_max; i++) {
28667         if (is_collecting() || GC_nprocs == 1)
28668           goto yield;
28669         if (i < my_last_spins/2) {
28670             GC_pause();
28671             continue;
28672         }
28673         if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
28674             AO_store(&last_spins, (AO_t)i);
28675             AO_store(&spin_max, (AO_t)high_spin_max);
28676             return;
28677         }
28678     }
28679     AO_store(&spin_max, (AO_t)low_spin_max);
28680 yield:
28681     for (i = 0;; ++i) {
28682         if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
28683             return;
28684         }
28685 #define SLEEP_THRESHOLD 12
28686         if (i < SLEEP_THRESHOLD) {
28687             sched_yield();
28688         } else {
28689             struct timespec ts;
28690             if (i > 24) i = 24;
28691             ts.tv_sec = 0;
28692             ts.tv_nsec = 1 << i;
28693             nanosleep(&ts, 0);
28694         }
28695     }
28696 }
28697 #elif defined(USE_PTHREAD_LOCKS)
28698 #ifndef NO_PTHREAD_TRYLOCK
28699     GC_INNER void GC_lock(void)
28700     {
28701       if (1 == GC_nprocs || is_collecting()) {
28702         pthread_mutex_lock(&GC_allocate_ml);
28703       } else {
28704         GC_generic_lock(&GC_allocate_ml);
28705       }
28706     }
28707 #elif defined(GC_ASSERTIONS)
28708     GC_INNER void GC_lock(void)
28709     {
28710       pthread_mutex_lock(&GC_allocate_ml);
28711     }
28712 #endif
28713 #endif
28714 #ifdef PARALLEL_MARK
28715 #ifdef GC_ASSERTIONS
28716     STATIC unsigned long GC_mark_lock_holder = NO_THREAD;
28717 #define SET_MARK_LOCK_HOLDER \
28718                 (void)(GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
28719 #define UNSET_MARK_LOCK_HOLDER \
28720                 do { \
28721                   GC_ASSERT(GC_mark_lock_holder \
28722                                 == NUMERIC_THREAD_ID(pthread_self())); \
28723                   GC_mark_lock_holder = NO_THREAD; \
28724                 } while (0)
28725 #else
28726 #define SET_MARK_LOCK_HOLDER (void)0
28727 #define UNSET_MARK_LOCK_HOLDER (void)0
28728 #endif
28729 #ifdef GLIBC_2_1_MUTEX_HACK
28730   static pthread_mutex_t mark_mutex =
28731         {0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, {0, 0}};
28732 #else
28733   static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
28734 #endif
28735 static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;
28736 static void setup_mark_lock(void)
28737 {
28738 #ifdef GLIBC_2_19_TSX_BUG
28739     pthread_mutexattr_t mattr;
28740     int glibc_minor = -1;
28741     int glibc_major = GC_parse_version(&glibc_minor, gnu_get_libc_version());
28742     if (glibc_major > 2 || (glibc_major == 2 && glibc_minor >= 19)) {
28743       if (0 != pthread_mutexattr_init(&mattr)) {
28744         ABORT("pthread_mutexattr_init failed");
28745       }
28746       if (0 != pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL)) {
28747         ABORT("pthread_mutexattr_settype failed");
28748       }
28749       if (0 != pthread_mutex_init(&mark_mutex, &mattr)) {
28750         ABORT("pthread_mutex_init failed");
28751       }
28752       (void)pthread_mutexattr_destroy(&mattr);
28753     }
28754 #endif
28755 }
28756 GC_INNER void GC_acquire_mark_lock(void)
28757 {
28758 #if defined(NUMERIC_THREAD_ID_UNIQUE) && !defined(THREAD_SANITIZER)
28759       GC_ASSERT(GC_mark_lock_holder != NUMERIC_THREAD_ID(pthread_self()));
28760 #endif
28761     GC_generic_lock(&mark_mutex);
28762     SET_MARK_LOCK_HOLDER;
28763 }
28764 GC_INNER void GC_release_mark_lock(void)
28765 {
28766     UNSET_MARK_LOCK_HOLDER;
28767     if (pthread_mutex_unlock(&mark_mutex) != 0) {
28768         ABORT("pthread_mutex_unlock failed");
28769     }
28770 }
28771 STATIC void GC_wait_builder(void)
28772 {
28773     ASSERT_CANCEL_DISABLED();
28774     UNSET_MARK_LOCK_HOLDER;
28775     if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0) {
28776         ABORT("pthread_cond_wait failed");
28777     }
28778     GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
28779     SET_MARK_LOCK_HOLDER;
28780 }
28781 GC_INNER void GC_wait_for_reclaim(void)
28782 {
28783     GC_acquire_mark_lock();
28784     while (GC_fl_builder_count > 0) {
28785         GC_wait_builder();
28786     }
28787     GC_release_mark_lock();
28788 }
28789 GC_INNER void GC_notify_all_builder(void)
28790 {
28791     GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
28792     if (pthread_cond_broadcast(&builder_cv) != 0) {
28793         ABORT("pthread_cond_broadcast failed");
28794     }
28795 }
28796 GC_INNER void GC_wait_marker(void)
28797 {
28798     ASSERT_CANCEL_DISABLED();
28799     GC_ASSERT(GC_parallel);
28800     UNSET_MARK_LOCK_HOLDER;
28801     if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0) {
28802         ABORT("pthread_cond_wait failed");
28803     }
28804     GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
28805     SET_MARK_LOCK_HOLDER;
28806 }
28807 GC_INNER void GC_notify_all_marker(void)
28808 {
28809     GC_ASSERT(GC_parallel);
28810     if (pthread_cond_broadcast(&mark_cv) != 0) {
28811         ABORT("pthread_cond_broadcast failed");
28812     }
28813 }
28814 #endif
28815 #ifdef PTHREAD_REGISTER_CANCEL_WEAK_STUBS
28816   EXTERN_C_BEGIN
28817   extern void __pthread_register_cancel(void) __attribute__((__weak__));
28818   extern void __pthread_unregister_cancel(void) __attribute__((__weak__));
28819   EXTERN_C_END
28820   void __pthread_register_cancel(void) {}
28821   void __pthread_unregister_cancel(void) {}
28822 #endif
28823 #endif
28824 #if defined(USE_CUSTOM_SPECIFIC)
28825 static const tse invalid_tse = {INVALID_QTID, 0, 0, INVALID_THREADID};
28826 GC_INNER int GC_key_create_inner(tsd ** key_ptr)
28827 {
28828     int i;
28829     int ret;
28830     tsd * result;
28831     GC_ASSERT(I_HOLD_LOCK());
28832     GC_ASSERT((word)(&invalid_tse.next) % sizeof(tse *) == 0);
28833     result = (tsd *)MALLOC_CLEAR(sizeof(tsd));
28834     if (NULL == result) return ENOMEM;
28835     ret = pthread_mutex_init(&result->lock, NULL);
28836     if (ret != 0) return ret;
28837     for (i = 0; i < TS_CACHE_SIZE; ++i) {
28838       result -> cache[i] = ( tse *)&invalid_tse;
28839     }
28840 #ifdef GC_ASSERTIONS
28841       for (i = 0; i < TS_HASH_SIZE; ++i) {
28842         GC_ASSERT(result -> hash[i].p == 0);
28843       }
28844 #endif
28845     *key_ptr = result;
28846     return 0;
28847 }
28848 GC_INNER int GC_setspecific(tsd * key, void * value)
28849 {
28850     pthread_t self = pthread_self();
28851     unsigned hash_val = HASH(self);
28852     volatile tse * entry;
28853     GC_ASSERT(I_HOLD_LOCK());
28854     GC_ASSERT(self != INVALID_THREADID);
28855     GC_dont_gc++;
28856     entry = (volatile tse *)MALLOC_CLEAR(sizeof(tse));
28857     GC_dont_gc--;
28858     if (0 == entry) return ENOMEM;
28859     pthread_mutex_lock(&(key -> lock));
28860     entry -> next = key->hash[hash_val].p;
28861     entry -> thread = self;
28862     entry -> value = TS_HIDE_VALUE(value);
28863     GC_ASSERT(entry -> qtid == INVALID_QTID);
28864     AO_store_release(&key->hash[hash_val].ao, (AO_t)entry);
28865     GC_dirty(( void *)entry);
28866     GC_dirty(key->hash + hash_val);
28867     if (pthread_mutex_unlock(&key->lock) != 0)
28868       ABORT("pthread_mutex_unlock failed (setspecific)");
28869     return 0;
28870 }
28871 GC_INNER void GC_remove_specific_after_fork(tsd * key, pthread_t t)
28872 {
28873     unsigned hash_val = HASH(t);
28874     tse *entry;
28875     tse *prev = NULL;
28876 #ifdef CAN_HANDLE_FORK
28877       GC_ASSERT(I_HOLD_LOCK());
28878 #endif
28879     pthread_mutex_lock(&(key -> lock));
28880     entry = key->hash[hash_val].p;
28881     while (entry != NULL && !THREAD_EQUAL(entry->thread, t)) {
28882       prev = entry;
28883       entry = entry->next;
28884     }
28885     if (entry != NULL) {
28886       entry -> qtid = INVALID_QTID;
28887       if (NULL == prev) {
28888         key->hash[hash_val].p = entry->next;
28889         GC_dirty(key->hash + hash_val);
28890       } else {
28891         prev->next = entry->next;
28892         GC_dirty(prev);
28893       }
28894     }
28895 #ifdef LINT2
28896       GC_noop1((word)entry);
28897 #endif
28898     if (pthread_mutex_unlock(&key->lock) != 0)
28899       ABORT("pthread_mutex_unlock failed (remove_specific after fork)");
28900 }
28901 GC_INNER void * GC_slow_getspecific(tsd * key, word qtid,
28902                                     tse * volatile * cache_ptr)
28903 {
28904     pthread_t self = pthread_self();
28905     tse *entry = key->hash[HASH(self)].p;
28906     GC_ASSERT(qtid != INVALID_QTID);
28907     while (entry != NULL && !THREAD_EQUAL(entry->thread, self)) {
28908       entry = entry -> next;
28909     }
28910     if (entry == NULL) return NULL;
28911     entry -> qtid = (AO_t)qtid;
28912     *cache_ptr = entry;
28913     return TS_REVEAL_PTR(entry -> value);
28914 }
28915 #ifdef GC_ASSERTIONS
28916   void GC_check_tsd_marks(tsd *key)
28917   {
28918     int i;
28919     tse *p;
28920     if (!GC_is_marked(GC_base(key))) {
28921       ABORT("Unmarked thread-specific-data table");
28922     }
28923     for (i = 0; i < TS_HASH_SIZE; ++i) {
28924       for (p = key->hash[i].p; p != 0; p = p -> next) {
28925         if (!GC_is_marked(GC_base(p))) {
28926           ABORT_ARG1("Unmarked thread-specific-data entry",
28927                      " at %p", (void *)p);
28928         }
28929       }
28930     }
28931     for (i = 0; i < TS_CACHE_SIZE; ++i) {
28932       p = key -> cache[i];
28933       if (p != &invalid_tse && !GC_is_marked(GC_base(p))) {
28934         ABORT_ARG1("Unmarked cached thread-specific-data entry",
28935                    " at %p", (void *)p);
28936       }
28937     }
28938   }
28939 #endif
28940 #endif
28941 #if defined(GC_WIN32_THREADS)
28942 #ifdef THREAD_LOCAL_ALLOC
28943 #endif
28944 #if !defined(USE_PTHREAD_LOCKS)
28945   GC_INNER CRITICAL_SECTION GC_allocate_ml;
28946 #ifdef GC_ASSERTIONS
28947     GC_INNER DWORD GC_lock_holder = NO_THREAD;
28948 #endif
28949 #else
28950   GC_INNER pthread_mutex_t GC_allocate_ml = PTHREAD_MUTEX_INITIALIZER;
28951 #ifdef GC_ASSERTIONS
28952     GC_INNER unsigned long GC_lock_holder = NO_THREAD;
28953 #endif
28954 #endif
28955 #undef CreateThread
28956 #undef ExitThread
28957 #undef _beginthreadex
28958 #undef _endthreadex
28959 #ifdef GC_PTHREADS
28960 #include <errno.h>
28961 #undef pthread_create
28962 #undef pthread_join
28963 #undef pthread_detach
28964 #ifndef GC_NO_PTHREAD_SIGMASK
28965 #undef pthread_sigmask
28966 #endif
28967   STATIC void * GC_pthread_start(void * arg);
28968   STATIC void GC_thread_exit_proc(void *arg);
28969 #include <pthread.h>
28970 #ifdef CAN_CALL_ATFORK
28971 #include <unistd.h>
28972 #endif
28973 #elif !defined(MSWINCE)
28974 #include <process.h>
28975 #include <errno.h>
28976 #endif
28977 static ptr_t copy_ptr_regs(word *regs, const CONTEXT *pcontext);
28978 #if defined(I386)
28979 #ifdef WOW64_THREAD_CONTEXT_WORKAROUND
28980 #define PUSHED_REGS_COUNT 9
28981 #else
28982 #define PUSHED_REGS_COUNT 7
28983 #endif
28984 #elif defined(X86_64) || defined(SHx)
28985 #define PUSHED_REGS_COUNT 15
28986 #elif defined(ARM32)
28987 #define PUSHED_REGS_COUNT 13
28988 #elif defined(AARCH64)
28989 #define PUSHED_REGS_COUNT 30
28990 #elif defined(MIPS) || defined(ALPHA)
28991 #define PUSHED_REGS_COUNT 28
28992 #elif defined(PPC)
28993 #define PUSHED_REGS_COUNT 29
28994 #endif
28995 #if (defined(GC_DLL) || defined(GC_INSIDE_DLL)) && !defined(NO_CRT) \
28996         && !defined(GC_NO_THREADS_DISCOVERY) && !defined(MSWINCE) \
28997         && !defined(THREAD_LOCAL_ALLOC) && !defined(GC_PTHREADS)
28998 #ifdef GC_DISCOVER_TASK_THREADS
28999 #define GC_win32_dll_threads TRUE
29000 #else
29001     STATIC GC_bool GC_win32_dll_threads = FALSE;
29002 #endif
29003 #else
29004 #ifndef GC_NO_THREADS_DISCOVERY
29005 #define GC_NO_THREADS_DISCOVERY
29006 #endif
29007 #define GC_win32_dll_threads FALSE
29008 #undef MAX_THREADS
29009 #define MAX_THREADS 1
29010 #endif
29011 typedef LONG * IE_t;
29012 STATIC GC_bool GC_thr_initialized = FALSE;
29013 #ifndef GC_ALWAYS_MULTITHREADED
29014   GC_INNER GC_bool GC_need_to_lock = FALSE;
29015 #endif
29016 static GC_bool parallel_initialized = FALSE;
29017 GC_API void GC_CALL GC_use_threads_discovery(void)
29018 {
29019 #ifdef GC_NO_THREADS_DISCOVERY
29020     ABORT("GC DllMain-based thread registration unsupported");
29021 #else
29022     GC_ASSERT(!parallel_initialized);
29023 #ifndef GC_DISCOVER_TASK_THREADS
29024       GC_win32_dll_threads = TRUE;
29025 #endif
29026     GC_init_parallel();
29027 #endif
29028 }
29029 #define ADDR_LIMIT ((ptr_t)GC_WORD_MAX)
29030 struct GC_Thread_Rep {
29031   union {
29032 #ifndef GC_NO_THREADS_DISCOVERY
29033       volatile AO_t in_use;
29034       LONG long_in_use;
29035 #endif
29036     struct GC_Thread_Rep * next;
29037   } tm;
29038   DWORD id;
29039 #ifdef MSWINCE
29040 #define THREAD_HANDLE(t) (HANDLE)(word)(t)->id
29041 #else
29042     HANDLE handle;
29043 #define THREAD_HANDLE(t) (t)->handle
29044 #endif
29045   ptr_t stack_base;
29046   ptr_t last_stack_min;
29047 #ifdef IA64
29048     ptr_t backing_store_end;
29049     ptr_t backing_store_ptr;
29050 #elif defined(I386)
29051     ptr_t initial_stack_base;
29052 #endif
29053   ptr_t thread_blocked_sp;
29054   struct GC_traced_stack_sect_s *traced_stack_sect;
29055   unsigned short finalizer_skipped;
29056   unsigned char finalizer_nested;
29057   unsigned char suspended;
29058 #ifdef GC_PTHREADS
29059     unsigned char flags;
29060 #define FINISHED 1
29061 #define DETACHED 2
29062 #define KNOWN_FINISHED(t) (((t) -> flags) & FINISHED)
29063     pthread_t pthread_id;
29064     void *status;
29065 #else
29066 #define KNOWN_FINISHED(t) 0
29067 #endif
29068 #ifdef THREAD_LOCAL_ALLOC
29069     struct thread_local_freelists tlfs;
29070 #endif
29071 #ifdef RETRY_GET_THREAD_CONTEXT
29072     ptr_t context_sp;
29073     word context_regs[PUSHED_REGS_COUNT];
29074 #endif
29075 };
29076 typedef struct GC_Thread_Rep * GC_thread;
29077 typedef volatile struct GC_Thread_Rep * GC_vthread;
29078 #ifndef GC_NO_THREADS_DISCOVERY
29079   STATIC DWORD GC_main_thread = 0;
29080   STATIC volatile AO_t GC_attached_thread = FALSE;
29081   STATIC volatile GC_bool GC_please_stop = FALSE;
29082 #elif defined(GC_ASSERTIONS)
29083   STATIC GC_bool GC_please_stop = FALSE;
29084 #endif
29085 #if defined(WRAP_MARK_SOME) && !defined(GC_PTHREADS)
29086   GC_INNER GC_bool GC_started_thread_while_stopped(void)
29087   {
29088 #ifndef GC_NO_THREADS_DISCOVERY
29089       if (GC_win32_dll_threads) {
29090 #ifdef AO_HAVE_compare_and_swap_release
29091           if (AO_compare_and_swap_release(&GC_attached_thread, TRUE,
29092                                           FALSE ))
29093             return TRUE;
29094 #else
29095           AO_nop_full();
29096           if (AO_load(&GC_attached_thread)) {
29097             AO_store(&GC_attached_thread, FALSE);
29098             return TRUE;
29099           }
29100 #endif
29101       }
29102 #endif
29103     return FALSE;
29104   }
29105 #endif
29106 #ifndef MAX_THREADS
29107 #define MAX_THREADS 512
29108 #endif
29109 volatile struct GC_Thread_Rep dll_thread_table[MAX_THREADS];
29110 STATIC volatile LONG GC_max_thread_index = 0;
29111 #ifndef THREAD_TABLE_SZ
29112 #define THREAD_TABLE_SZ 256
29113 #endif
29114 #define THREAD_TABLE_INDEX(id)  \
29115                 (int)((((id) >> 8) ^ (id)) % THREAD_TABLE_SZ)
29116 STATIC GC_thread GC_threads[THREAD_TABLE_SZ];
29117 static struct GC_Thread_Rep first_thread;
29118 static GC_bool first_thread_used = FALSE;
29119 STATIC GC_thread GC_new_thread(DWORD id)
29120 {
29121   int hv = THREAD_TABLE_INDEX(id);
29122   GC_thread result;
29123 #ifdef DEBUG_THREADS
29124     GC_log_printf("Creating thread 0x%lx\n", (long)id);
29125     if (GC_threads[hv] != NULL)
29126       GC_log_printf("Hash collision at GC_threads[%d]\n", hv);
29127 #endif
29128   GC_ASSERT(I_HOLD_LOCK());
29129   if (!EXPECT(first_thread_used, TRUE)) {
29130     result = &first_thread;
29131     first_thread_used = TRUE;
29132     GC_ASSERT(NULL == GC_threads[hv]);
29133   } else {
29134     GC_ASSERT(!GC_win32_dll_threads);
29135     result = (struct GC_Thread_Rep *)
29136                 GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep), NORMAL);
29137     if (result == 0) return(0);
29138   }
29139   result -> tm.next = GC_threads[hv];
29140   GC_threads[hv] = result;
29141 #ifdef GC_PTHREADS
29142     GC_ASSERT(result -> flags == 0);
29143 #endif
29144   GC_ASSERT(result -> thread_blocked_sp == NULL);
29145   if (EXPECT(result != &first_thread, TRUE))
29146     GC_dirty(result);
29147   return(result);
29148 }
29149 GC_INNER GC_bool GC_in_thread_creation = FALSE;
29150 GC_INLINE void GC_record_stack_base(GC_vthread me,
29151                                     const struct GC_stack_base *sb)
29152 {
29153   me -> stack_base = (ptr_t)sb->mem_base;
29154 #ifdef IA64
29155     me -> backing_store_end = (ptr_t)sb->reg_base;
29156 #elif defined(I386)
29157     me -> initial_stack_base = (ptr_t)sb->mem_base;
29158 #endif
29159   if (me -> stack_base == NULL)
29160     ABORT("Bad stack base in GC_register_my_thread");
29161 }
29162 STATIC GC_thread GC_register_my_thread_inner(const struct GC_stack_base *sb,
29163                                              DWORD thread_id)
29164 {
29165   GC_vthread me;
29166 #if defined(MPROTECT_VDB) && !defined(CYGWIN32)
29167     if (GC_auto_incremental
29168 #ifdef GWW_VDB
29169           && !GC_gww_dirty_init()
29170 #endif
29171         )
29172       GC_set_write_fault_handler();
29173 #endif
29174 #ifndef GC_NO_THREADS_DISCOVERY
29175     if (GC_win32_dll_threads) {
29176       int i;
29177       for (i = 0;
29178            InterlockedExchange(&dll_thread_table[i].tm.long_in_use, 1) != 0;
29179            i++) {
29180         if (i == MAX_THREADS - 1)
29181           ABORT("Too many threads");
29182       }
29183       while (i > GC_max_thread_index) {
29184         InterlockedIncrement((IE_t)&GC_max_thread_index);
29185       }
29186       if (GC_max_thread_index >= MAX_THREADS) {
29187         GC_max_thread_index = MAX_THREADS - 1;
29188       }
29189       me = dll_thread_table + i;
29190     } else
29191 #endif
29192     {
29193     GC_ASSERT(I_HOLD_LOCK());
29194     GC_in_thread_creation = TRUE;
29195     me = GC_new_thread(thread_id);
29196     GC_in_thread_creation = FALSE;
29197     if (me == 0)
29198       ABORT("Failed to allocate memory for thread registering");
29199   }
29200 #ifdef GC_PTHREADS
29201     me -> pthread_id = pthread_self();
29202 #endif
29203 #ifndef MSWINCE
29204     if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
29205                         GetCurrentProcess(),
29206                         (HANDLE*)&(me -> handle),
29207                         0 , FALSE ,
29208                         DUPLICATE_SAME_ACCESS)) {
29209         ABORT_ARG1("DuplicateHandle failed",
29210                    ": errcode= 0x%X", (unsigned)GetLastError());
29211     }
29212 #endif
29213   me -> last_stack_min = ADDR_LIMIT;
29214   GC_record_stack_base(me, sb);
29215   me -> id = thread_id;
29216 #if defined(THREAD_LOCAL_ALLOC)
29217     GC_init_thread_local((GC_tlfs)(&(me->tlfs)));
29218 #endif
29219 #ifndef GC_NO_THREADS_DISCOVERY
29220     if (GC_win32_dll_threads) {
29221       if (GC_please_stop) {
29222         AO_store(&GC_attached_thread, TRUE);
29223         AO_nop_full();
29224       }
29225     } else
29226 #endif
29227    {
29228     GC_ASSERT(!GC_please_stop);
29229   }
29230   return (GC_thread)(me);
29231 }
29232 GC_INLINE LONG GC_get_max_thread_index(void)
29233 {
29234   LONG my_max = GC_max_thread_index;
29235   if (my_max >= MAX_THREADS) return MAX_THREADS - 1;
29236   return my_max;
29237 }
29238 STATIC GC_thread GC_lookup_thread_inner(DWORD thread_id)
29239 {
29240 #ifndef GC_NO_THREADS_DISCOVERY
29241     if (GC_win32_dll_threads) {
29242       int i;
29243       LONG my_max = GC_get_max_thread_index();
29244       for (i = 0; i <= my_max &&
29245                   (!AO_load_acquire(&dll_thread_table[i].tm.in_use)
29246                   || dll_thread_table[i].id != thread_id);
29247            i++) {
29248       }
29249       return i <= my_max ? (GC_thread)(dll_thread_table + i) : NULL;
29250     } else
29251 #endif
29252    {
29253     GC_thread p = GC_threads[THREAD_TABLE_INDEX(thread_id)];
29254     GC_ASSERT(I_HOLD_LOCK());
29255     while (p != 0 && p -> id != thread_id) p = p -> tm.next;
29256     return(p);
29257   }
29258 }
29259 #ifdef LINT2
29260 #define CHECK_LOOKUP_MY_THREAD(me) \
29261         if (!(me)) ABORT("GC_lookup_thread_inner(GetCurrentThreadId) failed")
29262 #else
29263 #define CHECK_LOOKUP_MY_THREAD(me)
29264 #endif
29265 GC_INNER void GC_reset_finalizer_nested(void)
29266 {
29267   GC_thread me = GC_lookup_thread_inner(GetCurrentThreadId());
29268   CHECK_LOOKUP_MY_THREAD(me);
29269   me->finalizer_nested = 0;
29270 }
29271 GC_INNER unsigned char *GC_check_finalizer_nested(void)
29272 {
29273   GC_thread me = GC_lookup_thread_inner(GetCurrentThreadId());
29274   unsigned nesting_level;
29275   CHECK_LOOKUP_MY_THREAD(me);
29276   nesting_level = me->finalizer_nested;
29277   if (nesting_level) {
29278     if (++me->finalizer_skipped < (1U << nesting_level)) return NULL;
29279     me->finalizer_skipped = 0;
29280   }
29281   me->finalizer_nested = (unsigned char)(nesting_level + 1);
29282   return &me->finalizer_nested;
29283 }
29284 #if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
29285   GC_bool GC_is_thread_tsd_valid(void *tsd)
29286   {
29287     GC_thread me;
29288     DCL_LOCK_STATE;
29289     LOCK();
29290     me = GC_lookup_thread_inner(GetCurrentThreadId());
29291     UNLOCK();
29292     return (word)tsd >= (word)(&me->tlfs)
29293             && (word)tsd < (word)(&me->tlfs) + sizeof(me->tlfs);
29294   }
29295 #endif
29296 GC_API int GC_CALL GC_thread_is_registered(void)
29297 {
29298     DWORD thread_id = GetCurrentThreadId();
29299     GC_thread me;
29300     DCL_LOCK_STATE;
29301     LOCK();
29302     me = GC_lookup_thread_inner(thread_id);
29303     UNLOCK();
29304     return me != NULL;
29305 }
29306 GC_API void GC_CALL GC_register_altstack(void *stack GC_ATTR_UNUSED,
29307                                          GC_word stack_size GC_ATTR_UNUSED,
29308                                          void *altstack GC_ATTR_UNUSED,
29309                                          GC_word altstack_size GC_ATTR_UNUSED)
29310 {
29311 }
29312 #if defined(MPROTECT_VDB)
29313 #define UNPROTECT_THREAD(t) \
29314     if (!GC_win32_dll_threads && GC_auto_incremental \
29315         && t != &first_thread) { \
29316       GC_ASSERT(SMALL_OBJ(GC_size(t))); \
29317       GC_remove_protection(HBLKPTR(t), 1, FALSE); \
29318     } else (void)0
29319 #else
29320 #define UNPROTECT_THREAD(t) (void)0
29321 #endif
29322 #ifdef CYGWIN32
29323 #define GC_PTHREAD_PTRVAL(pthread_id) pthread_id
29324 #elif defined(GC_WIN32_PTHREADS) || defined(GC_PTHREADS_PARAMARK)
29325 #include <pthread.h>
29326 #if defined(__WINPTHREADS_VERSION_MAJOR)
29327 #define GC_PTHREAD_PTRVAL(pthread_id) pthread_id
29328 #else
29329 #define GC_PTHREAD_PTRVAL(pthread_id) pthread_id.p
29330 #endif
29331 #endif
29332 STATIC void GC_delete_gc_thread_no_free(GC_vthread t)
29333 {
29334 #ifndef MSWINCE
29335     CloseHandle(t->handle);
29336 #endif
29337 #ifndef GC_NO_THREADS_DISCOVERY
29338     if (GC_win32_dll_threads) {
29339       t -> stack_base = 0;
29340       t -> id = 0;
29341       t -> suspended = FALSE;
29342 #ifdef RETRY_GET_THREAD_CONTEXT
29343         t -> context_sp = NULL;
29344 #endif
29345       AO_store_release(&t->tm.in_use, FALSE);
29346     } else
29347 #endif
29348    {
29349     DWORD id = ((GC_thread)t) -> id;
29350     int hv = THREAD_TABLE_INDEX(id);
29351     GC_thread p = GC_threads[hv];
29352     GC_thread prev = NULL;
29353     GC_ASSERT(I_HOLD_LOCK());
29354     while (p != (GC_thread)t) {
29355       prev = p;
29356       p = p -> tm.next;
29357     }
29358     if (prev == 0) {
29359       GC_threads[hv] = p -> tm.next;
29360     } else {
29361       GC_ASSERT(prev != &first_thread);
29362       prev -> tm.next = p -> tm.next;
29363       GC_dirty(prev);
29364     }
29365   }
29366 }
29367 STATIC void GC_delete_thread(DWORD id)
29368 {
29369   if (GC_win32_dll_threads) {
29370     GC_vthread t = GC_lookup_thread_inner(id);
29371     if (0 == t) {
29372       WARN("Removing nonexistent thread, id= %" WARN_PRIdPTR "\n", id);
29373     } else {
29374       GC_delete_gc_thread_no_free(t);
29375     }
29376   } else {
29377     int hv = THREAD_TABLE_INDEX(id);
29378     GC_thread p = GC_threads[hv];
29379     GC_thread prev = NULL;
29380     GC_ASSERT(I_HOLD_LOCK());
29381     while (p -> id != id) {
29382       prev = p;
29383       p = p -> tm.next;
29384     }
29385 #ifndef MSWINCE
29386       CloseHandle(p->handle);
29387 #endif
29388     if (prev == 0) {
29389       GC_threads[hv] = p -> tm.next;
29390     } else {
29391       GC_ASSERT(prev != &first_thread);
29392       prev -> tm.next = p -> tm.next;
29393       GC_dirty(prev);
29394     }
29395     if (EXPECT(p != &first_thread, TRUE)) {
29396       GC_INTERNAL_FREE(p);
29397     }
29398   }
29399 }
29400 GC_API void GC_CALL GC_allow_register_threads(void)
29401 {
29402   GC_ASSERT(GC_lookup_thread_inner(GetCurrentThreadId()) != 0);
29403 #if !defined(GC_ALWAYS_MULTITHREADED) && !defined(PARALLEL_MARK) \
29404      && !defined(GC_NO_THREADS_DISCOVERY)
29405     parallel_initialized = TRUE;
29406 #endif
29407   set_need_to_lock();
29408 }
29409 GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base *sb)
29410 {
29411   GC_thread me;
29412   DWORD thread_id = GetCurrentThreadId();
29413   DCL_LOCK_STATE;
29414   if (GC_need_to_lock == FALSE)
29415     ABORT("Threads explicit registering is not previously enabled");
29416   LOCK();
29417   me = GC_lookup_thread_inner(thread_id);
29418   if (me == 0) {
29419 #ifdef GC_PTHREADS
29420       me = GC_register_my_thread_inner(sb, thread_id);
29421 #if defined(CPPCHECK)
29422         GC_noop1(me->flags);
29423 #endif
29424       me -> flags |= DETACHED;
29425 #else
29426       GC_register_my_thread_inner(sb, thread_id);
29427 #endif
29428     UNLOCK();
29429     return GC_SUCCESS;
29430   } else
29431 #ifdef GC_PTHREADS
29432        if ((me -> flags & FINISHED) != 0) {
29433         GC_record_stack_base(me, sb);
29434         me -> flags &= ~FINISHED;
29435 #ifdef THREAD_LOCAL_ALLOC
29436           GC_init_thread_local((GC_tlfs)(&me->tlfs));
29437 #endif
29438         UNLOCK();
29439         return GC_SUCCESS;
29440       } else
29441 #endif
29442    {
29443     UNLOCK();
29444     return GC_DUPLICATE;
29445   }
29446 }
29447 STATIC void GC_wait_for_gc_completion(GC_bool wait_for_all)
29448 {
29449   GC_ASSERT(I_HOLD_LOCK());
29450   if (GC_incremental && GC_collection_in_progress()) {
29451     word old_gc_no = GC_gc_no;
29452     do {
29453       ENTER_GC();
29454       GC_in_thread_creation = TRUE;
29455       GC_collect_a_little_inner(1);
29456       GC_in_thread_creation = FALSE;
29457       EXIT_GC();
29458       UNLOCK();
29459       Sleep(0);
29460       LOCK();
29461     } while (GC_incremental && GC_collection_in_progress()
29462              && (wait_for_all || old_gc_no == GC_gc_no));
29463   }
29464 }
29465 GC_API int GC_CALL GC_unregister_my_thread(void)
29466 {
29467   DCL_LOCK_STATE;
29468 #ifdef DEBUG_THREADS
29469     GC_log_printf("Unregistering thread 0x%lx\n", (long)GetCurrentThreadId());
29470 #endif
29471   if (GC_win32_dll_threads) {
29472 #if defined(THREAD_LOCAL_ALLOC)
29473       GC_ASSERT(FALSE);
29474 #else
29475       GC_delete_thread(GetCurrentThreadId());
29476 #endif
29477   } else {
29478 #if defined(THREAD_LOCAL_ALLOC) || defined(GC_PTHREADS)
29479       GC_thread me;
29480 #endif
29481     DWORD thread_id = GetCurrentThreadId();
29482     LOCK();
29483     GC_wait_for_gc_completion(FALSE);
29484 #if defined(THREAD_LOCAL_ALLOC) || defined(GC_PTHREADS)
29485       me = GC_lookup_thread_inner(thread_id);
29486       CHECK_LOOKUP_MY_THREAD(me);
29487       GC_ASSERT(!KNOWN_FINISHED(me));
29488 #endif
29489 #if defined(THREAD_LOCAL_ALLOC)
29490       GC_ASSERT(GC_getspecific(GC_thread_key) == &me->tlfs);
29491       GC_destroy_thread_local(&(me->tlfs));
29492 #endif
29493 #ifdef GC_PTHREADS
29494       if ((me -> flags & DETACHED) == 0) {
29495         me -> flags |= FINISHED;
29496       } else
29497 #endif
29498      {
29499       GC_delete_thread(thread_id);
29500     }
29501 #if defined(THREAD_LOCAL_ALLOC)
29502       GC_remove_specific(GC_thread_key);
29503 #endif
29504     UNLOCK();
29505   }
29506   return GC_SUCCESS;
29507 }
29508 GC_INNER void GC_do_blocking_inner(ptr_t data, void * context GC_ATTR_UNUSED)
29509 {
29510   struct blocking_data * d = (struct blocking_data *) data;
29511   DWORD thread_id = GetCurrentThreadId();
29512   GC_thread me;
29513 #ifdef IA64
29514     ptr_t stack_ptr = GC_save_regs_in_stack();
29515 #endif
29516   DCL_LOCK_STATE;
29517   LOCK();
29518   me = GC_lookup_thread_inner(thread_id);
29519   CHECK_LOOKUP_MY_THREAD(me);
29520   GC_ASSERT(me -> thread_blocked_sp == NULL);
29521 #ifdef IA64
29522     me -> backing_store_ptr = stack_ptr;
29523 #endif
29524   me -> thread_blocked_sp = (ptr_t) &d;
29525   UNLOCK();
29526   d -> client_data = (d -> fn)(d -> client_data);
29527   LOCK();
29528 #if defined(CPPCHECK)
29529     GC_noop1((word)me->thread_blocked_sp);
29530 #endif
29531   me -> thread_blocked_sp = NULL;
29532   UNLOCK();
29533 }
29534 GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type fn,
29535                                              void * client_data)
29536 {
29537   struct GC_traced_stack_sect_s stacksect;
29538   DWORD thread_id = GetCurrentThreadId();
29539   GC_thread me;
29540   DCL_LOCK_STATE;
29541   LOCK();
29542   me = GC_lookup_thread_inner(thread_id);
29543   CHECK_LOOKUP_MY_THREAD(me);
29544   GC_ASSERT(me -> stack_base != NULL);
29545   if ((word)me->stack_base < (word)(&stacksect)) {
29546     me -> stack_base = (ptr_t)(&stacksect);
29547 #if defined(I386)
29548       me -> initial_stack_base = me -> stack_base;
29549 #endif
29550   }
29551   if (me -> thread_blocked_sp == NULL) {
29552     UNLOCK();
29553     client_data = fn(client_data);
29554     GC_noop1(COVERT_DATAFLOW(&stacksect));
29555     return client_data;
29556   }
29557   stacksect.saved_stack_ptr = me -> thread_blocked_sp;
29558 #ifdef IA64
29559     stacksect.backing_store_end = GC_save_regs_in_stack();
29560     stacksect.saved_backing_store_ptr = me -> backing_store_ptr;
29561 #endif
29562   stacksect.prev = me -> traced_stack_sect;
29563   me -> thread_blocked_sp = NULL;
29564   me -> traced_stack_sect = &stacksect;
29565   UNLOCK();
29566   client_data = fn(client_data);
29567   GC_ASSERT(me -> thread_blocked_sp == NULL);
29568   GC_ASSERT(me -> traced_stack_sect == &stacksect);
29569   LOCK();
29570 #if defined(CPPCHECK)
29571     GC_noop1((word)me->traced_stack_sect);
29572 #endif
29573   me -> traced_stack_sect = stacksect.prev;
29574 #ifdef IA64
29575     me -> backing_store_ptr = stacksect.saved_backing_store_ptr;
29576 #endif
29577   me -> thread_blocked_sp = stacksect.saved_stack_ptr;
29578   UNLOCK();
29579   return client_data;
29580 }
29581 GC_API void GC_CALL GC_set_stackbottom(void *gc_thread_handle,
29582                                        const struct GC_stack_base *sb)
29583 {
29584   GC_thread t = (GC_thread)gc_thread_handle;
29585   GC_ASSERT(sb -> mem_base != NULL);
29586   if (!EXPECT(GC_is_initialized, TRUE)) {
29587     GC_ASSERT(NULL == t);
29588     GC_stackbottom = (char *)sb->mem_base;
29589 #ifdef IA64
29590       GC_register_stackbottom = (ptr_t)sb->reg_base;
29591 #endif
29592     return;
29593   }
29594   GC_ASSERT(I_HOLD_LOCK());
29595   if (NULL == t) {
29596     t = GC_lookup_thread_inner(GetCurrentThreadId());
29597     CHECK_LOOKUP_MY_THREAD(t);
29598   }
29599   GC_ASSERT(!KNOWN_FINISHED(t));
29600   GC_ASSERT(NULL == t -> thread_blocked_sp
29601             && NULL == t -> traced_stack_sect);
29602   t -> stack_base = (ptr_t)sb->mem_base;
29603   t -> last_stack_min = ADDR_LIMIT;
29604 #ifdef IA64
29605     t -> backing_store_end = (ptr_t)sb->reg_base;
29606 #endif
29607 }
29608 GC_API void * GC_CALL GC_get_my_stackbottom(struct GC_stack_base *sb)
29609 {
29610   DWORD thread_id = GetCurrentThreadId();
29611   GC_thread me;
29612   DCL_LOCK_STATE;
29613   LOCK();
29614   me = GC_lookup_thread_inner(thread_id);
29615   CHECK_LOOKUP_MY_THREAD(me);
29616   sb -> mem_base = me -> stack_base;
29617 #ifdef IA64
29618     sb -> reg_base = me -> backing_store_end;
29619 #endif
29620   UNLOCK();
29621   return (void *)me;
29622 }
29623 #ifdef GC_PTHREADS
29624 #define PTHREAD_MAP_SIZE 512
29625   DWORD GC_pthread_map_cache[PTHREAD_MAP_SIZE] = {0};
29626 #define PTHREAD_MAP_INDEX(pthread_id) \
29627                 ((NUMERIC_THREAD_ID(pthread_id) >> 5) % PTHREAD_MAP_SIZE)
29628 #define SET_PTHREAD_MAP_CACHE(pthread_id, win32_id) \
29629       (void)(GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)] = (win32_id))
29630 #define GET_PTHREAD_MAP_CACHE(pthread_id) \
29631           GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)]
29632   STATIC GC_thread GC_lookup_pthread(pthread_t id)
29633   {
29634 #ifndef GC_NO_THREADS_DISCOVERY
29635       if (GC_win32_dll_threads) {
29636         int i;
29637         LONG my_max = GC_get_max_thread_index();
29638         for (i = 0; i <= my_max &&
29639                     (!AO_load_acquire(&dll_thread_table[i].tm.in_use)
29640                     || THREAD_EQUAL(dll_thread_table[i].pthread_id, id));
29641              i++) {
29642         }
29643         return i <= my_max ? (GC_thread)(dll_thread_table + i) : NULL;
29644       } else
29645 #endif
29646      {
29647       DWORD win32_id = GET_PTHREAD_MAP_CACHE(id);
29648       int hv_guess = THREAD_TABLE_INDEX(win32_id);
29649       int hv;
29650       GC_thread p;
29651       DCL_LOCK_STATE;
29652       LOCK();
29653       for (p = GC_threads[hv_guess]; 0 != p; p = p -> tm.next) {
29654         if (THREAD_EQUAL(p -> pthread_id, id))
29655           goto foundit;
29656       }
29657       for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
29658         for (p = GC_threads[hv]; 0 != p; p = p -> tm.next) {
29659           if (THREAD_EQUAL(p -> pthread_id, id))
29660             goto foundit;
29661         }
29662       }
29663       p = 0;
29664      foundit:
29665       UNLOCK();
29666       return p;
29667     }
29668   }
29669 #endif
29670 #ifdef CAN_HANDLE_FORK
29671     STATIC void GC_remove_all_threads_but_me(void)
29672     {
29673       int hv;
29674       GC_thread me = NULL;
29675       DWORD thread_id;
29676       pthread_t pthread_id = pthread_self();
29677       GC_ASSERT(!GC_win32_dll_threads);
29678       for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
29679         GC_thread p, next;
29680         for (p = GC_threads[hv]; 0 != p; p = next) {
29681           next = p -> tm.next;
29682           if (THREAD_EQUAL(p -> pthread_id, pthread_id)
29683               && me == NULL) {
29684             me = p;
29685             p -> tm.next = 0;
29686           } else {
29687 #ifdef THREAD_LOCAL_ALLOC
29688               if ((p -> flags & FINISHED) == 0) {
29689                 GC_remove_specific_after_fork(GC_thread_key, p -> pthread_id);
29690               }
29691 #endif
29692             if (&first_thread != p)
29693               GC_INTERNAL_FREE(p);
29694           }
29695         }
29696         GC_threads[hv] = NULL;
29697       }
29698       GC_ASSERT(me != NULL);
29699       thread_id = GetCurrentThreadId();
29700       GC_threads[THREAD_TABLE_INDEX(thread_id)] = me;
29701       me -> id = thread_id;
29702 #ifndef MSWINCE
29703         if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
29704                         GetCurrentProcess(), (HANDLE *)&me->handle,
29705                         0 , FALSE ,
29706                         DUPLICATE_SAME_ACCESS))
29707           ABORT("DuplicateHandle failed");
29708 #endif
29709 #if defined(THREAD_LOCAL_ALLOC) && !defined(USE_CUSTOM_SPECIFIC)
29710         if (GC_setspecific(GC_thread_key, &me->tlfs) != 0)
29711           ABORT("GC_setspecific failed (in child)");
29712 #endif
29713     }
29714     static void fork_prepare_proc(void)
29715     {
29716       LOCK();
29717 #ifdef PARALLEL_MARK
29718         if (GC_parallel)
29719           GC_wait_for_reclaim();
29720 #endif
29721       GC_wait_for_gc_completion(TRUE);
29722 #ifdef PARALLEL_MARK
29723         if (GC_parallel)
29724           GC_acquire_mark_lock();
29725 #endif
29726     }
29727     static void fork_parent_proc(void)
29728     {
29729 #ifdef PARALLEL_MARK
29730         if (GC_parallel)
29731           GC_release_mark_lock();
29732 #endif
29733       UNLOCK();
29734     }
29735     static void fork_child_proc(void)
29736     {
29737 #ifdef PARALLEL_MARK
29738         if (GC_parallel) {
29739           GC_release_mark_lock();
29740           GC_parallel = FALSE;
29741         }
29742 #endif
29743       GC_remove_all_threads_but_me();
29744       UNLOCK();
29745     }
29746   GC_API void GC_CALL GC_atfork_prepare(void)
29747   {
29748     if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
29749     if (GC_handle_fork <= 0)
29750       fork_prepare_proc();
29751   }
29752   GC_API void GC_CALL GC_atfork_parent(void)
29753   {
29754     if (GC_handle_fork <= 0)
29755       fork_parent_proc();
29756   }
29757   GC_API void GC_CALL GC_atfork_child(void)
29758   {
29759     if (GC_handle_fork <= 0)
29760       fork_child_proc();
29761   }
29762 #endif
29763 void GC_push_thread_structures(void)
29764 {
29765   GC_ASSERT(I_HOLD_LOCK());
29766 #ifndef GC_NO_THREADS_DISCOVERY
29767     if (GC_win32_dll_threads) {
29768     } else
29769 #endif
29770    {
29771     GC_PUSH_ALL_SYM(GC_threads);
29772   }
29773 #if defined(THREAD_LOCAL_ALLOC)
29774     GC_PUSH_ALL_SYM(GC_thread_key);
29775 #endif
29776 }
29777 #ifdef WOW64_THREAD_CONTEXT_WORKAROUND
29778 #ifndef CONTEXT_EXCEPTION_ACTIVE
29779 #define CONTEXT_EXCEPTION_ACTIVE    0x08000000
29780 #define CONTEXT_EXCEPTION_REQUEST   0x40000000
29781 #define CONTEXT_EXCEPTION_REPORTING 0x80000000
29782 #endif
29783   static BOOL isWow64;
29784 #define GET_THREAD_CONTEXT_FLAGS (isWow64 \
29785                         ? CONTEXT_INTEGER | CONTEXT_CONTROL \
29786                           | CONTEXT_EXCEPTION_REQUEST | CONTEXT_SEGMENTS \
29787                         : CONTEXT_INTEGER | CONTEXT_CONTROL)
29788 #else
29789 #define GET_THREAD_CONTEXT_FLAGS (CONTEXT_INTEGER | CONTEXT_CONTROL)
29790 #endif
29791 STATIC void GC_suspend(GC_thread t)
29792 {
29793 #ifndef MSWINCE
29794     DWORD exitCode;
29795 #ifdef RETRY_GET_THREAD_CONTEXT
29796       int retry_cnt;
29797 #define MAX_SUSPEND_THREAD_RETRIES (1000 * 1000)
29798 #endif
29799 #endif
29800 #ifdef DEBUG_THREADS
29801     GC_log_printf("Suspending 0x%x\n", (int)t->id);
29802 #endif
29803   UNPROTECT_THREAD(t);
29804   GC_acquire_dirty_lock();
29805 #ifdef MSWINCE
29806     while (SuspendThread(THREAD_HANDLE(t)) == (DWORD)-1) {
29807       GC_release_dirty_lock();
29808       Sleep(10);
29809       GC_acquire_dirty_lock();
29810     }
29811 #elif defined(RETRY_GET_THREAD_CONTEXT)
29812     for (retry_cnt = 0;;) {
29813       if (GetExitCodeThread(t -> handle, &exitCode)
29814           && exitCode != STILL_ACTIVE) {
29815         GC_release_dirty_lock();
29816 #ifdef GC_PTHREADS
29817           t -> stack_base = 0;
29818 #else
29819           GC_ASSERT(GC_win32_dll_threads);
29820           GC_delete_gc_thread_no_free(t);
29821 #endif
29822         return;
29823       }
29824       if (SuspendThread(t->handle) != (DWORD)-1) {
29825         CONTEXT context;
29826         context.ContextFlags = GET_THREAD_CONTEXT_FLAGS;
29827         if (GetThreadContext(t->handle, &context)) {
29828           t->context_sp = copy_ptr_regs(t->context_regs, &context);
29829           break;
29830         }
29831         if (ResumeThread(t->handle) == (DWORD)-1)
29832           ABORT("ResumeThread failed in suspend loop");
29833       }
29834       if (retry_cnt > 1) {
29835         GC_release_dirty_lock();
29836         Sleep(0);
29837         GC_acquire_dirty_lock();
29838       }
29839       if (++retry_cnt >= MAX_SUSPEND_THREAD_RETRIES)
29840         ABORT("SuspendThread loop failed");
29841     }
29842 #else
29843     if (GetExitCodeThread(t -> handle, &exitCode)
29844         && exitCode != STILL_ACTIVE) {
29845       GC_release_dirty_lock();
29846 #ifdef GC_PTHREADS
29847         t -> stack_base = 0;
29848 #else
29849         GC_ASSERT(GC_win32_dll_threads);
29850         GC_delete_gc_thread_no_free(t);
29851 #endif
29852       return;
29853     }
29854     if (SuspendThread(t -> handle) == (DWORD)-1)
29855       ABORT("SuspendThread failed");
29856 #endif
29857   t -> suspended = (unsigned char)TRUE;
29858   GC_release_dirty_lock();
29859   if (GC_on_thread_event)
29860     GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, THREAD_HANDLE(t));
29861 }
29862 #if defined(GC_ASSERTIONS) \
29863     && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
29864   GC_INNER GC_bool GC_write_disabled = FALSE;
29865 #endif
29866 GC_INNER void GC_stop_world(void)
29867 {
29868   DWORD thread_id = GetCurrentThreadId();
29869   if (!GC_thr_initialized)
29870     ABORT("GC_stop_world() called before GC_thr_init()");
29871   GC_ASSERT(I_HOLD_LOCK());
29872 #ifdef PARALLEL_MARK
29873     if (GC_parallel) {
29874       GC_acquire_mark_lock();
29875       GC_ASSERT(GC_fl_builder_count == 0);
29876     }
29877 #endif
29878 #if !defined(GC_NO_THREADS_DISCOVERY) || defined(GC_ASSERTIONS)
29879     GC_please_stop = TRUE;
29880 #endif
29881 #if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
29882     GC_ASSERT(!GC_write_disabled);
29883     EnterCriticalSection(&GC_write_cs);
29884 #ifdef GC_ASSERTIONS
29885       GC_write_disabled = TRUE;
29886 #endif
29887 #endif
29888 #ifndef GC_NO_THREADS_DISCOVERY
29889     if (GC_win32_dll_threads) {
29890       int i;
29891       int my_max;
29892       AO_store(&GC_attached_thread, FALSE);
29893       my_max = (int)GC_get_max_thread_index();
29894       for (i = 0; i <= my_max; i++) {
29895         GC_vthread t = dll_thread_table + i;
29896         if (t -> stack_base != 0 && t -> thread_blocked_sp == NULL
29897             && t -> id != thread_id) {
29898           GC_suspend((GC_thread)t);
29899         }
29900       }
29901     } else
29902 #endif
29903    {
29904     GC_thread t;
29905     int i;
29906     for (i = 0; i < THREAD_TABLE_SZ; i++) {
29907       for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
29908         if (t -> stack_base != 0 && t -> thread_blocked_sp == NULL
29909             && !KNOWN_FINISHED(t) && t -> id != thread_id) {
29910           GC_suspend(t);
29911         }
29912       }
29913     }
29914   }
29915 #if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
29916 #ifdef GC_ASSERTIONS
29917       GC_write_disabled = FALSE;
29918 #endif
29919     LeaveCriticalSection(&GC_write_cs);
29920 #endif
29921 #ifdef PARALLEL_MARK
29922     if (GC_parallel)
29923       GC_release_mark_lock();
29924 #endif
29925 }
29926 GC_INNER void GC_start_world(void)
29927 {
29928 #ifdef GC_ASSERTIONS
29929     DWORD thread_id = GetCurrentThreadId();
29930 #endif
29931   GC_ASSERT(I_HOLD_LOCK());
29932   if (GC_win32_dll_threads) {
29933     LONG my_max = GC_get_max_thread_index();
29934     int i;
29935     for (i = 0; i <= my_max; i++) {
29936       GC_thread t = (GC_thread)(dll_thread_table + i);
29937       if (t -> suspended) {
29938 #ifdef DEBUG_THREADS
29939           GC_log_printf("Resuming 0x%x\n", (int)t->id);
29940 #endif
29941         GC_ASSERT(t -> stack_base != 0 && t -> id != thread_id);
29942         if (ResumeThread(THREAD_HANDLE(t)) == (DWORD)-1)
29943           ABORT("ResumeThread failed");
29944         t -> suspended = FALSE;
29945         if (GC_on_thread_event)
29946           GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, THREAD_HANDLE(t));
29947       }
29948     }
29949   } else {
29950     GC_thread t;
29951     int i;
29952     for (i = 0; i < THREAD_TABLE_SZ; i++) {
29953       for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
29954         if (t -> suspended) {
29955 #ifdef DEBUG_THREADS
29956             GC_log_printf("Resuming 0x%x\n", (int)t->id);
29957 #endif
29958           GC_ASSERT(t -> stack_base != 0 && t -> id != thread_id);
29959           if (ResumeThread(THREAD_HANDLE(t)) == (DWORD)-1)
29960             ABORT("ResumeThread failed");
29961           UNPROTECT_THREAD(t);
29962           t -> suspended = FALSE;
29963           if (GC_on_thread_event)
29964             GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, THREAD_HANDLE(t));
29965         } else {
29966 #ifdef DEBUG_THREADS
29967             GC_log_printf("Not resuming thread 0x%x as it is not suspended\n",
29968                           (int)t->id);
29969 #endif
29970         }
29971       }
29972     }
29973   }
29974 #if !defined(GC_NO_THREADS_DISCOVERY) || defined(GC_ASSERTIONS)
29975     GC_please_stop = FALSE;
29976 #endif
29977 }
29978 #ifdef MSWINCE
29979 #define GC_wince_evaluate_stack_min(s) \
29980                         (ptr_t)(((word)(s) - 1) & ~(word)0xFFFF)
29981 #elif defined(GC_ASSERTIONS)
29982 #define GC_dont_query_stack_min FALSE
29983 #endif
29984 static ptr_t last_address = 0;
29985 static MEMORY_BASIC_INFORMATION last_info;
29986 STATIC ptr_t GC_get_stack_min(ptr_t s)
29987 {
29988   ptr_t bottom;
29989   GC_ASSERT(I_HOLD_LOCK());
29990   if (s != last_address) {
29991     VirtualQuery(s, &last_info, sizeof(last_info));
29992     last_address = s;
29993   }
29994   do {
29995     bottom = (ptr_t)last_info.BaseAddress;
29996     VirtualQuery(bottom - 1, &last_info, sizeof(last_info));
29997     last_address = bottom - 1;
29998   } while ((last_info.Protect & PAGE_READWRITE)
29999            && !(last_info.Protect & PAGE_GUARD));
30000   return(bottom);
30001 }
30002 static GC_bool may_be_in_stack(ptr_t s)
30003 {
30004   GC_ASSERT(I_HOLD_LOCK());
30005   if (s != last_address) {
30006     VirtualQuery(s, &last_info, sizeof(last_info));
30007     last_address = s;
30008   }
30009   return (last_info.Protect & PAGE_READWRITE)
30010           && !(last_info.Protect & PAGE_GUARD);
30011 }
30012 static ptr_t copy_ptr_regs(word *regs, const CONTEXT *pcontext) {
30013     ptr_t sp;
30014     int cnt = 0;
30015 #define context (*pcontext)
30016 #define PUSH1(reg) (regs[cnt++] = (word)pcontext->reg)
30017 #define PUSH2(r1,r2) (PUSH1(r1), PUSH1(r2))
30018 #define PUSH4(r1,r2,r3,r4) (PUSH2(r1,r2), PUSH2(r3,r4))
30019 #if defined(I386)
30020 #ifdef WOW64_THREAD_CONTEXT_WORKAROUND
30021         PUSH2(ContextFlags, SegFs);
30022 #endif
30023       PUSH4(Edi,Esi,Ebx,Edx), PUSH2(Ecx,Eax), PUSH1(Ebp);
30024       sp = (ptr_t)context.Esp;
30025 #elif defined(X86_64)
30026       PUSH4(Rax,Rcx,Rdx,Rbx); PUSH2(Rbp, Rsi); PUSH1(Rdi);
30027       PUSH4(R8, R9, R10, R11); PUSH4(R12, R13, R14, R15);
30028       sp = (ptr_t)context.Rsp;
30029 #elif defined(ARM32)
30030       PUSH4(R0,R1,R2,R3),PUSH4(R4,R5,R6,R7),PUSH4(R8,R9,R10,R11);
30031       PUSH1(R12);
30032       sp = (ptr_t)context.Sp;
30033 #elif defined(AARCH64)
30034       PUSH4(X0,X1,X2,X3),PUSH4(X4,X5,X6,X7),PUSH4(X8,X9,X10,X11);
30035       PUSH4(X12,X13,X14,X15),PUSH4(X16,X17,X18,X19),PUSH4(X20,X21,X22,X23);
30036       PUSH4(X24,X25,X26,X27),PUSH1(X28);
30037       PUSH1(Lr);
30038       sp = (ptr_t)context.Sp;
30039 #elif defined(SHx)
30040       PUSH4(R0,R1,R2,R3), PUSH4(R4,R5,R6,R7), PUSH4(R8,R9,R10,R11);
30041       PUSH2(R12,R13), PUSH1(R14);
30042       sp = (ptr_t)context.R15;
30043 #elif defined(MIPS)
30044       PUSH4(IntAt,IntV0,IntV1,IntA0), PUSH4(IntA1,IntA2,IntA3,IntT0);
30045       PUSH4(IntT1,IntT2,IntT3,IntT4), PUSH4(IntT5,IntT6,IntT7,IntS0);
30046       PUSH4(IntS1,IntS2,IntS3,IntS4), PUSH4(IntS5,IntS6,IntS7,IntT8);
30047       PUSH4(IntT9,IntK0,IntK1,IntS8);
30048       sp = (ptr_t)context.IntSp;
30049 #elif defined(PPC)
30050       PUSH4(Gpr0, Gpr3, Gpr4, Gpr5),  PUSH4(Gpr6, Gpr7, Gpr8, Gpr9);
30051       PUSH4(Gpr10,Gpr11,Gpr12,Gpr14), PUSH4(Gpr15,Gpr16,Gpr17,Gpr18);
30052       PUSH4(Gpr19,Gpr20,Gpr21,Gpr22), PUSH4(Gpr23,Gpr24,Gpr25,Gpr26);
30053       PUSH4(Gpr27,Gpr28,Gpr29,Gpr30), PUSH1(Gpr31);
30054       sp = (ptr_t)context.Gpr1;
30055 #elif defined(ALPHA)
30056       PUSH4(IntV0,IntT0,IntT1,IntT2), PUSH4(IntT3,IntT4,IntT5,IntT6);
30057       PUSH4(IntT7,IntS0,IntS1,IntS2), PUSH4(IntS3,IntS4,IntS5,IntFp);
30058       PUSH4(IntA0,IntA1,IntA2,IntA3), PUSH4(IntA4,IntA5,IntT8,IntT9);
30059       PUSH4(IntT10,IntT11,IntT12,IntAt);
30060       sp = (ptr_t)context.IntSp;
30061 #elif defined(CPPCHECK)
30062       sp = (ptr_t)(word)cnt;
30063 #else
30064 #error Architecture is not supported
30065 #endif
30066 #undef context
30067     GC_ASSERT(cnt == PUSHED_REGS_COUNT);
30068     return sp;
30069 }
30070 STATIC word GC_push_stack_for(GC_thread thread, DWORD me)
30071 {
30072   ptr_t sp, stack_min;
30073   struct GC_traced_stack_sect_s *traced_stack_sect =
30074                                       thread -> traced_stack_sect;
30075   if (thread -> id == me) {
30076     GC_ASSERT(thread -> thread_blocked_sp == NULL);
30077     sp = GC_approx_sp();
30078   } else if ((sp = thread -> thread_blocked_sp) == NULL) {
30079 #ifdef RETRY_GET_THREAD_CONTEXT
30080       word *regs = thread->context_regs;
30081       if (thread->suspended) {
30082         sp = thread->context_sp;
30083       } else
30084 #else
30085       word regs[PUSHED_REGS_COUNT];
30086 #endif
30087        {
30088         CONTEXT context;
30089         context.ContextFlags = GET_THREAD_CONTEXT_FLAGS;
30090         if (GetThreadContext(THREAD_HANDLE(thread), &context)) {
30091           sp = copy_ptr_regs(regs, &context);
30092         } else {
30093 #ifdef RETRY_GET_THREAD_CONTEXT
30094             sp = thread->context_sp;
30095             if (NULL == sp) {
30096               return 0;
30097             }
30098 #else
30099             ABORT("GetThreadContext failed");
30100 #endif
30101         }
30102       }
30103 #ifdef THREAD_LOCAL_ALLOC
30104       GC_ASSERT(thread->suspended || !GC_world_stopped);
30105 #endif
30106 #ifndef WOW64_THREAD_CONTEXT_WORKAROUND
30107       GC_push_many_regs(regs, PUSHED_REGS_COUNT);
30108 #else
30109       GC_push_many_regs(regs + 2, PUSHED_REGS_COUNT - 2);
30110       if (isWow64) {
30111         DWORD ContextFlags = (DWORD)regs[0];
30112         WORD SegFs = (WORD)regs[1];
30113         if ((ContextFlags & CONTEXT_EXCEPTION_REPORTING) != 0
30114             && (ContextFlags & (CONTEXT_EXCEPTION_ACTIVE
30115                                 )) != 0) {
30116           LDT_ENTRY selector;
30117           PNT_TIB tib;
30118           if (!GetThreadSelectorEntry(THREAD_HANDLE(thread), SegFs, &selector))
30119             ABORT("GetThreadSelectorEntry failed");
30120           tib = (PNT_TIB)(selector.BaseLow
30121                           | (selector.HighWord.Bits.BaseMid << 16)
30122                           | (selector.HighWord.Bits.BaseHi << 24));
30123 #ifdef DEBUG_THREADS
30124             GC_log_printf("TIB stack limit/base: %p .. %p\n",
30125                           (void *)tib->StackLimit, (void *)tib->StackBase);
30126 #endif
30127           GC_ASSERT(!((word)thread->stack_base
30128                       COOLER_THAN (word)tib->StackBase));
30129           if (thread->stack_base != thread->initial_stack_base
30130               && ((word)thread->stack_base <= (word)tib->StackLimit
30131                   || (word)tib->StackBase < (word)thread->stack_base)) {
30132             WARN("GetThreadContext might return stale register values"
30133                  " including ESP= %p\n", sp);
30134           } else {
30135             sp = (ptr_t)tib->StackLimit;
30136           }
30137         }
30138 #ifdef DEBUG_THREADS
30139           else {
30140             static GC_bool logged;
30141             if (!logged
30142                 && (ContextFlags & CONTEXT_EXCEPTION_REPORTING) == 0) {
30143               GC_log_printf("CONTEXT_EXCEPTION_REQUEST not supported\n");
30144               logged = TRUE;
30145             }
30146           }
30147 #endif
30148       }
30149 #endif
30150   }
30151   if (thread -> last_stack_min == ADDR_LIMIT) {
30152 #ifdef MSWINCE
30153       if (GC_dont_query_stack_min) {
30154         stack_min = GC_wince_evaluate_stack_min(traced_stack_sect != NULL ?
30155                       (ptr_t)traced_stack_sect : thread -> stack_base);
30156       } else
30157 #endif
30158      {
30159       stack_min = GC_get_stack_min(traced_stack_sect != NULL ?
30160                       (ptr_t)traced_stack_sect : thread -> stack_base);
30161       UNPROTECT_THREAD(thread);
30162       thread -> last_stack_min = stack_min;
30163     }
30164   } else {
30165     if (traced_stack_sect != NULL &&
30166         (word)thread->last_stack_min > (word)traced_stack_sect) {
30167       UNPROTECT_THREAD(thread);
30168       thread -> last_stack_min = (ptr_t)traced_stack_sect;
30169     }
30170     if ((word)sp < (word)thread->stack_base
30171         && (word)sp >= (word)thread->last_stack_min) {
30172       stack_min = sp;
30173     } else {
30174       if (may_be_in_stack(thread -> id == me &&
30175                           (word)sp < (word)thread->last_stack_min ?
30176                           sp : thread -> last_stack_min)) {
30177         stack_min = (ptr_t)last_info.BaseAddress;
30178         if ((word)sp < (word)stack_min
30179             || (word)sp >= (word)thread->stack_base)
30180           stack_min = GC_get_stack_min(thread -> last_stack_min);
30181       } else {
30182         stack_min = GC_get_stack_min(thread -> stack_base);
30183       }
30184       UNPROTECT_THREAD(thread);
30185       thread -> last_stack_min = stack_min;
30186     }
30187   }
30188   GC_ASSERT(GC_dont_query_stack_min
30189             || stack_min == GC_get_stack_min(thread -> stack_base)
30190             || ((word)sp >= (word)stack_min
30191                 && (word)stack_min < (word)thread->stack_base
30192                 && (word)stack_min
30193                         > (word)GC_get_stack_min(thread -> stack_base)));
30194   if ((word)sp >= (word)stack_min && (word)sp < (word)thread->stack_base) {
30195 #ifdef DEBUG_THREADS
30196       GC_log_printf("Pushing stack for 0x%x from sp %p to %p from 0x%x\n",
30197                     (int)thread->id, (void *)sp, (void *)thread->stack_base,
30198                     (int)me);
30199 #endif
30200     GC_push_all_stack_sections(sp, thread->stack_base, traced_stack_sect);
30201   } else {
30202     if (thread -> id == me || (word)sp >= (word)thread->stack_base
30203         || (word)(sp + GC_page_size) < (word)stack_min)
30204       WARN("Thread stack pointer %p out of range, pushing everything\n",
30205            sp);
30206 #ifdef DEBUG_THREADS
30207       GC_log_printf("Pushing stack for 0x%x from (min) %p to %p from 0x%x\n",
30208                     (int)thread->id, (void *)stack_min,
30209                     (void *)thread->stack_base, (int)me);
30210 #endif
30211     GC_push_all_stack(stack_min, thread->stack_base);
30212   }
30213   return thread->stack_base - sp;
30214 }
30215 GC_INNER void GC_push_all_stacks(void)
30216 {
30217   DWORD thread_id = GetCurrentThreadId();
30218   GC_bool found_me = FALSE;
30219 #ifndef SMALL_CONFIG
30220     unsigned nthreads = 0;
30221 #endif
30222   word total_size = 0;
30223 #ifndef GC_NO_THREADS_DISCOVERY
30224     if (GC_win32_dll_threads) {
30225       int i;
30226       LONG my_max = GC_get_max_thread_index();
30227       for (i = 0; i <= my_max; i++) {
30228         GC_thread t = (GC_thread)(dll_thread_table + i);
30229         if (t -> tm.in_use && t -> stack_base) {
30230 #ifndef SMALL_CONFIG
30231             ++nthreads;
30232 #endif
30233           total_size += GC_push_stack_for(t, thread_id);
30234           if (t -> id == thread_id) found_me = TRUE;
30235         }
30236       }
30237     } else
30238 #endif
30239    {
30240     int i;
30241     for (i = 0; i < THREAD_TABLE_SZ; i++) {
30242       GC_thread t;
30243       for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
30244         if (!KNOWN_FINISHED(t) && t -> stack_base) {
30245 #ifndef SMALL_CONFIG
30246             ++nthreads;
30247 #endif
30248           total_size += GC_push_stack_for(t, thread_id);
30249           if (t -> id == thread_id) found_me = TRUE;
30250         }
30251       }
30252     }
30253   }
30254 #ifndef SMALL_CONFIG
30255     GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks%s\n", nthreads,
30256                           GC_win32_dll_threads ?
30257                                 " based on DllMain thread tracking" : "");
30258 #endif
30259   if (!found_me && !GC_in_thread_creation)
30260     ABORT("Collecting from unknown thread");
30261   GC_total_stacksize = total_size;
30262 }
30263 #ifdef PARALLEL_MARK
30264 #ifndef MAX_MARKERS
30265 #define MAX_MARKERS 16
30266 #endif
30267   static ptr_t marker_sp[MAX_MARKERS - 1];
30268 #ifdef IA64
30269     static ptr_t marker_bsp[MAX_MARKERS - 1];
30270 #endif
30271   static ptr_t marker_last_stack_min[MAX_MARKERS - 1];
30272 #endif
30273 GC_INNER void GC_get_next_stack(char *start, char *limit,
30274                                 char **lo, char **hi)
30275 {
30276   int i;
30277   char * current_min = ADDR_LIMIT;
30278   ptr_t *plast_stack_min = NULL;
30279   GC_thread thread = NULL;
30280   if (GC_win32_dll_threads) {
30281     LONG my_max = GC_get_max_thread_index();
30282     for (i = 0; i <= my_max; i++) {
30283       ptr_t s = (ptr_t)(dll_thread_table[i].stack_base);
30284       if ((word)s > (word)start && (word)s < (word)current_min) {
30285         plast_stack_min = (ptr_t * )
30286                             &dll_thread_table[i].last_stack_min;
30287         current_min = s;
30288 #if defined(CPPCHECK)
30289           thread = (GC_thread)&dll_thread_table[i];
30290 #endif
30291       }
30292     }
30293   } else {
30294     for (i = 0; i < THREAD_TABLE_SZ; i++) {
30295       GC_thread t;
30296       for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
30297         ptr_t s = t -> stack_base;
30298         if ((word)s > (word)start && (word)s < (word)current_min) {
30299           plast_stack_min = &t -> last_stack_min;
30300           thread = t;
30301           current_min = s;
30302         }
30303       }
30304     }
30305 #ifdef PARALLEL_MARK
30306       for (i = 0; i < GC_markers_m1; ++i) {
30307         ptr_t s = marker_sp[i];
30308 #ifdef IA64
30309 #endif
30310         if ((word)s > (word)start && (word)s < (word)current_min) {
30311           GC_ASSERT(marker_last_stack_min[i] != NULL);
30312           plast_stack_min = &marker_last_stack_min[i];
30313           current_min = s;
30314           thread = NULL;
30315         }
30316       }
30317 #endif
30318   }
30319   *hi = current_min;
30320   if (current_min == ADDR_LIMIT) {
30321       *lo = ADDR_LIMIT;
30322       return;
30323   }
30324   GC_ASSERT((word)current_min > (word)start && plast_stack_min != NULL);
30325 #ifdef MSWINCE
30326     if (GC_dont_query_stack_min) {
30327       *lo = GC_wince_evaluate_stack_min(current_min);
30328       return;
30329     }
30330 #endif
30331   if ((word)current_min > (word)limit && !may_be_in_stack(limit)) {
30332     *lo = ADDR_LIMIT;
30333     return;
30334   }
30335   if (*plast_stack_min == ADDR_LIMIT
30336       || !may_be_in_stack(*plast_stack_min)) {
30337     *lo = GC_get_stack_min(current_min);
30338   } else {
30339     *lo = GC_get_stack_min(*plast_stack_min);
30340   }
30341   if (thread != NULL) {
30342     UNPROTECT_THREAD(thread);
30343   }
30344   *plast_stack_min = *lo;
30345 }
30346 #ifdef PARALLEL_MARK
30347 #if defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK)
30348 #if !defined(__MINGW32__)
30349 #define GC_PTHREADS_PARAMARK
30350 #endif
30351 #endif
30352 #if !defined(GC_PTHREADS_PARAMARK)
30353     STATIC HANDLE GC_marker_cv[MAX_MARKERS - 1] = {0};
30354     STATIC DWORD GC_marker_Id[MAX_MARKERS - 1] = {0};
30355 #endif
30356 #if defined(GC_PTHREADS) && defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID)
30357     static void set_marker_thread_name(unsigned id)
30358     {
30359       char name_buf[16];
30360       int len = sizeof("GC-marker-") - 1;
30361       BCOPY("GC-marker-", name_buf, len);
30362       if (id >= 10)
30363         name_buf[len++] = (char)('0' + (id / 10) % 10);
30364       name_buf[len] = (char)('0' + id % 10);
30365       name_buf[len + 1] = '\0';
30366       if (pthread_setname_np(pthread_self(), name_buf) != 0)
30367         WARN("pthread_setname_np failed\n", 0);
30368     }
30369 #elif !defined(MSWINCE)
30370     static FARPROC setThreadDescription_fn;
30371     static void set_marker_thread_name(unsigned id)
30372     {
30373       WCHAR name_buf[16];
30374       int len = sizeof(L"GC-marker-") / sizeof(WCHAR) - 1;
30375       HRESULT hr;
30376       if (!setThreadDescription_fn) return;
30377       BCOPY(L"GC-marker-", name_buf, len * sizeof(WCHAR));
30378       if (id >= 10)
30379         name_buf[len++] = (WCHAR)('0' + (id / 10) % 10);
30380       name_buf[len] = (WCHAR)('0' + id % 10);
30381       name_buf[len + 1] = 0;
30382       hr = (*(HRESULT (WINAPI *)(HANDLE, const WCHAR *))
30383             (word)setThreadDescription_fn)(GetCurrentThread(), name_buf);
30384       if (FAILED(hr))
30385         WARN("SetThreadDescription failed\n", 0);
30386     }
30387 #else
30388 #define set_marker_thread_name(id) (void)(id)
30389 #endif
30390 #ifdef GC_PTHREADS_PARAMARK
30391     STATIC void * GC_mark_thread(void * id)
30392 #elif defined(MSWINCE)
30393     STATIC DWORD WINAPI GC_mark_thread(LPVOID id)
30394 #else
30395     STATIC unsigned __stdcall GC_mark_thread(void * id)
30396 #endif
30397   {
30398     word my_mark_no = 0;
30399     if ((word)id == GC_WORD_MAX) return 0;
30400     set_marker_thread_name((unsigned)(word)id);
30401     marker_sp[(word)id] = GC_approx_sp();
30402 #ifdef IA64
30403       marker_bsp[(word)id] = GC_save_regs_in_stack();
30404 #endif
30405 #if !defined(GC_PTHREADS_PARAMARK)
30406       GC_marker_Id[(word)id] = GetCurrentThreadId();
30407 #endif
30408     GC_acquire_mark_lock();
30409     if (0 == --GC_fl_builder_count)
30410       GC_notify_all_builder();
30411     for (;; ++my_mark_no) {
30412       if (my_mark_no - GC_mark_no > (word)2) {
30413         my_mark_no = GC_mark_no;
30414       }
30415 #ifdef DEBUG_THREADS
30416         GC_log_printf("Starting mark helper for mark number %lu\n",
30417                       (unsigned long)my_mark_no);
30418 #endif
30419       GC_help_marker(my_mark_no);
30420     }
30421   }
30422 #ifndef GC_ASSERTIONS
30423 #define SET_MARK_LOCK_HOLDER (void)0
30424 #define UNSET_MARK_LOCK_HOLDER (void)0
30425 #endif
30426 #ifdef CAN_HANDLE_FORK
30427     static int available_markers_m1 = 0;
30428 #else
30429 #define available_markers_m1 GC_markers_m1
30430 #endif
30431 #ifdef GC_PTHREADS_PARAMARK
30432 #include <pthread.h>
30433 #if defined(GC_ASSERTIONS) && !defined(NUMERIC_THREAD_ID)
30434 #define NUMERIC_THREAD_ID(id) (unsigned long)(word)GC_PTHREAD_PTRVAL(id)
30435 #endif
30436 #ifdef CAN_HANDLE_FORK
30437       static pthread_cond_t mark_cv;
30438 #else
30439       static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;
30440 #endif
30441     GC_INNER void GC_start_mark_threads_inner(void)
30442     {
30443       int i;
30444       pthread_attr_t attr;
30445       pthread_t new_thread;
30446 #ifndef NO_MARKER_SPECIAL_SIGMASK
30447         sigset_t set, oldset;
30448 #endif
30449       GC_ASSERT(I_DONT_HOLD_LOCK());
30450       if (available_markers_m1 <= 0) return;
30451 #ifdef CAN_HANDLE_FORK
30452         if (GC_parallel) return;
30453         {
30454           pthread_cond_t mark_cv_local = PTHREAD_COND_INITIALIZER;
30455           BCOPY(&mark_cv_local, &mark_cv, sizeof(mark_cv));
30456         }
30457 #endif
30458       GC_ASSERT(GC_fl_builder_count == 0);
30459       if (0 != pthread_attr_init(&attr)) ABORT("pthread_attr_init failed");
30460       if (0 != pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
30461         ABORT("pthread_attr_setdetachstate failed");
30462 #ifndef NO_MARKER_SPECIAL_SIGMASK
30463         if (sigfillset(&set) != 0)
30464           ABORT("sigfillset failed");
30465         if (pthread_sigmask(SIG_BLOCK, &set, &oldset) < 0) {
30466           WARN("pthread_sigmask set failed, no markers started,"
30467                " errno= %" WARN_PRIdPTR "\n", errno);
30468           GC_markers_m1 = 0;
30469           (void)pthread_attr_destroy(&attr);
30470           return;
30471         }
30472 #endif
30473 #ifdef CAN_HANDLE_FORK
30474         GC_markers_m1 = available_markers_m1;
30475 #endif
30476       for (i = 0; i < available_markers_m1; ++i) {
30477         marker_last_stack_min[i] = ADDR_LIMIT;
30478         if (0 != pthread_create(&new_thread, &attr,
30479                                 GC_mark_thread, (void *)(word)i)) {
30480           WARN("Marker thread creation failed\n", 0);
30481           GC_markers_m1 = i;
30482           break;
30483         }
30484       }
30485 #ifndef NO_MARKER_SPECIAL_SIGMASK
30486         if (pthread_sigmask(SIG_SETMASK, &oldset, NULL) < 0) {
30487           WARN("pthread_sigmask restore failed, errno= %" WARN_PRIdPTR "\n",
30488                errno);
30489         }
30490 #endif
30491       (void)pthread_attr_destroy(&attr);
30492       GC_wait_for_markers_init();
30493       GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
30494     }
30495 #ifdef GC_ASSERTIONS
30496       STATIC unsigned long GC_mark_lock_holder = NO_THREAD;
30497 #define SET_MARK_LOCK_HOLDER \
30498                 (void)(GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
30499 #define UNSET_MARK_LOCK_HOLDER \
30500                 do { \
30501                   GC_ASSERT(GC_mark_lock_holder \
30502                                 == NUMERIC_THREAD_ID(pthread_self())); \
30503                   GC_mark_lock_holder = NO_THREAD; \
30504                 } while (0)
30505 #endif
30506     static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
30507     static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;
30508 #ifdef LOCK_STATS
30509       volatile AO_t GC_block_count = 0;
30510 #endif
30511     GC_INNER void GC_acquire_mark_lock(void)
30512     {
30513 #if defined(NUMERIC_THREAD_ID_UNIQUE) && !defined(THREAD_SANITIZER)
30514         GC_ASSERT(GC_mark_lock_holder != NUMERIC_THREAD_ID(pthread_self()));
30515 #endif
30516       if (pthread_mutex_lock(&mark_mutex) != 0) {
30517         ABORT("pthread_mutex_lock failed");
30518       }
30519 #ifdef LOCK_STATS
30520         (void)AO_fetch_and_add1(&GC_block_count);
30521 #endif
30522       SET_MARK_LOCK_HOLDER;
30523     }
30524     GC_INNER void GC_release_mark_lock(void)
30525     {
30526       UNSET_MARK_LOCK_HOLDER;
30527       if (pthread_mutex_unlock(&mark_mutex) != 0) {
30528         ABORT("pthread_mutex_unlock failed");
30529       }
30530     }
30531     STATIC void GC_wait_builder(void)
30532     {
30533       UNSET_MARK_LOCK_HOLDER;
30534       if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0) {
30535         ABORT("pthread_cond_wait failed");
30536       }
30537       GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
30538       SET_MARK_LOCK_HOLDER;
30539     }
30540     GC_INNER void GC_wait_for_reclaim(void)
30541     {
30542       GC_acquire_mark_lock();
30543       while (GC_fl_builder_count > 0) {
30544         GC_wait_builder();
30545       }
30546       GC_release_mark_lock();
30547     }
30548     GC_INNER void GC_notify_all_builder(void)
30549     {
30550       GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
30551       if (pthread_cond_broadcast(&builder_cv) != 0) {
30552         ABORT("pthread_cond_broadcast failed");
30553       }
30554     }
30555     GC_INNER void GC_wait_marker(void)
30556     {
30557       GC_ASSERT(GC_parallel);
30558       UNSET_MARK_LOCK_HOLDER;
30559       if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0) {
30560         ABORT("pthread_cond_wait failed");
30561       }
30562       GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
30563       SET_MARK_LOCK_HOLDER;
30564     }
30565     GC_INNER void GC_notify_all_marker(void)
30566     {
30567       GC_ASSERT(GC_parallel);
30568       if (pthread_cond_broadcast(&mark_cv) != 0) {
30569         ABORT("pthread_cond_broadcast failed");
30570       }
30571     }
30572 #else
30573 #ifndef MARK_THREAD_STACK_SIZE
30574 #define MARK_THREAD_STACK_SIZE 0
30575 #endif
30576     static HANDLE mark_mutex_event = (HANDLE)0;
30577     static HANDLE builder_cv = (HANDLE)0;
30578     static HANDLE mark_cv = (HANDLE)0;
30579     GC_INNER void GC_start_mark_threads_inner(void)
30580     {
30581       int i;
30582       GC_ASSERT(I_DONT_HOLD_LOCK());
30583       if (available_markers_m1 <= 0) return;
30584       GC_ASSERT(GC_fl_builder_count == 0);
30585       for (i = 0; i < GC_markers_m1; ++i) {
30586         if ((GC_marker_cv[i] = CreateEvent(NULL ,
30587                                         TRUE ,
30588                                         FALSE ,
30589                                         NULL )) == (HANDLE)0)
30590           ABORT("CreateEvent failed");
30591       }
30592       for (i = 0; i < GC_markers_m1; ++i) {
30593 #if defined(MSWINCE) || defined(MSWIN_XBOX1)
30594           HANDLE handle;
30595           DWORD thread_id;
30596           marker_last_stack_min[i] = ADDR_LIMIT;
30597           handle = CreateThread(NULL ,
30598                                 MARK_THREAD_STACK_SIZE ,
30599                                 GC_mark_thread, (LPVOID)(word)i,
30600                                 0 , &thread_id);
30601           if (handle == NULL) {
30602             WARN("Marker thread creation failed\n", 0);
30603             break;
30604           } else {
30605             CloseHandle(handle);
30606           }
30607 #else
30608           GC_uintptr_t handle;
30609           unsigned thread_id;
30610           marker_last_stack_min[i] = ADDR_LIMIT;
30611           handle = _beginthreadex(NULL ,
30612                                 MARK_THREAD_STACK_SIZE, GC_mark_thread,
30613                                 (void *)(word)i, 0 , &thread_id);
30614           if (!handle || handle == (GC_uintptr_t)-1L) {
30615             WARN("Marker thread creation failed\n", 0);
30616             break;
30617           } else {
30618           }
30619 #endif
30620       }
30621       while (GC_markers_m1 > i) {
30622         GC_markers_m1--;
30623         CloseHandle(GC_marker_cv[GC_markers_m1]);
30624       }
30625       GC_wait_for_markers_init();
30626       GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
30627       if (i == 0) {
30628         CloseHandle(mark_cv);
30629         CloseHandle(builder_cv);
30630         CloseHandle(mark_mutex_event);
30631       }
30632     }
30633 #ifdef GC_ASSERTIONS
30634       STATIC DWORD GC_mark_lock_holder = NO_THREAD;
30635 #define SET_MARK_LOCK_HOLDER \
30636                 (void)(GC_mark_lock_holder = GetCurrentThreadId())
30637 #define UNSET_MARK_LOCK_HOLDER \
30638                 do { \
30639                   GC_ASSERT(GC_mark_lock_holder == GetCurrentThreadId()); \
30640                   GC_mark_lock_holder = NO_THREAD; \
30641                 } while (0)
30642 #endif
30643     STATIC  LONG GC_mark_mutex_state = 0;
30644 #ifdef LOCK_STATS
30645       volatile AO_t GC_block_count = 0;
30646       volatile AO_t GC_unlocked_count = 0;
30647 #endif
30648     GC_INNER void GC_acquire_mark_lock(void)
30649     {
30650 #ifndef THREAD_SANITIZER
30651         GC_ASSERT(GC_mark_lock_holder != GetCurrentThreadId());
30652 #endif
30653       if (InterlockedExchange(&GC_mark_mutex_state, 1 ) != 0) {
30654 #ifdef LOCK_STATS
30655           (void)AO_fetch_and_add1(&GC_block_count);
30656 #endif
30657         while (InterlockedExchange(&GC_mark_mutex_state,
30658                                    -1 ) != 0) {
30659           if (WaitForSingleObject(mark_mutex_event, INFINITE) == WAIT_FAILED)
30660             ABORT("WaitForSingleObject failed");
30661         }
30662       }
30663 #ifdef LOCK_STATS
30664         else {
30665           (void)AO_fetch_and_add1(&GC_unlocked_count);
30666         }
30667 #endif
30668       GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
30669       SET_MARK_LOCK_HOLDER;
30670     }
30671     GC_INNER void GC_release_mark_lock(void)
30672     {
30673       UNSET_MARK_LOCK_HOLDER;
30674       if (InterlockedExchange(&GC_mark_mutex_state, 0 ) < 0) {
30675         if (SetEvent(mark_mutex_event) == FALSE)
30676           ABORT("SetEvent failed");
30677       }
30678     }
30679     GC_INNER void GC_wait_for_reclaim(void)
30680     {
30681       GC_ASSERT(builder_cv != 0);
30682       for (;;) {
30683         GC_acquire_mark_lock();
30684         if (GC_fl_builder_count == 0)
30685           break;
30686         if (ResetEvent(builder_cv) == FALSE)
30687           ABORT("ResetEvent failed");
30688         GC_release_mark_lock();
30689         if (WaitForSingleObject(builder_cv, INFINITE) == WAIT_FAILED)
30690           ABORT("WaitForSingleObject failed");
30691       }
30692       GC_release_mark_lock();
30693     }
30694     GC_INNER void GC_notify_all_builder(void)
30695     {
30696       GC_ASSERT(GC_mark_lock_holder == GetCurrentThreadId());
30697       GC_ASSERT(builder_cv != 0);
30698       GC_ASSERT(GC_fl_builder_count == 0);
30699       if (SetEvent(builder_cv) == FALSE)
30700         ABORT("SetEvent failed");
30701     }
30702     GC_INNER void GC_wait_marker(void)
30703     {
30704       HANDLE event = mark_cv;
30705       DWORD thread_id = GetCurrentThreadId();
30706       int i = GC_markers_m1;
30707       while (i-- > 0) {
30708         if (GC_marker_Id[i] == thread_id) {
30709           event = GC_marker_cv[i];
30710           break;
30711         }
30712       }
30713       if (ResetEvent(event) == FALSE)
30714         ABORT("ResetEvent failed");
30715       GC_release_mark_lock();
30716       if (WaitForSingleObject(event, INFINITE) == WAIT_FAILED)
30717         ABORT("WaitForSingleObject failed");
30718       GC_acquire_mark_lock();
30719     }
30720     GC_INNER void GC_notify_all_marker(void)
30721     {
30722       DWORD thread_id = GetCurrentThreadId();
30723       int i = GC_markers_m1;
30724       while (i-- > 0) {
30725         if (SetEvent(GC_marker_Id[i] != thread_id ? GC_marker_cv[i] :
30726                      mark_cv) == FALSE)
30727           ABORT("SetEvent failed");
30728       }
30729     }
30730 #endif
30731   static unsigned required_markers_cnt = 0;
30732 #endif
30733   typedef struct {
30734     LPTHREAD_START_ROUTINE start;
30735     LPVOID param;
30736   } thread_args;
30737   STATIC void * GC_CALLBACK GC_win32_start_inner(struct GC_stack_base *sb,
30738                                                  void *arg)
30739   {
30740     void * ret;
30741     LPTHREAD_START_ROUTINE start = ((thread_args *)arg)->start;
30742     LPVOID param = ((thread_args *)arg)->param;
30743     GC_register_my_thread(sb);
30744 #ifdef DEBUG_THREADS
30745       GC_log_printf("thread 0x%lx starting...\n", (long)GetCurrentThreadId());
30746 #endif
30747     GC_free(arg);
30748 #if !defined(__GNUC__) && !defined(NO_CRT)
30749       ret = NULL;
30750       __try
30751 #endif
30752     {
30753       ret = (void *)(word)(*start)(param);
30754     }
30755 #if !defined(__GNUC__) && !defined(NO_CRT)
30756       __finally
30757 #endif
30758     {
30759       GC_unregister_my_thread();
30760     }
30761 #ifdef DEBUG_THREADS
30762       GC_log_printf("thread 0x%lx returned from start routine\n",
30763                     (long)GetCurrentThreadId());
30764 #endif
30765     return ret;
30766   }
30767   STATIC DWORD WINAPI GC_win32_start(LPVOID arg)
30768   {
30769     return (DWORD)(word)GC_call_with_stack_base(GC_win32_start_inner, arg);
30770   }
30771   GC_API HANDLE WINAPI GC_CreateThread(
30772                         LPSECURITY_ATTRIBUTES lpThreadAttributes,
30773                         GC_WIN32_SIZE_T dwStackSize,
30774                         LPTHREAD_START_ROUTINE lpStartAddress,
30775                         LPVOID lpParameter, DWORD dwCreationFlags,
30776                         LPDWORD lpThreadId)
30777   {
30778     if (!EXPECT(parallel_initialized, TRUE))
30779       GC_init_parallel();
30780 #ifdef DEBUG_THREADS
30781       GC_log_printf("About to create a thread from 0x%lx\n",
30782                     (long)GetCurrentThreadId());
30783 #endif
30784     if (GC_win32_dll_threads) {
30785       return CreateThread(lpThreadAttributes, dwStackSize, lpStartAddress,
30786                           lpParameter, dwCreationFlags, lpThreadId);
30787     } else {
30788       thread_args *args =
30789                 (thread_args *)GC_malloc_uncollectable(sizeof(thread_args));
30790       HANDLE thread_h;
30791       if (NULL == args) {
30792         SetLastError(ERROR_NOT_ENOUGH_MEMORY);
30793         return NULL;
30794       }
30795       args -> start = lpStartAddress;
30796       args -> param = lpParameter;
30797       GC_dirty(args);
30798       REACHABLE_AFTER_DIRTY(lpParameter);
30799       set_need_to_lock();
30800       thread_h = CreateThread(lpThreadAttributes, dwStackSize, GC_win32_start,
30801                               args, dwCreationFlags, lpThreadId);
30802       if (thread_h == 0) GC_free(args);
30803       return thread_h;
30804     }
30805   }
30806   GC_API DECLSPEC_NORETURN void WINAPI GC_ExitThread(DWORD dwExitCode)
30807   {
30808     GC_unregister_my_thread();
30809     ExitThread(dwExitCode);
30810   }
30811 #if !defined(CYGWIN32) && !defined(MSWINCE) && !defined(MSWIN_XBOX1) \
30812      && !defined(NO_CRT)
30813     GC_API GC_uintptr_t GC_CALL GC_beginthreadex(
30814                                   void *security, unsigned stack_size,
30815                                   unsigned (__stdcall *start_address)(void *),
30816                                   void *arglist, unsigned initflag,
30817                                   unsigned *thrdaddr)
30818     {
30819       if (!EXPECT(parallel_initialized, TRUE))
30820         GC_init_parallel();
30821 #ifdef DEBUG_THREADS
30822         GC_log_printf("About to create a thread from 0x%lx\n",
30823                       (long)GetCurrentThreadId());
30824 #endif
30825       if (GC_win32_dll_threads) {
30826         return _beginthreadex(security, stack_size, start_address,
30827                               arglist, initflag, thrdaddr);
30828       } else {
30829         GC_uintptr_t thread_h;
30830         thread_args *args =
30831                 (thread_args *)GC_malloc_uncollectable(sizeof(thread_args));
30832         if (NULL == args) {
30833           errno = EAGAIN;
30834           return 0;
30835         }
30836         args -> start = (LPTHREAD_START_ROUTINE)start_address;
30837         args -> param = arglist;
30838         GC_dirty(args);
30839         REACHABLE_AFTER_DIRTY(arglist);
30840         set_need_to_lock();
30841         thread_h = _beginthreadex(security, stack_size,
30842                         (unsigned (__stdcall *)(void *))GC_win32_start,
30843                         args, initflag, thrdaddr);
30844         if (thread_h == 0) GC_free(args);
30845         return thread_h;
30846       }
30847     }
30848     GC_API void GC_CALL GC_endthreadex(unsigned retval)
30849     {
30850       GC_unregister_my_thread();
30851       _endthreadex(retval);
30852     }
30853 #endif
30854 #ifdef GC_WINMAIN_REDIRECT
30855 #if defined(MSWINCE) && defined(UNDER_CE)
30856 #define WINMAIN_LPTSTR LPWSTR
30857 #else
30858 #define WINMAIN_LPTSTR LPSTR
30859 #endif
30860 #undef WinMain
30861   int WINAPI GC_WinMain(HINSTANCE, HINSTANCE, WINMAIN_LPTSTR, int);
30862   typedef struct {
30863     HINSTANCE hInstance;
30864     HINSTANCE hPrevInstance;
30865     WINMAIN_LPTSTR lpCmdLine;
30866     int nShowCmd;
30867   } main_thread_args;
30868   static DWORD WINAPI main_thread_start(LPVOID arg)
30869   {
30870     main_thread_args * args = (main_thread_args *) arg;
30871     return (DWORD)GC_WinMain(args->hInstance, args->hPrevInstance,
30872                              args->lpCmdLine, args->nShowCmd);
30873   }
30874   STATIC void * GC_waitForSingleObjectInfinite(void * handle)
30875   {
30876     return (void *)(word)WaitForSingleObject((HANDLE)handle, INFINITE);
30877   }
30878 #ifndef WINMAIN_THREAD_STACK_SIZE
30879 #define WINMAIN_THREAD_STACK_SIZE 0
30880 #endif
30881   int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
30882                      WINMAIN_LPTSTR lpCmdLine, int nShowCmd)
30883   {
30884     DWORD exit_code = 1;
30885     main_thread_args args = {
30886                 hInstance, hPrevInstance, lpCmdLine, nShowCmd
30887     };
30888     HANDLE thread_h;
30889     DWORD thread_id;
30890     GC_INIT();
30891     thread_h = GC_CreateThread(NULL ,
30892                         WINMAIN_THREAD_STACK_SIZE ,
30893                         main_thread_start, &args, 0 ,
30894                         &thread_id);
30895     if (thread_h != NULL) {
30896       if ((DWORD)(word)GC_do_blocking(GC_waitForSingleObjectInfinite,
30897                                       (void *)thread_h) == WAIT_FAILED)
30898         ABORT("WaitForSingleObject(main_thread) failed");
30899       GetExitCodeThread (thread_h, &exit_code);
30900       CloseHandle (thread_h);
30901     } else {
30902       ABORT("GC_CreateThread(main_thread) failed");
30903     }
30904 #ifdef MSWINCE
30905       GC_deinit();
30906 #endif
30907     return (int) exit_code;
30908   }
30909 #endif
30910 GC_API void GC_CALL GC_set_markers_count(unsigned markers GC_ATTR_UNUSED)
30911 {
30912 #ifdef PARALLEL_MARK
30913     required_markers_cnt = markers < MAX_MARKERS ? markers : MAX_MARKERS;
30914 #endif
30915 }
30916 GC_INNER void GC_thr_init(void)
30917 {
30918   struct GC_stack_base sb;
30919 #if (!defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) && !defined(MSWINCE) \
30920       && defined(PARALLEL_MARK)) || defined(WOW64_THREAD_CONTEXT_WORKAROUND)
30921     HMODULE hK32 = GetModuleHandle(TEXT("kernel32.dll"));
30922 #endif
30923   GC_ASSERT(I_HOLD_LOCK());
30924   if (GC_thr_initialized) return;
30925   GC_ASSERT((word)&GC_threads % sizeof(word) == 0);
30926 #ifdef GC_NO_THREADS_DISCOVERY
30927 #define GC_main_thread GetCurrentThreadId()
30928 #else
30929     GC_main_thread = GetCurrentThreadId();
30930 #endif
30931   GC_thr_initialized = TRUE;
30932 #ifdef CAN_HANDLE_FORK
30933     if (GC_handle_fork) {
30934 #ifdef CAN_CALL_ATFORK
30935         if (pthread_atfork(fork_prepare_proc, fork_parent_proc,
30936                            fork_child_proc) == 0) {
30937           GC_handle_fork = 1;
30938         } else
30939 #endif
30940        if (GC_handle_fork != -1)
30941         ABORT("pthread_atfork failed");
30942     }
30943 #endif
30944 #ifdef WOW64_THREAD_CONTEXT_WORKAROUND
30945       if (hK32) {
30946         FARPROC pfn = GetProcAddress(hK32, "IsWow64Process");
30947         if (pfn
30948             && !(*(BOOL (WINAPI*)(HANDLE, BOOL*))(word)pfn)(
30949                                         GetCurrentProcess(), &isWow64))
30950           isWow64 = FALSE;
30951       }
30952 #endif
30953   sb.mem_base = GC_stackbottom;
30954   GC_ASSERT(sb.mem_base != NULL);
30955 #ifdef IA64
30956     sb.reg_base = GC_register_stackbottom;
30957 #endif
30958 #if defined(PARALLEL_MARK)
30959     {
30960       char * markers_string = GETENV("GC_MARKERS");
30961       int markers = required_markers_cnt;
30962       if (markers_string != NULL) {
30963         markers = atoi(markers_string);
30964         if (markers <= 0 || markers > MAX_MARKERS) {
30965           WARN("Too big or invalid number of mark threads: %" WARN_PRIdPTR
30966                "; using maximum threads\n", (signed_word)markers);
30967           markers = MAX_MARKERS;
30968         }
30969       } else if (0 == markers) {
30970 #ifdef MSWINCE
30971           markers = (int)GC_sysinfo.dwNumberOfProcessors;
30972 #else
30973 #ifdef _WIN64
30974             DWORD_PTR procMask = 0;
30975             DWORD_PTR sysMask;
30976 #else
30977             DWORD procMask = 0;
30978             DWORD sysMask;
30979 #endif
30980           int ncpu = 0;
30981           if (
30982 #ifdef __cplusplus
30983               GetProcessAffinityMask(GetCurrentProcess(), &procMask, &sysMask)
30984 #else
30985               GetProcessAffinityMask(GetCurrentProcess(),
30986                                      (void *)&procMask, (void *)&sysMask)
30987 #endif
30988               && procMask) {
30989             do {
30990               ncpu++;
30991             } while ((procMask &= procMask - 1) != 0);
30992           }
30993           markers = ncpu;
30994 #endif
30995 #if defined(GC_MIN_MARKERS) && !defined(CPPCHECK)
30996           if (markers < GC_MIN_MARKERS)
30997             markers = GC_MIN_MARKERS;
30998 #endif
30999         if (markers > MAX_MARKERS)
31000           markers = MAX_MARKERS;
31001       }
31002       available_markers_m1 = markers - 1;
31003     }
31004       if (GC_win32_dll_threads || available_markers_m1 <= 0) {
31005         GC_parallel = FALSE;
31006         GC_COND_LOG_PRINTF(
31007                 "Single marker thread, turning off parallel marking\n");
31008       } else {
31009 #ifndef GC_PTHREADS_PARAMARK
31010           mark_mutex_event = CreateEvent(NULL ,
31011                                 FALSE ,
31012                                 FALSE , NULL );
31013           builder_cv = CreateEvent(NULL ,
31014                                 TRUE ,
31015                                 FALSE , NULL );
31016           mark_cv = CreateEvent(NULL , TRUE ,
31017                                 FALSE , NULL );
31018           if (mark_mutex_event == (HANDLE)0 || builder_cv == (HANDLE)0
31019               || mark_cv == (HANDLE)0)
31020             ABORT("CreateEvent failed");
31021 #endif
31022 #if !defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) && !defined(MSWINCE)
31023           if (hK32)
31024             setThreadDescription_fn = GetProcAddress(hK32,
31025                                                      "SetThreadDescription");
31026 #endif
31027       }
31028 #endif
31029   GC_ASSERT(0 == GC_lookup_thread_inner(GC_main_thread));
31030   GC_register_my_thread_inner(&sb, GC_main_thread);
31031 #undef GC_main_thread
31032 }
31033 #ifdef GC_PTHREADS
31034   struct start_info {
31035     void *(*start_routine)(void *);
31036     void *arg;
31037     GC_bool detached;
31038   };
31039   GC_API int GC_pthread_join(pthread_t pthread_id, void **retval)
31040   {
31041     int result;
31042 #ifndef GC_WIN32_PTHREADS
31043       GC_thread t;
31044 #endif
31045     DCL_LOCK_STATE;
31046     GC_ASSERT(!GC_win32_dll_threads);
31047 #ifdef DEBUG_THREADS
31048       GC_log_printf("thread %p(0x%lx) is joining thread %p\n",
31049                     (void *)GC_PTHREAD_PTRVAL(pthread_self()),
31050                     (long)GetCurrentThreadId(),
31051                     (void *)GC_PTHREAD_PTRVAL(pthread_id));
31052 #endif
31053 #ifndef GC_WIN32_PTHREADS
31054       while ((t = GC_lookup_pthread(pthread_id)) == 0)
31055         Sleep(10);
31056 #endif
31057     result = pthread_join(pthread_id, retval);
31058     if (0 == result) {
31059 #ifdef GC_WIN32_PTHREADS
31060         GC_thread t = GC_lookup_pthread(pthread_id);
31061         if (NULL == t) ABORT("Thread not registered");
31062 #endif
31063       LOCK();
31064       if ((t -> flags & FINISHED) != 0) {
31065         GC_delete_gc_thread_no_free(t);
31066         GC_INTERNAL_FREE(t);
31067       }
31068       UNLOCK();
31069     }
31070 #ifdef DEBUG_THREADS
31071       GC_log_printf("thread %p(0x%lx) join with thread %p %s\n",
31072                     (void *)GC_PTHREAD_PTRVAL(pthread_self()),
31073                     (long)GetCurrentThreadId(),
31074                     (void *)GC_PTHREAD_PTRVAL(pthread_id),
31075                     result != 0 ? "failed" : "succeeded");
31076 #endif
31077     return result;
31078   }
31079   GC_API int GC_pthread_create(pthread_t *new_thread,
31080                                GC_PTHREAD_CREATE_CONST pthread_attr_t *attr,
31081                                void *(*start_routine)(void *), void *arg)
31082   {
31083     int result;
31084     struct start_info * si;
31085     if (!EXPECT(parallel_initialized, TRUE))
31086       GC_init_parallel();
31087     GC_ASSERT(!GC_win32_dll_threads);
31088       si = (struct start_info *)GC_malloc_uncollectable(
31089                                                 sizeof(struct start_info));
31090       if (NULL == si)
31091         return EAGAIN;
31092       si -> start_routine = start_routine;
31093       si -> arg = arg;
31094       GC_dirty(si);
31095       REACHABLE_AFTER_DIRTY(arg);
31096       if (attr != 0 &&
31097           pthread_attr_getdetachstate(attr, &si->detached)
31098           == PTHREAD_CREATE_DETACHED) {
31099         si->detached = TRUE;
31100       }
31101 #ifdef DEBUG_THREADS
31102         GC_log_printf("About to create a thread from %p(0x%lx)\n",
31103                       (void *)GC_PTHREAD_PTRVAL(pthread_self()),
31104                       (long)GetCurrentThreadId());
31105 #endif
31106       set_need_to_lock();
31107       result = pthread_create(new_thread, attr, GC_pthread_start, si);
31108       if (result) {
31109           GC_free(si);
31110       }
31111       return(result);
31112   }
31113   STATIC void * GC_CALLBACK GC_pthread_start_inner(struct GC_stack_base *sb,
31114                                                    void * arg)
31115   {
31116     struct start_info * si = (struct start_info *)arg;
31117     void * result;
31118     void *(*start)(void *);
31119     void *start_arg;
31120     DWORD thread_id = GetCurrentThreadId();
31121     pthread_t pthread_id = pthread_self();
31122     GC_thread me;
31123     DCL_LOCK_STATE;
31124 #ifdef DEBUG_THREADS
31125       GC_log_printf("thread %p(0x%x) starting...\n",
31126                     (void *)GC_PTHREAD_PTRVAL(pthread_id), (int)thread_id);
31127 #endif
31128     GC_ASSERT(!GC_win32_dll_threads);
31129     LOCK();
31130     me = GC_register_my_thread_inner(sb, thread_id);
31131     SET_PTHREAD_MAP_CACHE(pthread_id, thread_id);
31132     GC_ASSERT(me != &first_thread);
31133     me -> pthread_id = pthread_id;
31134     if (si->detached) me -> flags |= DETACHED;
31135     UNLOCK();
31136     start = si -> start_routine;
31137     start_arg = si -> arg;
31138     GC_free(si);
31139     pthread_cleanup_push(GC_thread_exit_proc, (void *)me);
31140     result = (*start)(start_arg);
31141     me -> status = result;
31142     GC_dirty(me);
31143     pthread_cleanup_pop(1);
31144 #ifdef DEBUG_THREADS
31145       GC_log_printf("thread %p(0x%x) returned from start routine\n",
31146                     (void *)GC_PTHREAD_PTRVAL(pthread_id), (int)thread_id);
31147 #endif
31148     return(result);
31149   }
31150   STATIC void * GC_pthread_start(void * arg)
31151   {
31152     return GC_call_with_stack_base(GC_pthread_start_inner, arg);
31153   }
31154   STATIC void GC_thread_exit_proc(void *arg)
31155   {
31156     GC_thread me = (GC_thread)arg;
31157     DCL_LOCK_STATE;
31158     GC_ASSERT(!GC_win32_dll_threads);
31159 #ifdef DEBUG_THREADS
31160       GC_log_printf("thread %p(0x%lx) called pthread_exit()\n",
31161                     (void *)GC_PTHREAD_PTRVAL(pthread_self()),
31162                     (long)GetCurrentThreadId());
31163 #endif
31164     LOCK();
31165     GC_wait_for_gc_completion(FALSE);
31166 #if defined(THREAD_LOCAL_ALLOC)
31167       GC_ASSERT(GC_getspecific(GC_thread_key) == &me->tlfs);
31168       GC_destroy_thread_local(&(me->tlfs));
31169 #endif
31170     if (me -> flags & DETACHED) {
31171       GC_delete_thread(GetCurrentThreadId());
31172     } else {
31173       me -> flags |= FINISHED;
31174     }
31175 #if defined(THREAD_LOCAL_ALLOC)
31176       GC_remove_specific(GC_thread_key);
31177 #endif
31178     UNLOCK();
31179   }
31180 #ifndef GC_NO_PTHREAD_SIGMASK
31181     GC_API int GC_pthread_sigmask(int how, const sigset_t *set,
31182                                   sigset_t *oset)
31183     {
31184       return pthread_sigmask(how, set, oset);
31185     }
31186 #endif
31187   GC_API int GC_pthread_detach(pthread_t thread)
31188   {
31189     int result;
31190     GC_thread t;
31191     DCL_LOCK_STATE;
31192     GC_ASSERT(!GC_win32_dll_threads);
31193     while ((t = GC_lookup_pthread(thread)) == NULL)
31194       Sleep(10);
31195     result = pthread_detach(thread);
31196     if (result == 0) {
31197       LOCK();
31198       t -> flags |= DETACHED;
31199       if ((t -> flags & FINISHED) != 0) {
31200         GC_delete_gc_thread_no_free(t);
31201         GC_INTERNAL_FREE(t);
31202       }
31203       UNLOCK();
31204     }
31205     return result;
31206   }
31207 #elif !defined(GC_NO_THREADS_DISCOVERY)
31208 #ifdef GC_INSIDE_DLL
31209     GC_API
31210 #else
31211 #define GC_DllMain DllMain
31212 #endif
31213   BOOL WINAPI GC_DllMain(HINSTANCE inst GC_ATTR_UNUSED, ULONG reason,
31214                          LPVOID reserved GC_ATTR_UNUSED)
31215   {
31216       DWORD thread_id;
31217       if (!GC_win32_dll_threads && parallel_initialized) return TRUE;
31218       switch (reason) {
31219        case DLL_THREAD_ATTACH:
31220 #ifdef PARALLEL_MARK
31221           if (GC_parallel) {
31222             break;
31223           }
31224 #endif
31225        case DLL_PROCESS_ATTACH:
31226         thread_id = GetCurrentThreadId();
31227         if (parallel_initialized && GC_main_thread != thread_id) {
31228 #ifdef PARALLEL_MARK
31229             ABORT("Cannot initialize parallel marker from DllMain");
31230 #else
31231             struct GC_stack_base sb;
31232 #ifdef GC_ASSERTIONS
31233               int sb_result =
31234 #endif
31235                         GC_get_stack_base(&sb);
31236             GC_ASSERT(sb_result == GC_SUCCESS);
31237             GC_register_my_thread_inner(&sb, thread_id);
31238 #endif
31239         }
31240         break;
31241        case DLL_THREAD_DETACH:
31242         GC_ASSERT(parallel_initialized);
31243         if (GC_win32_dll_threads) {
31244           GC_delete_thread(GetCurrentThreadId());
31245         }
31246         break;
31247        case DLL_PROCESS_DETACH:
31248         if (GC_win32_dll_threads) {
31249           int i;
31250           int my_max = (int)GC_get_max_thread_index();
31251           for (i = 0; i <= my_max; ++i) {
31252            if (AO_load(&(dll_thread_table[i].tm.in_use)))
31253              GC_delete_gc_thread_no_free(&dll_thread_table[i]);
31254           }
31255           GC_deinit();
31256         }
31257         break;
31258       }
31259       return TRUE;
31260   }
31261 #endif
31262 GC_INNER void GC_init_parallel(void)
31263 {
31264 #if defined(THREAD_LOCAL_ALLOC)
31265     GC_thread me;
31266     DCL_LOCK_STATE;
31267 #endif
31268   if (parallel_initialized) return;
31269   parallel_initialized = TRUE;
31270   if (!GC_is_initialized) GC_init();
31271 #if defined(CPPCHECK) && !defined(GC_NO_THREADS_DISCOVERY)
31272     GC_noop1((word)&GC_DllMain);
31273 #endif
31274   if (GC_win32_dll_threads) {
31275     set_need_to_lock();
31276   }
31277 #if defined(THREAD_LOCAL_ALLOC)
31278     LOCK();
31279     me = GC_lookup_thread_inner(GetCurrentThreadId());
31280     CHECK_LOOKUP_MY_THREAD(me);
31281     GC_init_thread_local(&me->tlfs);
31282     UNLOCK();
31283 #endif
31284 }
31285 #if defined(USE_PTHREAD_LOCKS)
31286   GC_INNER void GC_lock(void)
31287   {
31288     pthread_mutex_lock(&GC_allocate_ml);
31289   }
31290 #endif
31291 #if defined(THREAD_LOCAL_ALLOC)
31292   GC_INNER void GC_mark_thread_local_free_lists(void)
31293   {
31294     int i;
31295     GC_thread p;
31296     for (i = 0; i < THREAD_TABLE_SZ; ++i) {
31297       for (p = GC_threads[i]; 0 != p; p = p -> tm.next) {
31298         if (!KNOWN_FINISHED(p)) {
31299 #ifdef DEBUG_THREADS
31300             GC_log_printf("Marking thread locals for 0x%x\n", (int)p -> id);
31301 #endif
31302           GC_mark_thread_local_fls_for(&(p->tlfs));
31303         }
31304       }
31305     }
31306   }
31307 #if defined(GC_ASSERTIONS)
31308     void GC_check_tls(void)
31309     {
31310         int i;
31311         GC_thread p;
31312         for (i = 0; i < THREAD_TABLE_SZ; ++i) {
31313           for (p = GC_threads[i]; 0 != p; p = p -> tm.next) {
31314             if (!KNOWN_FINISHED(p))
31315               GC_check_tls_for(&(p->tlfs));
31316           }
31317         }
31318 #if defined(USE_CUSTOM_SPECIFIC)
31319           if (GC_thread_key != 0)
31320             GC_check_tsd_marks(GC_thread_key);
31321 #endif
31322     }
31323 #endif
31324 #endif
31325 #ifndef GC_NO_THREAD_REDIRECTS
31326 #define CreateThread GC_CreateThread
31327 #define ExitThread GC_ExitThread
31328 #undef _beginthreadex
31329 #define _beginthreadex GC_beginthreadex
31330 #undef _endthreadex
31331 #define _endthreadex GC_endthreadex
31332 #endif
31333 #endif
31334 #ifndef GC_PTHREAD_START_STANDALONE
31335 #if defined(__GNUC__) && defined(__linux__)
31336 #undef __EXCEPTIONS
31337 #endif
31338 #if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)
31339 #include <pthread.h>
31340 #include <sched.h>
31341 GC_INNER_PTHRSTART void * GC_CALLBACK GC_inner_start_routine(
31342                                         struct GC_stack_base *sb, void *arg)
31343 {
31344   void * (*start)(void *);
31345   void * start_arg;
31346   void * result;
31347   volatile GC_thread me =
31348                 GC_start_rtn_prepare_thread(&start, &start_arg, sb, arg);
31349 #ifndef NACL
31350     pthread_cleanup_push(GC_thread_exit_proc, me);
31351 #endif
31352   result = (*start)(start_arg);
31353 #if defined(DEBUG_THREADS) && !defined(GC_PTHREAD_START_STANDALONE)
31354     GC_log_printf("Finishing thread %p\n", (void *)pthread_self());
31355 #endif
31356   me -> status = result;
31357   GC_end_stubborn_change(me);
31358 #ifndef NACL
31359     pthread_cleanup_pop(1);
31360 #endif
31361   return result;
31362 }
31363 #endif
31364 #endif
31365 #ifndef GC_NO_THREAD_REDIRECTS
31366 #define GC_PTHREAD_REDIRECTS_ONLY
31367 #endif