More effort in avoiding errors in finalizers

Before calling a finalizer, Lua not only checks stack limits, but
actually ensures that a minimum number of slots are already allocated
for the call. (If it cannot ensure that, it postpones the finalizer.)
That avoids finalizers not running due to memory errors that the
programmer cannot control.

testes/gc.lua

@@ -707,4 +707,46 @@ end
 
 collectgarbage(oldmode)
 
+
+if T then
+  print("testing stack issues when calling finalizers")
+
+  local X
+  local obj
+
+  local function initobj ()
+    X = false
+    obj = setmetatable({}, {__gc = function ()  X = true end})
+  end
+
+  local function loop (n)
+    if n > 0 then loop(n - 1) end
+  end
+
+  -- should not try to call finalizer without a CallInfo available
+  initobj()
+  loop(20)   -- ensure stack space
+  T.resetCI()   -- remove extra CallInfos
+  T.alloccount(0)   -- cannot allocate more CallInfos
+  obj = nil
+  collectgarbage()   -- will not call finalizer
+  T.alloccount()
+  assert(X == false)
+  collectgarbage()   -- now will call finalizer (it was still pending)
+  assert(X == true)
+
+  -- should not try to call finalizer without stack space available
+  initobj()
+  loop(5)     -- ensure enough CallInfos
+  T.reallocstack(0)   -- remove extra stack slots
+  T.alloccount(0)   -- cannot reallocate stack
+  obj = nil
+  collectgarbage()   -- will not call finalizer
+  T.alloccount()
+  assert(X == false)
+  collectgarbage()   -- now will call finalizer (it was still pending)
+  assert(X == true)
+end
+
+
 print('OK')

testes/memerr.lua

@@ -282,6 +282,25 @@ testamem("growing stack", function ()
   return foo(100)
 end)
 
+
+collectgarbage()
+collectgarbage()
+global io, T, setmetatable, collectgarbage, print
+
+local Count = 0
+testamem("finalizers", function ()
+  local X = false
+  local obj = setmetatable({}, {__gc = function ()  X = true end})
+  obj = nil
+  T.resetCI()   -- remove extra CallInfos
+  T.reallocstack(18)   -- remove extra stack slots
+  Count = Count + 1
+  io.stderr:write(Count, "\n")
+  T.trick(io)
+  collectgarbage()
+  return X
+end)
+
 -- }==================================================================
 
 

testes/tracegc.lua

@@ -1,10 +1,15 @@
 -- track collections
 
+
 local M = {}
 
 -- import list
-local setmetatable, stderr, collectgarbage =
-         setmetatable, io.stderr, collectgarbage
+local stderr, collectgarbage = io.stderr, collectgarbage
+
+-- the debug version of setmetatable does not create any object (such as
+-- a '__metatable' string), and so it is more appropriate to be used in
+-- a finalizer
+local setmetatable = require"debug".setmetatable
 
 global none