Rework memory manager to make stack slot update faster

Interop/JsRuntime/src/main/kotlin/jsinterop.kt

@@ -27,15 +27,15 @@ typealias Pointer = Int
 /**
  * @Retain annotation is required to preserve functions from internalization and DCE.
  */
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_allocateArena")
 external public fun allocateArena(): Arena
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_freeArena")
 external public fun freeArena(arena: Arena)
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_pushIntToArena")
 external public fun pushIntToArena(arena: Arena, value: Int)
 
@@ -49,15 +49,15 @@ fun doubleUpper(value: Double): Int =
 fun doubleLower(value: Double): Int =
     (value.toBits() and 0x00000000ffffffff) .toInt()
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("ReturnSlot_getDouble")
 external public fun ReturnSlot_getDouble(): Double
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Kotlin_String_utf16pointer")
 external public fun stringPointer(message: String): Pointer
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Kotlin_String_utf16length")
 external public fun stringLengthBytes(message: String): Int
 
@@ -68,7 +68,7 @@ fun <R> wrapFunction(func: KtFunction<R>): Int {
     return ptr.toInt() // TODO: LP64 unsafe.
 }
 
-@Retain
+@RetainForTarget("wasm")
 @ExportForCppRuntime("Konan_js_runLambda")
 fun runLambda(pointer: Int, argumentsArena: Arena, argumentsArenaSize: Int): Int {
     val arguments = arrayListOf<JsValue>()
@@ -104,19 +104,19 @@ open class JsArray(arena: Arena, index: Object): JsValue(arena, index) {
         get() = this.getInt("length")
 }
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_getInt")
 external public fun getInt(arena: Arena, obj: Object, propertyPtr: Pointer, propertyLen: Int): Int;
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_getProperty")
 external public fun Konan_js_getProperty(arena: Arena, obj: Object, propertyPtr: Pointer, propertyLen: Int): Int;
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_setFunction")
 external public fun setFunction(arena: Arena, obj: Object, propertyName: Pointer, propertyLength: Int , function: Int)
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_setString")
 external public fun setString(arena: Arena, obj: Object, propertyName: Pointer, propertyLength: Int, stringPtr: Pointer, stringLength: Int )
 
@@ -140,4 +140,4 @@ fun JsValue.setter(property: String, string: String) {
 object ArenaManager {
     val globalArena = allocateArena()
     var currentArena = globalArena
-}
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt

@@ -850,31 +850,47 @@ internal class CAdapterGenerator(
         |
         |extern "C" {
         |void UpdateHeapRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW;
+        |void UpdateStackRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW;
         |KObjHeader* AllocInstance(const KTypeInfo*, KObjHeader**) RUNTIME_NOTHROW;
         |KObjHeader* DerefStablePointer(void*, KObjHeader**) RUNTIME_NOTHROW;
         |void* CreateStablePointer(KObjHeader*) RUNTIME_NOTHROW;
         |void DisposeStablePointer(void*) RUNTIME_NOTHROW;
         |int IsInstance(const KObjHeader*, const KTypeInfo*) RUNTIME_NOTHROW;
+        |void EnterFrame(KObjHeader** start, int parameters, int count) RUNTIME_NOTHROW;
+        |void LeaveFrame(KObjHeader** start, int parameters, int count) RUNTIME_NOTHROW;
         |void Kotlin_initRuntimeIfNeeded();
         |
         |KObjHeader* CreateStringFromCString(const char*, KObjHeader**);
         |char* CreateCStringFromString(const KObjHeader*);
         |void DisposeCString(char* cstring);
         |}  // extern "C"
         |
+        |struct ${prefix}_FrameOverlay {
+        |  void* arena;
+        |  ${prefix}_FrameOverlay* previous;
+        |  ${prefix}_KInt parameters;
+        |  ${prefix}_KInt count;
+        |};
+        |
         |class KObjHolder {
         |public:
-        |  KObjHolder() : obj_(nullptr) {}
+        |  KObjHolder() : obj_(nullptr) {
+        |    EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*));
+        |  }
         |  explicit KObjHolder(const KObjHeader* obj) : obj_(nullptr) {
-        |    UpdateHeapRef(&obj_, obj);
+        |    EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*));
+        |    UpdateStackRef(&obj_, obj);
         |  }
         |  ~KObjHolder() {
-        |    UpdateHeapRef(&obj_, nullptr);
+        |    LeaveFrame(frame(), 0, sizeof(*this)/sizeof(void*));
         |  }
         |  KObjHeader* obj() { return obj_; }
         |  KObjHeader** slot() { return &obj_; }
         | private:
+        |  ${prefix}_FrameOverlay frame_;
         |  KObjHeader* obj_;
+        |
+        |  KObjHeader** frame() { return reinterpret_cast<KObjHeader**>(&frame_); }
         |};
         |static void DisposeStablePointerImpl(${prefix}_KNativePtr ptr) {
         |  DisposeStablePointer(ptr);

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -996,7 +996,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
 
     private val kotlinExceptionRtti: ConstPointer
         get() = constPointer(importGlobal(
-                "_ZTI9ObjHolder", // typeinfo for ObjHolder
+                "_ZTI18ExceptionObjHolder", // typeinfo for ObjHolder
                 int8TypePtr,
                 origin = context.stdlibModule.llvmSymbolOrigin
         )).bitcast(int8TypePtr)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.backend.konan.descriptors.*
 import org.jetbrains.kotlin.backend.konan.ir.*
 import org.jetbrains.kotlin.backend.konan.llvm.coverage.*
+import org.jetbrains.kotlin.backend.konan.llvm.objcexport.is64Bit
 import org.jetbrains.kotlin.backend.konan.optimizations.*
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.builtins.UnsignedType
@@ -681,7 +682,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
         }
 
 
-        if (declaration.descriptor.retainAnnotation) {
+        if (declaration.descriptor.retainAnnotation(context.config.target)) {
             context.llvm.usedFunctions.add(codegen.llvmFunction(declaration))
         }
 
@@ -2190,7 +2191,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
         val constructedClass = functionGenerationContext.constructedClass!!
         val thisPtr = currentCodeContext.genGetValue(constructedClass.thisReceiver!!)
 
-        if (constructor.constructedClass.isExternalObjCClass()) {
+        if (constructor.constructedClass.isExternalObjCClass() || constructor.constructedClass.isAny()) {
             assert(args.isEmpty())
             return codegen.theUnitInstanceRef.llvm
         }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RetainAnnotation.kt

@@ -5,13 +5,18 @@
 
 package org.jetbrains.kotlin.backend.konan.llvm
 
-import org.jetbrains.kotlin.descriptors.FunctionDescriptor
-import org.jetbrains.kotlin.descriptors.annotations.*
+import org.jetbrains.kotlin.backend.konan.descriptors.getStringValueOrNull
+import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.name.FqName
 
-private val annotationName = FqName("kotlin.native.Retain")
+private val retainAnnotationName = FqName("kotlin.native.Retain")
+private val retainForTargetAnnotationName = FqName("kotlin.native.RetainForTarget")
 
-internal val FunctionDescriptor.retainAnnotation: Boolean
-    get() {
-        return (this.annotations.findAnnotation(annotationName) != null)
-    }
+
+internal fun FunctionDescriptor.retainAnnotation(target: KonanTarget): Boolean {
+    if (this.annotations.findAnnotation(retainAnnotationName) != null) return true
+    val forTarget = this.annotations.findAnnotation(retainForTargetAnnotationName)
+    if (forTarget != null && forTarget.getStringValueOrNull("target") == target.name) return true
+    return false
+}

backend.native/tests/build.gradle

@@ -2065,7 +2065,7 @@ task extend_exception(type: RunKonanTest) {
 }
 
 task check_stacktrace_format(type: RunStandaloneKonanTest) {
-    disabled = !isAppleTarget(project)
+    disabled = true // !isAppleTarget(project)
     flags = ['-g']
     source = "runtime/exceptions/check_stacktrace_format.kt"
 }
@@ -3184,6 +3184,7 @@ task produce_dynamic(type: DynamicKonanTest) {
                 "enum100 = 100\n" +
                 "object = 42\n" +
                 "singleton = I am single\n" +
+                "mutable = foo\n" +
                 "topLevel = 777 3\n"
 }
 

backend.native/tests/interop/objc/smoke.kt

@@ -182,6 +182,8 @@ fun testWeakReference(block: () -> NSObject) {
         createAndTestWeakReference(block)
     }
 
+    kotlin.native.internal.GC.collect()
+
     assertNull(ref.get())
 }
 

backend.native/tests/produce_dynamic/simple/hello.kt

@@ -14,6 +14,10 @@ fun hello() {
 
 fun getString() = "Kotlin/Native"
 
+data class Data(var string: String)
+
+fun getMutable() = Data("foo")
+
 // Class with inheritance.
 open class Base {
     open fun foo() = println("Base.foo")

backend.native/tests/produce_dynamic/simple/main.c

@@ -16,9 +16,11 @@ int main(void) {
     T_(I) casted_impl2 = { .pinned = impl2.pinned };
     T_(Enum) enum1 = __ kotlin.root.Enum.HUNDRED.get();
     T_(Codeable) object1 = __ kotlin.root.get_an_object();
+    T_(Data) data = __ kotlin.root.getMutable();
 
     const char* string1 = __ kotlin.root.getString();
     const char* string2 = __ kotlin.root.Singleton.toString(singleton);
+    const char* string3 = __ kotlin.root.Data.get_string(data);
 
     __ kotlin.root.hello();
     __ kotlin.root.Base.foo(base);
@@ -40,6 +42,8 @@ int main(void) {
 
     printf("singleton = %s\n",  string2);
 
+    printf("mutable = %s\n",  string3);
+
     topLevelFunctionVoidFromC(42, 0);
     __ kotlin.root.topLevelFunctionVoid(42, 0);
     printf("topLevel = %d %d\n", topLevelFunctionFromC(780, 3), __ kotlin.root.topLevelFunctionFromCShort(5, 2));
@@ -49,6 +53,7 @@ int main(void) {
     __ DisposeStablePointer(singleton.pinned);
     __ DisposeString(string1);
     __ DisposeString(string2);
+    __ DisposeString(string3);
     __ DisposeStablePointer(base.pinned);
     __ DisposeStablePointer(child.pinned);
     __ DisposeStablePointer(impl1.pinned);

backend.native/tests/runtime/memory/weak0.kt

@@ -13,7 +13,6 @@ data class Data(val s: String)
 fun localWeak(): WeakReference<Data>  {
     val x = Data("Hello")
     val weak = WeakReference(x)
-
     println(weak.get())
     return weak
 }
@@ -29,10 +28,14 @@ fun multiWeak(): Array<WeakReference<Data>>  {
 
 @Test fun runTest() {
     val weak = localWeak()
+    kotlin.native.internal.GC.collect()
     val value = weak.get()
     println(value?.toString())
 
     val weaks = multiWeak()
+
+    kotlin.native.internal.GC.collect()
+
     weaks.forEach {
         it -> if (it.get()?.s != null) throw Error("not null")
     }

runtime/src/launcher/cpp/launcher.cpp

@@ -29,8 +29,9 @@ OBJ_GETTER(setupArgs, int argc, const char** argv) {
   ObjHeader* result = AllocArrayInstance(theArrayTypeInfo, argc - 1, OBJ_RESULT);
   ArrayHeader* array = result->array();
   for (int index = 1; index < argc; index++) {
-    CreateStringFromCString(
-      argv[index], ArrayAddressOfElementAt(array, index - 1));
+    ObjHolder result;
+    CreateStringFromCString(argv[index], result.slot());
+    UpdateHeapRef(ArrayAddressOfElementAt(array, index - 1), result.obj());
   }
   return result;
 }

runtime/src/main/cpp/Atomic.cpp

@@ -208,7 +208,7 @@ OBJ_GETTER(Kotlin_AtomicReference_get, KRef thiz) {
     // rescheduled unluckily, between the moment value is read from the field and RC is incremented,
     // object may go away.
     AtomicReferenceLayout* ref = asAtomicReference(thiz);
-    RETURN_RESULT_OF(ReadRefLocked, &ref->value_, &ref->lock_);
+    RETURN_RESULT_OF(ReadHeapRefLocked, &ref->value_, &ref->lock_);
 }
 
 }  // extern "C"

runtime/src/main/cpp/Common.h

@@ -24,6 +24,7 @@
 #define RUNTIME_USED __attribute__((used))
 
 #define ALWAYS_INLINE __attribute__((always_inline))
+#define NO_INLINE __attribute__((noinline))
 
 #if KONAN_NO_THREADS
 #define THREAD_LOCAL_VARIABLE

runtime/src/main/cpp/Exceptions.cpp

@@ -156,7 +156,9 @@ OBJ_GETTER0(GetCurrentStackTrace) {
 OBJ_GETTER(GetStackTraceStrings, KConstRef stackTrace) {
 #if OMIT_BACKTRACE
   ObjHeader* result = AllocArrayInstance(theArrayTypeInfo, 1, OBJ_RESULT);
-  CreateStringFromCString("<UNIMPLEMENTED>", ArrayAddressOfElementAt(result->array(), 0));
+  ObjHolder holder;
+  CreateStringFromCString("<UNIMPLEMENTED>", holder.slot());
+  UpdateHeapRef(ArrayAddressOfElementAt(result->array(), 0), holder.obj());
   return result;
 #else
   uint32_t size = stackTrace->array()->count_;
@@ -172,7 +174,9 @@ OBJ_GETTER(GetStackTraceStrings, KConstRef stackTrace) {
     }
     char line[512];
     konan::snprintf(line, sizeof(line) - 1, "%s (%p)", symbol, (void*)(intptr_t)address);
-    CreateStringFromCString(line, ArrayAddressOfElementAt(strings->array(), index));
+    ObjHolder holder;
+    CreateStringFromCString(line, holder.slot());
+    UpdateHeapRef(ArrayAddressOfElementAt(strings->array(), index), holder.obj());
   }
 #else
   if (size > 0) {
@@ -195,7 +199,9 @@ OBJ_GETTER(GetStackTraceStrings, KConstRef stackTrace) {
       } else {
         result = symbol;
       }
-      CreateStringFromCString(result, ArrayAddressOfElementAt(strings->array(), index));
+      ObjHolder holder;
+      CreateStringFromCString(result, holder.slot());
+      UpdateHeapRef(ArrayAddressOfElementAt(strings->array(), index), holder.obj());
     }
   }
 #endif
@@ -210,7 +216,7 @@ void ThrowException(KRef exception) {
   PrintThrowable(exception);
   RuntimeCheck(false, "Exceptions unsupported");
 #else
-  throw ObjHolder(exception);
+  throw ExceptionObjHolder(exception);
 #endif
 }
 
@@ -276,7 +282,7 @@ static void KonanTerminateHandler() {
   } else {
     try {
       std::rethrow_exception(currentException);
-    } catch (ObjHolder& e) {
+    } catch (ExceptionObjHolder& e) {
       TerminateWithUnhandledException(e.obj());
     } catch (...) {
       // Not a Kotlin exception.

runtime/src/main/cpp/KString.cpp

@@ -1176,23 +1176,22 @@ OBJ_GETTER(Kotlin_String_subSequence, KString thiz, KInt startIndex, KInt endInd
     RETURN_RESULT_OF0(TheEmptyString);
   }
   KInt length = endIndex - startIndex;
-  ArrayHeader* result = AllocArrayInstance(
-    theStringTypeInfo, length, OBJ_RESULT)->array();
+  ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, length, OBJ_RESULT)->array();
   memcpy(CharArrayAddressOfElementAt(result, 0),
          CharArrayAddressOfElementAt(thiz, startIndex),
          length * sizeof(KChar));
   RETURN_OBJ(result->obj());
 }
 
 const KChar* Kotlin_String_utf16pointer(KString message) {
-    RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string");
-    const KChar* utf16 = CharArrayAddressOfElementAt(message, 0);
-    return utf16;
+  RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string");
+  const KChar* utf16 = CharArrayAddressOfElementAt(message, 0);
+  return utf16;
 }
 
 KInt Kotlin_String_utf16length(KString message) {
-    RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string");
-    return message->count_ * sizeof(KChar);
+  RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string");
+  return message->count_ * sizeof(KChar);
 }