Fix/object allocation

src/array.c

@@ -471,13 +471,6 @@ JL_DLLEXPORT jl_value_t *jl_array_to_string(jl_array_t *a)
          ((a->maxsize + sizeof(void*) + 1 <= GC_MAX_SZCLASS) == (len + sizeof(void*) + 1 <= GC_MAX_SZCLASS)))) {
         jl_value_t *o = jl_array_data_owner(a);
         if (jl_is_string(o)) {
-#ifdef MMTKHEAP
-            // since we need the size of the string to be accurate according to its allocation size, we simply allocate a new string here
-            // instead of changing its size to len as in `*(size_t*)o = len`
-            o = jl_gc_realloc_string(o, len);
-            jl_value_t** owner_addr = (a + jl_array_data_owner_offset(jl_array_ndims(a)));
-            owner_addr = o;
-#endif
             a->flags.isshared = 1;
             *(size_t*)o = len;
             a->nrows = 0;
@@ -503,29 +496,32 @@ JL_DLLEXPORT jl_value_t *jl_alloc_string(size_t len)
     jl_value_t *s;
     jl_ptls_t ptls = ct->ptls;
     const size_t allocsz = sz + sizeof(jl_taggedvalue_t);
-    if (sz <= GC_MAX_SZCLASS) {
 #ifndef MMTKHEAP
+    if (sz <= GC_MAX_SZCLASS) {
         int pool_id = jl_gc_szclass_align8(allocsz);
         jl_gc_pool_t *p = &ptls->heap.norm_pools[pool_id];
         int osize = jl_gc_sizeclasses[pool_id];
         // We call `jl_gc_pool_alloc_noinline` instead of `jl_gc_pool_alloc` to avoid double-counting in
         // the Allocations Profiler. (See https://github.com/JuliaLang/julia/pull/43868 for more details.)
         s = jl_gc_pool_alloc_noinline(ptls, (char*)p - (char*)ptls, osize);
-#else
-        int pool_id = jl_gc_szclass_align8(allocsz);
-        int osize = jl_gc_sizeclasses[pool_id];
-        s = jl_mmtk_gc_alloc_default(ptls, pool_id, osize, jl_string_type);
-#endif
     }
     else {
         if (allocsz < sz) // overflow in adding offs, size was "negative"
             jl_throw(jl_memory_exception);
-#ifndef MMTKHEAP
+
         s = jl_gc_big_alloc_noinline(ptls, allocsz);
+    }
 #else
+    if (allocsz < sz) // overflow in adding offs, size was "negative"
+        jl_throw(jl_memory_exception);
+
+    if (allocsz < MAX_STANDARD_OBJECT_SIZE) {
+        s = jl_mmtk_gc_alloc_default(ptls, allocsz, jl_string_type);
+    } else {
         s = jl_mmtk_gc_alloc_big(ptls, allocsz);
-#endif
     }
+#endif
+
     jl_set_typeof(s, jl_string_type);
     maybe_record_alloc_to_profile(s, len, jl_string_type);
     *(size_t*)s = len;

src/julia_internal.h

@@ -239,7 +239,8 @@ jl_value_t *jl_gc_pool_alloc_noinline(jl_ptls_t ptls, int pool_offset,
                                    int osize);
 jl_value_t *jl_gc_big_alloc_noinline(jl_ptls_t ptls, size_t allocsz);
 #ifdef MMTKHEAP
-JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int pool_offset, int osize, void* ty);
+extern size_t MAX_STANDARD_OBJECT_SIZE;
+JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, size_t size, void* ty);
 JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_big(jl_ptls_t ptls, size_t allocsz);
 #endif
 JL_DLLEXPORT int jl_gc_classify_pools(size_t sz, int *osize);
@@ -364,29 +365,30 @@ STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty)
 {
     jl_value_t *v;
     const size_t allocsz = sz + sizeof(jl_taggedvalue_t);
-    if (sz <= GC_MAX_SZCLASS) {
 #ifndef MMTKHEAP
+    if (sz <= GC_MAX_SZCLASS) {
         int pool_id = jl_gc_szclass(allocsz);
         jl_gc_pool_t *p = &ptls->heap.norm_pools[pool_id];
         int osize = jl_gc_sizeclasses[pool_id];
         // We call `jl_gc_pool_alloc_noinline` instead of `jl_gc_pool_alloc` to avoid double-counting in
         // the Allocations Profiler. (See https://github.com/JuliaLang/julia/pull/43868 for more details.)
         v = jl_gc_pool_alloc_noinline(ptls, (char*)p - (char*)ptls, osize);
-#else
-        int pool_id = jl_gc_szclass(allocsz);
-        int osize = jl_gc_sizeclasses[pool_id];
-        v = jl_mmtk_gc_alloc_default(ptls, pool_id, osize, ty);
-#endif
     }
     else {
         if (allocsz < sz) // overflow in adding offs, size was "negative"
             jl_throw(jl_memory_exception);
-#ifndef MMTKHEAP
         v = jl_gc_big_alloc_noinline(ptls, allocsz);
+    }
 #else
+    if (allocsz < sz) // overflow in adding offs, size was "negative"
+        jl_throw(jl_memory_exception);
+
+    if (allocsz < (MAX_STANDARD_OBJECT_SIZE - 8)) { // buffer may take 8 bytes extra
+        v = jl_mmtk_gc_alloc_default(ptls, allocsz, ty);
+    } else {
         v = jl_mmtk_gc_alloc_big(ptls, allocsz);
-#endif
     }
+#endif
     jl_set_typeof(v, ty);
     maybe_record_alloc_to_profile(v, sz, (jl_datatype_t*)ty);
     return v;

src/llvm-final-gc-lowering.cpp

@@ -171,25 +171,39 @@ Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
 {
     assert(target->arg_size() == 2);
     auto sz = (size_t)cast<ConstantInt>(target->getArgOperand(1))->getZExtValue();
-    // This is strongly architecture and OS dependent
-    int osize;
-    int offset = jl_gc_classify_pools(sz, &osize);
+
     IRBuilder<> builder(target);
     builder.SetCurrentDebugLocation(target->getDebugLoc());
     auto ptls = target->getArgOperand(0);
     CallInst *newI;
+#ifndef MMTKHEAP
+    // This is strongly architecture and OS dependent
+    int osize;
+    int offset = jl_gc_classify_pools(sz, &osize);
     if (offset < 0) {
         newI = builder.CreateCall(
             bigAllocFunc,
             { ptls, ConstantInt::get(T_size, sz + sizeof(void*)) });
     }
     else {
-#ifndef MMTKHEAP
         auto pool_offs = ConstantInt::get(Type::getInt32Ty(F.getContext()), offset);
         auto pool_osize = ConstantInt::get(Type::getInt32Ty(F.getContext()), osize);
         newI = builder.CreateCall(poolAllocFunc, { ptls, pool_offs, pool_osize });
+    }
+    newI->setAttributes(newI->getCalledFunction()->getAttributes());
+    newI->takeName(target);
+    return newI;
 #else
-        auto pool_osize = ConstantInt::get(Type::getInt64Ty(F.getContext()), osize);
+    if (sz + sizeof(void*) >= MAX_STANDARD_OBJECT_SIZE) {
+        newI = builder.CreateCall(
+                bigAllocFunc,
+                { ptls, ConstantInt::get(T_size, sz + sizeof(void*)) });
+
+        newI->setAttributes(newI->getCalledFunction()->getAttributes());
+        newI->takeName(target);
+        return newI;
+    } else {
+        auto pool_osize = ConstantInt::get(T_size, sz + sizeof(void*));
         auto cursor_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, cursor));
         auto limit_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()),  offsetof(jl_tls_states_t, limit));
 
@@ -253,11 +267,8 @@ Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
         phiNode->takeName(target);
 
         return phiNode;
-#endif
     }
-    newI->setAttributes(newI->getCalledFunction()->getAttributes());
-    newI->takeName(target);
-    return newI;
+#endif
 }
 
 bool FinalLowerGC::doInitialization(Module &M) {

src/llvm-pass-helpers.cpp

@@ -247,7 +247,7 @@ namespace jl_well_known {
 #ifndef MMTKHEAP
                     { Type::getInt8PtrTy(context.getLLVMContext()), Type::getInt32Ty(context.getLLVMContext()), Type::getInt32Ty(context.getLLVMContext()) },
 #else
-                    { Type::getInt32Ty(context.getLLVMContext()), Type::getInt64Ty(context.getLLVMContext()) },
+                    { Type::getInt32Ty(context.getLLVMContext()), context.T_size },
 #endif
                     false),
                 Function::ExternalLinkage,