Change rust_vec to have a 16-byte header, to 16-byte-align vec-body data. Major perf win.

Author: graydon

src/boot/be/abi.ml

@@ -93,7 +93,8 @@ let tydesc_field_stateflag = 11;;
 let vec_elt_rc = 0;;
 let vec_elt_alloc = 1;;
 let vec_elt_fill = 2;;
-let vec_elt_data = 3;;
+let vec_elt_pad = 3;;
+let vec_elt_data = 4;;
 
 let calltup_elt_out_ptr = 0;;
 let calltup_elt_task_ptr = 1;;

src/boot/me/semant.ml

@@ -2382,7 +2382,7 @@ and fn_rty (cx:ctxt) (opaque_box_body:bool) : Il.referent_ty =
 and vec_sty (word_bits:Il.bits) : Il.scalar_ty =
   let word = word_rty word_bits in
   let ptr = Il.ScalarTy (Il.AddrTy Il.OpaqueTy) in
-    Il.AddrTy (Il.StructTy [| word; word; word; ptr |])
+    Il.AddrTy (Il.StructTy [| word; word; word; word; ptr |])
 
 and referent_type
     ?parent_tags:parent_tags
@@ -2442,7 +2442,7 @@ and referent_type
       | Ast.TY_mach (TY_i64)
       | Ast.TY_mach (TY_f64) -> sv Il.Bits64
 
-      | Ast.TY_str -> sp (Il.StructTy [| word; word; word; ptr |])
+      | Ast.TY_str -> sp (Il.StructTy [| word; word; word; word; ptr |])
       | Ast.TY_vec _ -> s (vec_sty word_bits)
       | Ast.TY_tup tt -> tup tt
       | Ast.TY_rec tr -> tup (Array.map snd tr)

src/boot/me/trans.ml

@@ -1549,6 +1549,7 @@ let trans_visitor
                              Asm.IMM Abi.const_refcount);
                    Asm.WORD (word_ty_mach, Asm.IMM init_sz);
                    Asm.WORD (word_ty_mach, Asm.IMM init_sz);
+                   Asm.WORD (word_ty_mach, Asm.IMM 0L);
                    Asm.ZSTRING s
                  |]))
           (referent_type cx Ast.TY_str)
@@ -2850,15 +2851,16 @@ let trans_visitor
 
   (*
    * A vec is implicitly boxed: every slot vec[T] is 1 word and
-   * points to a refcounted structure. That structure has 3 words with
+   * points to a refcounted structure. That structure has 4 words with
    * defined meaning at the beginning; data follows the header.
    *
    *   word 0: refcount or gc control word
    *   word 1: allocated size of data
    *   word 2: initialised size of data
-   *   word 3...N: data
+   *   word 3: padding word to hit even multiple of 16
+   *   word 4...N: data
    * 
-   * This 3-word prefix is shared with strings, we factor the common
+   * This 4-word prefix is shared with strings, we factor the common
    * part out for reuse in string code.
    *)
 

src/comp/back/abi.rs

@@ -26,7 +26,8 @@ const int general_code_alignment = 16;
 const int vec_elt_rc = 0;
 const int vec_elt_alloc = 1;
 const int vec_elt_fill = 2;
-const int vec_elt_data = 3;
+const int vec_elt_pad = 3;
+const int vec_elt_data = 4;
 
 const int tydesc_field_first_param = 0;
 const int tydesc_field_size = 1;

src/comp/middle/trans.rs

@@ -372,6 +372,7 @@ fn T_vec(TypeRef t) -> TypeRef {
     ret T_struct(vec(T_int(),       // Refcount
                      T_int(),       // Alloc
                      T_int(),       // Fill
+                     T_int(),       // Pad
                      T_array(t, 0u) // Body elements
                      ));
 }
@@ -870,6 +871,7 @@ fn C_str(@crate_ctxt cx, str s) -> ValueRef {
     auto box = C_struct(vec(C_int(abi.const_refcount as int),
                             C_int(len + 1u as int), // 'alloc'
                             C_int(len + 1u as int), // 'fill'
+                            C_int(0),               // 'pad'
                             llvm.LLVMConstString(_str.buf(s),
                                                  len, False)));
     auto g = llvm.LLVMAddGlobal(cx.llmod, val_ty(box),

src/rt/rust_util.h

@@ -172,6 +172,7 @@ rust_vec : public rc_base<rust_vec>
 {
     size_t alloc;
     size_t fill;
+    size_t pad; // Pad to align data[0] to 16 bytes.
     uint8_t data[];
     rust_vec(rust_dom *dom, size_t alloc, size_t fill, uint8_t const *d) :
         alloc(alloc),