Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

experiment: precise heap tags #4544

Open
wants to merge 31 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
6be52e1
subdivide ambiguous tags
crusso May 17, 2024
7b03506
WIP: rts for precise tagging
crusso May 17, 2024
cc85f80
used correct tags in compile.ml
crusso May 17, 2024
99b2985
ugly attempt to tag allocations
crusso May 17, 2024
0b0976a
implement refined tagging; refine Blob Const (WIP); todo: refine Cons…
crusso May 18, 2024
00e5895
copy, don't alias, in blob_to_text;restrict blob_of_text; generslize …
crusso May 22, 2024
6bbfdea
refine Const.Array into Const.Array and Const.Tuple to reflect precis…
crusso May 22, 2024
4981b3a
adapt incremental array slicing; fix rts tests
crusso May 23, 2024
788f403
rust format
crusso May 23, 2024
30c91ee
adapt generational array slicing; updeate bench numbers
crusso May 24, 2024
28d98e0
fix blob comparistion in patterns
crusso May 30, 2024
4310118
refactor slicing
crusso May 30, 2024
448962f
hide encoding of array tags in helpers
crusso May 30, 2024
2cb651a
refactor array_slicing
crusso May 30, 2024
3de6cac
Merge branch 'master' into claudio/precise-heap-tags
crusso May 31, 2024
b44b133
extern array_slicing test
crusso May 31, 2024
8dbe885
add basic tag checking on unboxing of new tags
crusso May 31, 2024
ac88615
refine tags for actors and shared functions, TAG_BLOB_A and TAB_ARRAY_S
crusso May 31, 2024
6f581c6
format
crusso May 31, 2024
9f86d91
implement required coercions
crusso Jun 1, 2024
ba137c7
fix bugs; extend interpreter; rename prims; add __LINE__ to Tagged.sa…
crusso Jun 3, 2024
4d41889
Update rts/motoko-rts/src/debug.rs
crusso Jun 4, 2024
0fdbc12
document tags
crusso Jun 4, 2024
0ed4fe6
reformat; implement and use blob_of_ptr_size, blob_of_str; fix base32…
crusso Jun 5, 2024
41f8917
remove unused pattern match code from compile.ml
crusso Jun 5, 2024
1e8d0f6
use binary, not text, blob for IDL decoding check
crusso Jun 5, 2024
e8c4c19
add more assertions
crusso Jun 5, 2024
f5e97a9
assert array tag difference is even
crusso Jun 5, 2024
afde9e1
fix one broken assert (missing TAG_BLOB_A case); refactor other asserts
crusso Jun 5, 2024
34dd29d
Update rts/motoko-rts-tests/src/gc/heap.rs
crusso Jun 6, 2024
0082f1d
Update rts/motoko-rts-tests/src/gc/heap.rs
crusso Jun 7, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions rts/motoko-rts-tests/src/continuation_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use motoko_rts::continuation_table::{
continuation_count, recall_continuation, remember_continuation,
};
use motoko_rts::memory::alloc_blob;
use motoko_rts::types::{size_of, Array, Blob, Bytes, Value, Words};
use motoko_rts::types::{size_of, Array, Blob, Bytes, Value, Words, TAG_BLOB_B};

pub unsafe fn test() {
println!("Testing continuation table ...");
Expand All @@ -22,7 +22,7 @@ pub unsafe fn test() {
+ N as u32 * size_of::<Blob>().to_bytes().as_u32(),
));

let pointers: [Value; N] = from_fn(|_| alloc_blob(&mut heap, Bytes(0)));
let pointers: [Value; N] = from_fn(|_| alloc_blob(&mut heap, TAG_BLOB_B, Bytes(0)));

let mut references: [u32; N] = [0; N];
for i in 0..N {
Expand Down
13 changes: 9 additions & 4 deletions rts/motoko-rts-tests/src/gc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ fn check_dynamic_heap(

let is_forwarded = forward != make_pointer(address as u32);

if incremental && tag == TAG_BLOB {
if incremental && tag == TAG_BLOB_B {
assert!(!is_forwarded);
// in-heap mark stack blobs
let length = read_word(heap, offset);
Expand All @@ -285,9 +285,14 @@ fn check_dynamic_heap(
.as_usize();
} else {
if incremental {
assert!(tag == TAG_ARRAY || tag >= TAG_ARRAY_SLICE_MIN);
assert!(
tag == TAG_ARRAY_I
|| tag == TAG_ARRAY_M
|| tag == TAG_ARRAY_T
|| tag >= TAG_ARRAY_SLICE_MIN
);
} else {
assert_eq!(tag, TAG_ARRAY);
assert!(tag == TAG_ARRAY_I || tag == TAG_ARRAY_M || tag == TAG_ARRAY_T)
}

if is_forwarded {
Expand Down Expand Up @@ -431,7 +436,7 @@ fn check_continuation_table(mut offset: usize, continuation_table: &[ObjectIdx],
let incremental = cfg!(feature = "incremental_gc");

let table_addr = heap.as_ptr() as usize + offset;
assert_eq!(read_word(heap, offset), TAG_ARRAY);
assert_eq!(read_word(heap, offset), TAG_ARRAY_M);
offset += WORD_SIZE;

if incremental {
Expand Down
18 changes: 13 additions & 5 deletions rts/motoko-rts-tests/src/gc/compacting/mark_stack.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,26 @@ fn test_push_pop() {
.unwrap();
}

static TAGS: [Tag; 14] = [
static TAGS: [Tag; 22] = [
TAG_OBJECT,
TAG_OBJ_IND,
TAG_ARRAY,
TAG_BITS64,
TAG_ARRAY_I,
TAG_ARRAY_M,
TAG_ARRAY_T,
TAG_BITS64_U,
TAG_BITS64_S,
TAG_BITS64_F,
TAG_MUTBOX,
TAG_CLOSURE,
TAG_SOME,
TAG_VARIANT,
TAG_BLOB,
TAG_BLOB_B,
TAG_BLOB_T,
TAG_BLOB_P,
TAG_FWD_PTR,
TAG_BITS32,
TAG_BITS32_U,
TAG_BITS32_S,
TAG_BITS32_F,
TAG_BIGINT,
TAG_CONCAT,
TAG_NULL,
Expand Down
6 changes: 3 additions & 3 deletions rts/motoko-rts-tests/src/gc/heap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,7 @@ fn create_dynamic_heap(

// Store object header
let address = u32::try_from(heap_start + heap_offset).unwrap();
write_word(dynamic_heap, heap_offset, TAG_ARRAY);
write_word(dynamic_heap, heap_offset, TAG_ARRAY_M);
heap_offset += WORD_SIZE;

if incremental {
Expand Down Expand Up @@ -497,7 +497,7 @@ fn create_dynamic_heap(
let mut heap_offset = continuation_table_offset;

let continuation_table_address = u32::try_from(heap_start + heap_offset).unwrap();
write_word(dynamic_heap, heap_offset, TAG_ARRAY);
write_word(dynamic_heap, heap_offset, TAG_ARRAY_M); // TODO: TBR
crusso marked this conversation as resolved.
Show resolved Hide resolved
heap_offset += WORD_SIZE;

if incremental {
Expand Down Expand Up @@ -570,7 +570,7 @@ fn create_static_heap(
// root.
let array_addr = u32::try_from(heap.as_ptr() as usize).unwrap();
let mut offset = 0;
write_word(heap, offset, TAG_ARRAY);
write_word(heap, offset, TAG_ARRAY_M); // TODO: TBR
crusso marked this conversation as resolved.
Show resolved Hide resolved
crusso marked this conversation as resolved.
Show resolved Hide resolved
offset += WORD_SIZE;

if incremental {
Expand Down
30 changes: 17 additions & 13 deletions rts/motoko-rts-tests/src/gc/incremental/array_slicing.rs
Original file line number Diff line number Diff line change
@@ -1,33 +1,37 @@
use motoko_rts::{
gc::incremental::array_slicing::slice_array,
memory::alloc_array,
types::{Words, TAG_ARRAY, TAG_ARRAY_SLICE_MIN},
types::{Tag, Words, TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_SLICE_MIN, TAG_ARRAY_T},
};

use crate::memory::TestMemory;

pub unsafe fn test() {
println!(" Testing array slicing...");

let mut mem = TestMemory::new(Words(1024 * 1024));
// multiple of slice increment
test_array_slicing(&mut mem, 4096);
// odd remainder of slice increment
test_array_slicing(&mut mem, 3999);
// small array
test_array_slicing(&mut mem, 10);
// empty array
test_array_slicing(&mut mem, 0);
let tags = vec![TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_T];

for tag in tags.into_iter() {
let mut mem = TestMemory::new(Words(1024 * 1024));
// multiple of slice increment
test_array_slicing(&mut mem, tag, 4096);
// odd remainder of slice increment
test_array_slicing(&mut mem, tag, 3999);
// small array
test_array_slicing(&mut mem, tag, 10);
// empty array
test_array_slicing(&mut mem, tag, 0);
}
}

unsafe fn test_array_slicing(mem: &mut TestMemory, array_length: u32) {
let array = alloc_array(mem, array_length).as_array();
unsafe fn test_array_slicing(mem: &mut TestMemory, tag: Tag, array_length: u32) {
let array = alloc_array(mem, tag, array_length).as_array();
let mut last_offset = 0;
loop {
let new_offset = slice_array(array);
assert!(new_offset > last_offset || array.len() == 0 && new_offset == 0);
last_offset = new_offset;
if (*array).header.tag == TAG_ARRAY {
if (*array).header.tag == tag {
break;
}
assert!((*array).header.tag >= TAG_ARRAY_SLICE_MIN);
Expand Down
17 changes: 11 additions & 6 deletions rts/motoko-rts-tests/src/gc/incremental/partitioned_heap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ use motoko_rts::{
time::BoundedTime,
},
memory::{alloc_array, alloc_blob, Memory},
types::{Array, Blob, Bytes, Obj, Tag, Value, Words, TAG_ARRAY, TAG_BLOB},
types::{
Array, Blob, Bytes, Obj, Tag, Value, Words, TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_T,
TAG_BLOB_B, TAG_BLOB_P, TAG_BLOB_T,
},
};

use crate::{gc::utils::WORD_SIZE, memory::TestMemory};
Expand Down Expand Up @@ -308,7 +311,7 @@ unsafe fn iterate_large_partition(
let mut time = BoundedTime::new(0);
while iterator.has_object() {
let object = iterator.current_object();
assert_eq!(object.tag(), TAG_BLOB);
assert_eq!(object.tag(), TAG_BLOB_B);
let size = block_size(object as *const Tag);
detected_sizes.push(size);
time.tick();
Expand Down Expand Up @@ -396,7 +399,7 @@ impl PartitionedTestHeap {

pub fn allocate_array(&mut self, elements: &[Value]) -> Value {
unsafe {
let array = alloc_array(self, elements.len() as u32);
let array = alloc_array(self, TAG_ARRAY_M, elements.len() as u32);
for index in 0..elements.len() {
let raw_array = array.as_array();
raw_array.set_scalar(index as u32, elements[index]);
Expand All @@ -406,16 +409,18 @@ impl PartitionedTestHeap {
}

pub fn allocate_blob(&mut self, size: usize) -> Value {
unsafe { alloc_blob(self, Bytes(size as u32)) }
unsafe { alloc_blob(self, TAG_BLOB_B, Bytes(size as u32)) }
}
}

unsafe fn block_size(block: *const Tag) -> usize {
match *block {
TAG_ARRAY => {
TAG_ARRAY_I | TAG_ARRAY_M | TAG_ARRAY_T => {
size_of::<Array>() + (block as *const Array).len() as usize * WORD_SIZE as usize
}
TAG_BLOB => size_of::<Blob>() + (block as *const Blob).len().as_usize(),
TAG_BLOB_B | TAG_BLOB_T | TAG_BLOB_P => {
size_of::<Blob>() + (block as *const Blob).len().as_usize()
}
_ => unimplemented!(),
}
}
Expand Down
6 changes: 3 additions & 3 deletions rts/motoko-rts-tests/src/principal_id.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::memory::TestMemory;

use motoko_rts::principal_id::{blob_of_principal, principal_of_blob};
use motoko_rts::text::{text_compare, text_of_ptr_size, text_of_str};
use motoko_rts::text::{blob_compare, text_compare, text_of_ptr_size, text_of_str};
use motoko_rts::types::{Bytes, Words};

pub unsafe fn test() {
Expand Down Expand Up @@ -36,11 +36,11 @@ pub unsafe fn test() {

let text = text_of_str(&mut heap, "aaaaa-aa");
let principal = blob_of_principal(&mut heap, text);
assert_eq!(text_compare(principal, text_of_str(&mut heap, ""),), 0);
assert_eq!(blob_compare(principal, text_of_str(&mut heap, ""),), 0);

let text = text_of_str(&mut heap, "bfozs-kwa73-7nadi");
assert_eq!(
text_compare(
blob_compare(
crusso marked this conversation as resolved.
Show resolved Hide resolved
blob_of_principal(&mut heap, text),
text_of_ptr_size(&mut heap, b"\xC0\xFE\xFE\xD0\x0D".as_ptr(), Bytes(5))
),
Expand Down
4 changes: 2 additions & 2 deletions rts/motoko-rts-tests/src/text.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use motoko_rts::text::{
text_singleton, text_size,
};
use motoko_rts::text_iter::{text_iter, text_iter_done, text_iter_next};
use motoko_rts::types::{Bytes, Value, Words, TAG_BLOB};
use motoko_rts::types::{Bytes, Value, Words, TAG_BLOB_T};

use std::convert::TryFrom;

Expand Down Expand Up @@ -68,7 +68,7 @@ pub unsafe fn test() {
for i in 0..8 {
let str = &STR[0..i + 1];
let text = text_of_str(&mut mem, str);
assert_eq!(text.tag(), TAG_BLOB);
assert_eq!(text.tag(), TAG_BLOB_T);
let iter = TextIter::from_text(&mut mem, text);
assert_eq!(iter.collect::<String>(), str);
}
Expand Down
5 changes: 3 additions & 2 deletions rts/motoko-rts/src/allocator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use alloc::alloc::{GlobalAlloc, Layout};
//use core::ptr::null_mut;
use crate::memory::{alloc_blob, ic};
use crate::types::Bytes;
use crate::types::{Bytes, TAG_BLOB_B};

pub struct EphemeralAllocator;

Expand All @@ -24,7 +24,8 @@ unsafe impl GlobalAlloc for EphemeralAllocator {
let min_align = (align + word_size - 1) / word_size * word_size;
let blob_size = size + min_align - word_size;
let blob =
alloc_blob::<ic::IcMemory>(&mut ic::IcMemory, Bytes(blob_size as u32)).as_blob_mut();
alloc_blob::<ic::IcMemory>(&mut ic::IcMemory, TAG_BLOB_B, Bytes(blob_size as u32))
.as_blob_mut();
let payload_address = blob.payload_addr() as usize;
let aligned_address = (payload_address + min_align - 1) / min_align * min_align;

Expand Down
4 changes: 2 additions & 2 deletions rts/motoko-rts/src/blob_iter.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use crate::{
barriers::allocation_barrier,
types::{size_of, Array, Bytes, Value, Words, TAG_ARRAY},
types::{size_of, Array, Bytes, Value, Words, TAG_ARRAY_T},
};

use motoko_rts_macros::ic_mem_fn;
Expand All @@ -16,7 +16,7 @@ unsafe fn blob_iter<M: crate::memory::Memory>(mem: &mut M, blob: Value) -> Value

// NB. cannot use as_array() here as we didn't write the header yet
let iter_array = iter_ptr.get_ptr() as *mut Array;
(*iter_array).header.tag = TAG_ARRAY;
(*iter_array).header.tag = TAG_ARRAY_T;
luc-blaeser marked this conversation as resolved.
Show resolved Hide resolved
(*iter_array).header.init_forward(iter_ptr);
(*iter_array).len = 2;

Expand Down
6 changes: 3 additions & 3 deletions rts/motoko-rts/src/continuation_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
use crate::barriers::allocation_barrier;
use crate::memory::{alloc_array, Memory};
use crate::rts_trap_with;
use crate::types::Value;
use crate::types::{Value, TAG_ARRAY_M};

use motoko_rts_macros::ic_mem_fn;

Expand All @@ -42,7 +42,7 @@ static mut N_CONTINUATIONS: u32 = 0;
static mut FREE_SLOT: u32 = 0;

unsafe fn create_continuation_table<M: Memory>(mem: &mut M) {
TABLE = alloc_array(mem, INITIAL_SIZE);
TABLE = alloc_array(mem, TAG_ARRAY_M, INITIAL_SIZE);
FREE_SLOT = 0;
N_CONTINUATIONS = 0;

Expand All @@ -61,7 +61,7 @@ unsafe fn double_continuation_table<M: Memory>(mem: &mut M) {

let new_size = old_size * 2;

TABLE = alloc_array(mem, new_size);
TABLE = alloc_array(mem, TAG_ARRAY_M, new_size);
let new_array = TABLE.as_array();

for i in 0..old_size {
Expand Down
8 changes: 4 additions & 4 deletions rts/motoko-rts/src/debug.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ pub(crate) unsafe fn print_boxed_object(buf: &mut WriteBuf, p: usize) {
let obj_ind = obj as *const ObjInd;
let _ = write!(buf, "<ObjInd field={:#x}>", (*obj_ind).field.get_raw());
}
TAG_ARRAY | TAG_ARRAY_SLICE_MIN.. => {
TAG_ARRAY_I | TAG_ARRAY_M | TAG_ARRAY_T | TAG_ARRAY_SLICE_MIN.. => {
let array = obj as *mut Array;
let _ = write!(buf, "<Array len={:#x}", (*array).len);

Expand All @@ -179,7 +179,7 @@ pub(crate) unsafe fn print_boxed_object(buf: &mut WriteBuf, p: usize) {
let _ = write!(buf, ">");
}
}
TAG_BITS64 => {
TAG_BITS64_U | TAG_BITS64_S | TAG_BITS64_F => {
let bits64 = obj as *const Bits64;
let _ = write!(buf, "<Bits64 {:#x}>", (*bits64).bits());
}
Expand All @@ -204,15 +204,15 @@ pub(crate) unsafe fn print_boxed_object(buf: &mut WriteBuf, p: usize) {
(*variant).field.get_raw()
);
}
TAG_BLOB => {
TAG_BLOB_B | TAG_BLOB_T | TAG_BLOB_P => {
crusso marked this conversation as resolved.
Show resolved Hide resolved
crusso marked this conversation as resolved.
Show resolved Hide resolved
let blob = obj.as_blob();
let _ = write!(buf, "<Blob len={:#x}>", blob.len().as_u32());
}
TAG_FWD_PTR => {
let ind = obj as *const FwdPtr;
let _ = write!(buf, "<Forwarding to {:#x}>", (*ind).fwd.get_raw());
}
TAG_BITS32 => {
TAG_BITS32_U | TAG_BITS32_S | TAG_BITS32_F => {
let bits32 = obj as *const Bits32;
let _ = write!(buf, "<Bits32 {:#x}>", (*bits32).bits);
}
Expand Down
5 changes: 3 additions & 2 deletions rts/motoko-rts/src/gc/generational.rs
Original file line number Diff line number Diff line change
Expand Up @@ -296,15 +296,16 @@ impl<'a, M: Memory> GenerationalGC<'a, M> {
|gc, slice_start, array| {
const SLICE_INCREMENT: u32 = 255;
debug_assert!(SLICE_INCREMENT >= TAG_ARRAY_SLICE_MIN);
let base_tag = array.base_tag();
if array.len() - slice_start > SLICE_INCREMENT {
let new_start = slice_start + SLICE_INCREMENT;
// Remember to visit the array suffix later, store the next visit offset in the tag.
(*array).header.tag = new_start;
array.set_slice_start(base_tag, new_start);
push_mark_stack(gc.heap.mem, array as usize);
new_start
} else {
// No further visits of this array. Restore the tag.
(*array).header.tag = TAG_ARRAY;
array.restore_tag(base_tag); // restore original tag
array.len()
}
},
Expand Down
4 changes: 2 additions & 2 deletions rts/motoko-rts/src/gc/generational/mark_stack.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
//! Simplified to only store object pointers without tags.

use crate::memory::{alloc_blob, Memory};
use crate::types::{Blob, Words};
use crate::types::{Blob, Words, TAG_BLOB_B};

use core::ptr::null_mut;

Expand All @@ -27,7 +27,7 @@ pub unsafe fn alloc_mark_stack<M: Memory>(mem: &mut M) {

// Allocating an actual object here to not break dump_heap
// No post allocation barrier as this RTS-internal blob will be collected by the GC.
STACK_BLOB_PTR = alloc_blob(mem, INIT_STACK_SIZE.to_bytes()).get_ptr() as *mut Blob;
STACK_BLOB_PTR = alloc_blob(mem, TAG_BLOB_B, INIT_STACK_SIZE.to_bytes()).get_ptr() as *mut Blob;
STACK_BASE = STACK_BLOB_PTR.payload_addr() as *mut usize;
STACK_PTR = STACK_BASE;
STACK_TOP = STACK_BASE.add(INIT_STACK_SIZE.as_usize());
Expand Down
Loading