Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Working heap overflow detection. #61

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Working heap overflow detection. #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 30 additions & 8 deletions villain/compile.rkt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
(define rsi 'rsi) ; arg2
(define r10 'r10) ; scratch in compile-prim3, make-string, string-set!, compile-vector, vector-set!
; compile-define, fl<=
(define r15 'r15) ; Holds pointer to the end of the heap
(define rcx 'rcx) ; arity indicator
(define al 'al) ; low byte of rax ; open-input-file
(define xmm0 'xmm0) ; registers to hold double precision floating numbers
Expand All @@ -38,13 +39,20 @@
(Extern 'str_to_symbol)
(Label 'entry)
(Mov rbx rdi) ; recv heap pointer
(Mov r15 rsi) ; pointer to end of heap
(Mov rsi 0)
(compile-e-tail e '())
(Mov rdx rbx) ; return heap pointer in second return register
(Ret)
(compile-defines ds)
(Label 'raise_error_align)
(Sub rsp 8)
(Jmp 'raise_error))]))
;; Asm
(define check-heap-ptr
(seq
(Cmp rbx r15)
(Jg 'raise_error)))

;; Expr -> Asm
(define (compile-library p)
Expand Down Expand Up @@ -110,6 +118,7 @@
(Mov (Offset rbx 8) rax)
(Mov rax rbx)
(Add rbx 16)
check-heap-ptr
(Or rax type-cons)
(Sub rcx (imm->bits 1))
(Jmp loop)
Expand Down Expand Up @@ -161,7 +170,8 @@
(Mov (Offset rbx 0) rax)
(Mov rax rbx)
(Or rax type-flonum)
(Add rbx 8))
(Add rbx 8)
check-heap-ptr)
)

;; String -> Asm
Expand All @@ -173,7 +183,8 @@
(compile-str-chars (string->list s) 3 0 1)
(Mov rax rbx)
(Or rax type-string)
(Add rbx (* 8 (add1 (ceiling (/ len 3))))))))
(Add rbx (* 8 (add1 (ceiling (/ len 3)))))
check-heap-ptr)))

;; Vec CEnv -> Asm
(define (compile-vector ds c)
Expand All @@ -182,8 +193,10 @@
(Mov (Offset rbx 0) r9) ;;write length in first word, will also store rbx location
(Mov r10 rbx)
(Add rbx 8)
check-heap-ptr
(Mov r9 rbx) ;;r9 will be used in compile-vec-elems as a temporary heap pointer
(Add rbx (* 8 len)) ;; rbx now points to next open space on heap for future calls
check-heap-ptr
(compile-vec-elems ds c)
(Mov rax r10)
(Or rax type-vector))))
Expand Down Expand Up @@ -392,7 +405,8 @@
(seq (Mov (Offset rbx 0) rax)
(Mov rax rbx)
(Or rax type-box)
(Add rbx 8))]
(Add rbx 8)
check-heap-ptr)]
['unbox
(seq (assert-box rax c)
(Xor rax type-box)
Expand Down Expand Up @@ -458,6 +472,7 @@
;; heap alignment.
(Add rbx (+ (- 8 (modulo (+ 11 port-buffer-bytes) 8)) 11
port-buffer-bytes))
check-heap-ptr
(Mov rax r8)
(Or rax type-port))]
['close-input-port
Expand Down Expand Up @@ -620,11 +635,13 @@
(Mov (Offset rbx 0) r8) ;should r8
;(Mov (Offset rbx 8) rax)
(Add rbx 8) ;advances heap pointer
check-heap-ptr
(Label l1)
(Cmp r8 0)
(Je l2) ;(While r8 > 0){
(Mov (Offset rbx 0) rax) ;;should rax ;Copies the value into the spot on the heap
(Add rbx 8)
check-heap-ptr
(Sub r8 1) ;r8--;
(Jmp l1) ;}
(Label l2) ;done writing
Expand Down Expand Up @@ -657,6 +674,7 @@
(Mov r10 rbx) ; save heap pointer
(Mov (Offset rbx 0) r8) ; write length in word 0
(Add rbx 8) ; advance heap pointer
check-heap-ptr
(Cmp r8 (imm->bits 1))
(Jl l3)
(Mov r9 rax) ; r9 = char arg
Expand All @@ -675,6 +693,7 @@
(Je l4)
(Mov (Offset rbx 0) r8)
(Add rbx 8) ; advance the heap pointer
check-heap-ptr
(Sub rax 1)
(Jmp l1)
(Label l4)
Expand All @@ -699,35 +718,38 @@
(Mov (Offset rbx 8) rax)
(Mov rax rbx)
(Or rax type-cons)
(Add rbx 16))]
(Add rbx 16)
check-heap-ptr)]

['fl+ (seq
(Pop r8)
(assert-flonum r8 c)
(assert-flonum rax c)
(Xor rax type-flonum)
(Xor rax type-flonum)
(Xor r8 type-flonum)
(Movapd xmm0 (Offset r8 0))
(Addsd xmm0 (Offset rax 0))
(Addsd xmm0 (Offset rax 0))
(Movapd (Offset rbx 0) xmm0)
(Mov rax rbx)
(Or rax type-flonum)
(Add rbx 8)
check-heap-ptr
)
]
['fl- (seq
(Pop r8)
(assert-flonum r8 c)
(assert-flonum rax c)
(Xor rax type-flonum)

(Xor r8 type-flonum)
(Movapd xmm0 (Offset r8 0))
(Subsd xmm0 (Offset rax 0))
(Subsd xmm0 (Offset rax 0))
(Movapd (Offset rbx 0) xmm0)
(Mov rax rbx)
(Or rax type-flonum)
(Add rbx 8)
check-heap-ptr
)
]

Expand Down
3 changes: 2 additions & 1 deletion villain/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,8 @@ int main(int argc, char** argv)
error_handler = &error_exit;
heap = vl_calloc(8 * heap_size, 1);

result = entry(heap);
result = entry(heap, heap + (heap_size - heap_buffer));
// result = entry(heap);

print_result(result);
if (vl_typeof(result) != VL_VOID)
Expand Down
2 changes: 2 additions & 0 deletions villain/runtime.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ extern void (*error_handler)();

// in words
#define heap_size 10000
// maximum number of words we expect to allocate before checking heap ptr.
#define heap_buffer 2
extern int64_t *heap;

#endif