Merge pull request 'process_mem' (#10) from process_mem into main

Reviewed-on: #10

.gitignore

@@ -6,4 +6,7 @@ iso_root
 *.iso
 *.gch
 */*.gch
-*/*/*.gch
+*/*/*.gch
+.gdb_history
+symbols.map
+symbols.S

DOOM.txt

@@ -0,0 +1,24 @@
+up to doom:
+
+- Return from pedicel_main() normally (to idle)
+
+** Checkpoint: ring0 process working
+
+- VFS layer (open/read/write/...) with USTar filesystem (for initrd)
+
+** Checkpoint: files not linked to but accessible by the kernel
+
+- Ring3 memory mappings 
+- Ring3 privilege switch
+
+** Checkpoint: welcome to userland
+
+- Syscall interface
+- Implement syscalls needed for doom
+
+** Checkpoint: can run simple programs, ring 3, loaded from filesystem 
+
+- Properly handle the keyboard interrupt (keyboard buffer)
+- Port DOOMgeneric (few functions with Framebuffer/ticks/etc.)
+
+** Achievement: It runs doom!

Makefile

@@ -1,4 +1,4 @@
-SOURCES = src/boot/boot.c src/sched/scheduler.c src/sched/process.c src/mem/heap/kheap.c src/mem/paging/vmm.c src/mem/paging/paging.c src/mem/paging/pmm.c src/string/string.c src/io/kbd/ps2.c src/io/serial/serial.c src/io/term/printf.c src/io/term/term.c src/idt/idt.c src/mem/gdt/gdt.c src/mem/misc/utils.c src/time/timer.c src/kmain.c
+SOURCES = src/debug/panic.c src/debug/stacktrace.c src/boot/boot.c src/sched/scheduler.c src/sched/process.c src/mem/heap/kheap.c src/mem/paging/vmm.c src/mem/paging/paging.c src/mem/paging/pmm.c src/string/string.c src/io/kbd/ps2.c src/io/serial/serial.c src/io/term/printf.c src/io/term/term.c src/idt/idt.c src/mem/gdt/gdt.c src/mem/misc/utils.c src/time/timer.c src/kmain.c
 
 PROBLEMATIC_FLAGS=-Wno-unused-parameter -Wno-unused-variable
 
@@ -7,6 +7,11 @@ build:
 	x86_64-elf-gcc -g -c -Isrc $(SOURCES) $(PROBLEMATIC_FLAGS) -Wall -Wextra -std=gnu99 -nostdlib -ffreestanding -fno-stack-protector -fno-omit-frame-pointer -fno-stack-check -fno-PIC -ffunction-sections -fdata-sections -mcmodel=kernel
 	objcopy -O elf64-x86-64 -B i386 -I binary zap-light16.psf zap-light16.o
 	nasm -f elf64 src/idt/idt.S -o idt_stub.o
+	nasm -f elf64 src/entry.S -o entry.o
+	x86_64-elf-ld -o pepperk -T linker.ld *.o
+	nm -n pepperk | awk '$$2 ~ /[TtDdBbRr]/ {print $$1, $$3}' > symbols.map
+	python3 symbols.py
+	nasm -f elf64 symbols.S -o symbols.o
 	x86_64-elf-ld -o pepperk -T linker.ld *.o
 
 limine/limine:
@@ -32,11 +37,15 @@ build-iso: limine/limine build
 	./limine/limine bios-install pepper.iso
 
 debug:
-	qemu-system-x86_64 -drive file=pepper.iso -s -S -d int -no-reboot -no-shutdown &
+	/usr/bin/qemu-system-x86_64 -drive file=pepper.iso -s -S -d int -no-reboot -no-shutdown &
 	gdb pepperk --command=debug.gdb
 
+debug2:
+	/usr/bin/qemu-system-x86_64 -drive file=pepper.iso -s -S -d int -no-reboot -no-shutdown &
+	pwndbg pepperk --command=debug.gdb
+
 run: build-iso
-	qemu-system-x86_64 -cdrom pepper.iso -serial stdio
+	/usr/bin/qemu-system-x86_64 -cdrom pepper.iso -serial stdio
 
 clean:
-	rm -rf *.o pepperk iso_root pepper.iso limine
+	rm -rf *.o symbols.map symbols.S pepperk iso_root pepper.iso limine

README.md

@@ -2,7 +2,7 @@
 
 ## Trying the kernel
 
-First install the dependencies: `sudo apt install xorriso make qemu-system`
+First install the dependencies: `sudo apt install python3 xorriso make qemu-system`
 
 Then, to compile the kernel and make an ISO image file: `make build-iso`
 To run it with QEMU, `make run`

debug.gdb

@@ -1,3 +1,3 @@
 target remote localhost:1234
 set disassembly-flavor intel
-display/8i $rip
+display/4i $rip

linker.ld

@@ -1,6 +1,6 @@
 OUTPUT_FORMAT(elf64-x86-64)
 
-ENTRY(kmain)
+ENTRY(_start)
 
 PHDRS
 {

src/config.h

@@ -10,12 +10,14 @@
 /* version */
 #define PEPPEROS_VERSION_MAJOR "0"
 #define PEPPEROS_VERSION_MINOR "0"
-#define PEPPEROS_VERSION_PATCH "1"
+#define PEPPEROS_VERSION_PATCH "58"
 #define PEPPEROS_SPLASH "pepperOS version "PEPPEROS_VERSION_MAJOR"."PEPPEROS_VERSION_MINOR"."PEPPEROS_VERSION_PATCH"\n"
 
 /* process */
 #define PROCESS_NAME_MAX 64
 #define PROCESS_STACK_SIZE 0x10000 // 64kb
+#define PROCESS_BASE 0x400000
+#define PROCESS_STACK_BASE 0x1000000
 
 /* sched */
 // 1 tick = 1 ms => quantum = 10ms
@@ -25,6 +27,7 @@
 #define KERNEL_BASE 0xFFFFFFFF80000000ULL
 // 2 MB should be enough (as of now, the whole kernel ELF is around 75kb)
 #define KERNEL_SIZE 0x200000
+#define KERNEL_STACK_SIZE 65536
 
 /* heap */
 #define KHEAP_SIZE (16*1024*1024)
@@ -32,4 +35,4 @@
 /* term */
 #define TERM_HISTORY_MAX_LINES 256
 
-#endif
+#endif

src/debug/panic.c

@@ -0,0 +1,23 @@
+#include <stddef.h>
+#include "idt/idt.h"
+#include "io/serial/serial.h"
+#include "kernel.h"
+
+void panic(struct cpu_status_t* ctx, const char* str)
+{
+	CLEAR_INTERRUPTS;
+	if (ctx == NULL)
+	{
+		DEBUG("\x1b[38;5;231m\x1b[48;5;196mKernel panic!!!\x1b[0m Something went horribly wrong! (no cpu ctx)");
+		DIE_DEBUG(str);
+		skputc('\n');
+		DEBUG("\x1b[38;5;231m\x1b[48;5;196mend Kernel panic - halting...\x1b[0m");
+		hcf();
+	}
+	DEBUG("\x1b[38;5;231m\x1b[48;5;196mKernel panic!!!\x1b[0m at rip=%p\nSomething went horribly wrong! vect=0x%.2x errcode=0x%x\n\rrax=%p rbx=%p rcx=%p rdx=%p\n\rrsi=%p rdi=%p  r8=%p  r9=%p\n\rr10=%p r11=%p r12=%p r13=%p\n\rr14=%p r15=%p\n\n\rflags=%p\n\rHalting...",
+		ctx->iret_rip,
+		ctx->vector_number, ctx->error_code, ctx->rax, ctx->rbx, ctx->rcx, ctx->rdx, ctx->rsi, ctx->rdi,
+		ctx->r8, ctx->r9, ctx->r10, ctx->r11, ctx->r12, ctx->r13, ctx->r14, ctx->r15, ctx->iret_flags);
+	debug_stack_trace(100);
+	hcf();
+}

src/debug/stacktrace.c

@@ -0,0 +1,75 @@
+#include <stdint.h>
+#include "kernel.h"
+
+void debug_stack_trace(unsigned int max_frames)
+{
+    DEBUG("*** begin stack trace ***");
+    // Thanks GCC :)
+    uintptr_t* rbp = (uintptr_t*)__builtin_frame_address(0);
+
+    for (unsigned int frame=0; frame<max_frames && rbp != NULL; frame++)
+    {
+        // Return address, 1 word above saved rbp
+        uintptr_t rip = rbp[1];
+        uintptr_t offset = 0;
+        const char* name = debug_find_symbol(rip, &offset);
+        DEBUG("[%u] <0x%p> (%s+0x%x)", frame, (void*)rip, name, offset);
+
+        uintptr_t* next_rbp = (uintptr_t*)rbp[0];
+
+        // invalid rbp or we're at the end
+        if (next_rbp <= rbp || next_rbp == NULL)
+        {
+            break;
+        }
+
+        rbp = next_rbp;
+    }
+    DEBUG("*** end stack trace ***");
+}
+
+typedef struct
+{
+    uint64_t addr;
+    const char *name;
+} __attribute__((packed)) kernel_symbol_t;
+
+__attribute__((weak)) extern kernel_symbol_t symbol_table[];
+__attribute__((weak)) extern uint64_t symbol_count;
+
+// binary search
+const char* debug_find_symbol(uintptr_t rip, uintptr_t* offset)
+{
+    if (!symbol_table || symbol_count == 0)
+    {
+        if (offset) *offset = 0;
+        return "???";
+    }
+
+    int low = 0, high = (int)symbol_count - 1;
+    int best = -1;
+
+    while (low <= high)
+    {
+        int mid = (low + high) / 2;
+        if (symbol_table[mid].addr <= rip)
+        {
+            best = mid;
+            low = mid + 1;
+        } else {
+            high = mid - 1;
+        }
+    }
+
+    if (best != -1)
+    {
+        if (offset)
+        {
+            *offset = rip - symbol_table[best].addr;
+        }
+        return symbol_table[best].name;
+    }
+
+    if (offset) *offset = 0;
+    return "unknown";
+}

src/entry.S

@@ -0,0 +1,23 @@
+bits 64
+global _start
+
+extern kmain
+extern kernel_stack
+
+KERNEL_STACK_SIZE equ 65536
+
+section .text
+
+_start:
+    cli
+
+    ; load kernel stack
+    lea rsp, [kernel_stack+KERNEL_STACK_SIZE]
+
+    ; rbp=0 so last frame in stack trace
+    xor rbp, rbp
+
+    ; 16 byte align
+    and rsp, -16
+
+    call kmain

src/idt/idt.c

@@ -13,6 +13,7 @@
 #include <stdbool.h>
 #include "sched/scheduler.h"
 #include "config.h"
+#include "sched/process.h"
 
 struct interrupt_descriptor idt[256];
 struct idtr idt_reg;
@@ -87,11 +88,7 @@ static void page_fault_handler(struct cpu_status_t* ctx)
     CHECK_BIT(ctx->error_code, 7) ? " SGX_VIOLATION" : "",
     cr2);
 
-    /* if (CHECK_BIT(ctx->error_code, 0))
-    {
-        panic(ctx);
-    } */
-    panic(ctx);
+    panic(ctx, "page fault");
 }
 
 static void gp_fault_handler(struct cpu_status_t* ctx)
@@ -117,11 +114,16 @@ static void gp_fault_handler(struct cpu_status_t* ctx)
         index);
     }
 
-    panic(ctx);
+    panic(ctx, "gp fault");
 }
 
 struct cpu_status_t* interrupt_dispatch(struct cpu_status_t* context)
 {
+    if (context == NULL)
+    {
+        panic(NULL, "Interrupt dispatch recieved NULL context!");
+    }
+
     switch(context->vector_number)
     {
         case 0:
@@ -144,6 +146,7 @@ struct cpu_status_t* interrupt_dispatch(struct cpu_status_t* context)
             break;
         case 6:
             DEBUG("Invalid Opcode!");
+            panic(context, "Invalid Opcode!");
             break;
         case 7:
             DEBUG("Device Not Available!");
@@ -194,16 +197,17 @@ struct cpu_status_t* interrupt_dispatch(struct cpu_status_t* context)
 
         case 32: // Timer Interrupt
             ticks++;
+            // Send an EOI so that we can continue having interrupts
+            outb(0x20, 0x20);
 
             if (ticks % SCHEDULER_QUANTUM == 0)
             {
-                CLEAR_INTERRUPTS;
-                scheduler_schedule();
-                SET_INTERRUPTS;
+                return scheduler_schedule(context);
+                //struct cpu_status_t* current_ctx = scheduler_schedule(context);
+                //process_switch(current_ctx->iret_rsp, current_ctx->iret_rip);
+                //SET_INTERRUPTS;
             }
 
-            // Send an EOI so that we can continue having interrupts
-            outb(0x20, 0x20);
             break;
 
         case 33:
@@ -217,4 +221,4 @@ struct cpu_status_t* interrupt_dispatch(struct cpu_status_t* context)
     }
 
     return context;
-}
+}

src/kernel.h

@@ -22,12 +22,19 @@ enum ErrorCodes
 
 #define DEBUG(log, ...) fctprintf((void*)&skputc, 0, "debug: [%s]: " log "\r\n", __FILE__, ##__VA_ARGS__)
 
+#define DIE_DEBUG(str) fctprintf((void*)&skputc, 0, str)
+
 #define CHECK_BIT(var,pos) ((var) & (1<<(pos)))
 
 // printf("debug: [%s]: " log "\n", __FILE__, ##__VA_ARGS__);
 
-void panic(struct cpu_status_t* ctx);
+void panic(struct cpu_status_t* ctx, const char* str);
 void hcf();
+void idle();
+
+void debug_stack_trace(unsigned int max_frames);
+const char* debug_find_symbol(uintptr_t rip, uintptr_t* offset);
+
 #define assert(check) do { if(!(check)) hcf(); } while(0)
 
 struct boot_context

src/kmain.c

@@ -35,17 +35,9 @@ void hcf()
 }
 
 // Doing nothing (can be interrupted)
-void idle() {for(;;)asm("hlt");}
+void idle() {SET_INTERRUPTS; for(;;)asm("hlt");}
 
-void panic(struct cpu_status_t* ctx)
-{
-	DEBUG("\x1b[38;5;231m\x1b[48;5;196mKernel panic!!!\x1b[0m at rip=%p\nSomething went horribly wrong! vect=0x%.2x errcode=0x%x\n\rrax=%p rbx=%p rcx=%p rdx=%p\n\rrsi=%p rdi=%p  r8=%p  r9=%p\n\rr10=%p r11=%p r12=%p r13=%p\n\rr14=%p r15=%p\n\n\rflags=%p\n\rstack at rbp=%p\n\rHalting...",
-		ctx->iret_rip,
-		ctx->vector_number, ctx->error_code, ctx->rax, ctx->rbx, ctx->rcx, ctx->rdx, ctx->rsi, ctx->rdi,
-		ctx->r8, ctx->r9, ctx->r10, ctx->r11, ctx->r12, ctx->r13, ctx->r14, ctx->r15, ctx->iret_flags,
-		ctx->rbp);
-	hcf();
-}
+uint8_t kernel_stack[KERNEL_STACK_SIZE] __attribute__((aligned(16)));
 
 struct boot_context boot_ctx;
 
@@ -59,17 +51,17 @@ extern struct process_t* current_process;
 
 void pedicel_main(void* arg)
 {
-	
+	printf("Hello, world from a KERNEL PROCESS!");
 }
 
 void two_main(void* arg)
 {
-
+	printf("...process 2 speaking!!!");
 }
 
-void three_main(void* arg)
+void idle_main(void* arg)
 {
-
+	for(;;)asm("hlt");
 }
 
 // This is our entry point
@@ -103,13 +95,17 @@ void kmain()
 
 	vmm_init();
 	
+	struct process_t* idle_proc = process_create("idle", (void*)idle_main, 0);
 	struct process_t* pedicel = process_create("pedicel", (void*)pedicel_main, 0);
 	struct process_t* two = process_create("two", (void*)two_main, 0);
-	struct process_t* three = process_create("three", (void*)three_main, 0);
+	
 	process_display_list(processes_list);
 
 	scheduler_init();
 
+	current_process = idle_proc; 
+    current_process->status = RUNNING;
+
 	SET_INTERRUPTS;
 
 	keyboard_init(FR);

src/mem/heap/kheap.c

@@ -142,6 +142,6 @@ void kfree(void* ptr)
 // Should return a pointer to top of the stack (as stack grows DOWNWARDS)
 void* kalloc_stack()
 {
-    uint8_t* ptr = kmalloc(PROCESS_STACK_SIZE);
+    uint8_t* ptr = kmalloc(PROCESS_STACK_SIZE); // As it's out of kmalloc, stack is already mapped into kernel space
     return ptr ? ptr+PROCESS_STACK_SIZE : NULL;
 }

src/mem/paging/paging.c

@@ -24,7 +24,7 @@ If we use 1GB huge pages: PML4 -> PDPT -> 1gb pages
 4KB (regular size): PML4 -> PDPT -> PD -> PT -> 4kb pages
 */
 
-static inline void load_cr3(uint64_t value) {
+void load_cr3(uint64_t value) {
     asm volatile ("mov %0, %%cr3" :: "r"(value) : "memory");
 }
 

src/mem/paging/paging.h

@@ -16,6 +16,9 @@
 void paging_init();
 void paging_map_page(uint64_t* root_table, uint64_t virt, uint64_t phys, uint64_t flags);
 
+// To swap root page tables
+void load_cr3(uint64_t value);
+
 extern uint64_t hhdm_off;
 
 #define PHYS_TO_VIRT(x) ((void*)((uintptr_t)(x) + hhdm_off))

src/mem/paging/pmm.c

@@ -66,7 +66,10 @@ static uintptr_t g_freelist = 0;
 
 uintptr_t pmm_alloc()
 {
-    if (!g_freelist) return 0;
+    if (!g_freelist)
+	{
+		panic(NULL, "PMM is out of memory!");		
+	}
     uintptr_t addr = g_freelist;
     g_freelist = *(uintptr_t*) PHYS_TO_VIRT(g_freelist);
     return addr;
@@ -104,4 +107,4 @@ void pmm_init(struct limine_memmap_response* memmap, struct limine_hhdm_response
     // Now we have biggest USABLE region,
     // so to populate the free list we just iterate through it
     pmm_init_freelist();
-}
+}

src/sched/process.c

@@ -11,10 +11,13 @@
 #include "string/string.h"
 #include "mem/gdt/gdt.h"
 #include "config.h"
+#include "io/serial/serial.h"
 
 struct process_t* processes_list;
 struct process_t* current_process;
 
+extern uint64_t *kernel_pml4;
+
 size_t next_free_pid = 0;
 
 void process_init()
@@ -52,14 +55,22 @@ struct process_t* process_create(char* name, void(*function)(void*), void* arg)
     proc->pid = next_free_pid++;
     proc->status = READY;
 
+    uint64_t* stack_top = (uint64_t*)kalloc_stack();
+    // push return address to the stack so when "ret" hits we jmp to exit instead of idk what
+    // stack grows DOWNWARDS!!
+    *(--stack_top) = (uint64_t)process_exit;
+
     proc->context = ctx;
     proc->context->iret_ss = KERNEL_DATA_SEGMENT; // process will live in kernel mode
-    proc->context->iret_rsp = (uint64_t)kalloc_stack();
+    proc->context->iret_rsp = (uint64_t)stack_top;
     proc->context->iret_flags = 0x202; //bit 2 and 9 set (Interrupt Flag)
     proc->context->iret_cs = KERNEL_CODE_SEGMENT;
     proc->context->iret_rip = (uint64_t)function; // beginning of executable code
     proc->context->rdi = (uint64_t)arg; // 1st arg is in rdi (as per x64 calling convention)
-    proc->context->rbp = 0;
+    proc->context->rbp = 0; 
+
+    // Kernel PML4 as it already maps code/stack (when switching to userland we'll have to change that)
+    proc->root_page_table = kernel_pml4;
 
     proc->next = 0;
 
@@ -126,4 +137,40 @@ struct process_t* process_get_next(struct process_t* process)
 {
     if (!process) return NULL;
     return process->next;
+}
+
+// (from gdt) This will switch tasks ONLY in ring 0
+// KERNEL CS = 0x08
+// KERNEL SS = 0x10
+__attribute__((naked, noreturn))
+void process_switch(uint64_t stack_addr, uint64_t code_addr)
+{
+    asm volatile(" \
+        push $0x10 \n\
+        push %0 \n\
+        push $0x202 \n\
+        push $0x08 \n\
+        push %1 \n\
+        iretq \n\
+        " :: "r"(stack_addr), "r"(code_addr));
+}
+
+
+// Will be used to clean up resources (if any, when we implement it)
+// Just mark as DEAD then idle. Scheduler will delete process at next timer interrupt % quantum.
+void process_exit()
+{
+    DEBUG("Exiting from process '%s'", current_process->name);
+    CLEAR_INTERRUPTS;
+    if (current_process)
+    {
+        current_process->status = DEAD;
+    }
+    SET_INTERRUPTS;
+    
+    outb(0x20, 0x20);
+    for (;;)
+    {
+        asm("hlt");
+    }
 }

src/sched/process.h

@@ -9,6 +9,7 @@
 
 #include <stddef.h>
 #include "config.h"
+#include <stdint.h>
 
 typedef enum 
 {
@@ -24,6 +25,7 @@ struct process_t
 
     status_t status;
     struct cpu_status_t* context;
+	void* root_page_table; // Process PML4 (should contain kernel PML4 in higher half [256-511]
     struct process_t* next;
 };
 
@@ -32,6 +34,8 @@ struct process_t* process_create(char* name, void(*function)(void*), void* arg);
 void process_add(struct process_t** processes_list, struct process_t* process);
 void process_delete(struct process_t** processes_list, struct process_t* process);
 struct process_t* process_get_next(struct process_t* process);
+void process_switch(uint64_t stack_addr, uint64_t code_addr);
+void process_exit();
 
 void process_display_list(struct process_t* processes_list);
 

src/sched/scheduler.c

@@ -6,6 +6,9 @@
 
 #include "kernel.h"
 #include "process.h"
+#include "mem/paging/paging.h"
+#include <stdint.h>
+#include "io/serial/serial.h"
 
 extern struct process_t* processes_list;
 extern struct process_t* current_process;
@@ -41,6 +44,12 @@ struct cpu_status_t* scheduler_schedule(struct cpu_status_t* context)
         }
     }
     
-    DEBUG("current_process={pid=%u name='%s'}", current_process->pid, current_process->name);
+
+    // Current_process gets wrong context?? (iret_rip points to other stuff than process function; like putpixel() for example)
+    DEBUG("current_process={pid=%u, name='%s', root_page_table[virt]=%p}", current_process->pid, current_process->name, current_process->root_page_table);
+
+    load_cr3(VIRT_TO_PHYS((uint64_t)current_process->root_page_table));
+    DEBUG("loaded process pml4 into cr3");
+
     return current_process->context;
 }

src/sched/scheduler.h

@@ -7,7 +7,7 @@
 #ifndef SCHEDULER_H
 #define SCHEDULER_H
 
-void scheduler_schedule();
+struct cpu_status_t* scheduler_schedule(struct cpu_status_t* context);
 void scheduler_init();
 
 #endif

src/sched/task.S

@@ -0,0 +1,7 @@
+; Task (process) switching
+
+bits 64
+
+global switch_to_task
+switch_to_task:
+

symbols.py

@@ -0,0 +1,33 @@
+# Make assembly file from ELF symbols map
+# Then link it to kernel so it's aware of symbol names
+# then we can use that for the stack trace.
+
+print("Extracting symbols from map file to assembly...")
+
+with open("symbols.map", "r") as f:
+    lines = f.readlines()
+
+    symbols = []
+    for line in lines:
+        parts = line.split()
+        # output is formed like "address name"
+        symbols.append((parts[0], parts[1]))
+
+with open("symbols.S", "w") as f:
+    f.write("section .rodata\n")
+    f.write("global symbol_table\n")
+    f.write("global symbol_count\n")
+    f.write("symbol_table:\n")
+
+    for i, (addr, name) in enumerate(symbols):
+        f.write(f"    dq 0x{addr}\n")
+        f.write(f"    dq sym_name_{i}\n")
+
+    f.write("\nsymbol_count: dq " + str(len(symbols)) + "\n\n")
+
+    for i, (addr, name) in enumerate(symbols):
+        # escaping quotes
+        safe_name = name.replace('"', '\\"')
+        f.write(f'sym_name_{i}: db "{safe_name}", 0\n')
+
+print("Done!")