From b1934fb29ed318cab4d9ed5aa5d18e84440d96f8 Mon Sep 17 00:00:00 2001 From: Jan Altenberg Date: Thu, 10 Feb 2011 11:37:06 +0100 Subject: First version of a kernel debugging chapter. Currently covers basic information about: - printk - Serial Console / Net Console - Oops / addr2line - ftrace / ftrace_dump_on_oops - Qemu gdbserver interface - KGDB - User Mode Linux --- .../kernel-debugging/pres_kernel-debugging_en.tex | 356 +++++++++++++++++++++ 1 file changed, 356 insertions(+) create mode 100644 kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex (limited to 'kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex') diff --git a/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex b/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex new file mode 100644 index 0000000..f875368 --- /dev/null +++ b/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex @@ -0,0 +1,356 @@ +\input{configpres} + +\title{Kernel-Debugging} +\maketitle +\section{printk} +\begin{frame}[fragile] +\frametitle{printk is your friend!!} +Usage is similar to printf() in userspace. Different loglevels: +\small +\begin{verbatim} +KERN_EMERG "<0>" /* system is unusable */ +KERN_ALERT "<1>" /* action must be taken immediately */ +KERN_CRIT "<2>" /* critical conditions */ +KERN_ERR "<3>" /* error conditions */ +KERN_WARNING "<4>" /* warning conditions */ +KERN_NOTICE "<5>" /* normal but significant condition */ +KERN_INFO "<6>" /* informational */ +KERN_DEBUG "<7>" /* debug-level messages */ +\end{verbatim} +\end{frame} + +\section{printk} +\begin{frame}[fragile] +\frametitle{Example} +\begin{verbatim} +printk(KERN_EMERG "Fatal error!\n"); +\end{verbatim} +Setting the loglevel via kernel commandline: +\begin{verbatim} +loglevel=7 +\end{verbatim} +Loglevel in procfs: +\begin{verbatim} +# console_loglevel, default_message_loglevel, +# minimum_console_level and default_console_loglevel +$ cat /proc/sys/kernel/printk +4 4 1 7 +\end{verbatim} +\end{frame} + +\section{Logging messages} +\begin{frame}[fragile] +\frametitle{Serial Console} +Kernel configuration (example): +\begin{verbatim} +Device Drivers ---> +Character devices ---> +Serial drivers ---> +[*] Console on 8250/16550 and + compatible serial port +\end{verbatim} +Kernel commandline (examples) +\begin{verbatim} +console=ttyS0,115200 +\end{verbatim} +\begin{verbatim} +console=ttyS0,115200n8 +\end{verbatim} +\begin{verbatim} +console=ttyAM0,115200 +\end{verbatim} +\end{frame} + +\begin{frame}[fragile] +\frametitle{Netconsole} +Kernel commandline: +\begin{verbatim} +netconsole=[src-port]@[src-ip]/[], + [tgt-port]@/[tgt-macaddr] +\end{verbatim} +src-port defaults to 6665. tgt-port defaults to 6666. +\begin{verbatim} +netconsole=@/,@10.10.0.2/ +\end{verbatim} +On the host side: +\begin{verbatim} +$ nc -l -u -p 6666 +# or (depending on your netcat version) +$ nc -l -u 6666 +\end{verbatim} +\end{frame} + +\section{Analyzing Backtraces} +\begin{frame}[fragile] +\frametitle{Oops...Something went wrong ;-)} +Note: PC is at c0009378 +\tiny +\begin{verbatim} +CPU: 0 Not tainted (2.6.37 #9) +PC is at prepare_namespace+0x170/0x1d4 +LR is at do_unlinkat+0x10c/0x14c +pc : [] lr : [] psr: 80000013 +sp : c783dfc8 ip : c783df20 fp : c783dfe0 +r10: 00000000 r9 : 00000000 r8 : 00000000 +r7 : 00000013 r6 : c0054a1c r5 : c0022995 r4 : c03969a4 +r3 : 00000000 r2 : 00000000 r1 : c78c7000 r0 : 00000000 +Flags: Nzcv IRQs on FIQs on Mode SVC_32 ISA ARM Segment kernel +Control: 00093177 Table: 00004000 DAC: 00000017 +Process swapper (pid: 1, stack limit = 0xc783c260) +Stack: (0xc783dfc8 to 0xc783e000) +dfc0: c783dfd4 c0396940 c00084fc c783dff4 c783dfe4 c0008610 +dfe0: c0009214 00000000 00000000 c783dff8 c0054a1c c0008508 00000000 00000000 +Backtrace: +[] (prepare_namespace+0x0/0x1d4) from [] (kernel_init+0x114/0x154) + r5:c00084fc r4:c0396940 +[] (kernel_init+0x0/0x154) from [] (do_exit+0x0/0x660) + r4:00000000 +Code: e3500000 13a03601 15843000 e3a03000 (e5d31000) +---[ end trace 4ed5c061b76895d8 ]--- +\end{verbatim} +\end{frame} + +\begin{frame}[fragile] +\frametitle{Analyzing Backtraces: addr2line} +If you compiled your kernel with debug info, you can use addr2line to decode +the address: +\small +\begin{verbatim} +$ arm-none-linux-gnueabi-addr2line -e vmlinux c0009378 +linux-2.6.37/init/do_mounts.c:488 +\end{verbatim} +\end{frame} + +\section{Debugging early crashes} +\begin{frame}[fragile] +\frametitle{Early printk} +\begin{verbatim} +Kernel hacking ---> +[*] Kernel low-level debugging functions +[*] Early printk +\end{verbatim} +\begin{verbatim} +earlyprintk=serial,ttyAMA0,115200,keep \ +console=ttyAMA0,115200 +\end{verbatim} +\end{frame} + +\section{Magic SysRQ} +\begin{frame} +\frametitle{Magic Sysrequest Keys} +Sysrequest (Alt-Print) or break signal (via serial console) +followed by a specific key: +\begin{itemize} +\item t: Show task states +\item p: Show registers +\item q: Show all timers +\item b: Force reboot +\item e: Terminate all tasks +\item l: Show backtrace of all active CPUs +\item s: Sync +\item h: Help +\end{itemize} +\end{frame} + +\section{ftrace} + +\begin{frame}[fragile] +\frametitle{trace\_printk()} +\begin{itemize} +\item Writes to the tracing ring buffer +\item Can be used in any context +\item Useful for debugging ''high volume areas'' +\item Syntax similar to printk(): +\begin{verbatim} +trace_printk("my_var -> %d\n", my_var); +\end{verbatim} +\end{itemize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{ftrace\_dump\_on\_oops} +Example: +\begin{verbatim} +# Kernel Commandline: +ftrace=sched_switch ftrace_dump_on_oops +\end{verbatim} +\tiny +\begin{verbatim} +Unable to handle kernel NULL pointer dereference at virtual address 00000000 +pgd = c0004000 +[00000000] *pgd=00000000 +Internal error: Oops: 35 [#1] +last sysfs file: +Dumping ftrace buffer: +--------------------------------- + swapper-1 0dN... 9034us : 1:120:R + [000] 2:120:S kthreadd + swapper-1 0..... 10044us : 1:120:D ==> [000] 2:120:R kthreadd +kthreadd-2 0d.... 10529us : 2:120:R + [000] 15:120:R kswapd0 +kthreadd-2 0..... 10577us : 2:120:S ==> [000] 14:120:R kworker/0:1 +kworker/-14 0d.... 10623us : 14:120:D + [000] 4:120:D kworker/0:0 +kworker/-14 0..... 10656us : 14:120:D ==> [000] 4:120:R kworker/0:0 +kworker/-4 0d.... 10848us : 4:120:R + [000] 14:120:D kworker/0:1 +kworker/-4 0..... 11943us : 4:120:S ==> [000] 14:120:R kworker/0:1 +kworker/-14 0..... 12004us : 14:120:S ==> [000] 15:120:R kswapd0 +[...] +\end{verbatim} +\end{frame} + +\section{Qemu} +\begin{frame}[fragile] +\frametitle{Kerneldebugging with Qemu} +\begin{verbatim} +$ qemu-system-arm -M versatilepb -m 128 \ +-S -s -kernel zImage +\end{verbatim} +\begin{verbatim} +$ arm-none-linux-gnueabi-gdb vmlinux +(gdb) target remote localhost:1234 +Remote debugging using localhost:1234 +0x00000000 in ?? () +(gdb) break start_kernel +(gdb) c +Continuing. + +Breakpoint 1, start_kernel () + at linux-2.6.37-rc4/init/main.c:539 +539 smp_setup_processor_id(); +\end{verbatim} +\end{frame} + +\section{KGDB} +\begin{frame} +\frametitle{KGDB} +\begin{itemize} +\item KDB: Simplistic shell style interface: Inspect registers, process list, ... +\item KGDB: Source-level debugging +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{KDB} +\begin{itemize} +\item NOT a source level debugger!! +\item Enter KDB with SysRQ-g: echo g $>$ /proc/sysrq-trigger +\end{itemize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{KGDB} +Kernel configuration: +\begin{verbatim} +Kernel hacking ---> +[*] KGDB: kernel debugging with remote gdb ---> +<*> KGDB: use kgdb over the serial console (NEW) +\end{verbatim} +Kernel Commandline: +\begin{verbatim} +kgdboc=ttyAMA0 kgdbwait +\end{verbatim} +\end{frame} + +\begin{frame}[fragile] +\frametitle{KGDB} +1)\\ +\begin{verbatim} +qemu-system-arm -M versatilepb -m 128 \ + -serial tcp:localhost:2345,server \ + -kernel zImage -append "kgdboc=ttyAMA0 kgdbwait" +\end{verbatim} +2)\\ +telnet localhost 2345\\ +\begin{verbatim} +kgdb: Wait for connection from remote gdb... +\end{verbatim} +CTRL-] +quit +\end{frame} + +\begin{frame}[fragile] +\frametitle{KGDB} +3)\\ +\begin{verbatim} +arm-none-linux-gnueabi-gdb vmlinux +(gdb) target remote localhost:2345 +Remote debugging using localhost:2345 + +kgdb_breakpoint () + at linux-2.6.37-rc4/kernel/debug/debug_core.c:959 +959 arch_kgdb_breakpoint(); +(gdb) +\end{verbatim} +\end{frame} + +\section{User Mode Linux} +\begin{frame} +\frametitle{UML: User Mode Linux} +\begin{alertblock}{What is User Mode Linux?} +User Mode Linux allows you to run the Linux kernel as a normal userspace +process. +\end{alertblock} +\end{frame} + +\begin{frame}[fragile] +\frametitle{UML: Build and boot a UML kernel} +\begin{lstlisting}[language=bash] +$ make ARCH=um defconfig +$ make ARCH=um +$ file linux + linux: ELF 32-bit LSB executable, Intel 80386, + version 1 (SYSV), dynamically linked (uses shared libs), + for GNU/Linux 2.6.8, not stripped +$ ./linux +\end{lstlisting} +\end{frame} + +\begin{frame}[fragile] +\frametitle{UML: Parameters} +\begin{lstlisting}[language=bash] +$ ./linux ubd0=my_disk.img \ + eth0=slirp,,slirp \ + hostfs=${HOME} +\end{lstlisting} +\begin{itemize} +\item ubd0: The ubdX parameter maps a file in the underlying filesystem to a +device. This is used to specify the filesystem(s) +\item eth0=slirp,,slirp: This enables limited access to the network, without +having root permissions +\item hostfs: Tell UML which host directories can be mounted inside the UML +environment. For example: ''mount none /mnt/myhost -t hostfs'' inside UML will +mount your hosts / to /mnt/myhost +\end{itemize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{Running UML in gdb} +\begin{lstlisting}[language=bash] +$ gdb ./linux +(gdb) handle SIGSEGV pass nostop noprint +Signal Stop Print Pass to program Description +SIGSEGV No No Yes Segmentation fault +(gdb) b start_kernel +Breakpoint 1 at 0x80493ca: file /home/devel/images/linux-2.6.37/init/main.c, +line 539. +(gdb) r +Starting program: /home/devel/images/build_um/linux +Locating the bottom of the address space ... 0x1000 +Locating the top of the address space ... 0xc0000000 +Core dump limits : + soft - 0 + hard - NONE +[...] +Adding 10047488 bytes to physical memory to account for exec-shield gap + +Breakpoint 1, start_kernel () at +/home/devel/images/linux-2.6.37/init/main.c:539 +539 smp_setup_processor_id(); +(gdb) c +Continuing. +Linux version 2.6.37 (devel@ltx) (gcc version 4.3.2 (Debian 4.3.2-1.1) ) +Thu Feb 10 06:25:23 UTC 2011 +Built 1 zonelists in Zone order, mobility grouping on. Total pages: 10561 +[...] +\end{lstlisting} +\end{frame} + +\input{tailpres} -- cgit v1.2.3 From 6899aa44836314fcb2f387de7a9eff72e7631ed1 Mon Sep 17 00:00:00 2001 From: Jan Altenberg Date: Thu, 10 Feb 2011 12:32:00 +0100 Subject: Kernel debugging: Mention /proc/sysrq-trigger --- kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex') diff --git a/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex b/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex index f875368..cc1824f 100644 --- a/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex +++ b/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex @@ -150,6 +150,8 @@ followed by a specific key: \item s: Sync \item h: Help \end{itemize} +A sysrequest can also be triggered by echoing the values to +/proc/sysrq-trigger. \end{frame} \section{ftrace} -- cgit v1.2.3