path: root/application-devel/debugging-tools/pres_debugging-tools_en.tex

\input{configpres}

\title{\lq Debugging Tools\rq}
\maketitle

\subsection{Simple Debugging Tools}

\begin{frame}
\frametitle{strace}
\begin{alertblock}{What is strace?}
strace is a powerful diagnosing tool. It traces system calls and signals for a specified program.
\end{alertblock}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{strace Example}
\begin{verbatim}
$ strace /bin/ls
execve("/bin/ls", ["/bin/ls"], [/* 32 vars */]) = 0
brk(0)                                  = 0xa6a000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=73614, ...}) = 0
mmap(NULL, 73614, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f68e06d6000
[...]
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Important strace Options}
\begin{itemize}
\item \textbf{-f}: Follow Forks
\item \textbf{-v}: Verbose mode
\item \textbf{-T}: Print out time which is spent in each syscall
\item \textbf{-p} PID: Attach to PID
\end{itemize}
\end{frame}

\subsection{The GNU Debugger: gdb}

\begin{frame}[containsverbatim]
\frametitle{Debug Test Program}
build the program with debug symbols
\begin{verbatim}
gcc -g -ohello hello.c
\end{verbatim}
start the debugger
\begin{verbatim}
gdb ./hello
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Important gdb Commands}
run the program
\begin{verbatim}
(gdb) run
Starting program: /home/devel/work/hello
Hello, world!

[Inferior 1 (process 935) exited normally]
(gdb)
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Important gdb Commands}
set a breakpoint
\begin{verbatim}
(gdb) list
1       #include <stdio.h>
2
3       int main(void)
4       {
5               printf("Hello, world!\n");
6               return 0;
7       }
(gdb) break 5
Breakpoint 1 at 0x400528: file hello.c, line 5.
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Important gdb Commands}
run the program again (now with breakpoint set)
\begin{verbatim}
(gdb) run
Starting program: /home/jan/work/examples/hello

Breakpoint 1, main () at hello.c:5
5               printf("Hello, world!\n");
\end{verbatim}
\begin{verbatim}
step over printf
(gdb) next
Hello, world!
6               return 0;
\end{verbatim}
let the program run to end
\begin{verbatim}
(gdb) continue
Continuing.
[...]
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{gdb Commands: Overview}
\begin{tabular}{|c|c|p{5cm}|}
\hline
\textbf{Command} & \textbf{Short-Form} & \textbf{Description} \\
\hline
run & r & start running program \\
\hline
continue & c & continue running program \\
\hline
break X & b & set a breakpoint at line X \\
\hline
step & s & step \textbf{in} the current function \\
\hline
next & n & step \textbf{over} the current function \\
\hline
print V & -- & show the value of variable V \\
\hline
display V & -- & show the value of variable V every time the program stops \\
\hline
\end{tabular}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{gdb Commands: Overview}
\begin{tabular}{|c|c|p{5cm}|}
\hline
\textbf{Command} & \textbf{Short-Form} & \textbf{Description} \\
\hline
backtrace & bt & display the current backtrace (stack) \\
\hline
frame X & f & change display to frame X of the current stack \\
\hline
quit & q & end gdb \\
\hline
\end{tabular}
\end{frame}

\subsection{Post-Mortem Debugging (Core Files)}

\begin{frame}[containsverbatim]
\frametitle{Create a Crashing Program}
\begin{verbatim}
/* hello_crash.c */
#include <stdio.h>

int main(void)
{
        char *p = NULL;
        printf("Hello, crash! %c\n", *p);
        return 0;
}
\end{verbatim}
build the program with debug symbols
\begin{verbatim}
gcc -g -ohello_crash hello_crash.c
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Run Program}
enable core files
\begin{verbatim}
$ ulimit -c unlimited
\end{verbatim}
run program
\begin{verbatim}
$ ./hello_segfault
Segmentation fault (core dumped)
\end{verbatim}
core file created
\begin{verbatim}
$ ls -l core
-rw------- 1 devel devel 249856 Jan 1 00:00 core
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Analyzing Core Files with gdb}
start the debugger
\begin{verbatim}
$ gdb ./hello_crash ./core
[...]
Reading symbols from ./hello_crash...done.
[New LWP 1239]
Core was generated by `./hello_crash'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x000000000040051a in main () at hello_crash.c:7
7               printf("Hello, crash! %c\n", *p);
\end{verbatim}
look at the full backtrace
\begin{verbatim}
(gdb) bt
#0  0x000000000040051a in main () at hello_crash.c:7
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Important Commands for Post-Mortem Debugging}
activate core dumps and set a maximum size
\begin{verbatim}
ulimit -c N
\end{verbatim}
view the current filename pattern for core files
\begin{verbatim}
cat /proc/sys/kernel/core_pattern
\end{verbatim}
set the filename pattern for core files
\begin{verbatim}
echo core-%p > /proc/sys/kernel/core_pattern
\end{verbatim}
view the core dump with gdb
\begin{verbatim}
gdb ./exe ./corefile
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Symbol Tables}
\begin{alertblock}{What if release software may not contain debug information?}
\end{alertblock}
build the program with debug symbols
\begin{verbatim}
gcc -g -ohello_crash hello_crash.c
\end{verbatim}
copy out debug information
\begin{verbatim}
objcopy --only-keep-debug hello_crash hello_crash.dbg
\end{verbatim}
remove the debug symbols from the binary for release
\begin{verbatim}
strip --strip-all hello_crash
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Symbol Tables}
cannot debug using only the release binary
\begin{verbatim}
$ gdb ./hello_crash ./core
[...]
Reading symbols from ./hello_crash...(no debugging symbols found)...done.
[New LWP 1555]
Core was generated by `./hello_crash'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00005645f239a6c4 in ?? ()
\end{verbatim}
start gdb specifying symbol file
\begin{verbatim}
$ gdb --symbols=./hello_crash.dbg --exec=./hello_crash --core=./core
[...]
Reading symbols from ./hello_crash.dbg...done.
[New LWP 1555]
Core was generated by `./hello_crash'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00005645f239a6c4 in main () at hello_crash.c:7
7               printf("Hello, crash! %c\n", *p);
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Symbol Tables}
link debug symbols to the binary
\begin{verbatim}
objcopy --add-gnu-debuglink=./hello_crash.dbg ./hello_crash
\end{verbatim}
now the binary can be debugged (symbol file still used!)
\begin{verbatim}
$ gdb ./hello_crash ./core
[...]
Reading symbols from ./hello_crash...
Reading symbols from /home/devel/work/hello_crash.dbg...done.
done.
warning: exec file is newer than core file.
[New LWP 1555]
Core was generated by `./hello_crash'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00005645f239a6c4 in main () at hello_crash.c:7
7               printf("Hello, crash! %c\n", *p);
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Symbol Tables}
\begin{alertblock}{Always keep the debug symbols for release binaries!}
\end{alertblock}
verify that the debug symbols match the executable
\begin{verbatim}
$ file hello_crash hello_crash.dbg
hello_crash:     ... BuildID[sha1]=9bc82e1d6627ba58492ea24462fd6658726c4fc0
hello_crash.dbg: ... BuildID[sha1]=9bc82e1d6627ba58492ea24462fd6658726c4fc0
\end{verbatim}
\end{frame}

\subsection{Cross Debugging}

\begin{frame}[containsverbatim]
\frametitle{Building for the Target System}
build the program for armhf with debug symbols
\begin{verbatim}
$ arm-linux-gnueabihf-gcc -g -ohello hello.c
\end{verbatim}
check executable
\begin{verbatim}
$ file hello
hello: ELF 32-bit LSB executable, ARM, EABI5 version 1 (SYSV),...
\end{verbatim}
\end{frame}

\begin{frame}
\frametitle{QEMU as a Tool for Cross Development}
\begin{alertblock}{What is QEMU?}
QEMU is a high performance emulation and virtualization environment for all common CPU architectures.
gängigen CPU Architekturen.
\end{alertblock}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Testing Executables using QEMU User Emulation}
a system cannot directly execute foreign architecture executables
\begin{verbatim}
$ ./hello
bash: ./hello: cannot execute binary file: Exec format error
\end{verbatim}
but it can emulate them
\begin{verbatim}
$ qemu-arm -L /opt/.../arm-linux-gnueabihf/libc ./hello
Hello, world!
\end{verbatim}
\end{frame}

\begin{frame}\frametitle{Remote Debugging}
\begin{figure}[h]
\centering
\includegraphics[width=8cm]{images/remote_debug.png}
\end{figure}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Remote Debugging Session}
from the target:
\begin{verbatim}
$ gdbserver :2345 ./hello
Process ./hello created; pid = 310
Listening on port 2345
\end{verbatim}
from the host:
\begin{verbatim}
$ arm-linux-gnueabihf-gdb ./hello
(gdb) set solib-absolute-prefix /opt/.../arm-linux-gnueabihf/libc
(gdb) target remote 192.168.2.2:2345
Remote debugging using 192.168.2.2:2345
0x30016180 in _start() from /opt/.../libc/lib/ld-linux-armhf.so.3
(gdb) c
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Automatically Execute gdb Commands}
edit a gdb init script
\begin{verbatim}
vi gdbinit.txt
\end{verbatim}
add startup commands
\begin{verbatim}
set solib-absolute-prefix /opt/.../arm-linux-gnueabihf/libc
target remote 192.168.2.2:2345
\end{verbatim}
specify init script when starting gdb
\begin{verbatim}
arm-linux-gnueabihf-gdb -x gdbinit.txt ./hello
\end{verbatim}
\end{frame}

\subsection{Memory Debugging}

\begin{frame}
\frametitle{Common Problems}
\begin{itemize}
\item writing/reading beyond memory regions
\item memory leaks
\item ''use after free()''
\end{itemize}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{The glibc Mechanism: mtrace}
add support for mtrace to a program
\begin{verbatim}
#include <mcheck.h>
[...]
int main(void)
{
        mtrace();
[...]
\end{verbatim}
activate mtrace
\begin{verbatim}
MALLOC_TRACE=hello.trace ./hello
\end{verbatim}
view the mtrace results
\begin{verbatim}
mtrace ./hello.dbg hello.trace
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{The glibc Mechanism: mtrace}
create a test program with a memory leak
\begin{verbatim}
/* mem_leak.c */
#include <mcheck.h>
#include <stdio.h>
#include <stdlib.h>

int main(void)
{
        int i = 0;
        char *p = NULL;

        mtrace();

        for(i = 0; i < 50; i++)
                p = malloc(sizeof(char));

        free(p);
}
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{The glibc Mechanism: mtrace}
build the test program with debug symbols
\begin{verbatim}
$ gcc -fno-PIE -no-pie -g -omem_leak mem_leak.c
\end{verbatim}
run the test program with mtrace activated
\begin{verbatim}
$ MALLOC_TRACE=mem_leak.trace ./mem_leak
\end{verbatim}
view the trace results
\begin{verbatim}
$ mtrace ./mem_leak mem_leak.trace

Memory not freed:
-----------------
Address   Size   Caller
0x1536460 0x1 at /home/devel/work/mem_leak.c:13
0x1536480 0x1 at /home/devel/work/mem_leak.c:13
0x15364a0 0x1 at /home/devel/work/mem_leak.c:13
[...]
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{mtrace and PIE}
With a patched mtrace, PIE can be traced by disabling ASLR.
\begin{verbatim}
$ gcc -fPIE -pie -g -omem_leak mem_leak.c
$ setarch `uname -m` -R env MALLOC_TRACE=mem_leak.trace ./mem_leak
$ setarch `uname -m` -R ./mtrace-patched ./mem_leak mem_leak.trace

Memory not freed:
-----------------
Address   Size   Caller
0x0000555555756450 0x1 at /home/devel/work/mem_leak.c:13
0x0000555555756470 0x1 at /home/devel/work/mem_leak.c:13
0x0000555555756490 0x1 at /home/devel/work/mem_leak.c:13
[...]
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{glibc Hooks for malloc()}
\_\_malloc\_hook:
\begin{verbatim}
void *function (size_t size, const void *caller)
\end{verbatim}
\_\_realloc\_hook:
\begin{verbatim}
void *function (void *ptr, size_t size, const void *caller)
\end{verbatim}
\_\_free\_hook:
\begin{verbatim}
void *function (void *ptr, const void *caller)
\end{verbatim}
\_\_memalign\_hook:
\begin{verbatim}
void *function (size_t size, size_t alignment, const void *caller)
\end{verbatim}
Do not use them! Use mtrace instead!
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{libduma / electric fence}
create a test program with memory access beyond a region
\begin{verbatim}
/* array_access.c */
#include <stdio.h>
#include <stdlib.h>

int main(void)
{
        int *a;
        int i;

        a = calloc(10, sizeof(*a));

        for(i = 0; i < 11; i++)
                printf("%d ", a[i]);
        printf("\n");
        return 0;
}
\end{verbatim}
build and run the test program
\begin{verbatim}
$ gcc -g -oarray_access array_access.c
$ ./array_access
0 0 0 0 0 0 0 0 0 0 135121
\end{verbatim}
Invalid access and nobody cared!
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{libduma / electric fence}
enable core dumps
\begin{verbatim}
$ ulimit -c unlimited
\end{verbatim}
run the program with libduma active
\begin{verbatim}
$ LD_PRELOAD=libduma.so.0 ./array_access
[...]
Segmentation fault (core dumped)
\end{verbatim}
investigate core dump
\begin{verbatim}
$ gdb ./array_access core
[...]
Core was generated by `./array_access'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00000000004005ce in main () at array_access.c:13
13                      printf("%d ", a[i]);
(gdb) print i
$1 = 10
\end{verbatim}
\end{frame}

\begin{frame}
\frametitle{Valgrind}
\begin{alertblock}{Advantages}
\begin{itemize}
\item high success rate
\item lots of functionality and features
\end{itemize}
\end{alertblock}
\begin{alertblock}{Disadvantage}
\begin{itemize}
\item runtime dramatically affected
\end{itemize}
\end{alertblock}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Valgrind}
\begin{verbatim}
$ valgrind --leak-check=full ./mem_leak
[...]
==8457== HEAP SUMMARY:
==8457==     in use at exit: 49 bytes in 49 blocks
==8457==   total heap usage: 50 allocs, 1 frees, 50 bytes allocated
==8457==
==8457== 49 bytes in 49 blocks are definitely lost in loss record 1 of 1
==8457==    at 0x4C28C20: malloc (in /usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so)
==8457==    by 0x4005C4: main (mem_leak.c:14)
==8457==
==8457== LEAK SUMMARY:
==8457==    definitely lost: 49 bytes in 49 blocks
[...]
\end{verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Valgrind}
\begin{verbatim}
$ valgrind --leak-check=full ./array_access
[...]
==8493== Invalid read of size 4
==8493==    at 0x4005CE: main (array_access.c:13)
==8493==  Address 0x51e0068 is 0 bytes after a block of size 40 alloc'd
==8493==    at 0x4C2AD10: calloc (in /usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so)
==8493==    by 0x4005AC: main (array_access.c:10)
==8493==
0 0 0 0 0 0 0 0 0 0 0
==8493==
==8493== HEAP SUMMARY:
==8493==     in use at exit: 40 bytes in 1 blocks
==8493==   total heap usage: 1 allocs, 0 frees, 40 bytes allocated
==8493==
==8493== 40 bytes in 1 blocks are definitely lost in loss record 1 of 1
==8493==    at 0x4C2AD10: calloc (in /usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so)
==8493==    by 0x4005AC: main (array_access.c:10)
==8493==
==8493== LEAK SUMMARY:
==8493==    definitely lost: 40 bytes in 1 blocks
[...]
\end{verbatim}
\end{frame}

\input{tailpres}