path: root/misc/pres_minicoredumper_en.tex

\input{configpres}
\date{2016-10-12}

\section{minicoredumper}

\title{Creating Efficient Small Core Dumps\newline for Embedded Systems}
\author{John Ogness}

\maketitle

\newcommand\verbbf[1]{\textcolor[rgb]{0,0,0}{\textbf{#1}}}

\subsection{background}

\begin{frame}[containsverbatim]
\frametitle{What are core dumps?}
\begin{Verbatim}[commandchars=\\\{\}]
$ man 5 core

CORE(5)              Linux Programmer's Manual              CORE(5)

NAME
       core - core dump file

DESCRIPTION
       The  default action of certain signals is to cause a process
       to terminate and produce a  core  dump  file,  \verbbf{a  disk  file}
       \verbbf{containing  an  image of the process's memory at the time of}
       \verbbf{termination}.  This image can be used in  a  debugger  (e.g.,
       gdb(1)) to inspect the state of the program at the time that
       it terminated.  A list of the signals which cause a  process
       to dump core can be found in signal(7).
\end{Verbatim}
\vskip10pt
Core files utilize the ELF file format to organize the various elements of
the process image.
\end{frame}

\begin{frame}
\frametitle{Core Dumps}
\begin{alertblock}{advantages}
\begin{itemize}
\item functionality provided by the kernel
\item all process data available (registers, stacks, heap, ...)
\item post-mortem debugging
\item offline debugging
\end{itemize}
\end{alertblock}
\pause
\begin{alertblock}{disadvantages}
\begin{itemize}
\item large storage requirements
\item debugging tools required for analysis
\item no information about other processes
\end{itemize}
\end{alertblock}
\end{frame}

\subsection{overview}

\begin{frame}
\frametitle{The minicoredumper Project}
\begin{alertblock}{Primary Goals}
\begin{itemize}
\item minimal core dumps
\item custom core dumps
\item state snapshots
\end{itemize}
\end{alertblock}
\pause
\begin{alertblock}{Main Components}
\begin{itemize}
\item minicoredumper
\item libminicoredumper
\item live dumps
\end{itemize}
\end{alertblock}
\end{frame}

\subsection{minicoredumper}

\begin{frame}
\frametitle{What is the minicoredumper?}
\begin{itemize}
\item userspace application to extend the Linux core dump facility
\item configuration files to specify desired data
\item per-application configuration files
\item in-memory compression features
\item few dependencies
\item no kernel patches required
\end{itemize}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{How is this possible from userspace?}
\begin{Verbatim}[commandchars=\\\{\}]
$ man 5 core

[...]

   Naming of core dump files
       By  default,  a  core  dump  file  is  named  core,  but \verbbf{the}
       \verbbf{/proc/sys/kernel/core_pattern  file}  (since  Linux  2.6  and
       2.4.21) \verbbf{can be set to define a template that is used to name}
       \verbbf{core dump files}.  The  template  can  contain  %  specifiers
       which  are  substituted  by the following values when a core
       file is created:
[...]

   Piping core dumps to a program
       Since  kernel 2.6.19, Linux supports an alternate syntax for
       the  /proc/sys/kernel/core_pattern  file.   \verbbf{If   the   first}
       \verbbf{character  of  this  file  is  a  pipe  symbol (|), then the}
       \verbbf{remainder of the line is interpreted  as  a  program  to  be}
       \verbbf{executed.}  Instead of being written to a disk file, the core
       dump is given as standard input to the  program.
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{/proc/sys/kernel/core\_pattern}
Inform the kernel to use the minicoredumper to handle core dumps,
specifying how it is called.
\vskip10pt
\begin{Verbatim}[commandchars=\\\{\}]
$ echo '|/usr/sbin/minicoredumper %P %u %g %s %t %h %e' \textbackslash
                 | sudo tee /proc/sys/kernel/core_pattern

$ man 5 core

[...]
           %P  PID of dumped process, as seen in  the  initial  PID
               namespace (since Linux 3.12)
           %u  (numeric) real UID of dumped process
           %g  (numeric) real GID of dumped process
           %s  number of signal causing dump
           %t  time  of dump, expressed as seconds since the Epoch,
               1970-01-01 00:00:00 +0000 (UTC)
           %h  hostname (same as nodename returned by uname(2))
           %e  executable filename (without path prefix)
\end{Verbatim}
\end{frame}

\begin{frame}
\frametitle{Configuration}
\begin{alertblock}{configuration file}
\begin{itemize}
\item JSON format
\item specifies dump path
\item specifies matching rules for "recepts" (application-specific dump configurations)
\end{itemize}
\end{alertblock}
\pause
\begin{alertblock}{recept file}
\begin{itemize}
\item JSON format
\item general features (stacks, threads, ...)
\item specific memory mappings
\item specific symbols
\item compression options
\end{itemize}
\end{alertblock}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{minicoredumper.cfg.json}
Configuration file example:
\vskip10pt
\begin{Verbatim}[commandchars=\\\{\}]
\{
    "base_dir": "/var/crash/minicoredumper",
    "watch": [
        \{
            "exe": "*/real_example_app",
            "recept": "/etc/minicoredumper/example.recept.json"
        \},
        \{
            "comm": "example_app"
            "recept": "/etc/minicoredumper/example.recept.json"
        \},
        \{
            "exe": "/bin/*"
        \},
        \{
            "recept": "/etc/minicoredumper/generic.recept.json"
        \}
    ]
\}
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{example.recept.json}
\begin{Verbatim}[commandchars=\\\{\}]
\{
    "stacks": \{
        "dump_stacks": true,
        "first_thread_only": true,
        "max_stack_size": 16384
    \},
    "maps": \{
        "dump_by_name": [
            "[vdso]"
        ]
    \},
    "buffers": [
        \{
            "symname": "my_allocated_struct",
            "follow_ptr": true,
            "data_len": 42
        \}
    ],
    "compression": \{
        "compressor": "gzip",
        "extension": "gz",
        "in_tar": true
    \},
    "write_proc_info": true
\}
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{How It Works}
\begin{alertblock}{identify process data}
\begin{itemize}
\item ELF header from \verb|stdin| (virtual memory allocations, symbols, shared objects, relocation, debug objects, ...)
\item \verb|/proc/N/maps| (memory maps)
\item \verb|/proc/N/stat| (stack pointers)
\item \verb|/proc/N/auxv| (auxiliary vector)
\item \verb|/proc/N/mem | (memory access)
\end{itemize}
\end{alertblock}
\begin{alertblock}{dump process data}
\begin{itemize}
\item write core as sparse file
\item append custom ELF section note
\item in-memory compression (with tar format support)
\end{itemize}
\end{alertblock}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Simulate Core Dump}
\begin{figure}[h]
\centering
\includegraphics[width=10cm]{images/tracingsummit.png}
\end{figure}
\vskip10pt
\begin{Verbatim}[commandchars=\\\{\}]
$ kill -SEGV `pidof firefox-esr`
\end{Verbatim}
\end{frame}

\setlength{\tabcolsep}{10pt}

\begin{frame}[containsverbatim]
\frametitle{Core Size Comparisons}
default = default Linux core dump facility settings\newline
minicore/* = default minicoredumper settings\newline
minicore/1 = minicore/* changed to only first thread
\begin{center}
{\renewcommand{\arraystretch}{3}
\begin{tabular}{|l|r|r|r|}
\hline
\textbf{type} & \textbf{file size} & \textbf{disk usage} & \textbf{core.tar.gz} \\
\hline
default & 523,300 KB & 143,228 KB & 28,286 KB \\
\hline
minicore/* & 526,380 KB & 7,928 KB & 1,336 KB \\
\hline
minicore/1 & 522,412 KB & 724 KB & 31 KB \\
\hline
\end{tabular}}
\end{center}
The full backtrace of the crashed thread is available in all variations.
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Custom ELF Section Note}
The custom ELF section note contains a list of ranges within the core file
that are valid dump data.
\vskip10pt
\begin{Verbatim}[commandchars=\\\{\}]
$ eu-readelf -a core

[...]

Section Headers:
[Nr] Name                          Type     Addr     Off      Size
[ 0]                               NULL     00000000 00000000 00000000
[ 1] .shstrtab                     STRTAB   00000000 2020b14c 00000030
[ 2] .debug                        PROGBITS 00000000 00008540 20201ac0
[ 3] \verbbf{.note.minicoredumper.dumplist} NOTE     00000000 2020a000 0000114c

[...]

Note section [ 3] '.note.minicoredumper.dumplist' of 4428 bytes
at offset 0x2020a000:
  Owner          Data size  Type
  \verbbf{minicoredumper} 4400       \verbbf{<unknown>: 80}
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{gdb Support}
Non-dumped data always has a value of zero because of the sparse core.
\vskip10pt
\begin{Verbatim}[commandchars=\\\{\}]
$ \verbbf{gdb} /usr/bin/firefox-esr core

[...]

(gdb) print _edata
\verbbf{$1 = 0}
\end{Verbatim}
\vskip10pt
A proof-of-concept gdb fork to interpret the custom ELF section note is
available:
\begin{Verbatim}[commandchars=\\\{\}]
     https://github.com/Linutronix/binutils-gdb/
     (branch: minicoredumper-section-note)


$ \verbbf{gdb-linutronix} /usr/bin/firefox-esr core

[...]

(gdb) print _edata
\verbbf{$1 = <unavailable>}
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Dependencies}
With few dependencies, the minicoredumper can be added to
existing systems with a relatively low storage cost.
\vskip10pt
\begin{Verbatim}[commandchars=\\\{\}]
$ objdump -x /usr/sbin/minicoredumper | grep NEEDED

  NEEDED               libelf.so.1
  NEEDED               libjson-c.so.2
  NEEDED               libthread_db.so.1
  NEEDED               libpthread.so.0
  NEEDED               librt.so.1
  NEEDED               libc.so.6
\end{Verbatim}
\end{frame}

\begin{frame}
\frametitle{Summary}
The minicoredumper application itself is a very useful tool for providing
powerful post-mortem debugging capabilities for an embedded system.
\begin{itemize}
\item low storage overhead
\item no runtime overhead
\item simple configuration
\item useful crash data
\item very small dumps (even most EEPROM's would suffice!)
\end{itemize}
\pause
\vskip20pt
But wait! There's more...
\end{frame}

\subsection{libminicoredumper}

\begin{frame}
\frametitle{What is libminicoredumper?}
\begin{itemize}
\item userspace library that allows applications to register specific data for dumping
\item data can be dumped in-core and/or in external files
\item data can be text-formatted and placed in external files
\item data can be unregistered for dumping during runtime
\item few dependencies
\end{itemize}
\pause
\vskip10pt
\begin{alertblock}{Why is this interesting?}
\begin{itemize}
\item minimize dumped application data
\item dump internal application data
\item external dump files (text and binary) can provide insight into the problem without the need of a debugger
\end{itemize}
\end{alertblock}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{How It Works}
\begin{itemize}
\item libminicoredumper exports two special symbols
\begin{itemize}
\item \verb|mcd_dump_data_version| (data format version number)
\item \verb|mcd_dump_data_head| (linked list of dump registrations)
\end{itemize}
\item when an application crashes, the minicoredumper looks for these symbols
\item if the symbols are found, the minicoredumper can identify what and how the extra registered data is to be dumped
\end{itemize}
\vskip20pt
\begin{Verbatim}[commandchars=\\\{\}]
$ objdump -T /usr/lib/x86_64-linux-gnu/\verbbf{libminicoredumper.so.2.0.0} \textbackslash
          | grep '\textbackslash{}sDO\textbackslash{}s'

00201c40 g DO .data 00000004 Base \verbbf{mcd_dump_data_version}
00201cc8 g DO .bss  00000008 Base \verbbf{mcd_dump_data_head}
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{API}
\begin{Verbatim}[commandchars=\\\{\}]
int \verbbf{mcd_dump_data_register_bin}(const char *ident,
                               unsigned long dump_scope,
                               mcd_dump_data_t *save_ptr,
                               void *data_ptr, size_t data_size,
                               enum mcd_dump_data_flags flags);

int \verbbf{mcd_dump_data_register_text}(const char *ident,
                                unsigned long dump_scope,
                                mcd_dump_data_t *save_ptr,
                                const char *fmt, ...);

int \verbbf{mcd_vdump_data_register_text}(const char *ident,
                                 unsigned long dump_scope,
                                 mcd_dump_data_t *save_ptr,
                                 const char *fmt, va_list ap);

int \verbbf{mcd_dump_data_unregister}(mcd_dump_data_t dd);
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Example Application (mycrasher)}
\begin{Verbatim}[commandchars=\\\{\}]
int main(void)
\{
    mcd_dump_data_t d[3];
    char *x = NULL;
    char *s;
    int *i;

    s = strdup("my string");
    i = malloc(sizeof(*i));
    *i = 42;

    mcd_dump_data_register_bin(\verbbf{NULL}, 1024, &d[0], \verbbf{s}, strlen(s) + 1,
                               MCD_DATA_PTR_DIRECT | MCD_LENGTH_DIRECT);

    mcd_dump_data_register_bin("\verbbf{i.bin}", 1024, &d[1], \verbbf{i}, sizeof(*i),
                               MCD_DATA_PTR_DIRECT | MCD_LENGTH_DIRECT);

    mcd_dump_data_register_text("\verbbf{out.txt}", 1024, &d[2],
                                \verbbf{"s=\textbackslash{}"%s\textbackslash{}" *i=%d\textbackslash{}n", s, i});

    *x = 0; /* BOOM! */
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Example Application Debugging}
\begin{Verbatim}[commandchars=\\\{\}]
$ ./mycrasher 

Segmentation fault (core dumped)

$ sudo chown -R `id -u` /.../mycrasher.20161012.093000+0200.19481

$ cd /.../mycrasher.20161012.093000+0200.19481

$ find . -type f

./dumps/19481/\verbbf{i.bin}
./dumps/19481/\verbbf{out.txt}
./\verbbf{core.tar.gz}
./\verbbf{symbol.map}
\end{Verbatim}
The \verb|symbol.map| file contains the core file information for all
the external binary dumps.
\vskip20pt
\begin{Verbatim}[commandchars=\\\{\}]
$ cat dumps/19481/out.txt

\verbbf{s="my string" *i=42}
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Example Application Debugging (cont)}
\begin{Verbatim}[commandchars=\\\{\}]
$ tar -xzSf core.tar.gz

$ gdb-linutronix /.../mycrasher core

[...]

Core was generated by `./mycrasher'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00000000004008d2 in main () at mycrasher.c:26
26		*x = 0;
(gdb) print s
\verbbf{$1 = 0x11eb010 "my string"}
(gdb) print i
$2 = (int *) 0x11eb030
(gdb) print *i
\verbbf{$3 = <unavailable>}
\end{Verbatim}
\vskip10pt
Unlike for \verb|s|, the data pointed to by \verb|i| is not available
in the core file because it was stored externally in \verb|i.bin|.
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Example Application Debugging (cont)}
Using the coreinject tool, external binary dumps can be inserted into
the core files.
\vskip10pt
\begin{Verbatim}[commandchars=\\\{\}]
$ coreinject \verbbf{core} \verbbf{symbol.map} dumps/19481/\verbbf{i.bin}

injected: i.bin, 4 bytes, direct

$ gdb-linutronix /.../mycrasher core

[...]

Core was generated by `./mycrasher'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00000000004008d2 in main () at mycrasher.c:26
26		*x = 0;
(gdb) print s
$1 = 0x11eb010 "my string"
(gdb) print i
$2 = (int *) 0x11eb030
(gdb) print *i
\verbbf{$3 = 42}
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Dependencies}
With few dependencies, the libminicoredumper can be added to
custom applications with a relatively low storage cost.
\vskip10pt
\begin{Verbatim}[commandchars=\\\{\}]
$ objdump -x /usr/lib/x86_64-linux-gnu/libminicoredumper.so.2.0.0 \textbackslash
          | grep NEEDED

  NEEDED               libc.so.6
\end{Verbatim}
\end{frame}

\begin{frame}
\frametitle{Summary}
The libminicoredumper allows applications to provide very fine-tuned data
dumps at a minimal cost.
\begin{itemize}
\item low storage overhead
\item no runtime overhead, \textbf{but} be aware registration/unregistration invokes memory allocations, locking, list searching
\item simple API
\item precise data specification
\item runtime dump registration changes supported
\end{itemize}
\pause
\vskip20pt
But wait! There's more...
\end{frame}

\subsection{live dumps}

\begin{frame}
\frametitle{What are live dumps?}
\begin{itemize}
\item dump registered data for running applications
\item dumps can be triggered on crash
\item dumps can be triggered manually
\item few dependencies
\end{itemize}
\pause
\vskip10pt
\begin{alertblock}{Why is this interesting?}
\begin{itemize}
\item allows pseudo state snapshots
\end{itemize}
\end{alertblock}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{How It Works}
\begin{alertblock}{minicoredumper\_regd}
\begin{itemize}
\item creates UNIX local domain datagram socket with abstract address
\item socket receives credentials to identify sender PID
\item maintains a list of PID's in shared memory of applications with registered dumps
\end{itemize}
\end{alertblock}
\vskip10pt
\begin{Verbatim}[commandchars=\\\{\}]
$ netstat | grep minicoredumper

unix  2  [ ]  DGRAM  61620 @minicoredumper.24111
\verbbf{unix  2  [ ]  DGRAM  61619 @minicoredumper}

$ ls -l /dev/shm/minicoredumper.shm 

\verbbf{-rw------- 1 mcd mcd 56 Oct 12 09:30 /dev/shm/minicoredumper.shm}
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{How It Works (cont)}
\begin{alertblock}{libminicoredumper}
\begin{itemize}
\item registers itself with minicoredumper\_regd via UNIX local domain socket on first data dump registration
\item unregisters itself from minicoredumper\_regd via UNIX local domain socket on last data dump unregistration
\end{itemize}
\end{alertblock}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{How It Works (cont)}
\begin{alertblock}{minicoredumper (an application crashed)}
\begin{itemize}
\item read PID list from shared memory
\item for each thread associated with each PID, attach and freeze the task using \verb|PTRACE_SEIZE| and \verb|PTRACE_INTERRUPT|, respectively
\item for each PID, dump the registered data (via \verb|/proc/N/mem|)
\item for each thread associated with each PID, detach from the task using \verb|PTRACE_DETACH|
\item perform the dumps for the crashing application
\end{itemize}
\end{alertblock}
\end{frame}

\begin{frame}[containsverbatim]
\frametitle{Dependencies}
With few dependencies, the minicoredumper\_regd can be added to
existing systems with a relatively low storage cost.
\vskip10pt
\begin{Verbatim}[commandchars=\\\{\}]
$ objdump -x /usr/sbin/minicoredumper_regd | grep NEEDED

  NEEDED               libpthread.so.0
  NEEDED               librt.so.1
  NEEDED               libc.so.6
\end{Verbatim}
\end{frame}

\begin{frame}
\frametitle{Pseudo State Snapshots}
\begin{itemize}
\item latencies between dumps vary greatly depending on hardware, system load, application, number of registered applications, ...
\item expect latencies from 2ms to 30ms between crash event and the first dump
\item expect latencies from 30us to 4ms between all successive dumps
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Summary}
Live dumps can be useful for capturing a pseudo state snapshot of various
related applications if any one should crash or by manually triggering
it using the minicoredumper\_trigger tool.
\begin{itemize}
\item low storage overhead
\item dumps data for multiple applications, \textbf{but} be aware of latencies between dumps
\item no runtime overhead, \textbf{but} be aware of application freezing during dumps
\end{itemize}
\end{frame}

\subsection{status}

\begin{frame}
\frametitle{Project Status}
\begin{itemize}
\item about to release version 2.0.0 (presented here)
\item working on packaging for Debian/Stretch
\item working on Yocto layer for OpenEmbedded
\end{itemize}
\end{frame}

\subsection{}

\begin{frame}[containsverbatim]
\frametitle{Questions / Comments}
Thank you for your attention!
\vskip30pt
\begin{Verbatim}[commandchars=\\\{\}]
https://linutronix.de/minicoredumper


RCPT TO:<john.ogness@linutronix.de>
\end{Verbatim}
\end{frame}

\input{tailpres}