Merge ../projects/schulung

9 files changed, 808 insertions, 52 deletions

diff --git a/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex b/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex
index cc1824f..82dc25b 100644
--- a/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex
+++ b/kernel-devel/kernel-debugging/pres_kernel-debugging_en.tex
@@ -2,6 +2,73 @@
 
 \title{Kernel-Debugging}
 \maketitle
+
+\section{Kernel Configuration}
+\begin{frame}
+\frametitle{Kernel hacking / printk and dmesg options}
+\begin{itemize}
+\item Show timing information on printks
+\item Default message log level (1-7)
+\item Enable dynamic printk() support
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Kernel hacking / Compile-time checks and compiler options}
+\begin{itemize}
+\item Compile the kernel with debug info
+\item Debug Filesystem
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Kernel hacking / Memory Debugging}
+\begin{itemize}
+\item Kernel memory leak detector
+\end{itemize}
+reports leaks in /sys/kernel/debug/kmemleak
+
+see also: Documentation/kmemleak.txt
+\end{frame}
+
+\begin{frame}
+\frametitle{Kernel hacking / Debug Lockups and Hangs}
+\begin{itemize}
+\item Debug Lockups and Hangs
+\begin{description}
+\item[Softlockup] loop in kernel mode
+\item[Hardlockup] CPU loop in kernel mode without letting IRQs run
+\item[Hung task] task is uninterruptible (D state)
+\end{description}
+\end{itemize}
+stack trace is printed on detection
+\end{frame}
+
+\begin{frame}
+\frametitle{Kernel hacking / Lock Debugging}
+\begin{itemize}
+\item RT Mutex debugging, deadlock detection
+\item Lock debugging: prove locking correctness, see Documentation/lockdep-design.txt.
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Kernel hacking / Tracing}
+\begin{itemize}
+\item Kernel Function (Graph) Tracer
+\item Scheduling Latency Tracer
+\item Enable [k/u]probes-based dynamic events
+\item enable/disable function tracing dynamically
+\item Ring buffer benchmark stress tester (!!don't use it!!)
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Kernel hacking}
+\begin{description}
+\item[Remote debugging over FireWire] Documentation/debugging-via-ohci1394.txt
+\end{description}
+\end{frame}
 \section{printk}
 \begin{frame}[fragile]
 \frametitle{printk is your friend!!}
@@ -38,6 +105,25 @@ $ cat /proc/sys/kernel/printk
 \end{verbatim}
 \end{frame}
 
+\section{dynamic printk}
+\begin{frame}
+\frametitle{dynamic printk}
+controlled by debugfs: dynamic\_debug/control
+
+format: filename:lineno [module]function flags format
+\begin{description}
+\item[filename] source file of the debug statement
+\item[lineno] line number of the debug statement
+\item[module] module that contains the debug statement
+\item[function] function that contains the debug statement
+\item[flags] '=p' means the line is turned 'on' for printing
+\item[format] the format used for the debug statement
+\end{description}
+Use pr\_debug() and dev\_dbg() in your code.
+
+see also: Documentation/dynamic-debug-howto.txt
+\end{frame}
+
 \section{Logging messages}
 \begin{frame}[fragile]
 \frametitle{Serial Console}
diff --git a/kernel-devel/kernel-perf/Makefile b/kernel-devel/kernel-perf/Makefile
new file mode 100644
index 0000000..d641258
--- /dev/null
+++ b/kernel-devel/kernel-perf/Makefile
@@ -0,0 +1,9 @@
+all:
+	for pdf in `ls -1 *.tex` ; do \
+		TEXINPUTS=`pwd`/../..:.:..:$(TEXINPUTS) pdflatex $$pdf; \
+		TEXINPUTS=`pwd`/../..:.:..:$(TEXINPUTS) pdflatex $$pdf; \
+	done
+
+clean:
+	rm -f *.aux *.log *.pdf *.log *.snm *.toc *.vrb *.nav *.out
+
diff --git a/kernel-devel/kernel-perf/pres_kernel-perf_en.tex b/kernel-devel/kernel-perf/pres_kernel-perf_en.tex
new file mode 100644
index 0000000..b6edfc0
--- /dev/null
+++ b/kernel-devel/kernel-perf/pres_kernel-perf_en.tex
@@ -0,0 +1,135 @@
+\input{configpres}
+
+\title{perf}
+\section{Performance Analyzation}
+\maketitle
+
+\begin{frame}
+\frametitle{What is perf?}
+Perf is a profiler tool for Linux 2.6+ based systems.
+\begin{itemize}
+\item perf\_events interface of the Linux kernel
+\item perf userspace tool
+\item access perf events from own applications
+\end{itemize}
+Perf is an event counter, no tracer.
+\end{frame}
+
+\begin{frame}
+\frametitle{perf events}
+\begin{itemize}
+\item hardware performance counters (PMU of Host CPU)
+\item tracepoint events (from ftrace)
+\item dynamic probes (e.g. kprobes or uprobes)
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{perf subcommands I}
+\begin{description}
+\item[list]     list available events
+\item[stat]     measure total event count for single program or for system, for
+                  some time
+\item[top]      top-like dynamic view of hottest functions
+\item[record]   measure and save sampling data for single program
+\item[report]   analyze file generated by perf record;
+                  can generate flat, or graph profile
+\item[diff]     Read two perf.data files and display the differential profile
+\end{description}
+\end{frame}
+
+\begin{frame}
+\frametitle{perf subcommands II}
+\begin{description}
+\item[sched]    tracing/measuring of scheduler actions and latencies
+\item[lock]     analyze lock events
+\item[timechart]tool to visualize total system behavior during a workload
+\end{description}
+\end{frame}
+
+
+\begin{frame}
+\frametitle{selecting events}
+\begin{description}
+\item[-e] use the specified event
+\item[-f] filter events, e.g. -e irq:irq\_handler\_entry --filter irq==18
+\end{description}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Example: live-view of page-fault event}
+\begin{lstlisting}
+# perf stat chromium
+ Performance counter stats for 'chromium':
+
+       3733.112825 task-clock                #    0.716 CPUs utilized
+             8,696 context-switches          #    0.002 M/sec
+             1,063 cpu-migrations            #    0.285 K/sec
+           134,686 page-faults               #    0.036 M/sec
+     9,102,760,996 cycles                    #    2.438 GHz
+[83.59%]
+     6,217,623,412 stalled-cycles-frontend   #   68.30% frontend cycles idle
+[83.32%]
+     4,755,104,208 stalled-cycles-backend    #   52.24% backend  cycles idle
+[67.20%]
+     7,262,564,269 instructions              #    0.80  insns per cycle
+                                             #    0.86  stalled cycles per insn
+[83.72%]
+     1,495,191,394 branches                  #  400.521 M/sec
+[83.38%]
+        26,037,805 branch-misses             #    1.74% of all branches
+[83.04%]
+
+       5.214978731 seconds time elapsed
+\end{lstlisting}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Example: live-view of page-fault event}
+\begin{lstlisting}
+# sudo perf top -e page-faults
+Samples: 562  of event 'page-faults', Event count (approx.): 4557
+ 62.32%  libc-2.17.so              [.] __memcpy_ssse3_back
+ 20.36%  libc-2.17.so              [.] _int_malloc
+  3.16%  module.so                 [.] evas_gl_common_context_font_push
+  2.83%  libevas.so.1.7.7          [.] 0x0000000000083237
+  2.37%  libdricore9.0.2.so.1.0.0  [.] 0x0000000000126050
+  1.80%  libxul.so (deleted)       [.] 0x000000000182e48f
+  1.12%  [kernel]                  [k] file_read_actor
+  0.92%  libfontconfig.so.1.6.2    [.] 0x000000000001bd43
+  0.88%  libdrm_intel.so.1.0.0     [.] 0x0000000000008503
+  0.79%  libc-2.17.so              [.] __memset_sse2
+  0.75%  libeet.so.1.7.7           [.] eet_data_image_header_decode_cipher
+  0.61%  module.so                 [.] evas_gl_common_context_image_push
+  0.42%  libfreetype.so.6.10.0     [.] FT_Stream_ReadFields
+  0.42%  libecore_x.so.1.7.7       [.] ecore_x_netwm_icons_set
+  0.24%  libelementary.so.1.7.7    [.] 0x000000000005ae60
+\end{lstlisting}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{Example: count CPU cycles of a ls call}
+\begin{lstlisting}
+# perf record -e cpu-cycles ls
+<output of ls>
+[ perf record: Woken up 1 times to write data ]
+[ perf record: Captured and wrote 0.008 MB perf.data (~334 samples) ]
+# perf report
+Samples: 24  of event 'cpu-cycles', Event count (approx.): 12333055
+ 21.67%  ls  [kernel.kallsyms]  [k] get_unmapped_area
+ 20.77%  ls  ld-2.17.so         [.] _dl_map_object_deps
+ 17.21%  ls  [kernel.kallsyms]  [k] __slab_free
+ 15.73%  ls  [kernel.kallsyms]  [k] tty_insert_flip_string_fixed_flag
+ 12.65%  ls  [kernel.kallsyms]  [k] unlock_page
+  6.94%  ls  libc-2.17.so       [.] __strlen_sse2
+  3.41%  ls  [kernel.kallsyms]  [k] count.isra.17.constprop.28
+  1.02%  ls  [kernel.kallsyms]  [k] get_pageblock_flags_group
+  0.41%  ls  [kernel.kallsyms]  [k] perf_event_context_sched_in
+  0.13%  ls  [kernel.kallsyms]  [k] perf_ctx_unlock
+  0.07%  ls  [kernel.kallsyms]  [k] native_write_msr_safe
+\end{lstlisting}
+\end{frame}
+
+
+\input{tailpres}
diff --git a/kernel-devel/kernel-tracing/pres_kernel-tracing_en.tex b/kernel-devel/kernel-tracing/pres_kernel-tracing_en.tex
index d73a10f..d5a9686 100644
--- a/kernel-devel/kernel-tracing/pres_kernel-tracing_en.tex
+++ b/kernel-devel/kernel-tracing/pres_kernel-tracing_en.tex
@@ -1,19 +1,33 @@
 \input{configpres}
 
 \title{Kernel-Tracing}
-\section{FTRACE}
+\section{Overview}
 \maketitle
+
+\begin{frame}
+\frametitle{Overview}
+\tableofcontents
+\end{frame}
+
 \begin{frame}[fragile]
 \frametitle{Kerneltracing: Overview}
 \begin{itemize}
 \item DebugFS interface
-\item Different tracers: function, function\_graph, sched\_switch, ...
+\item Different tracers: function, function\_graph, wakeup, wakeup\_rt, \dots
 \item Custom trace events
 \item Graphical frontend (kernelshark)
 \end{itemize}
 \end{frame}
 
 \begin{frame}[fragile]
+\frametitle{Kerneltracing: Overview}
+\begin{figure}[h]
+\centering
+\includegraphics[width=10cm]{images/trace_overview.png}
+\end{figure}
+\end{frame}
+
+\begin{frame}[fragile]
 \frametitle{Kernel-Tracing: DebugFS}
 \begin{verbatim}
 $ mount -t debugfs debugfs /sys/kernel/debug
@@ -23,27 +37,103 @@ blk function_graph mmiotrace wakeup_rt wakeup
 \end{verbatim}
 \end{frame}
 
+\section{Event tracing}
+\begin{frame}[fragile]
+\frametitle{Event tracing}
+current\_tracer can be set to NOP
+\begin{verbatim}
+$ cd /sys/kernel/debug/tracing
+$ ls events/
+[...]
+irq
+sched
+scsi
+signal
+skb
+[...]
+\end{verbatim}
+\end{frame}
 
 \begin{frame}[fragile]
-\frametitle{Kerneltracing: Trivial example}
+\frametitle{Event tracing}
 \begin{verbatim}
-$ echo function_graph > current_tracer
-$ echo 1 > tracing_enabled
-$ sleep 1
-$ echo 0 > tracing_enabled
-$ less trace
-# tracer: function_graph
-# CPU  DURATION           FUNCTION CALLS
-# |     |   |              |   |   |   |
- 1)               |         enqueue_entity() {
- 1)               |           update_curr() {
- 1)   0.336 us    |             task_of();
- 1)   1.524 us    |           }
- 1)   0.420 us    |           place_entity();
+$ cd /sys/kernel/debug/tracing
+$ ls -1 events/sched/
+enable
+filter
+sched_kthread_stop
+sched_kthread_stop_ret
+sched_migrate_task
+sched_pi_setprio
+[...]
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Event tracing: Enable events}
+\begin{verbatim}
+$ cd /sys/kernel/debug/tracing
+# Enable ALL events of the group ''sched''
+$ echo 1 > events/sched/enable
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Record a trace}
+After enabling the events you want to see, do:
+\begin{verbatim}
+$ cd /sys/kernel/debug/tracing
+# Start recording to the ringbuffer
+$ echo 1 > tracing_on
+# Stop recording to the ringbuffer
+$ echo 0 > tracing_on
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Analyze a trace}
+You can even do this while recording!
+\begin{verbatim}
+$ cd /sys/kernel/debug/tracing
+# Just print the current content of the ring buffer
+$ cat trace
+# or: do a consuming read on the ring buffer
+$ cat trace_pipe
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Trace event format and filters}
+Each trace event has a specific format and parameters.
+You can put a filter on those parameters for recording a trace:
+\begin{verbatim}
+$ cat events/sched/sched_switch/format
+[...]
+field:__u8 prev_comm[15];
+field:pid_t prev_pid;
+field:int prev_prio;
+field:long prev_state;
+[...]
+$ echo 'next_comm == bash' \
+    > events/sched/sched_switch/filter
+$ echo 1 > events/sched/sched_switch/enable
+$ echo 1 > tracing_on
+$ cat trace
 \end{verbatim}
 \end{frame}
 
 \begin{frame}[fragile]
+\frametitle{Tracing on multicore}
+\begin{itemize}
+\item One ringbuffer per cpu
+\item trace contains ALL events
+\item the per\_cpu directory contains a trace for each cpu
+\item tracing\_cpumask can limit tracing to specific cores
+\end{itemize}
+\end{frame}
+
+\section{Tracers}
+\begin{frame}[fragile]
 \frametitle{Tracer: function}
 \begin{verbatim}
 # tracer: function
@@ -59,56 +149,60 @@ wnck-2022  [003]  5766.659918: kfree <-skb
 \end{frame} 
 
 \begin{frame}[fragile]
-\frametitle{Tracer: function / latency\_format}
+\frametitle{Tracer: function\_graph}
 \begin{verbatim}
-$ echo 1 > options/latency_format
+$ echo function_graph > current_tracer
+$ echo 1 > tracing_on
+$ sleep 1
+$ echo 0 > tracing_on
 $ less trace
-#                  _------=> CPU#            
-#                 / _-----=> irqs-off        
-#                | / _----=> need-resched    
-#                || / _---=> hardirq/softirq 
-#                ||| / _--=> preempt-depth   
-#                |||| /_--=> lock-depth       
-#                |||||/     delay             
-#  cmd     pid   |||||| time  |   caller      
-#     \   /      ||||||   \   |   /           
-wnck-app-2022    3..... 1237702us : skb_relea
-wnck-app-2022    3..... 1237703us : sock_wfre
+# tracer: function_graph
+# CPU  DURATION           FUNCTION CALLS
+# |     |   |              |   |   |   |
+ 1)               |         enqueue_entity() {
+ 1)               |           update_curr() {
+ 1)   0.336 us    |             task_of();
+ 1)   1.524 us    |           }
+ 1)   0.420 us    |           place_entity();
 \end{verbatim}
-\end{frame} 
+\end{frame}
 
 \begin{frame}[fragile]
-\frametitle{Tracer: function (custom tracepoint)}
+\frametitle{function\_graph: Set a trigger function}
+You can set a trigger function for the function\_graph tracer
+if you just want to record specific functions and their childs:
 \begin{verbatim}
-$ echo 1 > tracing_enabled
-$ echo "MARK" > trace_marker
-$ echo 0 > tracing_enabled
-$ less trace
-...
- bash-4328  [003]  5603.687935: get_slab
- bash-4328  [003]  5603.687935: _cond_re
- bash-4328  [003]  5603.687936: _cond_re
- bash-4328  [003]  5603.687939: 0: MARK
- bash-4328  [003]  5603.687939: kfree <-
-...
+echo do_IRQ > set_graph_function
+# Additionals triggers can be set with
+echo another_function >> set_graph_function
 \end{verbatim}
 \end{frame}
 
 \begin{frame}[fragile]
-\frametitle{Tracer: sched\_switch}
+\frametitle{Tracer: function / latency\_format}
 \tiny
 \begin{verbatim}
-# tracer: sched_switch
+$ echo 1 > options/latency_format
+# tracer: function
 #
-#  TASK-PID    CPU#    TIMESTAMP  FUNCTION
-#    | |       |          |         |
-  bash-4328  [003]  78.553966: 4328:120:S   + [003]  4328:120:S bash
-  bash-4328  [003]  78.553979: 4328:120:S ==> [003]     0:120:R <idle>
-<idle>-0     [003]  78.553986:    0:120:R   + [003]    13:120:R ksoftirqd/3
-<idle>-0     [003]  78.553988:    0:120:R ==> [003]    13:120:R ksoftirqd/3
-
+# function latency trace v1.1.5 on 3.9.4-x1-00124-g0bfd8ff
+# --------------------------------------------------------------------
+# latency: 0 us, #204955/25306195, CPU#0 | (M:desktop VP:0, KP:0, SP:0 HP:0 #P:4)
+#    -----------------
+#    | task: -0 (uid:0 nice:0 policy:0 rt_prio:0)
+#    -----------------
+#
+#                  _------=> CPU#
+#                 / _-----=> irqs-off
+#                | / _----=> need-resched
+#                || / _---=> hardirq/softirq
+#                ||| / _--=> preempt-depth
+#                |||| /     delay
+#  cmd     pid   ||||| time  |   caller
+#     \   /      |||||  \    |   /
+terminol-11964   1.... 11639243us : ep_read_events_proc <-ep_scan_ready_list.isra.8
 \end{verbatim}
-\end{frame}
+\end{frame} 
 
 \begin{frame}[fragile]
 \frametitle{Tracer: wakeup\_rt}
@@ -131,6 +225,44 @@ ls-4579    3dN...    5us : _raw_spin_unlock_irqrestore <-try_to_wake_up
 \end{verbatim}
 \end{frame}
 
+\section{Kernel function profiler}
+
+\begin{frame}[fragile]
+\frametitle{Kernel function profiler}
+\begin{verbatim}
+$ echo 1 > function_profile_enabled
+$ echo 1 > tracing_on
+# then do something...
+$ echo 0 > tracing_on
+$ less trace_stat/function0
+Function      Hit  Time          Avg        
+--------      ---  ----          ---        
+__schedule   7064  1958976725 us 277318.3 us 
+schedule     6961  1958965845 us 281420.1 us 
+[...]
+\end{verbatim}
+\end{frame}
+
+\section{trace\_marker}
+\begin{frame}[fragile]
+\frametitle{Custom application tracepoints: ''simple method''}
+\begin{verbatim}
+$ echo 1 > tracing_on
+$ echo "MARK" > trace_marker
+$ echo 0 > tracing_on
+$ less trace
+...
+ bash-4328  [003]  5603.687935: get_slab
+ bash-4328  [003]  5603.687935: _cond_re
+ bash-4328  [003]  5603.687936: _cond_re
+ bash-4328  [003]  5603.687939: 0: MARK
+ bash-4328  [003]  5603.687939: kfree <-
+...
+\end{verbatim}
+\end{frame}
+
+\section{trace\_printk}
+
 \begin{frame}[fragile]
 \frametitle{trace\_printk()}
 \begin{itemize}
@@ -158,6 +290,10 @@ Only trace specific functions.
 ftrace_notrace=
 \end{verbatim}
 Don't trace specific functions.
+\begin{verbatim}
+trace_event=
+\end{verbatim}
+Just enable trace events (comma separated list)
 \end{frame}
 
 \begin{frame}[fragile]
@@ -170,6 +306,20 @@ echo z > /proc/sysrq-trigger
 \end{verbatim}
 \end{frame}
 
+\begin{frame}[fragile]
+\frametitle{Trace instances}
+You can have separate trace instances with their own
+buffers and events:
+\begin{verbatim}
+$ cd /sys/kernel/debug/tracing
+$ mkdir instances/my_inst1
+$ cd instances/my_inst1
+$ echo 1 > events/sched/enable
+$ cat trace
+[...]
+\end{verbatim}
+\end{frame}
+
 \section{trace-cmd}
 
 \begin{frame}
@@ -326,9 +476,200 @@ $ kernelshark -i mytrace.dat
 \includegraphics[width=10cm]{images/kernelshark_zoom.png}
 \end{figure}
 \end{frame}
+
+\section{Useful things}
+
+\begin{frame}[fragile]
+\frametitle{Control a trace from your kernel code}
+\begin{verbatim}
+void my_kernel_function(void)
+{
+     tracing_on();
+     do_some_stuff_i_wanna_trace();
+     tracing_off();
+}
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Matching PIDs and process names of your trace(s)}
+\begin{verbatim}
+$ cat saved_cmdlines
+[...]
+5112 bash
+5223 ARTHUR_DENT
+5546 kworker/0:2
+8465 kworker/0:0
+[...]
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Where's my thread in the trace?!}
+To ''find'' a specific thread in your trace, you can use the
+following function to set the thread's name:
+\begin{verbatim}
+#define _GNU_SOURCE
+#include <pthread.h>
+
+pthread_setname_np(pthread_t thread,
+                     const char *name);
+\end{verbatim}
+Available since glibc version 2.12
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Thread names: Example}
+\begin{verbatim}
+/* pthread_example */
+pthread_t test_thread;
+[...]
+pthread_create(&test_thread, NULL,
+                 my_test_thread, NULL);
+[...]
+pthread_setname_np(test_thread, "ARTHUR_DENT");
+\end{verbatim}
+Check with the ps command:
+\begin{verbatim}
+$ ps H -C pthread_example -o 'pid tid cmd comm'
+ PID   TID CMD               COMMAND
+4515  4515 ./pthread_example pthread_example
+4515  4516 ./pthread_example ARTHUR_DENT
+\end{verbatim}
+\end{frame}
+
 \begin{frame}
+\frametitle{Thread name in kernelshark}
+\begin{figure}[h]
+\centering
+\includegraphics[width=10cm]{images/pthread_kernelshark.png}
+\end{figure}
+\end{frame}
+
+\section{kprobes}
+\begin{frame}[fragile]
+\frametitle{Dynamic kernel tracepoints: KPROBES}
+\begin{itemize}
+\item Similar to Tracepoints
+\item Can be added / removed dynamically
+\end{itemize}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Dynamic kernel tracepoints: KPROBES}
+\begin{verbatim}
+$ echo 'p:my_k_event do_IRQ' > kprobe_events
+$ echo 1 events/kprobes/my_k_event/enabled
+$ echo 1 > tracing_on
+$ cat trace
+<idle>-0 [000] d... 545.173709: my_k_event: (do_IRQ+0x0/0xc0)
+<idle>-0 [000] d... 545.331051: my_k_event: (do_IRQ+0x0/0xc0)
+<idle>-0 [000] d... 545.331490: my_k_event: (do_IRQ+0x0/0xc0)
+<idle>-0 [000] d... 545.490730: my_k_event: (do_IRQ+0x0/0xc0)
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Dynamic kernel tracepoints: KPROBES for custom modules}
+Let's assume we want to have a tracepoint for the function
+hello\_init in the module hello.ko
+\begin{verbatim}
+# Note: >> will append a new event
+$ echo 'p:my_mod_event hello:hello_init' \
+           >> kprobe_events
+$ echo 1 > events/kprobes/my_mod_event/enable
+$ insmod hello.ko
+$ cat trace
+insmod-9586  [000] d... 13278.003468: my_mod_event: (0xf878d080)
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Dynamic kernel tracepoints: Question / Exercise}
+What happens, if we add the following event. What's different?
+\begin{verbatim}
+# Note >> will append a new event
+$ echo 'r:my_exercise_event hello:hello_init' \
+           >> kprobe_events
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{KPROBES statistics}
+\begin{verbatim}
+$ cat kprobe_profile 
+my_mod_event_ret           2               0
+my_mod_event               2               0
+\end{verbatim}
+\end{frame}
+
+\section{uprobes}
+\begin{frame}[fragile]
+\frametitle{Dynamic Userspace Tracepoints: uprobes}
+\begin{itemize}
+\item Similar to kprobes
+\item For userspace applications
+\item A uprobe event is set on a specific offset in a userland process
+\item Powerful method to correlate your kernel and userland events!
+\end{itemize}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Dynamic Userspace Tracepoints: uprobes}
+\begin{verbatim}
+$ gcc -Wall -g -o pthread_example \
+      pthread_example.c -lpthread
+$ objdump -F -D -S pthread_example | less
+\end{verbatim}
+\begin{verbatim}
+08048594 <my_test_thread> (File Offset: 0x594):
+[...]
+void *my_test_thread(void *x_void_ptr)
+[...]
+        for (i = 0; i < 10; i++) {
+80485a1:  c7 45 f4 00 00 00 00    movl   $0x0,-0xc(%ebp)
+80485a8:  eb 1c                   jmp    80485c6 <my_test_thread+0x32> (File Offset: 0x5c6)
+          printf("The answer is 42!\n");
+80485aa:  c7 04 24 50 87 04 08    movl   $0x8048750,(%esp)
+\end{verbatim}
+So, the file offset for the printf call is 0x5aa !
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Dynamic Userspace Tracepoints: uprobes II}
+\begin{verbatim}
+echo \
+'p:my_ev /home/devel/pthread/pthread_example:0x5aa' \
+       >  /sys/kernel/debug/tracing/uprobe_events
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Dynamic Userspace Tracepoints: uprobes III}
+\begin{verbatim}
+$ cd /sys/kernel/debug/tracing/
+$ echo 1 > events/uprobes/my_ev/enable
+$ echo 1 > tracing on
+$ /home/devel/pthread_example/pthread_example
+$ echo 0 > tracing_on
+$ less trace
+#      TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
+#         | |       |   ||||       |         |
+ARTHUR_DENT-5223  [000] d...  5653.154822: my_ev: (0x80485aa)
+ARTHUR_DENT-5223  [000] d...  5654.155723: my_ev: (0x80485aa)
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{uprobes: statistics}
+\begin{verbatim}
+$  cat uprobe_profile 
+/home/devel/pthread/pthread_example my_ev    10
+\end{verbatim}
+\end{frame}
 
 \section{sources}
+\begin{frame}
 \frametitle{sources}
 \begin{thebibliography}{1}
 \bibitem{trace1} http://lwn.net/Articles/365835/
diff --git a/kernel-devel/kexec-and-crash-kernel/Makefile b/kernel-devel/kexec-and-crash-kernel/Makefile
new file mode 100644
index 0000000..d641258
--- /dev/null
+++ b/kernel-devel/kexec-and-crash-kernel/Makefile
@@ -0,0 +1,9 @@
+all:
+	for pdf in `ls -1 *.tex` ; do \
+		TEXINPUTS=`pwd`/../..:.:..:$(TEXINPUTS) pdflatex $$pdf; \
+		TEXINPUTS=`pwd`/../..:.:..:$(TEXINPUTS) pdflatex $$pdf; \
+	done
+
+clean:
+	rm -f *.aux *.log *.pdf *.log *.snm *.toc *.vrb *.nav *.out
+
diff --git a/kernel-devel/kexec-and-crash-kernel/crashkernel_patches/crashkernel.diff b/kernel-devel/kexec-and-crash-kernel/crashkernel_patches/crashkernel.diff
new file mode 100644
index 0000000..1d3e082
--- /dev/null
+++ b/kernel-devel/kexec-and-crash-kernel/crashkernel_patches/crashkernel.diff
@@ -0,0 +1,65 @@
+From: Magnus Damm <damm at opensource.se>
+
+Update the copy_oldmem_page() function to ioremap() only
+when accessing memory that is outside the regular range
+of system memory that is managed by the kernel.
+
+Without this patch a warning is triggered in the ARM-specific
+ioremap implementation, see WARN_ON(pfn_valid()) in ioremap.c
+
+The copy_oldmem_page() function is used by the secondary crash
+kernel to access memory using the /proc/vmcore code implemented
+in fs/proc/vmcore.c. To pass information from the first kernel
+to the secondary crash kernel a kernel command line option is
+used to point out where the elf core hdr is located.
+
+The crash kernel is loaded through kexec-tools which also contains
+code that reserves memory for the elfcorehdr= option. This memory
+block is reserved _inside_ the main system memory of the secondary
+kernel. The /proc/vmcore code in the secondary kernel is however
+using copy_oldmem_page() to access both this elfcorehdr area and
+the rest of the memory used by the the first kernel.
+
+So the copy_oldmem_page() function is used to access data that
+may be located in system memory, or it may be outside. Always
+using ioremap will not work, so this patch makes it conditional
+based on pfn_valid().
+
+For more details please look at the sh7372-based example here:
+http://permalink.gmane.org/gmane.linux.ports.sh.devel/11502
+
+Signed-off-by: Magnus Damm <damm at opensource.se>
+---
+
+ arch/arm/kernel/crash_dump.c |   14 ++++++++++----
+ 1 file changed, 10 insertions(+), 4 deletions(-)
+
+--- 0001/arch/arm/kernel/crash_dump.c
++++ work/arch/arm/kernel/crash_dump.c	2011-06-18 20:59:49.000000000 +0900
+@@ -39,9 +39,13 @@ ssize_t copy_oldmem_page(unsigned long p
+ 	if (!csize)
+ 		return 0;
+ 
+-	vaddr = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
+-	if (!vaddr)
+-		return -ENOMEM;
++	if (pfn_valid(pfn)) {
++		vaddr = phys_to_virt(pfn << PAGE_SHIFT);
++	} else {
++		vaddr = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
++		if (!vaddr)
++			return -ENOMEM;
++	}
+ 
+ 	if (userbuf) {
+ 		if (copy_to_user(buf, vaddr + offset, csize)) {
+@@ -52,6 +56,8 @@ ssize_t copy_oldmem_page(unsigned long p
+ 		memcpy(buf, vaddr + offset, csize);
+ 	}
+ 
+-	iounmap(vaddr);
++	if (!pfn_valid(pfn))
++		iounmap(vaddr);
++
+ 	return csize;
+ }
diff --git a/kernel-devel/kexec-and-crash-kernel/crashkernel_patches/kexec_fix_arm_braindamage.diff b/kernel-devel/kexec-and-crash-kernel/crashkernel_patches/kexec_fix_arm_braindamage.diff
new file mode 100644
index 0000000..877f5ea
--- /dev/null
+++ b/kernel-devel/kexec-and-crash-kernel/crashkernel_patches/kexec_fix_arm_braindamage.diff
@@ -0,0 +1,28 @@
+Index: kexec-tools-2.0.3/kexec/arch/arm/crashdump-arm.c
+===================================================================
+--- kexec-tools-2.0.3.orig/kexec/arch/arm/crashdump-arm.c	2011-10-03 00:56:38.000000000 +0200
++++ kexec-tools-2.0.3/kexec/arch/arm/crashdump-arm.c	2013-07-06 17:26:13.410309437 +0200
+@@ -204,12 +204,12 @@
+  * @cmdline. Note that @cmdline must be at least %COMMAND_LINE_SIZE bytes long
+  * (including %NUL).
+  */
+-static void cmdline_add_mem(char *cmdline, unsigned long size)
++static void cmdline_add_mem(char *cmdline, unsigned long size, unsigned long offset)
+ {
+ 	char buf[COMMAND_LINE_SIZE];
+ 	int buflen;
+ 
+-	buflen = snprintf(buf, sizeof(buf), "%s mem=%ldK", cmdline, size >> 10);
++	buflen = snprintf(buf, sizeof(buf), "%s mem=%ldK@0x%X", cmdline, size >> 10, offset);
+ 	if (buflen < 0)
+ 		die("Failed to construct mem= command line parameter\n");
+ 	if (buflen >= sizeof(buf))
+@@ -301,7 +301,7 @@
+ 	 * prevents the dump capture kernel from using any other memory regions
+ 	 * which belong to the primary kernel.
+ 	 */
+-	cmdline_add_mem(mod_cmdline, elfcorehdr - crash_reserved_mem.start);
++	cmdline_add_mem(mod_cmdline, elfcorehdr - crash_reserved_mem.start, crash_reserved_mem.start);
+ 
+ 	dump_memory_ranges();
+ 	dbgprintf("kernel command line: \"%s\"\n", mod_cmdline);
diff --git a/kernel-devel/kexec-and-crash-kernel/crashkernel_patches/zreladdr.diff b/kernel-devel/kexec-and-crash-kernel/crashkernel_patches/zreladdr.diff
new file mode 100644
index 0000000..867e13a
--- /dev/null
+++ b/kernel-devel/kexec-and-crash-kernel/crashkernel_patches/zreladdr.diff
@@ -0,0 +1,13 @@
+Index: linux-3.2/arch/arm/mach-omap2/Makefile.boot
+===================================================================
+--- linux-3.2.orig/arch/arm/mach-omap2/Makefile.boot	2012-01-05 00:55:44.000000000 +0100
++++ linux-3.2/arch/arm/mach-omap2/Makefile.boot	2013-07-06 17:00:18.718516198 +0200
+@@ -1,3 +1,8 @@
++ifeq ($(CONFIG_CRASH_DUMP),y)
++  zreladdr-y		+= 0x84008000
++params_phys-y		:= 0x84000100
++else
+   zreladdr-y		+= 0x80008000
+ params_phys-y		:= 0x80000100
++endif
+ initrd_phys-y		:= 0x80800000
diff --git a/kernel-devel/kexec-and-crash-kernel/pres_kexec_and_crashkernel_en.tex b/kernel-devel/kexec-and-crash-kernel/pres_kexec_and_crashkernel_en.tex
new file mode 100644
index 0000000..1791267
--- /dev/null
+++ b/kernel-devel/kexec-and-crash-kernel/pres_kexec_and_crashkernel_en.tex
@@ -0,0 +1,70 @@
+\input{configpres}
+
+\title{Kexec and Crashkernels}
+\maketitle
+\begin{frame}
+\frametitle{What is kexec?}
+Kexec is a mechanism to boot Linux from within Linux,
+without going through the BIOS / the Bootloader.
+\end{frame}
+
+\begin{frame}
+\frametitle{kexec-tools}
+http://horms.net/projects/kexec/
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Using kexec: Kernel configuration}
+\begin{verbatim}
+Boot options -->
+[*] Kexec system call
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Using kexec: Booting the new kernel}
+\begin{verbatim}
+# Load the kernel image and set the commandline
+$ kexec -l uImage --append=$(cat /proc/cmdline)
+# Start the new kernel
+$ kexec -e
+\end{verbatim}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{Using kexec: Crashkernel}
+\begin{itemize}
+\item Build production kernel (enable kexec system call!)
+\item Build ''crash kernel'' (enable kexec system call and crash dump kernel):
+\begin{verbatim}
+Boot options -->
+[*] Kexec system call
+[*] Build kdump crash kernel
+\end{verbatim}
+\item The crash kernel option should automatically select /proc/vmcore
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{Using kexec: Crashkernel}
+\begin{itemize}
+\item Boot production kernel (Commandline: crashkernel=256M@0x84000000)
+\item Check if crashkernel memory was reserved: 
+\begin{verbatim}
+cat /proc/iomem | grep Crash
+   84000000-93ffffff : Crash kernel
+\end{verbatim}
+\item Load crash kernel and let the system crash ;-)
+\begin{verbatim}
+kexec -p uImage-crash \
+  --append="$(cat /proc/cmdline | \
+    sed 's/crashkernel/bla/')"
+echo c > /proc/sysrq-trigger
+\end{verbatim}
+\item After crashing the crash kernel should boot up!
+\item Within the crashkernel the core file for the production kernel is available in /proc/vmcore
+\end{itemize}
+\end{frame}
+
+\input{tailpres}