move elbe-devel to elbe-internals

Signed-off-by: Manuel Traut <manut@linutronix.de>

1 files changed, 602 insertions, 0 deletions

diff --git a/distribution/elbe-internals/pres_elbe-devel_en.tex b/distribution/elbe-internals/pres_elbe-devel_en.tex
new file mode 100644
index 0000000..c57ad47
--- /dev/null
+++ b/distribution/elbe-internals/pres_elbe-devel_en.tex
@@ -0,0 +1,602 @@
+\input{configpres}
+
+\lstset{language=bash}
+
+\title{ELBE internals}
+\maketitle
+
+
+\subsection{source repository}
+
+\begin{frame}[fragile]
+\frametitle{source tree}
+
+ELBE development is hosted on github:
+\begin{lstlisting}
+https://github.com/Linutronix/elbe
+\end{lstlisting}
+
+Use git to retrieve the source:
+\begin{lstlisting}
+git clone git://github.com/Linutronix/elbe.git
+\end{lstlisting}
+
+\begin{lstlisting}
+.
+|-- debian/       # debian packaging
+|-- docs/         # man pages, dia files, overview
+|-- elbe          # python script calling several scripts from ./elbepack
+|-- elbepack/     # implementation of elbe subcommands like 'create'
+|  |-- mako/      # script and config file templates for the buildenv
+|-- examples/     # example XML files
+|-- Makefile      # calls dpkg-buildpackage to generate a debian package
+|-- README        # short introduction to ELBE
++-- setup.py      # python typical setup script
+\end{lstlisting}
+
+\end{frame}
+
+\subsection{elbepack/create.py}
+
+\begin{frame}
+\frametitle{create an elbe project}
+\begin{itemize}
+\item the 'elbe create' command is implemented
+
+at elbepack/create.py
+\item create.py has one mandotary parameter:
+
+path and name of an elbe xml file
+\item create.py accepts optional parameters
+
+that are parsed by OptionParser
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{xml validation}
+\begin{itemize}
+\item verifies given xml file against elbepack/dbsfed.xsd
+\item we try to do as much checks as posible with the schema
+\item normaly there are no further checks inside the python code
+\item validation can be skipped by --skip-validation parameter
+\item elbepack/validate.py is used to verify the schema
+\item elbepack/validate.py can be also called directly:
+
+'elbe validate examples/amd64-example.xml'
+\item elbepack/validate.py uses etree from lxml
+
+to verify the schema
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{create build directory}
+\begin{itemize}
+\item creates a directory named 'build'
+
+at the current working directory
+\item name can be overridden by --directory parameter
+\item the buildenvironment will be created inside this directory
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{the debian-installer (d-i)}
+\begin{itemize}
+\item the d-i needs a running linux system
+\item the debian project distributes the d-i in different packages:
+  e.g. as a bootable cd or for use with pxeboot
+\item all packages include at least
+\begin{itemize}
+\item one Linux kernel binary
+\item and an initial ramdisk with a minimal rfs and the d-i application
+\end{itemize}
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{d-i common use}
+\begin{itemize}
+\item d-i is started on the system debian should be installed on
+\item d-i does
+\begin{itemize}
+\item disk detection / partitioning
+\item sw installation
+\item system configuration
+\end{itemize}
+\end{itemize}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{elbe kinitrd packages}
+\begin{itemize}
+\item the elbe project has packaged the d-i used by elbe as binary debian packages
+\begin{lstlisting}
+elbe-bootstrap_0.1_armel
+|- opt
+|  +-elbe
+|    +-initrd
+|      +-initrd-cdrom.gz
+|      +-initrd.gz
+|      +-vmlinuz
+\end{lstlisting}
+\item the name of the debian package is specified by the 'kinitrd' tag inside the
+  xml file
+\item current packages are called elbe-bootstrap and are available at the debian
+repository:
+\begin{lstlisting}
+deb http://debian.linutronix.de/elbe wheezy main
+\end{lstlisting}
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{retrieve kinitrd}
+\begin{itemize}
+\item elbepack/pkgutils.py:copy\_kinitrd()
+
+is called by elbepack/create.py
+\item the pkgutils module uses python virtapt
+
+to retrieve the kinitrd package
+
+from a mirror or cdrom specified in the xml file
+\item virtapt takes care on using
+
+the correct debian architecture and suite
+\item the kinitrd package will be extracted
+
+and copied to the build directory
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{mako}
+Mako is a template library written in Python.
+
+It can be used to generate files from templates.
+
+It is normally used to generate webpages.
+
+Documentation and more informations can be found on
+\begin{lstlisting}
+http://www.makotemplates.org/
+\end{lstlisting}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{create Makefile, scripts and config files}
+ELBE uses
+\begin{lstlisting}
+d = {"opt": opt,
+     "xml": xml,
+     "prj": xml.node("/project"),
+     "tgt": xml.node("/target"),
+     "pkgs": xml.node("/target/pkg-list"),
+     "fine": xml.node("/finetuning"),
+     "preseed": get_preseed(xml) }
+...
+mako.Template(filename=fname).render(**d)
+\end{lstlisting}
+to insert several values from the XML file into scripts and configfiles.
+\begin{description}
+\item[fname] the function is called for each *.mako file in elbepack/mako
+\item[d] refers to several lxml.etree XML nodes
+\item[xml] is a etree initialized with the ELBE XML file
+\end{description}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{create Makefile, scripts and config files}
+For example a simple txt file including all package names defined in the
+$\langle$pkg-list$\rangle$ tag can be created by (pkg-list.mako):
+\begin{lstlisting}
+% for n in pkgs:
+% if n.tag == "pkg":
+${n.et.text}
+% endif
+% endfor
+\end{lstlisting}
+\end{frame}
+
+
+\begin{frame}
+\frametitle{preseeding}
+Preseeding provides a way to set answers to questions asked during the
+installation process, without having to manually enter the answers while
+the installation is running.
+\end{frame}
+
+\begin{frame}
+\frametitle{preseeding}
+Many pre-seeding rules are generated
+
+from the XML file with mako:
+
+./elbepack/mako/common.cfg.mako
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{preseeding}
+Project specific preseeding rules can be added to the xml file.
+These rules are then included by ./elbepack/mako/custom-preseed.cfg.mako
+\end{frame}
+
+
+\subsection{Makefile (pre-install)}
+
+\begin{frame}
+\frametitle{initrd.gz modification}
+\begin{itemize}
+\item a directory tmp-tree is created
+\item all files and scripts needed inside the installer
+
+are copied to the tmp-tree
+\item ungzip the original initrd.gz
+\item use cpio to append all files of tmp-tree
+
+to the unzipped initrd image
+\item gzip the new initrd image and remove the tmp-tree
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{create buildenv.img}
+\begin{itemize}
+\item qemu-img is used to create a buildenv.img file. This file will be used as
+harddisk of the emulated virtual machine.
+\item The debian-installer will install
+
+a full featured debian on this image.
+\item After the debian-installer was active,
+
+this image can be used as a virtual build-environment.
+\item The target rootfilesystem is normally
+
+just a subset of this image.
+\end{itemize}
+\end{frame}
+
+\begin{frame}[fragile]
+\frametitle{start emulator}
+\begin{itemize}
+\item the cmdline to start the emulator depends on several specifications made
+in the XML file:
+\begin{lstlisting}
+..
+    ${prj.text("buildimage/interpreter")}  \
+        -M ${prj.text("buildimage/machine")} \
+        -drive file=buildenv.img,if=${hd_type},bus=1,unit=0 \
+% if prj.has("mirror/cdrom"):
+  -drive file=${prj.text("mirror/cdrom")},if=${cdrom_type},media=cdrom,bus=1,unit=1 \
+% endif
+        -kernel vmlinuz \
+        -initrd initrd-preseeded.gz \
+        -append 'root=/dev/${hd_name} debconf_priority=critical console=${prj.text("buildimage/console")} DEBIAN_FRONTEND=newt' \
+        -no-reboot \
+        -net nic,vlan=1,model=${prj.text("buildimage/NIC/model")},macaddr="${prj.text('buildimage/NIC/MAC')}" \
+        -net user,vlan=1 \
+        -m $(MEMSIZE) \
+        -usb && reset
+..
+\end{lstlisting}
+\end{itemize}
+\end{frame}
+
+\subsection{debian-installer in emulator}
+
+\begin{frame}
+\frametitle{an ordinary debian installation}
+runs inside the virtual machine:
+\begin{itemize}
+\item the buildenv.img will be partitionated and formated
+\item the root partition will be mounted at
+
+/target inside the debian-installer rfs
+\item a minimal rootfilesystem will be installed to /target
+\item a chroot /target will be executed
+\item all packages will be installed and configured
+\item the /target chroot will be left
+\item the preseed hook
+
+d-i preseed/late\_command string /post-inst.sh
+
+will be executed
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+\frametitle{post-inst.sh}
+\begin{description}
+\item [cp-scipts-into-buildenv.sh] copy all (generated) scripts
+
+from the initrd (/)
+
+to the buildenv.img (/target/opt/elbe)
+
+\item [changeroot-into-buildenv.sh] chroot into /target again,
+
+trigger target (/opt/elbe/create-target-rfs.sh)
+
+and cdrom (/opt/elbe/mkcdrom.sh) generation.
+\end{description}
+\end{frame}
+
+\begin{frame}
+\frametitle{target generation (overview)}
+\begin{itemize}
+\item generate file and pkg lists
+\item create /target tree
+\item call dump.py to
+\begin{itemize}
+    \item validate packages / write logfile
+    \item add fullpkgs list to source.xml
+    \item add content of /etc/apt/sources.list to source.xml
+    \item add content of /etc/apt/preferences to source.xml
+    \item modify /target tree with finetuning.sh
+\end{itemize}
+\item generate licence information file
+\item call part-target.sh to generate images from /target tree
+\item (optional) generate cpio images and tarballs
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{create /target tree}
+\begin{itemize}
+\item depending on the ELBE mode, the filelist is generated.
+
+E.g. in normal mode:
+\begin{lstlisting}
+ls -A1 / | \
+    grep -v target | grep -v proc | grep -v sys | \
+    xargs find >> /opt/elbe/filelist
+\end{lstlisting}
+\item all files specified to be stored on the target rfs are copied:
+\begin{lstlisting}
+rsync -a --files-from=/opt/elbe/filelist / /target
+\end{lstlisting}
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{dump.py - check if all req. packages are installed}
+\begin{itemize}
+\item add lines about missing packages to validation.txt:
+\begin{lstlisting}
+    for name in [p.et.text for p in pkgs] + [kinitrd]:
+
+        if not name in cache:
+            elog.printo( "- package %s does not exist" % name )
+            errors += 1
+            continue
+
+        cp = cache[name]
+
+        if not cp.installed and name != kinitrd:
+            elog.printo( "- package %s is not installed" % name )
+            errors += 1
+            continue
+\end{lstlisting}
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{dump.py - check version of packages used}
+(only done if a fullpkgs tag was specified in the xml file)
+\begin{itemize}
+\item add lines about packages with wrong version to validation.txt:
+\begin{lstlisting}
+    for p in fullpkgs:
+        name = p.et.text
+        ver  = p.et.get('version')
+        md5  = p.et.get('md5')
+        pindex[name] = p
+        ...
+        if cpi.version != ver:
+            elog.printo("pack %s v%s != installed v%s" % (name, ver,  cpi.version))
+            errors += 1
+            continue
+
+        if cpi.md5 != md5:
+            elog.printo("pack %s md5 %s != installed md5 %s" % (name, md5,  cpi.md5) )
+            errors += 1
+        ...
+\end{lstlisting}
+\item there is also a check for packages that are installed but wasn't specified to be installed
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{dump.py - write fullpkglist to source.xml}
+\begin{itemize}
+\item all used packages are listed in the source.xml file
+
+(including version and md5sum):
+\begin{lstlisting}
+    paktree = xml.ensure_child( 'fullpkgs' )
+    paktree.clear()
+    pkgs = []
+    for p in cache:
+        if p.is_installed:
+            pkgs.append( (p.name, p.installed.version, p.is_auto_installed, p.installed.md5))
+    ...
+    for p in pkgs:
+        append_pkg_elem( paktree, p[0], p[1], p[2], p[3] )
+\end{lstlisting}
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+\frametitle{dump.py - modify /target tree with finetuning.sh}
+\begin{itemize}
+\item the script is executed from /target.
+
+    So files can be copied
+
+    from the debian rfs to the target rfs
+
+    by e.g. 'cp /etc/passwd etc/'
+
+\item the output of the finetuning.sh commands is appended to elbe-report.txt
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+\frametitle{dump.py - filelist generation}
+elbe-report.txt includes a filelist that indicates each file in the target rfs
+and its origin
+\begin{itemize}
+\item a time stamp is recorded before and after finetuning.
+\item files that show up with a modification time in that period
+
+can be identified as modified/created by finetuning.sh
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{generate licence information file}
+all licence informations that are stored in /usr/share/* are collected in one file:
+\begin{lstlisting}
+find /usr/share/doc -name copyright -exec \
+    /opt/elbe/print_licence.sh {} \; >> /opt/elbe/licence.txt
+\end{lstlisting}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{call part-target.sh to copy /target tree to (different) images}
+currently only UBI image support is functional.
+
+HD images are under development. They will be created by using python bindings
+of libparted and loopback mounts.
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{build ubifs images}
+\begin{itemize}
+\item mkfs.ubifs -r is used to generate a ubifs image
+
+from a subdirectory
+\item the name of the images is defined
+
+by the label in the fstab tag
+\item the image defined first in the fstab is created first
+\item the files copied to an image
+
+are moved temporarily away from /target:
+\begin{lstlisting}
+    /mountA/mountB # mountA shouldn't include files like mountB/test
+\end{lstlisting}
+\item /target is restored as all images were created
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{(optional) generate cpio images and tarballs}
+\begin{lstlisting}
+% if xml.has("target/package/tar"):
+tar cf /opt/elbe/target.tar -C /target .
+% endif
+
+% if xml.has("target/package/cpio"):
+cd /target
+find . -print | cpio -ov -H newc >/opt/elbe/target.cpio
+% endif
+\end{lstlisting}
+\end{frame}
+
+
+\begin{frame}
+\frametitle{binary cdrom generation}
+An ISO CD image will be created:
+\begin{itemize}
+\item CD should contain all packages
+
+needed to reproduce the current ELBE run
+\item the kinitrd package is installed into the buildenv
+
+to get it added to the cdrom like all other packages
+\item CD needs to be in a debian repository format
+
+otherwise it won't be accepted by d-i
+\item reprepo and genisoimage is used to create the CD
+\end{itemize}
+\end{frame}
+
+
+\begin{frame}[fragile]
+\frametitle{source cdrom generation}
+upon request (by 'elbe create --build-source')
+
+an ISO image containing all debian source packages is created:
+\begin{lstlisting}
+% if opt.buildsources:
+  mkdir -p /opt/elbe/source
+  dpkg --get-selections | awk '{print $1}' > /opt/elbe/pkg-list.actual
+  cd /opt/elbe/source
+  awk '{print "apt-get -d source "$1}' /opt/elbe/pkg-list.actual | sh
+  cd /opt/elbe
+  dpkg-scansources source /dev/null | gzip -9c > source/Sources.gz
+  genisoimage -o /opt/elbe/source.iso -J -R source
+% endif
+\end{lstlisting}
+\end{frame}
+
+
+\begin{frame}
+\frametitle{kinitrd package details}
+\begin{itemize}
+\item in xml the cdrom tag can be used instead of a mirror
+\item then d-i needs to load components (udeb) from cdrom instead of web
+\item this is the reason for kinitrd including two versions of the initrd:
+\begin{itemize}
+\item one with the d-i configured to load udebs from a web source
+\item a second with d-i configured to load udebs from a cdrom source
+\end{itemize}
+\item there is a switch in the 'elbe create' path to select the right initrd for
+  the project
+\end{itemize}
+\end{frame}
+
+
+\subsection{Makefile (post-install)}
+
+\begin{frame}[fragile]
+\frametitle{copy files to host}
+\begin{itemize}
+\item all files produced by elbe are stored inside buildenv.img:/opt/elbe
+\item they should be transfered into the build dir on the Host PC
+\item the emulator is shutdown,
+
+then e2cp can be used to retrive the files:
+\begin{lstlisting}
+e2cp buildenv.img?offset=32768:/opt/elbe/licence.txt .
+\end{lstlisting}
+\item this is done for all files by the Makefile, e.g:
+\begin{lstlisting}
+% if tgt.has("images"):
+%  for mtd in tgt.node("images"):
+%   if mtd.has("partitions"):
+    e2cp buildenv.img?offset=${loop_offset}:/opt/elbe/${mtd.text("name")} .
+%   endif
+%   if mtd.has("binary"):
+    e2cp buildenv.img?offset=${loop_offset}:${mtd.text("binary")} ${mtd.text("name")}
+\end{lstlisting}
+\end{itemize}
+\end{frame}
+
+
+\input{tailpres}