\input{configpres}

\lstset{language=bash}

\title{ELBE internals}
\maketitle


\subsection{source repository}

\begin{frame}[fragile]
\frametitle{source tree}

ELBE development is hosted on github:
\begin{lstlisting}
https://github.com/Linutronix/elbe
\end{lstlisting}

Use git to retrieve the source:
\begin{lstlisting}
git clone git://github.com/Linutronix/elbe.git
\end{lstlisting}

\begin{lstlisting}
.
|-- debian/       # debian packaging
|-- docs/         # man pages, dia files, overview
|-- elbe          # python script calling several scripts from ./elbepack
|-- elbepack/     # implementation of elbe subcommands like 'create'
|  |-- mako/      # script and config file templates for the buildenv
|-- examples/     # example XML files
|-- Makefile      # calls dpkg-buildpackage to generate a debian package
|-- README        # short introduction to ELBE
+-- setup.py      # python typical setup script
\end{lstlisting}

\end{frame}

\subsection{elbepack/create.py}

\begin{frame}
\frametitle{create an elbe project}
\begin{itemize}
\item the 'elbe create' command is implemented

at elbepack/create.py
\item create.py has one mandotary parameter:

path and name of an elbe xml file
\item create.py accepts optional parameters

that are parsed by OptionParser
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{xml validation}
\begin{itemize}
\item verifies given xml file against elbepack/dbsfed.xsd
\item we try to do as much checks as posible with the schema
\item normaly there are no further checks inside the python code
\item validation can be skipped by --skip-validation parameter
\item elbepack/validate.py is used to verify the schema
\item elbepack/validate.py can be also called directly:

'elbe validate examples/amd64-example.xml'
\item elbepack/validate.py uses etree from lxml

to verify the schema
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{create build directory}
\begin{itemize}
\item creates a directory named 'build'

at the current working directory
\item name can be overridden by --directory parameter
\item the buildenvironment will be created inside this directory
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{the debian-installer (d-i)}
\begin{itemize}
\item the d-i needs a running linux system
\item the debian project distributes the d-i in different packages:
  e.g. as a bootable cd or for use with pxeboot
\item all packages include at least
\begin{itemize}
\item one Linux kernel binary
\item and an initial ramdisk with a minimal rfs and the d-i application
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{d-i common use}
\begin{itemize}
\item d-i is started on the system debian should be installed on
\item d-i does
\begin{itemize}
\item disk detection / partitioning
\item sw installation
\item system configuration
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{elbe kinitrd packages}
\begin{itemize}
\item the elbe project has packaged the d-i used by elbe as binary debian packages
\begin{lstlisting}
testrd_0.6_armel
|- opt
|  +-elbe
|    +-initrd
|      +-initrd-cdrom.gz
|      +-initrd.gz
|      +-vmlinuz
\end{lstlisting}
\item the name of the debian package is specified by the 'kinitrd' tag inside the
  xml file
\item current packages are called testrd and are available at the debian
repository:
\begin{lstlisting}
deb http://debian.linutronix.de/elbe squeeze main
\end{lstlisting}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{retrieve kinitrd}
\begin{itemize}
\item elbepack/pkgutils.py:copy\_kinitrd()

is called by elbepack/create.py
\item the pkgutils module uses python virtapt

to retrieve the kinitrd package

from a mirror or cdrom specified in the xml file
\item virtapt takes care on using

the correct debian architecture and suite
\item the kinitrd package will be extracted

and copied to the build directory
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{mako}
Mako is a template library written in Python.

It can be used to generate files from templates.

It is normally used to generate webpages.

Documentation and more informations can be found on
\begin{lstlisting}
http://www.makotemplates.org/
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{create Makefile, scripts and config files}
ELBE uses
\begin{lstlisting}
d = {"opt": opt,
     "xml": xml,
     "prj": xml.node("/project"),
     "tgt": xml.node("/target"),
     "pkgs": xml.node("/target/pkg-list"),
     "fine": xml.node("/finetuning"),
     "preseed": get_preseed(xml) }
...
mako.Template(filename=fname).render(**d)
\end{lstlisting}
to insert several values from the XML file into scripts and configfiles.
\begin{description}
\item[fname] the function is called for each *.mako file in elbepack/mako
\item[d] refers to several lxml.etree XML nodes
\item[xml] is a etree initialized with the ELBE XML file
\end{description}
\end{frame}

\begin{frame}[fragile]
\frametitle{create Makefile, scripts and config files}
For example a simple txt file including all package names defined in the
$\langle$pkg-list$\rangle$ tag can be created by (pkg-list.mako):
\begin{lstlisting}
% for n in pkgs:
% if n.tag == "pkg":
${n.et.text}
% endif
% endfor
\end{lstlisting}
\end{frame}


\begin{frame}
\frametitle{preseeding}
Preseeding provides a way to set answers to questions asked during the
installation process, without having to manually enter the answers while
the installation is running.
\end{frame}

\begin{frame}
\frametitle{preseeding}
Many pre-seeding rules are generated

from the XML file with mako:

./elbepack/mako/common.cfg.mako
\end{frame}

\begin{frame}[fragile]
\frametitle{preseeding}
Project specific preseeding rules can be added to the xml file.
These rules are then included by ./elbepack/mako/custom-preseed.cfg.mako
\end{frame}


\subsection{Makefile (pre-install)}

\begin{frame}
\frametitle{initrd.gz modification}
\begin{itemize}
\item a directory tmp-tree is created
\item all files and scripts needed inside the installer

are copied to the tmp-tree
\item ungzip the original initrd.gz
\item use cpio to append all files of tmp-tree

to the unzipped initrd image
\item gzip the new initrd image and remove the tmp-tree
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{create buildenv.img}
\begin{itemize}
\item qemu-img is used to create a buildenv.img file. This file will be used as
harddisk of the emulated virtual machine.
\item The debian-installer will install

a full featured debian on this image.
\item After the debian-installer was active,

this image can be used as a virtual build-environment.
\item The target rootfilesystem is normally

just a subset of this image.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{start emulator}
\begin{itemize}
\item the cmdline to start the emulator depends on several specifications made
in the XML file:
\begin{lstlisting}
..
    ${prj.text("buildimage/interpreter")}  \
        -M ${prj.text("buildimage/machine")} \
        -drive file=buildenv.img,if=${hd_type},bus=1,unit=0 \
% if prj.has("mirror/cdrom"):
  -drive file=${prj.text("mirror/cdrom")},if=${cdrom_type},media=cdrom,bus=1,unit=1 \
% endif
        -kernel vmlinuz \
        -initrd initrd-preseeded.gz \
        -append 'root=/dev/${hd_name} debconf_priority=critical console=${prj.text("buildimage/console")} DEBIAN_FRONTEND=newt' \
        -no-reboot \
        -net nic,vlan=1,model=${prj.text("buildimage/NIC/model")},macaddr="${prj.text('buildimage/NIC/MAC')}" \
        -net user,vlan=1 \
        -m $(MEMSIZE) \
        -usb && reset
..
\end{lstlisting}
\end{itemize}
\end{frame}

\subsection{debian-installer in emulator}

\begin{frame}
\frametitle{an ordinary debian installation}
runs inside the virtual machine:
\begin{itemize}
\item the buildenv.img will be partitionated and formated
\item the root partition will be mounted at

/target inside the debian-installer rfs
\item a minimal rootfilesystem will be installed to /target
\item a chroot /target will be executed
\item all packages will be installed and configured
\item the /target chroot will be left
\item the preseed hook

d-i preseed/late\_command string /post-inst.sh

will be executed
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{post-inst.sh}
\begin{description}
\item [cp-scipts-into-buildenv.sh] copy all (generated) scripts

from the initrd (/)

to the buildenv.img (/target/opt/elbe)

\item [changeroot-into-buildenv.sh] chroot into /target again,

trigger target (/opt/elbe/create-target-rfs.sh)

and cdrom (/opt/elbe/mkcdrom.sh) generation.
\end{description}
\end{frame}

\begin{frame}
\frametitle{target generation (overview)}
\begin{itemize}
\item generate file and pkg lists
\item create /target tree
\item call dump.py to
\begin{itemize}
    \item validate packages / write logfile
    \item add fullpkgs list to source.xml
    \item add content of /etc/apt/sources.list to source.xml
    \item add content of /etc/apt/preferences to source.xml
    \item modify /target tree with finetuning.sh
\end{itemize}
\item generate licence information file
\item call part-target.sh to generate images from /target tree
\item (optional) generate cpio images and tarballs
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{create /target tree}
\begin{itemize}
\item depending on the ELBE mode, the filelist is generated.

E.g. in normal mode:
\begin{lstlisting}
ls -A1 / | \
    grep -v target | grep -v proc | grep -v sys | \
    xargs find >> /opt/elbe/filelist
\end{lstlisting}
\item all files specified to be stored on the target rfs are copied:
\begin{lstlisting}
rsync -a --files-from=/opt/elbe/filelist / /target
\end{lstlisting}
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{dump.py - check if all req. packages are installed}
\begin{itemize}
\item add lines about missing packages to validation.txt:
\begin{lstlisting}
    for name in [p.et.text for p in pkgs] + [kinitrd]:

        if not name in cache:
            elog.printo( "- package %s does not exist" % name )
            errors += 1
            continue

        cp = cache[name]

        if not cp.installed and name != kinitrd:
            elog.printo( "- package %s is not installed" % name )
            errors += 1
            continue
\end{lstlisting}
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{dump.py - check version of packages used}
(only done if a fullpkgs tag was specified in the xml file)
\begin{itemize}
\item add lines about packages with wrong version to validation.txt:
\begin{lstlisting}
    for p in fullpkgs:
        name = p.et.text
        ver  = p.et.get('version')
        md5  = p.et.get('md5')
        pindex[name] = p
        ...
        if cpi.version != ver:
            elog.printo("pack %s v%s != installed v%s" % (name, ver,  cpi.version))
            errors += 1
            continue

        if cpi.md5 != md5:
            elog.printo("pack %s md5 %s != installed md5 %s" % (name, md5,  cpi.md5) )
            errors += 1
        ...
\end{lstlisting}
\item there is also a check for packages that are installed but wasn't specified to be installed
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{dump.py - write fullpkglist to source.xml}
\begin{itemize}
\item all used packages are listed in the source.xml file

(including version and md5sum):
\begin{lstlisting}
    paktree = xml.ensure_child( 'fullpkgs' )
    paktree.clear()
    pkgs = []
    for p in cache:
        if p.is_installed:
            pkgs.append( (p.name, p.installed.version, p.is_auto_installed, p.installed.md5))
    ...
    for p in pkgs:
        append_pkg_elem( paktree, p[0], p[1], p[2], p[3] )
\end{lstlisting}
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{dump.py - modify /target tree with finetuning.sh}
\begin{itemize}
\item the script is executed from /target.

    So files can be copied

    from the debian rfs to the target rfs

    by e.g. 'cp /etc/passwd etc/'

\item the output of the finetuning.sh commands is appended to elbe-report.txt
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{dump.py - filelist generation}
elbe-report.txt includes a filelist that indicates each file in the target rfs
and its origin
\begin{itemize}
\item a time stamp is recorded before and after finetuning.
\item files that show up with a modification time in that period

can be identified as modified/created by finetuning.sh
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{generate licence information file}
all licence informations that are stored in /usr/share/* are collected in one file:
\begin{lstlisting}
find /usr/share/doc -name copyright -exec \
    /opt/elbe/print_licence.sh {} \; >> /opt/elbe/licence.txt
\end{lstlisting}
\end{frame}


\begin{frame}[fragile]
\frametitle{call part-target.sh to copy /target tree to (different) images}
currently only UBI image support is functional.

HD images are under development. They will be created by using python bindings
of libparted and loopback mounts.
\end{frame}


\begin{frame}[fragile]
\frametitle{build ubifs images}
\begin{itemize}
\item mkfs.ubifs -r is used to generate a ubifs image

from a subdirectory
\item the name of the images is defined

by the label in the fstab tag
\item the image defined first in the fstab is created first
\item the files copied to an image

are moved temporarily away from /target:
\begin{lstlisting}
    /mountA/mountB # mountA shouldn't include files like mountB/test
\end{lstlisting}
\item /target is restored as all images were created
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{(optional) generate cpio images and tarballs}
\begin{lstlisting}
% if xml.has("target/package/tar"):
tar cf /opt/elbe/target.tar -C /target .
% endif

% if xml.has("target/package/cpio"):
cd /target
find . -print | cpio -ov -H newc >/opt/elbe/target.cpio
% endif
\end{lstlisting}
\end{frame}


\begin{frame}
\frametitle{binary cdrom generation}
An ISO CD image will be created:
\begin{itemize}
\item CD should contain all packages

needed to reproduce the current ELBE run
\item the kinitrd package is installed into the buildenv

to get it added to the cdrom like all other packages
\item CD needs to be in a debian repository format

otherwise it won't be accepted by d-i
\item reprepo and genisoimage is used to create the CD
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{source cdrom generation}
upon request (by 'elbe create --build-source')

an ISO image containing all debian source packages is created:
\begin{lstlisting}
% if opt.buildsources:
  mkdir -p /opt/elbe/source
  dpkg --get-selections | awk '{print $1}' > /opt/elbe/pkg-list.actual
  cd /opt/elbe/source
  awk '{print "apt-get -d source "$1}' /opt/elbe/pkg-list.actual | sh
  cd /opt/elbe
  dpkg-scansources source /dev/null | gzip -9c > source/Sources.gz
  genisoimage -o /opt/elbe/source.iso -J -R source
% endif
\end{lstlisting}
\end{frame}


\begin{frame}
\frametitle{kinitrd package details}
\begin{itemize}
\item in xml the cdrom tag can be used instead of a mirror
\item then d-i needs to load components (udeb) from cdrom instead of web
\item this is the reason for kinitrd including two versions of the initrd:
\begin{itemize}
\item one with the d-i configured to load udebs from a web source
\item a second with d-i configured to load udebs from a cdrom source
\end{itemize}
\item there is a switch in the 'elbe create' path to select the right initrd for
  the project
\end{itemize}
\end{frame}


\subsection{Makefile (post-install)}

\begin{frame}[fragile]
\frametitle{copy files to host}
\begin{itemize}
\item all files produced by elbe are stored inside buildenv.img:/opt/elbe
\item they should be transfered into the build dir on the Host PC
\item the emulator is shutdown,

then e2cp can be used to retrive the files:
\begin{lstlisting}
e2cp buildenv.img?offset=32768:/opt/elbe/licence.txt .
\end{lstlisting}
\item this is done for all files by the Makefile, e.g:
\begin{lstlisting}
% if tgt.has("images"):
%  for mtd in tgt.node("images"):
%   if mtd.has("partitions"):
    e2cp buildenv.img?offset=${loop_offset}:/opt/elbe/${mtd.text("name")} .
%   endif
%   if mtd.has("binary"):
    e2cp buildenv.img?offset=${loop_offset}:${mtd.text("binary")} ${mtd.text("name")}
\end{lstlisting}
\end{itemize}
\end{frame}


\input{tailpres}