path: root/distribution/elbe-devel/pres_elbe-devel_en.tex

\input{configpres}

\title{ELBE internals}
\maketitle

\subsection{source repository}

\begin{frame}[fragile]
\frametitle{source tree}

ELBE development is hosted on github:
\begin{lstlisting}
https://github.com/Linutronix/elbe
\end{lstlisting}

Use git to retrieve the source:
\begin{lstlisting}
git clone git://github.com/Linutronix/elbe.git
\end{lstlisting}

\begin{lstlisting}
.
|-- debian/       # debian packaging
|-- docs/         # man pages, dia files, overview
|-- elbe          # python script calling several scripts from ./elbepack
|-- elbepack/     # implementation of elbe subcommands like 'create'
|  |-- mako/      # script and config file templates for the buildenv
|-- examples/     # example XML files
|-- Makefile      # calls dpkg-buildpackage to generate a debian package
|-- README        # short introduction to ELBE
+-- setup.py      # python typical setup script
\end{lstlisting}

\end{frame}

\subsection{elbepack/create.py}

\begin{frame}
\frametitle{create an elbe project}
\begin{itemize}
\item the 'elbe create' command calls the elbepack/elbe.py script
\item create.py has one mandotary parameter: path and name of an elbe xml file
\item create.py accepts optional parameters that are parsed by OptionParser
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{xml validation}
\begin{itemize}
\item verifies given xml file against elbepack/dbsfed.xsd
\item we try to do as much checks as posible with the schema
\item normaly there are no further checks inside the python code
\item validation can be skipped by --skip-validation parameter
\item elbepack/validate.py is used to verify the schema
\item elbepack/validate.py can be also called directly: 'elbe validate'
\item elbepack/validate.py uses etree from lxml to verify the schema
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{create build directory}
\begin{itemize}
\item creates a directory named 'build' at the current working directory
\item name can be overridden by --directory parameter
\item the buildenvironment will be created inside this directory
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{the debian-installer}
\begin{itemize}
\item the debian-installer is a Linux kernel and a initial ramdisk.
\item the initial ramdisk includes a minimal rootfilesystem and the d-i
  application
\item the debian project distributes the installer in different formats:
  e.g. as a bootable cd or for use with pxeboot
\item normally the debian-installer is booted on the system debian should
  be installed on
\item the debian-installer does disk detection / partitioning, sw installation
  and system configuration
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{debian-installer package}
\begin{itemize}
\item the elbe project has packaged the debian-installer as debian package
\item this was done to be able to verify the version of the installer
\item the name of the debian package is specified by the 'kinitrd' tag inside the
  xml file
\item current packages are called testrd and are available on the debian
repository:
\begin{lstlisting}
deb http://debian.linutronix.de/elbe squeeze main
\end{lstlisting}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{retrieve debian-installer package}
\begin{itemize}
\item elbepack/pkgutils.py:copy\_kinitrd() is called by create.py
\item the pkgutils module uses python virtapt to retrieve the debian package
  from a mirror or cdrom specified in the xml file
\item virtapt takes care on using the correct architecture and debian suite
\item then the package contents are extracted and copied to the build
  directory
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{mako}
Mako is a template library written in Python.

It can be used to generate files from templates.

It is normally used to generate webpages.

Documentation and more informations can be found on
\begin{lstlisting}
http://www.makotemplates.org/
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{create Makefile, scripts and config files}
ELBE uses
\begin{lstlisting}
mako.Template(filename=fname).render(**d)
\end{lstlisting}
to insert several values from the XML file into the scripts and configfiles
used in the buildenv. It is called several times, for each *.mako file in
elbepack/mako (fname).

d refers to several lxml.etree XML nodes. etree was initialized with the
ELBE XML file before.
\begin{lstlisting}
d = {"opt": opt,
     "xml": xml,
     "prj": xml.node("/project"),
     "tgt": xml.node("/target"),
     "pkgs": xml.node("/target/pkg-list"),
     "fine": xml.node("/finetuning"),
     "preseed": get_preseed(xml) }
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{create Makefile, scripts and config files}
For example a simple txt file including all package names defined in the
<pkg-list> tag can be created by (pkg-list.mako):
\begin{lstlisting}
% for n in pkgs:
% if n.tag == "pkg":
${n.et.text}
% endif
% endfor
\end{lstlisting}
\end{frame}


\begin{frame}
\frametitle{preseeding}
Preseeding provides a way to set answers to questions asked during the
installation process, without having to manually enter the answers while
the installation is running.
\end{frame}

\begin{frame}
\frametitle{preseeding}
Many pre-seeding rules are generated from the XML file with mako:
./elbepack/mako/commmon.cfg.mako
\end{frame}

\begin{frame}[fragile]
\frametitle{preseeding}
Project specific preseeding rules can be added to the xml file.
These rules are then included by ./elbepack/mako/custom-preseed.cfg.mako
\end{frame}


\subsection{Makefile (pre-install)}

\begin{frame}
\frametitle{initrd.gz modification}
\begin{itemize}
\item a directory tmp-tree is created
\item all files and scripts needed inside the installer
    are copied to the tmp-tree
\item ungzip the original initrd.gz
\item use cpio to append all files of tmp-tree to the unzipped initrd image
\item gzip the new initrd image and remove the tmp-tree
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{create buildenv.img}
\begin{itemize}
\item qemu-img is used to create a buildenv.img file. This file will be used as
harddisk of the emulated virtual machine.
\item The debian-installer will install a full featured debian on this image.
\item After the debian-installer was active, this image can be used as a virtual
build-environment.
\item The target rootfilesystem is normally just a subset of this image.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{start emulator}
\begin{itemize}
\item the cmdline to start the emulator depends on several specifications made
in the XML file:
\begin{lstlisting}
..
    ${prj.text("buildimage/interpreter")}  \
        -M ${prj.text("buildimage/machine")} \
        -drive file=buildenv.img,if=${hd_type},bus=1,unit=0 \
% if prj.has("mirror/cdrom"):
  -drive file=${prj.text("mirror/cdrom")},if=${cdrom_type},media=cdrom,bus=1,unit=1 \
% endif
        -kernel vmlinuz \
        -initrd initrd-preseeded.gz \
        -append 'root=/dev/${hd_name} debconf_priority=critical console=${prj.text("buildimage/console")} DEBIAN_FRONTEND=newt' \
        -no-reboot \
        -net nic,vlan=1,model=${prj.text("buildimage/NIC/model")},macaddr="${prj.text('buildimage/NIC/MAC')}" \
        -net user,vlan=1 \
        -m $(MEMSIZE) \
        -usb && reset
..
\end{lstlisting}
\end{itemize}
\end{frame}

\subsection{debian-installer in emulator}

\begin{frame}
\frametitle{an ordinary debian installation}
runs inside the virtual machine:
\begin{itemize}
\item the buildenv.img will be partitionated and formated
\item the root partition will be mounted at /target of the debian-installer rfs
\item a minimal rootfilesystem will be installed to /target
\item a chroot /target will be executed
\item all packages will be installed and configured
\item the /target chroot will be left
\item the hook d-i preseed/late\_command string /post-inst.sh will be executed
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{post-inst.sh}
\begin{description}
\item [cp-scipts-into-buildenv.sh] copy all (generated) scripts

from the initrd (/)

to the buildenv.img (/target/opt/elbe)

\item [changeroot-into-buildenv.sh] chroot into /target again,

trigger target (/opt/elbe/create-target-rfs.sh)

and cdrom (/opt/elbe/mkcdrom.sh) generation.
\end{description}
\end{frame}

\begin{frame}
\frametitle{target generation}
\begin{itemize}
\item generate file and pkg lists
\item create /target tree
\item call dump.py to
\begin{itemize}
    \item validate packages / write logfile
    \item add fullpkgs list to source.xml
    \item add content of /etc/apt/sources.list to source.xml
    \item add content of /etc/apt/preferences to source.xml
    \item modify /target tree with finetuning.sh
\end{itemize}
\item generate licence information file
\item call part-target.sh to copy /target tree to (different) images
\item (optional) generate cpio images and tarballs
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{create /target tree}
\begin{itemize}
\item depending on the elbe mode, a filelist is generated. e.g. in normal mode:
\begin{lstlisting}
ls -A1 / | \
    grep -v target | grep -v proc | grep -v sys | \
    xargs find >> /opt/elbe/filelist
\end{lstlisting}
\item then all files specified to be stored on the target rfs are copied:
\begin{lstlisting}
rsync -a --files-from=/opt/elbe/filelist / /target
\end{lstlisting}
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{dump.py - check if all req. packages are installed}
\begin{itemize}
\item add lines about missing packages to validation.txt:
\begin{lstlisting}
    for name in [p.et.text for p in pkgs] + [kinitrd]:

        if not name in cache:
            elog.printo( "- package %s does not exist" % name )
            errors += 1
            continue

        cp = cache[name]

        if not cp.installed and name != kinitrd:
            elog.printo( "- package %s is not installed" % name )
            errors += 1
            continue
\end{lstlisting}
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{dump.py - check version of packages used}
(only done if a fullpkgs tag was specified in the xml file)
\begin{itemize}
\item add lines about packages with wrong version to validation.txt:
\begin{lstlisting}
    for p in fullpkgs:
        name = p.et.text
        ver  = p.et.get('version')
        md5  = p.et.get('md5')
        pindex[name] = p
        ...
        if cpi.version != ver:
            elog.printo("pack %s v%s != installed v%s" % (name, ver,  cpi.version))
            errors += 1
            continue

        if cpi.md5 != md5:
            elog.printo("pack %s md5 %s != installed md5 %s" % (name, md5,  cpi.md5) )
            errors += 1
        ...
\end{lstlisting}
\item there is also a check for packages that are installed but wasn't specified to be installed
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{dump.py - write fullpkglist to source.xml}
\begin{itemize}
\item all used packages are mentioned in this list (including version and md5sum):
\begin{lstlisting}
    paktree = xml.ensure_child( 'fullpkgs' )
    paktree.clear()
    pkgs = []
    for p in cache:
        if p.is_installed:
            pkgs.append( (p.name, p.installed.version, p.is_auto_installed, p.installed.md5))
    ...
    for p in pkgs:
        append_pkg_elem( paktree, p[0], p[1], p[2], p[3] )
\end{lstlisting}
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{dump.py - modify /target tree with finetuning.sh}
\begin{itemize}
\item the script is executed while cwd is /target.

    So files can be copied

    from the debian rfs to the target rfs

    by e.g. 'cp /etc/passwd etc/'

\item the output of the finetuning.sh commands is appended to elbe-report.txt
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{dump.py - filelist generation}
elbe-report.txt includes a filelist that indicates each file in the target rfs
and its origin
\begin{itemize}
\item a time stamp is recorded before and after finetuning. Files that show up
    with a modification time in that period show up as modified/created by
    finetuning.sh
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{generate licence information file}
all licence informations that are stored in /usr/share/* are collected in one file:
\begin{lstlisting}
find /usr/share/doc -name copyright -exec \
    /opt/elbe/print_licence.sh {} \; >> /opt/elbe/licence.txt
\end{lstlisting}
\end{frame}


\begin{frame}[fragile]
\frametitle{call part-target.sh to copy /target tree to (different) images}
currently only UBI image support is functional.

HD images are under development. They will be created by using python bindings
of libparted and loopback mounts.
\end{frame}


\begin{frame}[fragile]
\frametitle{build ubifs images}
\begin{itemize}
\item mkfs.ubifs -r is used to generate a ubifs image from a subdirectory
\item the name of the images is defined by the label in the fstab tag.
\item the image defined first in the fstab is created first
\item the files copied to an image are moved temporarily away from /target:
\begin{lstlisting}
    /mountA/mountB # mountA shouldn't include files like mountB/test
\end{lstlisting}
\item /target is restored as all images were created
\end{itemize}
\end{frame}


\begin{frame}[fragile]
\frametitle{(optional) generate cpio images and tarballs}
\begin{lstlisting}
% if xml.has("target/package/tar"):
tar cf /opt/elbe/target.tar -C /target .
% endif

% if xml.has("target/package/cpio"):
cd /target
find . -print | cpio -ov -H newc >/opt/elbe/target.cpio
% endif
\end{lstlisting}
\end{frame}


\begin{frame}
\frametitle{cdrom generation}
\end{frame}


\subsection{Makefile (post-install)}

\begin{frame}
\frametitle{copy files to host}
\end{frame}


\input{tailpres}