diff options
Diffstat (limited to 'distribution')
| -rw-r--r-- | distribution/elbe-devel/pres_elbe-devel_en.tex | 596 |
1 files changed, 593 insertions, 3 deletions
diff --git a/distribution/elbe-devel/pres_elbe-devel_en.tex b/distribution/elbe-devel/pres_elbe-devel_en.tex index ecfdf34..c145d9d 100644 --- a/distribution/elbe-devel/pres_elbe-devel_en.tex +++ b/distribution/elbe-devel/pres_elbe-devel_en.tex @@ -1,12 +1,602 @@ \input{configpres} -\title{ELBE - Advanced usage and internals} +\lstset{language=bash} + +\title{ELBE internals} \maketitle -\subsection{ELBE internals} + +\subsection{source repository} + +\begin{frame}[fragile] +\frametitle{source tree} + +ELBE development is hosted on github: +\begin{lstlisting} +https://github.com/Linutronix/elbe +\end{lstlisting} + +Use git to retrieve the source: +\begin{lstlisting} +git clone git://github.com/Linutronix/elbe.git +\end{lstlisting} + +\begin{lstlisting} +. +|-- debian/ # debian packaging +|-- docs/ # man pages, dia files, overview +|-- elbe # python script calling several scripts from ./elbepack +|-- elbepack/ # implementation of elbe subcommands like 'create' +| |-- mako/ # script and config file templates for the buildenv +|-- examples/ # example XML files +|-- Makefile # calls dpkg-buildpackage to generate a debian package +|-- README # short introduction to ELBE ++-- setup.py # python typical setup script +\end{lstlisting} + +\end{frame} + +\subsection{elbepack/create.py} \begin{frame} -\frametitle{How ELBE works} +\frametitle{create an elbe project} +\begin{itemize} +\item the 'elbe create' command is implemented + +at elbepack/create.py +\item create.py has one mandotary parameter: + +path and name of an elbe xml file +\item create.py accepts optional parameters + +that are parsed by OptionParser +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{xml validation} +\begin{itemize} +\item verifies given xml file against elbepack/dbsfed.xsd +\item we try to do as much checks as posible with the schema +\item normaly there are no further checks inside the python code +\item validation can be skipped by --skip-validation parameter +\item elbepack/validate.py is used to verify the schema +\item elbepack/validate.py can be also called directly: + +'elbe validate examples/amd64-example.xml' +\item elbepack/validate.py uses etree from lxml + +to verify the schema +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{create build directory} +\begin{itemize} +\item creates a directory named 'build' + +at the current working directory +\item name can be overridden by --directory parameter +\item the buildenvironment will be created inside this directory +\end{itemize} \end{frame} +\begin{frame} +\frametitle{the debian-installer (d-i)} +\begin{itemize} +\item the d-i needs a running linux system +\item the debian project distributes the d-i in different packages: + e.g. as a bootable cd or for use with pxeboot +\item all packages include at least +\begin{itemize} +\item one Linux kernel binary +\item and an initial ramdisk with a minimal rfs and the d-i application +\end{itemize} +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{d-i common use} +\begin{itemize} +\item d-i is started on the system debian should be installed on +\item d-i does +\begin{itemize} +\item disk detection / partitioning +\item sw installation +\item system configuration +\end{itemize} +\end{itemize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{elbe kinitrd packages} +\begin{itemize} +\item the elbe project has packaged the d-i used by elbe as binary debian packages +\begin{lstlisting} +testrd_0.6_armel +|- opt +| +-elbe +| +-initrd +| +-initrd-cdrom.gz +| +-initrd.gz +| +-vmlinuz +\end{lstlisting} +\item the name of the debian package is specified by the 'kinitrd' tag inside the + xml file +\item current packages are called testrd and are available at the debian +repository: +\begin{lstlisting} +deb http://debian.linutronix.de/elbe squeeze main +\end{lstlisting} +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{retrieve kinitrd} +\begin{itemize} +\item elbepack/pkgutils.py:copy\_kinitrd() + +is called by elbepack/create.py +\item the pkgutils module uses python virtapt + +to retrieve the kinitrd package + +from a mirror or cdrom specified in the xml file +\item virtapt takes care on using + +the correct debian architecture and suite +\item the kinitrd package will be extracted + +and copied to the build directory +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{mako} +Mako is a template library written in Python. + +It can be used to generate files from templates. + +It is normally used to generate webpages. + +Documentation and more informations can be found on +\begin{lstlisting} +http://www.makotemplates.org/ +\end{lstlisting} +\end{frame} + +\begin{frame}[fragile] +\frametitle{create Makefile, scripts and config files} +ELBE uses +\begin{lstlisting} +d = {"opt": opt, + "xml": xml, + "prj": xml.node("/project"), + "tgt": xml.node("/target"), + "pkgs": xml.node("/target/pkg-list"), + "fine": xml.node("/finetuning"), + "preseed": get_preseed(xml) } +... +mako.Template(filename=fname).render(**d) +\end{lstlisting} +to insert several values from the XML file into scripts and configfiles. +\begin{description} +\item[fname] the function is called for each *.mako file in elbepack/mako +\item[d] refers to several lxml.etree XML nodes +\item[xml] is a etree initialized with the ELBE XML file +\end{description} +\end{frame} + +\begin{frame}[fragile] +\frametitle{create Makefile, scripts and config files} +For example a simple txt file including all package names defined in the +$\langle$pkg-list$\rangle$ tag can be created by (pkg-list.mako): +\begin{lstlisting} +% for n in pkgs: +% if n.tag == "pkg": +${n.et.text} +% endif +% endfor +\end{lstlisting} +\end{frame} + + +\begin{frame} +\frametitle{preseeding} +Preseeding provides a way to set answers to questions asked during the +installation process, without having to manually enter the answers while +the installation is running. +\end{frame} + +\begin{frame} +\frametitle{preseeding} +Many pre-seeding rules are generated + +from the XML file with mako: + +./elbepack/mako/common.cfg.mako +\end{frame} + +\begin{frame}[fragile] +\frametitle{preseeding} +Project specific preseeding rules can be added to the xml file. +These rules are then included by ./elbepack/mako/custom-preseed.cfg.mako +\end{frame} + + +\subsection{Makefile (pre-install)} + +\begin{frame} +\frametitle{initrd.gz modification} +\begin{itemize} +\item a directory tmp-tree is created +\item all files and scripts needed inside the installer + +are copied to the tmp-tree +\item ungzip the original initrd.gz +\item use cpio to append all files of tmp-tree + +to the unzipped initrd image +\item gzip the new initrd image and remove the tmp-tree +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{create buildenv.img} +\begin{itemize} +\item qemu-img is used to create a buildenv.img file. This file will be used as +harddisk of the emulated virtual machine. +\item The debian-installer will install + +a full featured debian on this image. +\item After the debian-installer was active, + +this image can be used as a virtual build-environment. +\item The target rootfilesystem is normally + +just a subset of this image. +\end{itemize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{start emulator} +\begin{itemize} +\item the cmdline to start the emulator depends on several specifications made +in the XML file: +\begin{lstlisting} +.. + ${prj.text("buildimage/interpreter")} \ + -M ${prj.text("buildimage/machine")} \ + -drive file=buildenv.img,if=${hd_type},bus=1,unit=0 \ +% if prj.has("mirror/cdrom"): + -drive file=${prj.text("mirror/cdrom")},if=${cdrom_type},media=cdrom,bus=1,unit=1 \ +% endif + -kernel vmlinuz \ + -initrd initrd-preseeded.gz \ + -append 'root=/dev/${hd_name} debconf_priority=critical console=${prj.text("buildimage/console")} DEBIAN_FRONTEND=newt' \ + -no-reboot \ + -net nic,vlan=1,model=${prj.text("buildimage/NIC/model")},macaddr="${prj.text('buildimage/NIC/MAC')}" \ + -net user,vlan=1 \ + -m $(MEMSIZE) \ + -usb && reset +.. +\end{lstlisting} +\end{itemize} +\end{frame} + +\subsection{debian-installer in emulator} + +\begin{frame} +\frametitle{an ordinary debian installation} +runs inside the virtual machine: +\begin{itemize} +\item the buildenv.img will be partitionated and formated +\item the root partition will be mounted at + +/target inside the debian-installer rfs +\item a minimal rootfilesystem will be installed to /target +\item a chroot /target will be executed +\item all packages will be installed and configured +\item the /target chroot will be left +\item the preseed hook + +d-i preseed/late\_command string /post-inst.sh + +will be executed +\end{itemize} +\end{frame} + + +\begin{frame} +\frametitle{post-inst.sh} +\begin{description} +\item [cp-scipts-into-buildenv.sh] copy all (generated) scripts + +from the initrd (/) + +to the buildenv.img (/target/opt/elbe) + +\item [changeroot-into-buildenv.sh] chroot into /target again, + +trigger target (/opt/elbe/create-target-rfs.sh) + +and cdrom (/opt/elbe/mkcdrom.sh) generation. +\end{description} +\end{frame} + +\begin{frame} +\frametitle{target generation (overview)} +\begin{itemize} +\item generate file and pkg lists +\item create /target tree +\item call dump.py to +\begin{itemize} + \item validate packages / write logfile + \item add fullpkgs list to source.xml + \item add content of /etc/apt/sources.list to source.xml + \item add content of /etc/apt/preferences to source.xml + \item modify /target tree with finetuning.sh +\end{itemize} +\item generate licence information file +\item call part-target.sh to generate images from /target tree +\item (optional) generate cpio images and tarballs +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{create /target tree} +\begin{itemize} +\item depending on the ELBE mode, the filelist is generated. + +E.g. in normal mode: +\begin{lstlisting} +ls -A1 / | \ + grep -v target | grep -v proc | grep -v sys | \ + xargs find >> /opt/elbe/filelist +\end{lstlisting} +\item all files specified to be stored on the target rfs are copied: +\begin{lstlisting} +rsync -a --files-from=/opt/elbe/filelist / /target +\end{lstlisting} +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{dump.py - check if all req. packages are installed} +\begin{itemize} +\item add lines about missing packages to validation.txt: +\begin{lstlisting} + for name in [p.et.text for p in pkgs] + [kinitrd]: + + if not name in cache: + elog.printo( "- package %s does not exist" % name ) + errors += 1 + continue + + cp = cache[name] + + if not cp.installed and name != kinitrd: + elog.printo( "- package %s is not installed" % name ) + errors += 1 + continue +\end{lstlisting} +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{dump.py - check version of packages used} +(only done if a fullpkgs tag was specified in the xml file) +\begin{itemize} +\item add lines about packages with wrong version to validation.txt: +\begin{lstlisting} + for p in fullpkgs: + name = p.et.text + ver = p.et.get('version') + md5 = p.et.get('md5') + pindex[name] = p + ... + if cpi.version != ver: + elog.printo("pack %s v%s != installed v%s" % (name, ver, cpi.version)) + errors += 1 + continue + + if cpi.md5 != md5: + elog.printo("pack %s md5 %s != installed md5 %s" % (name, md5, cpi.md5) ) + errors += 1 + ... +\end{lstlisting} +\item there is also a check for packages that are installed but wasn't specified to be installed +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{dump.py - write fullpkglist to source.xml} +\begin{itemize} +\item all used packages are listed in the source.xml file + +(including version and md5sum): +\begin{lstlisting} + paktree = xml.ensure_child( 'fullpkgs' ) + paktree.clear() + pkgs = [] + for p in cache: + if p.is_installed: + pkgs.append( (p.name, p.installed.version, p.is_auto_installed, p.installed.md5)) + ... + for p in pkgs: + append_pkg_elem( paktree, p[0], p[1], p[2], p[3] ) +\end{lstlisting} +\end{itemize} +\end{frame} + + +\begin{frame} +\frametitle{dump.py - modify /target tree with finetuning.sh} +\begin{itemize} +\item the script is executed from /target. + + So files can be copied + + from the debian rfs to the target rfs + + by e.g. 'cp /etc/passwd etc/' + +\item the output of the finetuning.sh commands is appended to elbe-report.txt +\end{itemize} +\end{frame} + + +\begin{frame} +\frametitle{dump.py - filelist generation} +elbe-report.txt includes a filelist that indicates each file in the target rfs +and its origin +\begin{itemize} +\item a time stamp is recorded before and after finetuning. +\item files that show up with a modification time in that period + +can be identified as modified/created by finetuning.sh +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{generate licence information file} +all licence informations that are stored in /usr/share/* are collected in one file: +\begin{lstlisting} +find /usr/share/doc -name copyright -exec \ + /opt/elbe/print_licence.sh {} \; >> /opt/elbe/licence.txt +\end{lstlisting} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{call part-target.sh to copy /target tree to (different) images} +currently only UBI image support is functional. + +HD images are under development. They will be created by using python bindings +of libparted and loopback mounts. +\end{frame} + + +\begin{frame}[fragile] +\frametitle{build ubifs images} +\begin{itemize} +\item mkfs.ubifs -r is used to generate a ubifs image + +from a subdirectory +\item the name of the images is defined + +by the label in the fstab tag +\item the image defined first in the fstab is created first +\item the files copied to an image + +are moved temporarily away from /target: +\begin{lstlisting} + /mountA/mountB # mountA shouldn't include files like mountB/test +\end{lstlisting} +\item /target is restored as all images were created +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{(optional) generate cpio images and tarballs} +\begin{lstlisting} +% if xml.has("target/package/tar"): +tar cf /opt/elbe/target.tar -C /target . +% endif + +% if xml.has("target/package/cpio"): +cd /target +find . -print | cpio -ov -H newc >/opt/elbe/target.cpio +% endif +\end{lstlisting} +\end{frame} + + +\begin{frame} +\frametitle{binary cdrom generation} +An ISO CD image will be created: +\begin{itemize} +\item CD should contain all packages + +needed to reproduce the current ELBE run +\item the kinitrd package is installed into the buildenv + +to get it added to the cdrom like all other packages +\item CD needs to be in a debian repository format + +otherwise it won't be accepted by d-i +\item reprepo and genisoimage is used to create the CD +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{source cdrom generation} +upon request (by 'elbe create --build-source') + +an ISO image containing all debian source packages is created: +\begin{lstlisting} +% if opt.buildsources: + mkdir -p /opt/elbe/source + dpkg --get-selections | awk '{print $1}' > /opt/elbe/pkg-list.actual + cd /opt/elbe/source + awk '{print "apt-get -d source "$1}' /opt/elbe/pkg-list.actual | sh + cd /opt/elbe + dpkg-scansources source /dev/null | gzip -9c > source/Sources.gz + genisoimage -o /opt/elbe/source.iso -J -R source +% endif +\end{lstlisting} +\end{frame} + + +\begin{frame} +\frametitle{kinitrd package details} +\begin{itemize} +\item in xml the cdrom tag can be used instead of a mirror +\item then d-i needs to load components (udeb) from cdrom instead of web +\item this is the reason for kinitrd including two versions of the initrd: +\begin{itemize} +\item one with the d-i configured to load udebs from a web source +\item a second with d-i configured to load udebs from a cdrom source +\end{itemize} +\item there is a switch in the 'elbe create' path to select the right initrd for + the project +\end{itemize} +\end{frame} + + +\subsection{Makefile (post-install)} + +\begin{frame}[fragile] +\frametitle{copy files to host} +\begin{itemize} +\item all files produced by elbe are stored inside buildenv.img:/opt/elbe +\item they should be transfered into the build dir on the Host PC +\item the emulator is shutdown, + +then e2cp can be used to retrive the files: +\begin{lstlisting} +e2cp buildenv.img?offset=32768:/opt/elbe/licence.txt . +\end{lstlisting} +\item this is done for all files by the Makefile, e.g: +\begin{lstlisting} +% if tgt.has("images"): +% for mtd in tgt.node("images"): +% if mtd.has("partitions"): + e2cp buildenv.img?offset=${loop_offset}:/opt/elbe/${mtd.text("name")} . +% endif +% if mtd.has("binary"): + e2cp buildenv.img?offset=${loop_offset}:${mtd.text("binary")} ${mtd.text("name")} +\end{lstlisting} +\end{itemize} +\end{frame} + + \input{tailpres} |