From f23a0f8295316d6cf556c6c3830e5fe3d9d0a5f4 Mon Sep 17 00:00:00 2001 From: Jan Altenberg Date: Fri, 25 Jan 2013 11:26:17 +0100 Subject: Skeleton for ELBE --- distribution/Makefile | 6 ++++++ distribution/elbe-devel/Makefile | 9 +++++++++ distribution/elbe-devel/pres_elbe-devel_en.tex | 12 ++++++++++++ distribution/elbe-usage/Makefile | 9 +++++++++ distribution/elbe-usage/pres_elbe-usage_en.tex | 12 ++++++++++++ distribution/section.tex | 1 + 6 files changed, 49 insertions(+) create mode 100644 distribution/Makefile create mode 100644 distribution/elbe-devel/Makefile create mode 100644 distribution/elbe-devel/pres_elbe-devel_en.tex create mode 100644 distribution/elbe-usage/Makefile create mode 100644 distribution/elbe-usage/pres_elbe-usage_en.tex create mode 100644 distribution/section.tex (limited to 'distribution') diff --git a/distribution/Makefile b/distribution/Makefile new file mode 100644 index 0000000..0a202ea --- /dev/null +++ b/distribution/Makefile @@ -0,0 +1,6 @@ +SUBDIRS = `ls -1 | grep -v *.tex | grep -v Makefile | grep -v TODO` + +all clean:: + for dir in $(SUBDIRS) ; do \ + (cd $$dir && make $@); \ + done diff --git a/distribution/elbe-devel/Makefile b/distribution/elbe-devel/Makefile new file mode 100644 index 0000000..d641258 --- /dev/null +++ b/distribution/elbe-devel/Makefile @@ -0,0 +1,9 @@ +all: + for pdf in `ls -1 *.tex` ; do \ + TEXINPUTS=`pwd`/../..:.:..:$(TEXINPUTS) pdflatex $$pdf; \ + TEXINPUTS=`pwd`/../..:.:..:$(TEXINPUTS) pdflatex $$pdf; \ + done + +clean: + rm -f *.aux *.log *.pdf *.log *.snm *.toc *.vrb *.nav *.out + diff --git a/distribution/elbe-devel/pres_elbe-devel_en.tex b/distribution/elbe-devel/pres_elbe-devel_en.tex new file mode 100644 index 0000000..ecfdf34 --- /dev/null +++ b/distribution/elbe-devel/pres_elbe-devel_en.tex @@ -0,0 +1,12 @@ +\input{configpres} + +\title{ELBE - Advanced usage and internals} +\maketitle + +\subsection{ELBE internals} + +\begin{frame} +\frametitle{How ELBE works} +\end{frame} + +\input{tailpres} diff --git a/distribution/elbe-usage/Makefile b/distribution/elbe-usage/Makefile new file mode 100644 index 0000000..d641258 --- /dev/null +++ b/distribution/elbe-usage/Makefile @@ -0,0 +1,9 @@ +all: + for pdf in `ls -1 *.tex` ; do \ + TEXINPUTS=`pwd`/../..:.:..:$(TEXINPUTS) pdflatex $$pdf; \ + TEXINPUTS=`pwd`/../..:.:..:$(TEXINPUTS) pdflatex $$pdf; \ + done + +clean: + rm -f *.aux *.log *.pdf *.log *.snm *.toc *.vrb *.nav *.out + diff --git a/distribution/elbe-usage/pres_elbe-usage_en.tex b/distribution/elbe-usage/pres_elbe-usage_en.tex new file mode 100644 index 0000000..01ffda8 --- /dev/null +++ b/distribution/elbe-usage/pres_elbe-usage_en.tex @@ -0,0 +1,12 @@ +\input{configpres} + +\title{ELBE - Embedded Linux Build Environment} +\maketitle + +\subsection{ELBE usage} + +\begin{frame} +\frametitle{What is ELBE?} +\end{frame} + +\input{tailpres} diff --git a/distribution/section.tex b/distribution/section.tex new file mode 100644 index 0000000..41fa146 --- /dev/null +++ b/distribution/section.tex @@ -0,0 +1 @@ +\section{ELBE} -- cgit v1.2.3 From 5943fbf9f9b0ed3aaf4c4a1a008320d5a170be5e Mon Sep 17 00:00:00 2001 From: Jan Altenberg Date: Fri, 25 Jan 2013 17:35:35 +0100 Subject: Initial version of ELBE usage slides --- distribution/elbe-usage/pres_elbe-usage_en.tex | 423 +++++++++++++++++++++++++ images/elbe-process.png | Bin 0 -> 37051 bytes images/emu-process.png | Bin 0 -> 11200 bytes 3 files changed, 423 insertions(+) create mode 100644 images/elbe-process.png create mode 100644 images/emu-process.png (limited to 'distribution') diff --git a/distribution/elbe-usage/pres_elbe-usage_en.tex b/distribution/elbe-usage/pres_elbe-usage_en.tex index 01ffda8..be63d54 100644 --- a/distribution/elbe-usage/pres_elbe-usage_en.tex +++ b/distribution/elbe-usage/pres_elbe-usage_en.tex @@ -7,6 +7,429 @@ \begin{frame} \frametitle{What is ELBE?} +ELBE is a Debian based system to generate root-filesystems for embedded devices \end{frame} +\begin{frame}[fragile] +\frametitle{How to install ELBE} +\begin{verbatim} +# On a Debian based system +$ echo 'deb http://debian.linutronix.de/elbe \ +squeeze main' >> /etc/apt/sources.list +$ aptitude install elbe +\end{verbatim} +Or get the latest source from github: +\begin{verbatim} +git clone https://github.com/Linutronix/elbe.git +\end{verbatim} +\end{frame} + +\begin{frame} +\frametitle{ELBE: Overview} +\begin{itemize} +\item The Elbe system consists of the program elbe which is implemented in Python +\item Similar to git, it can be called with several sub commands +\item An Elbe project consists of an xml file that describes the project +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{ELBE: Documentation} +\begin{itemize} +\item Have a look at /usr/share/doc/elbe/ +\item Please read /usr/share/doc/elbe/elbeoverview-en.html +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{How ELBE works} +\begin{figure}[h] +\centering +\includegraphics[width=10cm]{images/elbe-process.png} +\end{figure} +\end{frame} + +\begin{frame} +\frametitle{How ELBE works} +\begin{figure}[h] +\centering +\includegraphics[width=10cm]{images/emu-process.png} +\end{figure} +\end{frame} + +\begin{frame} +\frametitle{The ELBE workflow} +\begin{enumerate} +\item Create an XML file +\item Create a project based on that XML file +\item Change to the project directory an run make +\end{enumerate} +\end{frame} + +\begin{frame}[fragile] +\frametitle{ELBE: command overview} +Create a new project: +\begin{verbatim} +elbe create \ + --directory /home/user/example example.xml +\end{verbatim} +\end{frame} + +\begin{frame}[fragile] +\frametitle{ELBE: start build} +\begin{verbatim} +cd /home/user/example +# Note: the next step might take a while +make +\end{verbatim} +\end{frame} + +\begin{frame}[fragile] +\frametitle{ELBE: Using the VM} +\begin{verbatim} +cd /home/user/example +make run +# or (without graphics) +make run-con +\end{verbatim} +\end{frame} + +\begin{frame}[fragile] +\frametitle{The ELBE XML format} +/usr/share/doc/elbe/examples/ contains a couple of examples: +\begin{verbatim} +$ ls -1 /usr/share/doc/elbe/examples/ +amd64-example.xml +arm-complex-example.xml.gz +arm-example.xml +i386-example.xml +\end{verbatim} +\end{frame} + +\begin{frame}[fragile] +\frametitle{The ELBE XML format: Project setup} +\lstset{language=XML} +\begin{scriptsize} +\begin{lstlisting} + + ARMexample + 08.15 +... + +\end{lstlisting} +\end{scriptsize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{The ELBE XML format: Project setup} +\lstset{language=XML} +\begin{scriptsize} +\begin{lstlisting} + +... + + debian.tu-bs.de + /debian + http + + + http://debian.linutronix.de/elbe squeeze main + + + +... + +\end{lstlisting} +\end{scriptsize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{The ELBE XML format: Project setup} +\lstset{language=XML} +\begin{scriptsize} +\begin{lstlisting} + +... + + squeeze +... + +\end{lstlisting} +\end{scriptsize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{The ELBE XML format: Project setup} +\lstset{language=XML} +\begin{scriptsize} +\begin{lstlisting} + +... + + armel + 20G + 256 + qemu-system-arm + testrd + ttyAMA0,115200n1 + versatilepb + + smc91c111 + de:ad:be:ef:be:05 + + + + tcp + 22 + 5022 + + + + + +... + +\end{lstlisting} +\end{scriptsize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{The ELBE XML format: Defining a target} +\lstset{language=XML} +\begin{scriptsize} +\begin{lstlisting} + + myARM + tec.linutronix.de + foo + ttyS0,115200 + + + nfsroot.tar.gz + + + + + + build-essential + bash + less + git + debhelper + nfs-common + openssh-server + + +\end{lstlisting} +\end{scriptsize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{The ELBE XML format: finetuning} +\lstset{language=XML} +\begin{scriptsize} +\begin{lstlisting} + +... + + /usr/share/doc + /var_ro + /my/name/on/target + +... + +\end{lstlisting} +\end{scriptsize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{The ELBE XML format: finetuning} +\lstset{language=XML} +\begin{scriptsize} +\begin{lstlisting} + +... + + cp /etc/passwd etc/ + +... + +\end{lstlisting} +\end{scriptsize} +\end{frame} + +\begin{frame} +\frametitle{ELBE modes} +\begin{itemize} +\item default +\item norecommend (can be combined with any of the other modes): Don't install recommended packages +\item setsel: The resulting image will just contain dpkg. This mode offers a finegrained control on which packages should be installed +\item diet: Only copy files referenced in the package management. Only use this for small and simple root filesystems! +\end{itemize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{ELBE modes: Example} +\lstset{language=XML} +\begin{scriptsize} +\begin{lstlisting} + +... + + + + + bash + +... + +\end{lstlisting} +\end{scriptsize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{ELBE command overview: elbe-setsel} +\begin{enumerate} +\item Create package list: +\begin{verbatim} +dpkg --get-selections > selections.list +\end{verbatim} +\item Import the package list to your xml file: +\begin{verbatim} +elbe setsel selections.list +\end{verbatim} +\end{enumerate} +\end{frame} + +\begin{frame}[fragile] +\frametitle{Using a CDROM} +\begin{itemize} +\item Each ELBE build will create a CD-ROM image with the package (which have been used for the installation): install.iso +\item You can use this ISO image for future builds: +\begin{verbatim} + +/home/user/elbe-exmpl/install.iso + +\end{verbatim} +\item You can also use the elbe-setcdrom command +\end{itemize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{ELBE command overview: elbe-setcdrom} +\begin{verbatim} +$ elbe setcdrom mysqueeze.xml elbe-cd.iso +\end{verbatim} +This command removes all configured mirrors and additional urls from the XML file and sets the given ISO image as the only mirror. +\end{frame} + +\begin{frame}[fragile] +\frametitle{ELBE command overview: elbe-chg\_archive} +It is possible to include an archive into the xml file, that is unpacked into the target root-filesystem after the finetuning step: +\begin{verbatim} +elbe chg_archive archive.tar.bz2 +\end{verbatim} +NOTE: The file must be a bzip2 compressed tar (.tar.bz2) +\end{frame} + +\begin{frame}[fragile] +\frametitle{ELBE command overview: elbe-get\_archive} +It is also possible to extract an archive from an XML file: +\begin{verbatim} +elbe get_archive +\end{verbatim} +\end{frame} + +\begin{frame}[fragile] +\frametitle{ELBE command overview: elbe-show} +The elbe show command prints out some textual information about an ELBE xml file: +\begin{verbatim} +$ elbe show arm-example.xml +== arm-example.xml: ARMexample - version 08.15 == +Debian suite: squeeze/armel +[...] +\end{verbatim} +\end{frame} + +\begin{frame}[fragile] +\frametitle{ELBE command overview: elbe-xsdtoasciidoc} +Create an asciidoc documentation from an annotated XML Schema file +\begin{verbatim} +# Create asciidoc format +$ elbe xsdtoasciidoc --output=myasciidoc dbsfed.xsd +# Create HTML doc +$ asciidoc myasciidoc +\end{verbatim} +\end{frame} + +\begin{frame}[fragile] +\frametitle{Creating UBI images} +\lstset{language=XML} +\begin{scriptsize} +\begin{lstlisting} + + + linux.img 1 + 0 + 60MB + + + 2048 + 1533 + 126976 + 128KiB + + static + + 0 + 4MiB + /boot/vmlinuz + + + dynamic + + 1 + 26MiB + + + +\end{lstlisting} +\end{scriptsize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{Creating UBI images} +\lstset{language=XML} +\begin{scriptsize} +\begin{lstlisting} + + + + /opt + + ubifs + -x lzo + + rw + + + + / + + ubifs + -x lzo + + ro + + + proc + /proc + + proc + + + +\end{lstlisting} +\end{scriptsize} +\end{frame} \input{tailpres} diff --git a/images/elbe-process.png b/images/elbe-process.png new file mode 100644 index 0000000..3dedbee Binary files /dev/null and b/images/elbe-process.png differ diff --git a/images/emu-process.png b/images/emu-process.png new file mode 100644 index 0000000..f1b1107 Binary files /dev/null and b/images/emu-process.png differ -- cgit v1.2.3 From ad59b25e403b9dd9fbee29f44fd01f8655fa6479 Mon Sep 17 00:00:00 2001 From: Manuel Traut Date: Fri, 25 Jan 2013 17:39:46 +0100 Subject: fist work on elbe internals Signed-off-by: Manuel Traut --- distribution/elbe-devel/pres_elbe-devel_en.tex | 202 ++++++++++++++++++++++++- 1 file changed, 199 insertions(+), 3 deletions(-) (limited to 'distribution') diff --git a/distribution/elbe-devel/pres_elbe-devel_en.tex b/distribution/elbe-devel/pres_elbe-devel_en.tex index ecfdf34..44e29bb 100644 --- a/distribution/elbe-devel/pres_elbe-devel_en.tex +++ b/distribution/elbe-devel/pres_elbe-devel_en.tex @@ -1,12 +1,208 @@ \input{configpres} -\title{ELBE - Advanced usage and internals} +\title{ELBE internals} \maketitle -\subsection{ELBE internals} +\subsection{source repository} + +\begin{frame}[fragile] +\frametitle{source tree} + +ELBE development is hosted on github: +\begin{lstlisting} +https://github.com/Linutronix/elbe +\end{lstlisting} + +Use git to retrieve the source: +\begin{lstlisting} +git clone git://github.com/Linutronix/elbe.git +\end{lstlisting} + +\begin{lstlisting} +. +|-- debian/ # debian packaging +|-- docs/ # man pages, dia files, overview +|-- elbe # python script calling several scripts from ./elbepack +|-- elbepack/ # implementation of elbe subcommands like 'create' +| |-- mako/ # script and config file templates for the buildenv +|-- examples/ # example XML files +|-- Makefile # calls dpkg-buildpackage to generate a debian package +|-- README # short introduction to ELBE ++-- setup.py # python typical setup script +\end{lstlisting} + +\end{frame} + +\subsection{elbepack/create.py} + +\begin{frame} +\frametitle{create an elbe project} +\begin{itemize} +\item the 'elbe create' command calls the elbepack/elbe.py script +\item create.py has one mandotary parameter: path and name of an elbe xml file +\item create.py accepts optional parameters that are parsed by OptionParser +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{xml validation} +\begin{itemize} +\item verifies given xml file against elbepack/dbsfed.xsd +\item we try to do as much checks as posible with the schema +\item normaly there are no further checks inside the python code +\item validation can be skipped by --skip-validation parameter +\item elbepack/validate.py is used to verify the schema +\item elbepack/validate.py can be also called directly: 'elbe validate' +\item elbepack/validate.py uses etree from lxml to verify the schema +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{create build directory} +\begin{itemize} +\item creates a directory named 'build' at the current working directory +\item name can be overridden by --directory parameter +\item the buildenvironment will be created inside this directory +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{the debian-installer} +\begin{itemize} +\item the debian-installer is a Linux kernel and a initial ramdisk. +\item the initial ramdisk includes a minimal rootfilesystem and the d-i + application +\item the debian project distributes the installer in different formats: + e.g. as a bootable cd or for use with pxeboot +\item normally the debian-installer is booted on the system debian should + be installed on +\item the debian-installer does disk detection / partitioning, sw installation + and system configuration +\end{itemize} +\end{frame} + +\begin{frame}[fragile] +\frametitle{debian-installer package} +\begin{itemize} +\item the elbe project has packaged the debian-installer as debian package +\item this was done to be able to verify the version of the installer +\item the name of the debian package is specified by the 'kinitrd' tag inside the + xml file +\item current packages are called testrd and are available on the debian +repository: +\begin{lstlisting} +deb http://debian.linutronix.de/elbe squeeze main +\end{lstlisting} +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{retrieve debian-installer package} +\begin{itemize} +\item elbepack/pkgutils.py:copy\_kinitrd() is called by create.py +\item the pkgutils module uses python virtapt to retrieve the debian package + from a mirror or cdrom specified in the xml file +\item virtapt takes care on using the correct architecture and debian suite +\item then the package contents are extracted and copied to the build + directory +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{mako} +Mako is a template library written in Python. + +It can be used to generate files from templates. + +It is normally used to generate webpages. + +Documentation and more informations can be found on +\begin{lstlisting} +http://www.makotemplates.org/ +\end{lstlisting} +\end{frame} + +\begin{frame}[fragile] +\frametitle{create Makefile, scripts and config files} +ELBE uses +\begin{lstlisting} +mako.Template(filename=fname).render(**d) +\end{lstlisting} +to insert several values from the XML file into the scripts and configfiles +used in the buildenv. It is called several times, for each *.mako file in +elbepack/mako (fname). + +d refers to several lxml.etree XML nodes. etree was initialized with the +ELBE XML file before. +\begin{lstlisting} +d = {"opt": opt, + "xml": xml, + "prj": xml.node("/project"), + "tgt": xml.node("/target"), + "pkgs": xml.node("/target/pkg-list"), + "fine": xml.node("/finetuning"), + "preseed": get_preseed(xml) } +\end{lstlisting} +\end{frame} + +\begin{frame}[fragile] +\frametitle{create Makefile, scripts and config files} +For example a simple txt file including all package names defined in the + tag can be created by (pkg-list.mako): +\begin{lstlisting} +% for n in pkgs: +% if n.tag == "pkg": +${n.et.text} +% endif +% endfor +\end{lstlisting} +\end{frame} + + +\begin{frame} +\frametitle{preseeding} +\end{frame} + +\subsection{Makefile (pre-install)} + +\begin{frame} +\frametitle{initrd.gz modification} +\end{frame} + +\begin{frame} +\frametitle{create buildenv.img} +\end{frame} + +\begin{frame} +\frametitle{start emulator} +\end{frame} + +\subsection{debian-installer in emulator} + +\begin{frame} +\frametitle{preseeding} +\end{frame} + +\begin{frame} +\frametitle{post install} +cp-scipts-into-buildenv.sh +changeroot-into-buildenv.sh +\end{frame} \begin{frame} -\frametitle{How ELBE works} +\frametitle{target generation} \end{frame} +\begin{frame} +\frametitle{cdrom generation} +\end{frame} + +\subsection{Makefile (post-install)} + +\begin{frame} +\frametitle{copy files to host} +\end{frame} + + \input{tailpres} -- cgit v1.2.3 From cbc5342614b26482a9336ed35693d0a3cd42276c Mon Sep 17 00:00:00 2001 From: Manuel Traut Date: Wed, 30 Jan 2013 15:54:51 +0100 Subject: updte elbe-devel pres Signed-off-by: Manuel Traut --- distribution/elbe-devel/pres_elbe-devel_en.tex | 266 ++++++++++++++++++++++++- 1 file changed, 261 insertions(+), 5 deletions(-) (limited to 'distribution') diff --git a/distribution/elbe-devel/pres_elbe-devel_en.tex b/distribution/elbe-devel/pres_elbe-devel_en.tex index 44e29bb..7e3977c 100644 --- a/distribution/elbe-devel/pres_elbe-devel_en.tex +++ b/distribution/elbe-devel/pres_elbe-devel_en.tex @@ -162,42 +162,298 @@ ${n.et.text} \begin{frame} \frametitle{preseeding} +Preseeding provides a way to set answers to questions asked during the +installation process, without having to manually enter the answers while +the installation is running. \end{frame} +\begin{frame} +\frametitle{preseeding} +Many pre-seeding rules are generated from the XML file with mako: +./elbepack/mako/commmon.cfg.mako +\end{frame} + +\begin{frame}[fragile] +\frametitle{preseeding} +Project specific preseeding rules can be added to the xml file. +These rules are then included by ./elbepack/mako/custom-preseed.cfg.mako +\end{frame} + + \subsection{Makefile (pre-install)} \begin{frame} \frametitle{initrd.gz modification} +\begin{itemize} +\item a directory tmp-tree is created +\item all files and scripts needed inside the installer + are copied to the tmp-tree +\item ungzip the original initrd.gz +\item use cpio to append all files of tmp-tree to the unzipped initrd image +\item gzip the new initrd image and remove the tmp-tree +\end{itemize} \end{frame} \begin{frame} \frametitle{create buildenv.img} +\begin{itemize} +\item qemu-img is used to create a buildenv.img file. This file will be used as +harddisk of the emulated virtual machine. +\item The debian-installer will install a full featured debian on this image. +\item After the debian-installer was active, this image can be used as a virtual +build-environment. +\item The target rootfilesystem is normally just a subset of this image. +\end{itemize} \end{frame} -\begin{frame} +\begin{frame}[fragile] \frametitle{start emulator} +\begin{itemize} +\item the cmdline to start the emulator depends on several specifications made +in the XML file: +\begin{lstlisting} +.. + ${prj.text("buildimage/interpreter")} \ + -M ${prj.text("buildimage/machine")} \ + -drive file=buildenv.img,if=${hd_type},bus=1,unit=0 \ +% if prj.has("mirror/cdrom"): + -drive file=${prj.text("mirror/cdrom")},if=${cdrom_type},media=cdrom,bus=1,unit=1 \ +% endif + -kernel vmlinuz \ + -initrd initrd-preseeded.gz \ + -append 'root=/dev/${hd_name} debconf_priority=critical console=${prj.text("buildimage/console")} DEBIAN_FRONTEND=newt' \ + -no-reboot \ + -net nic,vlan=1,model=${prj.text("buildimage/NIC/model")},macaddr="${prj.text('buildimage/NIC/MAC')}" \ + -net user,vlan=1 \ + -m $(MEMSIZE) \ + -usb && reset +.. +\end{lstlisting} +\end{itemize} \end{frame} \subsection{debian-installer in emulator} \begin{frame} -\frametitle{preseeding} +\frametitle{an ordinary debian installation} +runs inside the virtual machine: +\begin{itemize} +\item the buildenv.img will be partitionated and formated +\item the root partition will be mounted at /target of the debian-installer rfs +\item a minimal rootfilesystem will be installed to /target +\item a chroot /target will be executed +\item all packages will be installed and configured +\item the /target chroot will be left +\item the hook d-i preseed/late\_command string /post-inst.sh will be executed +\end{itemize} \end{frame} + \begin{frame} -\frametitle{post install} -cp-scipts-into-buildenv.sh -changeroot-into-buildenv.sh +\frametitle{post-inst.sh} +\begin{description} +\item [cp-scipts-into-buildenv.sh] copy all (generated) scripts + +from the initrd (/) + +to the buildenv.img (/target/opt/elbe) + +\item [changeroot-into-buildenv.sh] chroot into /target again, + +trigger target (/opt/elbe/create-target-rfs.sh) + +and cdrom (/opt/elbe/mkcdrom.sh) generation. +\end{description} \end{frame} \begin{frame} \frametitle{target generation} +\begin{itemize} +\item generate file and pkg lists +\item create /target tree +\item call dump.py to +\begin{itemize} + \item validate packages / write logfile + \item add fullpkgs list to source.xml + \item add content of /etc/apt/sources.list to source.xml + \item add content of /etc/apt/preferences to source.xml + \item modify /target tree with finetuning.sh +\end{itemize} +\item generate licence information file +\item call part-target.sh to copy /target tree to (different) images +\item (optional) generate cpio images and tarballs +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{create /target tree} +\begin{itemize} +\item depending on the elbe mode, a filelist is generated. e.g. in normal mode: +\begin{lstlisting} +ls -A1 / | \ + grep -v target | grep -v proc | grep -v sys | \ + xargs find >> /opt/elbe/filelist +\end{lstlisting} +\item then all files specified to be stored on the target rfs are copied: +\begin{lstlisting} +rsync -a --files-from=/opt/elbe/filelist / /target +\end{lstlisting} +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{dump.py - check if all req. packages are installed} +\begin{itemize} +\item add lines about missing packages to validation.txt: +\begin{lstlisting} + for name in [p.et.text for p in pkgs] + [kinitrd]: + + if not name in cache: + elog.printo( "- package %s does not exist" % name ) + errors += 1 + continue + + cp = cache[name] + + if not cp.installed and name != kinitrd: + elog.printo( "- package %s is not installed" % name ) + errors += 1 + continue +\end{lstlisting} +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{dump.py - check version of packages used} +(only done if a fullpkgs tag was specified in the xml file) +\begin{itemize} +\item add lines about packages with wrong version to validation.txt: +\begin{lstlisting} + for p in fullpkgs: + name = p.et.text + ver = p.et.get('version') + md5 = p.et.get('md5') + pindex[name] = p + ... + if cpi.version != ver: + elog.printo("pack %s v%s != installed v%s" % (name, ver, cpi.version)) + errors += 1 + continue + + if cpi.md5 != md5: + elog.printo("pack %s md5 %s != installed md5 %s" % (name, md5, cpi.md5) ) + errors += 1 + ... +\end{lstlisting} +\item there is also a check for packages that are installed but wasn't specified to be installed +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{dump.py - write fullpkglist to source.xml} +\begin{itemize} +\item all used packages are mentioned in this list (including version and md5sum): +\begin{lstlisting} + paktree = xml.ensure_child( 'fullpkgs' ) + paktree.clear() + pkgs = [] + for p in cache: + if p.is_installed: + pkgs.append( (p.name, p.installed.version, p.is_auto_installed, p.installed.md5)) + ... + for p in pkgs: + append_pkg_elem( paktree, p[0], p[1], p[2], p[3] ) +\end{lstlisting} +\end{itemize} +\end{frame} + + +\begin{frame} +\frametitle{dump.py - modify /target tree with finetuning.sh} +\begin{itemize} +\item the script is executed while cwd is /target. + + So files can be copied + + from the debian rfs to the target rfs + + by e.g. 'cp /etc/passwd etc/' + +\item the output of the finetuning.sh commands is appended to elbe-report.txt +\end{itemize} +\end{frame} + + +\begin{frame} +\frametitle{dump.py - filelist generation} +elbe-report.txt includes a filelist that indicates each file in the target rfs +and its origin +\begin{itemize} +\item a time stamp is recorded before and after finetuning. Files that show up + with a modification time in that period show up as modified/created by + finetuning.sh +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{generate licence information file} +all licence informations that are stored in /usr/share/* are collected in one file: +\begin{lstlisting} +find /usr/share/doc -name copyright -exec \ + /opt/elbe/print_licence.sh {} \; >> /opt/elbe/licence.txt +\end{lstlisting} \end{frame} + +\begin{frame}[fragile] +\frametitle{call part-target.sh to copy /target tree to (different) images} +currently only UBI image support is functional. + +HD images are under development. They will be created by using python bindings +of libparted and loopback mounts. +\end{frame} + + +\begin{frame}[fragile] +\frametitle{build ubifs images} +\begin{itemize} +\item mkfs.ubifs -r is used to generate a ubifs image from a subdirectory +\item the name of the images is defined by the label in the fstab tag. +\item the image defined first in the fstab is created first +\item the files copied to an image are moved temporarily away from /target: +\begin{lstlisting} + /mountA/mountB # mountA shouldn't include files like mountB/test +\end{lstlisting} +\item /target is restored as all images were created +\end{itemize} +\end{frame} + + +\begin{frame}[fragile] +\frametitle{(optional) generate cpio images and tarballs} +\begin{lstlisting} +% if xml.has("target/package/tar"): +tar cf /opt/elbe/target.tar -C /target . +% endif + +% if xml.has("target/package/cpio"): +cd /target +find . -print | cpio -ov -H newc >/opt/elbe/target.cpio +% endif +\end{lstlisting} +\end{frame} + + \begin{frame} \frametitle{cdrom generation} \end{frame} + \subsection{Makefile (post-install)} \begin{frame} -- cgit v1.2.3 From 9f98d118df9c26e0508e452b3533b63021dd9512 Mon Sep 17 00:00:00 2001 From: Manuel Traut Date: Thu, 31 Jan 2013 16:41:14 +0100 Subject: add last frames to elbe-devel Signed-off-by: Manuel Traut --- distribution/elbe-devel/pres_elbe-devel_en.tex | 258 +++++++++++++++++++------ 1 file changed, 198 insertions(+), 60 deletions(-) (limited to 'distribution') diff --git a/distribution/elbe-devel/pres_elbe-devel_en.tex b/distribution/elbe-devel/pres_elbe-devel_en.tex index 7e3977c..c145d9d 100644 --- a/distribution/elbe-devel/pres_elbe-devel_en.tex +++ b/distribution/elbe-devel/pres_elbe-devel_en.tex @@ -1,8 +1,11 @@ \input{configpres} +\lstset{language=bash} + \title{ELBE internals} \maketitle + \subsection{source repository} \begin{frame}[fragile] @@ -38,9 +41,15 @@ git clone git://github.com/Linutronix/elbe.git \begin{frame} \frametitle{create an elbe project} \begin{itemize} -\item the 'elbe create' command calls the elbepack/elbe.py script -\item create.py has one mandotary parameter: path and name of an elbe xml file -\item create.py accepts optional parameters that are parsed by OptionParser +\item the 'elbe create' command is implemented + +at elbepack/create.py +\item create.py has one mandotary parameter: + +path and name of an elbe xml file +\item create.py accepts optional parameters + +that are parsed by OptionParser \end{itemize} \end{frame} @@ -52,43 +61,69 @@ git clone git://github.com/Linutronix/elbe.git \item normaly there are no further checks inside the python code \item validation can be skipped by --skip-validation parameter \item elbepack/validate.py is used to verify the schema -\item elbepack/validate.py can be also called directly: 'elbe validate' -\item elbepack/validate.py uses etree from lxml to verify the schema +\item elbepack/validate.py can be also called directly: + +'elbe validate examples/amd64-example.xml' +\item elbepack/validate.py uses etree from lxml + +to verify the schema \end{itemize} \end{frame} \begin{frame} \frametitle{create build directory} \begin{itemize} -\item creates a directory named 'build' at the current working directory +\item creates a directory named 'build' + +at the current working directory \item name can be overridden by --directory parameter \item the buildenvironment will be created inside this directory \end{itemize} \end{frame} \begin{frame} -\frametitle{the debian-installer} +\frametitle{the debian-installer (d-i)} \begin{itemize} -\item the debian-installer is a Linux kernel and a initial ramdisk. -\item the initial ramdisk includes a minimal rootfilesystem and the d-i - application -\item the debian project distributes the installer in different formats: +\item the d-i needs a running linux system +\item the debian project distributes the d-i in different packages: e.g. as a bootable cd or for use with pxeboot -\item normally the debian-installer is booted on the system debian should - be installed on -\item the debian-installer does disk detection / partitioning, sw installation - and system configuration +\item all packages include at least +\begin{itemize} +\item one Linux kernel binary +\item and an initial ramdisk with a minimal rfs and the d-i application +\end{itemize} +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{d-i common use} +\begin{itemize} +\item d-i is started on the system debian should be installed on +\item d-i does +\begin{itemize} +\item disk detection / partitioning +\item sw installation +\item system configuration +\end{itemize} \end{itemize} \end{frame} \begin{frame}[fragile] -\frametitle{debian-installer package} +\frametitle{elbe kinitrd packages} \begin{itemize} -\item the elbe project has packaged the debian-installer as debian package -\item this was done to be able to verify the version of the installer +\item the elbe project has packaged the d-i used by elbe as binary debian packages +\begin{lstlisting} +testrd_0.6_armel +|- opt +| +-elbe +| +-initrd +| +-initrd-cdrom.gz +| +-initrd.gz +| +-vmlinuz +\end{lstlisting} \item the name of the debian package is specified by the 'kinitrd' tag inside the xml file -\item current packages are called testrd and are available on the debian +\item current packages are called testrd and are available at the debian repository: \begin{lstlisting} deb http://debian.linutronix.de/elbe squeeze main @@ -97,14 +132,22 @@ deb http://debian.linutronix.de/elbe squeeze main \end{frame} \begin{frame} -\frametitle{retrieve debian-installer package} +\frametitle{retrieve kinitrd} \begin{itemize} -\item elbepack/pkgutils.py:copy\_kinitrd() is called by create.py -\item the pkgutils module uses python virtapt to retrieve the debian package - from a mirror or cdrom specified in the xml file -\item virtapt takes care on using the correct architecture and debian suite -\item then the package contents are extracted and copied to the build - directory +\item elbepack/pkgutils.py:copy\_kinitrd() + +is called by elbepack/create.py +\item the pkgutils module uses python virtapt + +to retrieve the kinitrd package + +from a mirror or cdrom specified in the xml file +\item virtapt takes care on using + +the correct debian architecture and suite +\item the kinitrd package will be extracted + +and copied to the build directory \end{itemize} \end{frame} @@ -127,15 +170,6 @@ http://www.makotemplates.org/ \frametitle{create Makefile, scripts and config files} ELBE uses \begin{lstlisting} -mako.Template(filename=fname).render(**d) -\end{lstlisting} -to insert several values from the XML file into the scripts and configfiles -used in the buildenv. It is called several times, for each *.mako file in -elbepack/mako (fname). - -d refers to several lxml.etree XML nodes. etree was initialized with the -ELBE XML file before. -\begin{lstlisting} d = {"opt": opt, "xml": xml, "prj": xml.node("/project"), @@ -143,13 +177,21 @@ d = {"opt": opt, "pkgs": xml.node("/target/pkg-list"), "fine": xml.node("/finetuning"), "preseed": get_preseed(xml) } +... +mako.Template(filename=fname).render(**d) \end{lstlisting} +to insert several values from the XML file into scripts and configfiles. +\begin{description} +\item[fname] the function is called for each *.mako file in elbepack/mako +\item[d] refers to several lxml.etree XML nodes +\item[xml] is a etree initialized with the ELBE XML file +\end{description} \end{frame} \begin{frame}[fragile] \frametitle{create Makefile, scripts and config files} For example a simple txt file including all package names defined in the - tag can be created by (pkg-list.mako): +$\langle$pkg-list$\rangle$ tag can be created by (pkg-list.mako): \begin{lstlisting} % for n in pkgs: % if n.tag == "pkg": @@ -169,8 +211,11 @@ the installation is running. \begin{frame} \frametitle{preseeding} -Many pre-seeding rules are generated from the XML file with mako: -./elbepack/mako/commmon.cfg.mako +Many pre-seeding rules are generated + +from the XML file with mako: + +./elbepack/mako/common.cfg.mako \end{frame} \begin{frame}[fragile] @@ -187,9 +232,12 @@ These rules are then included by ./elbepack/mako/custom-preseed.cfg.mako \begin{itemize} \item a directory tmp-tree is created \item all files and scripts needed inside the installer - are copied to the tmp-tree + +are copied to the tmp-tree \item ungzip the original initrd.gz -\item use cpio to append all files of tmp-tree to the unzipped initrd image +\item use cpio to append all files of tmp-tree + +to the unzipped initrd image \item gzip the new initrd image and remove the tmp-tree \end{itemize} \end{frame} @@ -199,10 +247,15 @@ These rules are then included by ./elbepack/mako/custom-preseed.cfg.mako \begin{itemize} \item qemu-img is used to create a buildenv.img file. This file will be used as harddisk of the emulated virtual machine. -\item The debian-installer will install a full featured debian on this image. -\item After the debian-installer was active, this image can be used as a virtual -build-environment. -\item The target rootfilesystem is normally just a subset of this image. +\item The debian-installer will install + +a full featured debian on this image. +\item After the debian-installer was active, + +this image can be used as a virtual build-environment. +\item The target rootfilesystem is normally + +just a subset of this image. \end{itemize} \end{frame} @@ -239,12 +292,18 @@ in the XML file: runs inside the virtual machine: \begin{itemize} \item the buildenv.img will be partitionated and formated -\item the root partition will be mounted at /target of the debian-installer rfs +\item the root partition will be mounted at + +/target inside the debian-installer rfs \item a minimal rootfilesystem will be installed to /target \item a chroot /target will be executed \item all packages will be installed and configured \item the /target chroot will be left -\item the hook d-i preseed/late\_command string /post-inst.sh will be executed +\item the preseed hook + +d-i preseed/late\_command string /post-inst.sh + +will be executed \end{itemize} \end{frame} @@ -267,7 +326,7 @@ and cdrom (/opt/elbe/mkcdrom.sh) generation. \end{frame} \begin{frame} -\frametitle{target generation} +\frametitle{target generation (overview)} \begin{itemize} \item generate file and pkg lists \item create /target tree @@ -280,7 +339,7 @@ and cdrom (/opt/elbe/mkcdrom.sh) generation. \item modify /target tree with finetuning.sh \end{itemize} \item generate licence information file -\item call part-target.sh to copy /target tree to (different) images +\item call part-target.sh to generate images from /target tree \item (optional) generate cpio images and tarballs \end{itemize} \end{frame} @@ -289,13 +348,15 @@ and cdrom (/opt/elbe/mkcdrom.sh) generation. \begin{frame}[fragile] \frametitle{create /target tree} \begin{itemize} -\item depending on the elbe mode, a filelist is generated. e.g. in normal mode: +\item depending on the ELBE mode, the filelist is generated. + +E.g. in normal mode: \begin{lstlisting} ls -A1 / | \ grep -v target | grep -v proc | grep -v sys | \ xargs find >> /opt/elbe/filelist \end{lstlisting} -\item then all files specified to be stored on the target rfs are copied: +\item all files specified to be stored on the target rfs are copied: \begin{lstlisting} rsync -a --files-from=/opt/elbe/filelist / /target \end{lstlisting} @@ -356,7 +417,9 @@ rsync -a --files-from=/opt/elbe/filelist / /target \begin{frame}[fragile] \frametitle{dump.py - write fullpkglist to source.xml} \begin{itemize} -\item all used packages are mentioned in this list (including version and md5sum): +\item all used packages are listed in the source.xml file + +(including version and md5sum): \begin{lstlisting} paktree = xml.ensure_child( 'fullpkgs' ) paktree.clear() @@ -375,7 +438,7 @@ rsync -a --files-from=/opt/elbe/filelist / /target \begin{frame} \frametitle{dump.py - modify /target tree with finetuning.sh} \begin{itemize} -\item the script is executed while cwd is /target. +\item the script is executed from /target. So files can be copied @@ -393,9 +456,10 @@ rsync -a --files-from=/opt/elbe/filelist / /target elbe-report.txt includes a filelist that indicates each file in the target rfs and its origin \begin{itemize} -\item a time stamp is recorded before and after finetuning. Files that show up - with a modification time in that period show up as modified/created by - finetuning.sh +\item a time stamp is recorded before and after finetuning. +\item files that show up with a modification time in that period + +can be identified as modified/created by finetuning.sh \end{itemize} \end{frame} @@ -422,10 +486,16 @@ of libparted and loopback mounts. \begin{frame}[fragile] \frametitle{build ubifs images} \begin{itemize} -\item mkfs.ubifs -r is used to generate a ubifs image from a subdirectory -\item the name of the images is defined by the label in the fstab tag. +\item mkfs.ubifs -r is used to generate a ubifs image + +from a subdirectory +\item the name of the images is defined + +by the label in the fstab tag \item the image defined first in the fstab is created first -\item the files copied to an image are moved temporarily away from /target: +\item the files copied to an image + +are moved temporarily away from /target: \begin{lstlisting} /mountA/mountB # mountA shouldn't include files like mountB/test \end{lstlisting} @@ -450,14 +520,82 @@ find . -print | cpio -ov -H newc >/opt/elbe/target.cpio \begin{frame} -\frametitle{cdrom generation} +\frametitle{binary cdrom generation} +An ISO CD image will be created: +\begin{itemize} +\item CD should contain all packages + +needed to reproduce the current ELBE run +\item the kinitrd package is installed into the buildenv + +to get it added to the cdrom like all other packages +\item CD needs to be in a debian repository format + +otherwise it won't be accepted by d-i +\item reprepo and genisoimage is used to create the CD +\end{itemize} \end{frame} -\subsection{Makefile (post-install)} +\begin{frame}[fragile] +\frametitle{source cdrom generation} +upon request (by 'elbe create --build-source') + +an ISO image containing all debian source packages is created: +\begin{lstlisting} +% if opt.buildsources: + mkdir -p /opt/elbe/source + dpkg --get-selections | awk '{print $1}' > /opt/elbe/pkg-list.actual + cd /opt/elbe/source + awk '{print "apt-get -d source "$1}' /opt/elbe/pkg-list.actual | sh + cd /opt/elbe + dpkg-scansources source /dev/null | gzip -9c > source/Sources.gz + genisoimage -o /opt/elbe/source.iso -J -R source +% endif +\end{lstlisting} +\end{frame} + \begin{frame} +\frametitle{kinitrd package details} +\begin{itemize} +\item in xml the cdrom tag can be used instead of a mirror +\item then d-i needs to load components (udeb) from cdrom instead of web +\item this is the reason for kinitrd including two versions of the initrd: +\begin{itemize} +\item one with the d-i configured to load udebs from a web source +\item a second with d-i configured to load udebs from a cdrom source +\end{itemize} +\item there is a switch in the 'elbe create' path to select the right initrd for + the project +\end{itemize} +\end{frame} + + +\subsection{Makefile (post-install)} + +\begin{frame}[fragile] \frametitle{copy files to host} +\begin{itemize} +\item all files produced by elbe are stored inside buildenv.img:/opt/elbe +\item they should be transfered into the build dir on the Host PC +\item the emulator is shutdown, + +then e2cp can be used to retrive the files: +\begin{lstlisting} +e2cp buildenv.img?offset=32768:/opt/elbe/licence.txt . +\end{lstlisting} +\item this is done for all files by the Makefile, e.g: +\begin{lstlisting} +% if tgt.has("images"): +% for mtd in tgt.node("images"): +% if mtd.has("partitions"): + e2cp buildenv.img?offset=${loop_offset}:/opt/elbe/${mtd.text("name")} . +% endif +% if mtd.has("binary"): + e2cp buildenv.img?offset=${loop_offset}:${mtd.text("binary")} ${mtd.text("name")} +\end{lstlisting} +\end{itemize} \end{frame} -- cgit v1.2.3