merging repos; build initramfs in packer; cleaning up the mess

This commit is contained in:
jalr 2018-02-20 01:36:23 +01:00
parent df69ddca4b
commit 38ed593a16
25 changed files with 421 additions and 271 deletions

View file

@ -1,3 +0,0 @@
pladde.img
initrd.gz
vmlinuz

8
.gitignore vendored
View file

@ -1,6 +1,4 @@
ansible/**/*.retry
/**/ansible/**/*.retry
bin
build
initrd.gz
pladde.img
vmlinuz
images
tmp

View file

@ -1,13 +0,0 @@
FROM debian:stretch
RUN apt-get update \
&& apt-get -y install \
initramfs-tools \
linux-image-amd64 \
aria2 \
xxd
COPY imagesync.sh /etc/initramfs-tools/scripts/local-premount/
RUN echo 'RESUME=none' > /etc/initramfs-tools/conf.d/resume \
&& mkinitramfs -o /tmp/initrd.gz $(find /boot -name 'vmlinuz-*' -printf '%f\n' | sed 's/^vmlinuz-//')

41
Makefile Normal file
View file

@ -0,0 +1,41 @@
PACKER_VERSION ?= 1.2.0
ANNOUNCE ?= udp://10.2.2.1:6969/announce
WEBSEED ?= http://10.2.2.1
PACKER_DOCKER_IMAGE = labsync-packer
MKTORRENT_DOCKER_IMAGE = labsync-mktorrent
.PHONY: clean
clean:
rm -f images/*
rm -f tmp/*
.PHONY: dockerimg
dockerimg:
docker build -t "$(PACKER_DOCKER_IMAGE)" --build-arg "PACKER_VERSION=$(PACKER_VERSION)" packer/docker
docker build -t "$(MKTORRENT_DOCKER_IMAGE)" mktorrent
images/debian-stretch: dockerimg
docker run \
-ti \
--rm \
-v /var/run/docker.sock:/var/run/docker.sock \
-v "${PWD}/packer:/workdir" \
-v "${PWD}/images:/images" \
-e "user=`id -u`" \
-e "group=`id -g`" \
"$(PACKER_DOCKER_IMAGE)" \
debian-stretch
#images/debian-stretch.squashfs
images/debian-stretch.torrent: dockerimg
docker run \
-ti \
--rm \
-v "${PWD}/images:/workdir" \
-e "user=`id -u`" \
-e "group=`id -g`" \
-e "ANNOUNCE=$(ANNOUNCE)" \
-e "WEBSEED=$(WEBSEED)" \
"$(MKTORRENT_DOCKER_IMAGE)" \
debian-stretch

View file

@ -1,3 +0,0 @@
#!/bin/sh
docker build -t initramfs:debian .
docker run --rm -v "$PWD:/artifacts" initramfs:debian sh -c 'cp /boot/vmlinuz-* /artifacts/vmlinuz && cp /tmp/initrd.gz /artifacts'

15
docker-compose.yml Normal file
View file

@ -0,0 +1,15 @@
version: "3.2"
services:
nginx:
image: nginx:alpine
ports:
- 10.2.2.1:80:80
volumes:
- ./images:/usr/share/nginx/html
opentracker:
build: opentracker
image: opentracker:alpine
ports:
- 10.2.2.1:6969:6969/udp

View file

@ -1,7 +0,0 @@
echo "building with packer…"
packer build packer.json
echo "compressing image…"
gzip -k -f rootfs.tar
echo "generate torrent…"
rm rootfs.torrent
mktorrent -a udp://tracker.sbruder.de:6969/announce -o rootfs.torrent -l 22 -w 'http://10.2.2.1:8081/rootfs.tar.gz' rootfs.tar.gz

View file

@ -1,227 +0,0 @@
#!/bin/sh
case $1 in
prereqs)
##
# GET EXECUTED WHEN BUILDING INITRAMFS
#
. /usr/share/initramfs-tools/hook-functions
copy_exec /usr/bin/aria2c
copy_exec /sbin/sfdisk
copy_exec /sbin/mke2fs
copy_exec /sbin/e2fsck
copy_file cert /etc/ssl/certs/ca-certificates.crt
exit 0
;;
esac
##
# FUNCTION DEFINITIONS
##
print_waiting() {
echo -en "\033[36m${1}... \033[0m"
}
print_warning() {
echo -e "\n\033[31mWARNING: ${1}\033[0m"
}
print_done() {
echo -e "\033[32mdone.\033[0m"
}
error_fatal() {
print_warning "$1; interrupting startup"
while sleep 3600;do sleep 3600;done
}
##
# SHOW BRANDING
##
echo -e '
Starting
""# #
# mmm #mmm mmm m m m mm mmm
# " # #" "# # " "m m" #" # #" "
# m"""# # # """m #m# # # #
"mm "mm"# ##m#" "mmm" "# # # "#mm"
m"
""
https://gitlab.jalr.de/fablab/initramfs
'
##
# READ CONFIGURATION PARAMETERS FROM KERNEL PARAMETERS
##
print_waiting "Reading configuration parameters"
set -- $(cat /proc/cmdline)
for x in "$@"; do
case "$(echo $x | cut -d= -f1)" in
disk)
disk="$(echo $x | cut -d= -f2)"
;;
root)
root="$(echo $x | cut -d= -f2)"
;;
partsize)
partsize="$(echo $x | cut -d= -f2)"
;;
torrent)
torrent="$(echo $x | cut -d= -f2)"
;;
qemu)
qemu="$(echo $x | cut -d= -f2)"
;;
esac
done
# Check if mandatory parameters are missing
if [ -z "$disk" ];then
error_fatal "The disk parameter is missing"
fi
if [ -z "$root" ];then
error_fatal "The root parameter is missing"
fi
if [ -z "$partsize" ];then
error_fatal "The partsize parameter is missing"
fi
if [ -z "$torrent" ];then
error_fatal "The torrent parameter is missing"
fi
print_done
##
# PARTITIONING OF DISK
##
size_part_1=$(($partsize * 1024 * 1024 / 512))
parttable="label: dos
label-id: 0xdeadbeef
device: $disk
unit: sectors
$root : start= 2048, size= $size_part_1, type=83"
readtable="$(sfdisk -d $disk)" # wtf? but it is needed
print_waiting "Checking partition tables"
if [ "$parttable" != "$readtable" ];then
print_done
print_waiting "Partition tables do not match; partitioning disk"
rewrite_rootfs=true
sfdisk -q $disk << PARTTABLE
$parttable
PARTTABLE
print_done
else
print_done
fi
print_waiting "Checking root filesystem"
e2fsck -y $root &> /dev/null
fsck=$?
if [ "$fsck" != 0 ] && [ "$fsck" != 1 ] || [ "$rewrite_rootfs" == "true" ];then
print_done
print_waiting "Creating root filesystem"
mke2fs -q -F -t ext4 $root
print_done
else
print_done
fi
##
# TEMPORARY NETWORK CONFIG (only for qemu)
##
if ! [ -z "$qemu" ];then
print_waiting "Setting up temporary network config"
ip addr add 10.2.2.2/24 dev ens3
ip link set ens3 up
print_done
else
print_waiting "Setting up interfaces with DHCP"
ipconfig all
print_done
fi
##
# MOUNT TEMPORARY ROOTFS
##
print_waiting "Mounting temporary root filesystem"
mkdir -p /tmp_root
mount -t ext4 $root /tmp_root
cd /tmp_root
print_done
##
# DOWNLOAD FS IMAGE
##
print_waiting "Downloading root filesystem image"
# --allow-overwrite: do not rename the torrent file
# --check-integrity: verify downloaded file and do not complain about missing controll file
# --seed-time=0: do not wait until ratio 1, we seed when were booted
# --summary-interval=0: do not show summary while downloading
# --console-log-level=error: Only show errors
# --download-result=hide: Do not show download results
download=true
while [ "$download" == true ];do
aria2c \
--allow-overwrite \
--check-integrity \
--seed-time=0 \
--summary-interval=0 \
--console-log-level=error \
--download-result=hide \
$torrent 2>&1 | grep -v 'DHT routing table'
if [ $? == 0 ];then
download=false
else
rm rootfs.tar.gz rootfs.torrent
fi
done
print_done
print_waiting "Comparing installed and remote image"
if [ -f /tmp_root/.build_uuid ];then
cd /tmp
tar xzf /tmp_root/rootfs.tar.gz .build_uuid
if [ "$(cat .build_uuid)" == "$(cat /tmp_root/.build_uuid)" ];then
sync=false
print_done
else
sync=true
print_done
print_waiting "Deleting old filesystem"
cd /tmp_root
rm -rf $(tar tzf rootfs.tar.gz|grep '^[^/]*/$' )
cd /tmp
print_done
fi
rm .build_uuid
cd /tmp_root
else
sync=true
print_done
fi
if [ "$sync" == "true" ];then
print_waiting "Extracting root filesystem image"
tar xzf rootfs.tar.gz
print_done
fi
##
# UNMOUNT TEMPORARY ROOTFS
##
print_waiting "Unmounting temporary root filesystem"
cd /
umount /tmp_root
print_done

11
mktorrent/Dockerfile Normal file
View file

@ -0,0 +1,11 @@
FROM alpine
RUN apk add --no-cache mktorrent \
&& mkdir /workdir
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
WORKDIR /workdir
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

25
mktorrent/entrypoint.sh Executable file
View file

@ -0,0 +1,25 @@
#!/bin/sh
set -e
NAME="$1"
if [ "$WEBSEED" = "" ]; then
echo '$WEBSEED not provided' >&2
exit 1
fi
if [ "$ANNOUNCE" = "" ]; then
echo '$ANNOUNCE not provided' >&2
exit 1
fi
TORRENT_FILE="$NAME.torrent"
WEBSEED_URL="$WEBSEED/$NAME"
if [ -e "$TORRENT_FILE" ]; then rm "$TORRENT_FILE"; fi
mktorrent-borg -a "$ANNOUNCE" -o "$TORRENT_FILE" -l 22 -w "$WEBSEED_URL" "$NAME"
if [ "$user" != "" ] && [ "$group" != "" ]; then
chown "$user:$group" "$TORRENT_FILE"
fi

5
opentracker/Dockerfile Normal file
View file

@ -0,0 +1,5 @@
FROM alpine
RUN apk add --no-cache opentracker
ENTRYPOINT ["/usr/bin/opentracker"]

View file

@ -20,6 +20,20 @@
}
],
"provisioners": [
{
"type": "shell",
"inline": [ "mkdir -p /etc/initramfs-tools/scripts/local-premount/" ]
},
{
"type": "file",
"source": "initramfs/labsync",
"destination": "/etc/initramfs-tools/scripts/"
},
{
"type": "file",
"source": "initramfs/labsync-prereqs",
"destination": "/etc/initramfs-tools/scripts/local-premount/"
},
{
"type": "shell",
"inline": [
@ -28,9 +42,17 @@
"echo \"commit tag: {{user `ci_commit_tag`}}\" >> /.build-info",
"echo \"commit ref name: {{user `ci_commit_ref_name`}}\" >> /.build-info",
"echo \"commit ref slug: {{user `ci_commit_ref_slug`}}\" >> /.build-info",
"apt-get update && apt-get -y install python openssh-server"
"apt-get update",
"apt-get -y dist-upgrade",
"apt-get -y install aria2 initramfs-tools linux-image-amd64 openssh-server python",
"echo 'RESUME=none' > /etc/initramfs-tools/conf.d/resume",
"echo squashfs >> /etc/initramfs-tools/modules",
"echo overlay >> /etc/initramfs-tools/modules",
"mkdir /artifacts",
"mkinitramfs -o /artifacts/initramfs $(find /boot -name 'vmlinuz-*' -printf '%f\\n' | sed 's/^vmlinuz-//')",
"cp $(find /boot -name 'vmlinuz-*' | sort -V | tail -n 1) /artifacts/linux"
]
},
},
{
"type": "ansible",
"playbook_file": "ansible/playbook.yml",

20
packer/docker/Dockerfile Normal file
View file

@ -0,0 +1,20 @@
FROM docker:dind
RUN apk add --no-cache \
ansible \
openssh-client \
squashfs-tools
ARG PACKER_VERSION
RUN mkdir -p /usr/local/bin \
&& wget -O /tmp/packer.zip -c https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip \
&& unzip /tmp/packer.zip -d /usr/local/bin \
&& rm -f /tmp/packer.zip \
&& mkdir /workdir
WORKDIR /workdir
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

49
packer/docker/entrypoint.sh Executable file
View file

@ -0,0 +1,49 @@
#!/bin/sh
set -e
NAME="$1"
if [ "$NAME" = "" ]; then
echo 'no name supplied, stopping.' >&2
exit 1
fi
PACKER_JSON="$NAME.json"
INITRAMFS_FILE="$NAME.initramfs"
LINUX_FILE="$NAME.linux"
SQUASHFS_FILE="$NAME"
packer build "$PACKER_JSON"
mkdir -p build/rootfs
tar xf build/rootfs.tar -C build/rootfs
mv build/rootfs/artifacts/initramfs "/images/$INITRAMFS_FILE"
mv build/rootfs/artifacts/linux "/images/$LINUX_FILE"
rm -rf build/rootfs/artifacts
rm -- \
build/rootfs/.dockerenv \
build/rootfs/initrd.* \
build/rootfs/vmlinuz*
if [ -e "/images/$SQUASHFS_FILE" ]; then
rm "/images/$SQUASHFS_FILE"
fi
mksquashfs build/rootfs "/images/$SQUASHFS_FILE" \
-comp lzo \
-Xcompression-level 9
rm -rf build
if [ "$user" != "" ] && [ "$group" != "" ]; then
chown $user:$group \
"/images/$INITRAMFS_FILE" \
"/images/$LINUX_FILE" \
"/images/$SQUASHFS_FILE"
fi

189
packer/initramfs/labsync Executable file
View file

@ -0,0 +1,189 @@
#!/bin/sh
labsync_mount_root() {
##
# FUNCTION DEFINITIONS
##
print_waiting() {
echo -en "\033[36m${1}... \033[0m"
}
print_warning() {
echo -e "\n\033[31mWARNING: ${1}\033[0m"
}
print_done() {
echo -e "\033[32mdone.\033[0m"
}
error_fatal() {
print_warning "$1; interrupting startup"
while sleep 3600;do sleep 3600;done
}
##
# SHOW BRANDING
##
echo -e '
Starting
""# #
# mmm #mmm mmm m m m mm mmm
# " # #" "# # " "m m" #" # #" "
# m"""# # # """m #m# # # #
"mm "mm"# ##m#" "mmm" "# # # "#mm"
m"
""
https://gitlab.jalr.de/fablab/initramfs
'
##
# READ CONFIGURATION PARAMETERS FROM KERNEL PARAMETERS
##
print_waiting "Reading configuration parameters"
set -- $(cat /proc/cmdline)
for x in "$@"; do
arg="$(echo $x | cut -d= -f1)"
case "$arg" in
disk|root|partsize|qemu)
eval $arg="$(echo $x | cut -d= -f2)"
;;
esac
done
# Check if mandatory parameters are missing
if [ -z "$disk" ];then
error_fatal "The disk parameter is missing"
fi
if [ -z "$partsize" ];then
error_fatal "The partsize parameter is missing"
fi
if [ -z "$torrent" ];then
error_fatal "The torrent parameter is missing"
fi
if [ -z "$torrent_file" ];then
error_fatal "The torrent_file parameter is missing"
fi
print_done
##
# PARTITIONING OF DISK
##
size_part_1=$(($partsize * 1024 * 1024 / 512))
parttable="label: dos
label-id: 0xdeadbeef
device: $disk
unit: sectors
$root : start= 2048, size= $size_part_1, type=83"
readtable="$(sfdisk -d $disk)" # wtf? but it is needed
print_waiting "Checking partition tables"
if [ "$parttable" != "$readtable" ];then
print_done
print_waiting "Partition tables do not match; partitioning disk"
rewrite_rootfs=true
sfdisk -q $disk << PARTTABLE
$parttable
PARTTABLE
print_done
else
print_done
fi
##
# TEMPORARY NETWORK CONFIG (only for qemu)
##
configure_networking
#if ! [ -z "$qemu" ];then
# print_waiting "Setting up temporary network config"
# ip addr add 10.2.2.2/24 dev ens3
# ip link set ens3 up
# print_done
#else
# print_waiting "Setting up interfaces with DHCP"
# ipconfig all
# print_done
#fi
##
# DOWNLOAD SQUASHFS
##
print_waiting "Downloading root filesystem image via $torrent"
# --allow-overwrite: do not rename the torrent file
# --check-integrity: verify downloaded file and do not complain about missing controll file
# --seed-time=0: do not wait until ratio 1, we seed when were booted
# --summary-interval=0: do not show summary while downloading
# --console-log-level=error: Only show errors
# --download-result=hide: Do not show download results
ip a
sleep 1
#wget -O /tmp/torrent "$torrent"
(cd /tmp
ln -s "${disk}1" "$torrent_file"
aria2c \
--allow-overwrite \
--check-integrity \
--seed-time=0 \
--summary-interval=10 \
--file-allocation=none \
--enable-dht=false \
"$torrent"
)
print_done
modprobe overlay
mkdir -p /root
mkdir /ro_root /rw_root
mount -t squashfs /dev/sda1 /ro_root
mke2fs -q -F -t ext4 /dev/sdb1
mount -t ext4 /dev/sdb1 /rw_root
mkdir -p /rw_root/upper /rw_root/work
mount -t overlay overlay -o lowerdir=/ro_root,upperdir=/rw_root/upper,workdir=/rw_root/work /root
sleep 10
}
labsync_top() {
:
}
labsync_premount() {
:
}
labsync_bottom() {
:
}
mountroot()
{
labsync_mount_root
}
mount_top()
{
labsync_top
}
mount_premount()
{
labsync_premount
}
mount_bottom()
{
labsync_bottom
}

View file

@ -0,0 +1,17 @@
#!/bin/sh
case $1 in
prereqs)
##
# GETS EXECUTED WHEN BUILDING INITRAMFS
##
. /usr/share/initramfs-tools/hook-functions
copy_exec /sbin/mke2fs
copy_exec /sbin/e2fsck
copy_exec /usr/bin/aria2c
copy_exec /sbin/sfdisk
copy_file cert /etc/ssl/certs/ca-certificates.crt
exit 0
;;
esac

33
qemu.sh
View file

@ -1,23 +1,34 @@
#!/bin/sh
ifname_tap=taptstsync
ifname_br=brtstsync
ifname_tap=taplabsync
ifname_br=brlabsync
[ -e pladde.img ] || qemu-img create pladde.img 4G
TORRENT="$1.torrent"
KERNEL="$1.linux"
INITRAMFS="$1.initramfs"
SQUASHFS="$1"
if [ "$1" = "" ]; then
echo "you must supply an image name!" >&2
exit 1
fi
mkdir -p ./tmp
[ -e tmp/qemu-disk.img ] || qemu-img create tmp/qemu-disk.img 4G
sudo ip tuntap add dev $ifname_tap mode tap user $USER
sudo brctl addbr $ifname_br
sudo brctl addif $ifname_br $ifname_tap
sudo ip addr add 10.2.2.1/24 dev brtstsync
sudo ip link set taptstsync up
sudo ip link set brtstsync up
sudo ip addr add 10.2.2.1/24 dev $ifname_br
sudo ip link set taplabsync up
sudo ip link set brlabsync up
qemu-system-x86_64 \
-kernel vmlinuz \
-initrd initrd.gz \
-drive format=raw,file=pladde.img \
-append "disk=/dev/sda root=/dev/sda1 partsize=3072 torrent=http://10.2.2.1:8081/rootfs.torrent qemu quiet" \
-curses \
-kernel "images/$KERNEL" \
-initrd "images/$INITRAMFS" \
-drive format=raw,file=tmp/qemu-disk.img \
-drive format=raw,file=tmp/qemu-disk2.img \
-append "boot=labsync disk=/dev/sda partsize=3072 torrent=http://10.2.2.1/$TORRENT torrent_file=$SQUASHFS quiet vga=792 ip=10.2.2.2:::255.255.255.0:qemumachine:ens3:off" \
-enable-kvm \
-m 1G \
-net nic \