From e43f34991d2bf8e1038fa7c5d9352e1a01563c80 Mon Sep 17 00:00:00 2001 From: Jakob Lechner Date: Tue, 7 Jan 2020 20:23:37 +0100 Subject: [PATCH] Use yaml for packer job description --- .gitignore | 1 + .gitlab-ci.yml | 1 - packer/debian-buster.json | 124 -------------------------------------- packer/debian-buster.yaml | 106 ++++++++++++++++++++++++++++++++ scripts/packer.sh | 2 + scripts/torrent.sh | 33 ---------- 6 files changed, 109 insertions(+), 158 deletions(-) delete mode 100644 packer/debian-buster.json create mode 100644 packer/debian-buster.yaml delete mode 100755 scripts/torrent.sh diff --git a/.gitignore b/.gitignore index 8236110..c842193 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /bin /images /tmp +packer/*.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2c4102..1ff4e29 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,7 +75,6 @@ security_scanner: - docker:dind script: - scripts/packer.sh debian-buster - - scripts/torrent.sh debian-buster.squashfs - find images -type f > images.txt artifacts: paths: diff --git a/packer/debian-buster.json b/packer/debian-buster.json deleted file mode 100644 index 7e60f8c..0000000 --- a/packer/debian-buster.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "variables": { - "name": "{{env `NAME`}}", - "squashfs_file": "{{env `NAME`}}.squashfs", - "initramfs_file": "{{env `NAME`}}.initramfs", - "linux_file": "{{env `NAME`}}.linux", - "dpkg_list_file": "{{env `NAME`}}.dpkg-list", - "compression_level": "{{env `COMPRESSION_LEVEL`}}", - "images": "{{env `IMAGES`}}", - "ci_job_id": "{{env `CI_JOB_ID`}}", - "ci_commit_sha": "{{env `CI_COMMIT_SHA`}}", - "ci_commit_tag": "{{env `CI_COMMIT_TAG`}}", - "ci_commit_ref_name": "{{env `CI_COMMIT_REF_NAME`}}", - "ci_commit_ref_slug": "{{env `CI_COMMIT_REF_SLUG`}}", - "ci_job_name": "{{env `CI_JOB_NAME`}}", - "ci_job_stage": "{{env `CI_JOB_STAGE`}}", - "ci_project_url": "{{env `CI_PROJECT_URL`}}", - "ci_pipeline_triggered": "{{env `PIPELINE_TRIGGERED`}}", - "ci_job_manual": "{{env `CI_JOB_MANUAL`}}" - }, - "builders": - [ - { - "type": "docker", - "image": "debian:buster", - "discard": true, - "run_command": [ - "-d", - "-i", - "-t", - "-v", "{{user `images`}}:/tmp/images", - "{{.Image}}", - "/bin/bash" - ] - } - ], - "provisioners": [ - { - "type": "shell", - "inline": [ "mkdir -p /etc/initramfs-tools/scripts/local-premount/" ] - }, - { - "type": "file", - "source": "initramfs/labsync", - "destination": "/etc/initramfs-tools/scripts/" - }, - { - "type": "shell", - "inline": [ - "echo \"LABSYNC_JOB_ID='{{user `ci_job_id`}}'\" >> /etc/environment", - "echo \"LABSYNC_COMMIT_SHA='{{user `ci_commit_sha`}}'\" >> /etc/environment", - "echo \"LABSYNC_COMMIT_TAG='{{user `ci_commit_tag`}}'\" >> /etc/environment", - "echo \"LABSYNC_COMMIT_REF_NAME='{{user `ci_commit_ref_name`}}'\" >> /etc/environment", - "echo \"LABSYNC_COMMIT_REF_SLUG='{{user `ci_commit_ref_slug`}}'\" >> /etc/environment", - "echo \"LABSYNC_PROJECT_URL='{{user `ci_project_url`}}'\" >> /etc/environment", - "sed -i 's#@@PROJECT_URL@@#{{user `ci_project_url`}}#' /etc/initramfs-tools/scripts/labsync" - ] - }, - { - "type": "file", - "source": "initramfs/labsync-prereqs", - "destination": "/etc/initramfs-tools/scripts/local-premount/" - }, - { - "type": "shell", - "inline": [ - "set -x", - "apt-get update", - "apt-get -y dist-upgrade", - "rmdir /boot && ln -s /usr/local/boot /boot", - "apt-get -y install initramfs-tools || true", - "echo squashfs >> /etc/initramfs-tools/modules", - "echo overlay >> /etc/initramfs-tools/modules", - "echo 'RESUME=none' > /etc/initramfs-tools/conf.d/resume", - "mkdir /usr/local/boot", - "apt-get -f -y install aria2 linux-image-amd64 lvm2 haveged", - "cp $(find /boot/ -name 'initrd.img-*' | sort -V | tail -n 1) '/tmp/images/{{user `initramfs_file`}}'", - "cp $(find /boot/ -name 'vmlinuz-*' | sort -V | tail -n 1) '/tmp/images/{{user `linux_file`}}'" - ] - }, - { - "type": "shell", - "inline": [ "apt-get -y install openssh-server python lsb-release" ] - }, - { - "type": "ansible", - "playbook_file": "ansible/playbook.yml", - "user": "root" - }, - { - "type": "shell", - "inline": [ "rm /boot && mkdir /boot" ] - }, - { - "type": "shell", - "inline": [ - "set -x", - "apt-get -y install squashfs-tools", - "dpkg -L squashfs-tools liblzo2-2 | while read f; do [ -f \"$f\" ] && echo \"$f\"; done > /tmp/ignore_files", - "dpkg -l > /tmp/images/{{ user `dpkg_list_file` }}", - "echo '/etc/resolv.conf' >> /tmp/ignore_files", - "echo '/etc/hostname' >> /tmp/ignore_files", - "echo '/etc/hosts' >> /tmp/ignore_files", - "echo '/var/lib/docker' >> /tmp/ignore_files", - "echo '/var/cache/apt/archives' >> /tmp/ignore_files", - "echo '/var/lib/apt' >> /tmp/ignore_files", - "mkdir -p /tmp/extra/tmp /tmp/extra/proc /tmp/extra/sys" - ] - }, - { - "type": "file", - "source": "hosts", - "destination": "/etc/hosts.template" - }, - { - "type": "shell", - "inline": [ - "if [ -e '/tmp/images/{{user `squashfs_file`}}' ]; then rm '/tmp/images/{{user `squashfs_file`}}'; fi", - "squashfs_content=\"$(find / -mindepth 1 -maxdepth 1 | grep -vE '^/(proc|sys|tmp|[.]dockerenv|packer-files)$')\"", - "mksquashfs $squashfs_content /tmp/extra/* '/tmp/images/{{user `squashfs_file`}}' -comp lzo -Xcompression-level {{user `compression_level`}} -ef /tmp/ignore_files" - ] - } - ] -} diff --git a/packer/debian-buster.yaml b/packer/debian-buster.yaml new file mode 100644 index 0000000..0b0d1f8 --- /dev/null +++ b/packer/debian-buster.yaml @@ -0,0 +1,106 @@ +variables: + ci_commit_ref_name: '{{env `CI_COMMIT_REF_NAME`}}' + ci_commit_ref_slug: '{{env `CI_COMMIT_REF_SLUG`}}' + ci_commit_sha: '{{env `CI_COMMIT_SHA`}}' + ci_commit_tag: '{{env `CI_COMMIT_TAG`}}' + ci_job_id: '{{env `CI_JOB_ID`}}' + ci_job_manual: '{{env `CI_JOB_MANUAL`}}' + ci_job_name: '{{env `CI_JOB_NAME`}}' + ci_job_stage: '{{env `CI_JOB_STAGE`}}' + ci_pipeline_triggered: '{{env `PIPELINE_TRIGGERED`}}' + ci_project_url: '{{env `CI_PROJECT_URL`}}' + compression_level: '{{env `COMPRESSION_LEVEL`}}' + dpkg_list_file: '{{env `NAME`}}.dpkg-list' + images: '{{env `IMAGES`}}' + initramfs_file: '{{env `NAME`}}.initramfs' + linux_file: '{{env `NAME`}}.linux' + name: '{{env `NAME`}}' + squashfs_file: '{{env `NAME`}}.squashfs' + torrent_file: '{{env `NAME`}}.torrent' + +builders: +- discard: true + image: debian:buster + type: docker + volumes: + '{{user `images`}}': /tmp/images + +provisioners: +- inline: + - mkdir -p /etc/initramfs-tools/scripts/local-premount/ + type: shell +- destination: /etc/initramfs-tools/scripts/ + source: initramfs/labsync + type: file +- inline: + - echo "LABSYNC_JOB_ID='{{user `ci_job_id`}}'" >> /etc/environment + - echo "LABSYNC_COMMIT_SHA='{{user `ci_commit_sha`}}'" >> /etc/environment + - echo "LABSYNC_COMMIT_TAG='{{user `ci_commit_tag`}}'" >> /etc/environment + - echo "LABSYNC_COMMIT_REF_NAME='{{user `ci_commit_ref_name`}}'" >> /etc/environment + - echo "LABSYNC_COMMIT_REF_SLUG='{{user `ci_commit_ref_slug`}}'" >> /etc/environment + - echo "LABSYNC_PROJECT_URL='{{user `ci_project_url`}}'" >> /etc/environment + - sed -i 's#@@PROJECT_URL@@#{{user `ci_project_url`}}#' /etc/initramfs-tools/scripts/labsync + type: shell +- destination: /etc/initramfs-tools/scripts/local-premount/ + source: initramfs/labsync-prereqs + type: file +- inline: + - set -x + - apt-get update + - apt-get -y dist-upgrade + - rmdir /boot && ln -s /usr/local/boot /boot + - apt-get -y install initramfs-tools || true + - echo squashfs >> /etc/initramfs-tools/modules + - echo overlay >> /etc/initramfs-tools/modules + - echo 'RESUME=none' > /etc/initramfs-tools/conf.d/resume + - mkdir /usr/local/boot + - apt-get -f -y install aria2 linux-image-amd64 lvm2 haveged + - mkdir -p /tmp/images + - cp $(find /boot/ -name 'initrd.img-*' | sort -V | tail -n 1) '/tmp/images/{{user `initramfs_file`}}' + - cp $(find /boot/ -name 'vmlinuz-*' | sort -V | tail -n 1) '/tmp/images/{{user `linux_file`}}' + type: shell +- inline: + - apt-get -y install openssh-server python lsb-release + type: shell +- playbook_file: ansible/playbook.yml + type: ansible + user: root +- inline: + - rm /boot && mkdir /boot + type: shell +- inline: + - set -x + - apt-get -y install squashfs-tools + - dpkg -L squashfs-tools liblzo2-2 | while read f; do [ -f "$f" ] && echo "$f"; + done > /tmp/ignore_files + - dpkg -l > /tmp/images/{{ user `dpkg_list_file` }} + - echo '/etc/resolv.conf' >> /tmp/ignore_files + - echo '/etc/hostname' >> /tmp/ignore_files + - echo '/etc/hosts' >> /tmp/ignore_files + - echo '/var/lib/docker' >> /tmp/ignore_files + - echo '/var/cache/apt/archives' >> /tmp/ignore_files + - echo '/var/lib/apt' >> /tmp/ignore_files + - mkdir -p /tmp/extra/tmp /tmp/extra/proc /tmp/extra/sys + type: shell +- destination: /etc/hosts.template + source: hosts + type: file +- inline: + - if [ -e '/tmp/images/{{user `squashfs_file`}}' ]; then rm '/tmp/images/{{user `squashfs_file`}}'; fi + - squashfs_content="$(find / -mindepth 1 -maxdepth 1 | grep -vE '^/(proc|sys|tmp|[.]dockerenv|packer-files)$')" + - > + mksquashfs $squashfs_content /tmp/extra/* '/tmp/images/{{user `squashfs_file`}}' + -comp lzo + -Xcompression-level {{user `compression_level`}} + -ef /tmp/ignore_files + type: shell + +post-processors: +- inline: + - > + mktorrent + -n '{{user `name`}}' + -o '{{user `images`}}/{{user `torrent_file`}}' + -l 22 + '{{user `images`}}/{{user `squashfs_file`}}' + type: shell-local diff --git a/scripts/packer.sh b/scripts/packer.sh index 8b51ce6..432633d 100755 --- a/scripts/packer.sh +++ b/scripts/packer.sh @@ -7,4 +7,6 @@ export IMAGES="$(realpath images)" cd packer +python3 -c 'import sys, yaml, json; json.dump(yaml.safe_load(sys.stdin), sys.stdout, indent=4)' < "$NAME.yaml" > "$NAME.json" + packer build "$NAME.json" diff --git a/scripts/torrent.sh b/scripts/torrent.sh deleted file mode 100755 index fdab0fc..0000000 --- a/scripts/torrent.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -set -o nounset - -cd images - -NAME="$1" - -if [ "$NAME" = "" ]; then - # remove file extension - NAME="$(echo "$1" | sed 's/\.[^.]*//')" -fi - -if [ "$WEBSEED" = "" ]; then - echo '$WEBSEED not provided' >&2 - exit 1 -fi -if [ "$ANNOUNCE" = "" ]; then - echo '$ANNOUNCE not provided' >&2 - exit 1 -fi - -TORRENT_FILE="$NAME.torrent" -WEBSEED_URL="$WEBSEED/$1" - -if [ -e "$TORRENT_FILE" ]; then rm "$TORRENT_FILE"; fi - -mktorrent \ - -n "$NAME" \ - -a "$ANNOUNCE" \ - -o "$TORRENT_FILE" \ - -l 22 \ - -w "$WEBSEED_URL" \ - "$1"