Merge branch 's3webseed' into 'master'

S3 webseed

See merge request fablab/labsync!58

.gitignore

@@ -2,4 +2,6 @@
 /**/ansible/inventories/test/
 /bin
 /images
+/images.txt
 /tmp
+packer/*.json

.gitlab-ci.yml

@@ -15,7 +15,6 @@ stages:
  - check
  - build
 
-
 dockerimage_builder:
   stage: prepare
   before_script:
@@ -34,7 +33,6 @@ dockerimage_builder:
     refs:
       - schedules
 
-
 dockerimage_security_scanner:
   stage: prepare
   before_script:
@@ -71,18 +69,21 @@ security_scanner:
     - dedicated
 
 .squashfs_template: &squashfs_template
+  image: $DOCKER_IMAGE_BUILDER
   stage: build
-  before_script:
-    - apk add --no-cache make
   services:
-   - docker:dind
+    - docker:dind
   script:
-   - make images/debian-buster.squashfs
-   - find images -type f > images.txt
+    - scripts/packer.sh debian-buster
+    - aws --endpoint-url "$AWS_ENDPOINT_URL" s3 cp images/ "s3://$AWS_BUCKET/$CI_COMMIT_REF_SLUG/$CI_JOB_ID/" --recursive --no-progress
   artifacts:
     paths:
-     - images
-     - images.txt
+      - images.txt
+      - images/*.dpkg-list
+      - images/*.initramfs
+      - images/*.linux
+      #- images/*.squashfs
+      - images/*.torrent
   tags:
     - fablab
     - ssd
@@ -90,7 +91,7 @@ security_scanner:
 squashfs_featurebranch:
   <<: *squashfs_template
   variables:
-    COMPRESSION_LEVEL: 5
+    COMPRESSION_LEVEL: 4
   except:
     variables:
       - $task == "security-scanner"
@@ -100,7 +101,7 @@ squashfs_featurebranch:
 squashfs_master:
   <<: *squashfs_template
   variables:
-    COMPRESSION_LEVEL: 7
+    COMPRESSION_LEVEL: 15
   only:
     refs:
       - master

Makefile

@@ -1,4 +1,5 @@
-PACKER_VERSION ?= 1.4.3
+PACKER_VERSION ?= 1.5.1
+ANSIBLE_VERSION ?= 2.9.2
 ANNOUNCE ?= http://10.2.2.1:6969/announce
 WEBSEED ?= http://10.2.2.1
 
@@ -44,7 +45,13 @@ clean:
 
 .PHONY: builderimg
 builderimg:
-	docker build --pull -t "$(DOCKER_IMAGE_BUILDER)" --cache-from "$(DOCKER_IMAGE_BUILDER)" --build-arg "PACKER_VERSION=$(PACKER_VERSION)" builder
+	docker build \
+		--pull \
+		-t "$(DOCKER_IMAGE_BUILDER)" \
+		--cache-from "$(DOCKER_IMAGE_BUILDER)" \
+		--build-arg "PACKER_VERSION=$(PACKER_VERSION)" \
+		--build-arg "ANSIBLE_VERSION=$(ANSIBLE_VERSION)" \
+		builder
 
 .PHONY: secscanimg
 secscanimg:
@@ -58,29 +65,26 @@ images/debian-buster.squashfs: images
 	docker run \
 		--rm \
 		-v /var/run/docker.sock:/var/run/docker.sock \
-		-v "${PWD}/packer:/workdir" \
-		-v "${PWD}/aria2c:/workdir/aria2c" \
-		-v "${PWD}/images:/images" \
-		-e "user=`id -u`" \
-		-e "group=`id -g`" \
+		-v "${PWD}:${PWD}" \
+		-w "${PWD}" \
 		-e "IMAGES=${PWD}/images" \
 		-e "ANNOUNCE=$(ANNOUNCE)" \
 		-e "WEBSEED=$(WEBSEED)" \
 		-e "COMPRESSION_LEVEL=$(COMPRESSION_LEVEL)" \
 		$(ci_environment) \
 		"$(DOCKER_IMAGE_BUILDER)" \
+		scripts/packer.sh \
 		debian-buster
 
 images/debian-buster.torrent: images
 	docker run \
 		--rm \
-		-v "${PWD}/images:/images" \
-		-e "user=`id -u`" \
-		-e "group=`id -g`" \
+		-v "${PWD}:${PWD}" \
+		-w "${PWD}" \
 		-e "ANNOUNCE=$(ANNOUNCE)" \
 		-e "WEBSEED=$(WEBSEED)" \
-		-e "TASK=torrent" \
 		"$(DOCKER_IMAGE_BUILDER)" \
+		scripts/torrent.sh \
 		debian-buster
 
 .PHONY: ansible
@@ -88,14 +92,14 @@ ansible:
 	docker run \
 		--rm \
 		-v "${PWD}/packer/ansible:/ansible" \
-		-e "user=`id -u`" \
-		-e "group=`id -g`" \
+		-u `id -u`:`id -g` \
 		-e "ANNOUNCE=$(ANNOUNCE)" \
 		-e "WEBSEED=$(WEBSEED)" \
-		-e "TASK=ansible" \
 		-v "${SSH_AUTH_SOCK}:/var/run/ssh_auth_sock" \
 		-e "SSH_AUTH_SOCK=/var/run/ssh_auth_sock" \
+		-w /ansible \
 		"$(DOCKER_IMAGE_BUILDER)" \
+			/usr/bin/ansible-playbook \
 			-i inventories \
 			$(if $(ANSIBLE_TAGS),-t $(ANSIBLE_TAGS),) \
 			-l $(ANSIBLE_LIMIT) \

builder/Dockerfile

@@ -1,5 +1,7 @@
 FROM docker
 
+ARG ANSIBLE_VERSION
+
 RUN apk add --no-cache \
 		ca-certificates \
 		coreutils \
@@ -19,7 +21,8 @@ RUN apk add --no-cache \
 		openssl-dev \
 		python3-dev \
 	&& pip3 install --upgrade pip \
-	&& pip3 install ansible==2.8.0 \
+	&& pip3 install ansible=="$ANSIBLE_VERSION" \
+	&& pip3 install awscli \
 	&& apk del .build-deps
 
 ARG PACKER_VERSION
@@ -46,7 +49,4 @@ RUN gpg --import /usr/local/share/hashicorp.asc \
 
 WORKDIR /workdir
 
-COPY entrypoint.sh /usr/local/bin/entrypoint.sh
-
-ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 # vim: ts=4 sw=4 sts=4 noet:

builder/entrypoint.sh

@@ -1,66 +0,0 @@
-#!/bin/sh
-set -e
-
-make_torrent() {
-	(cd /images
-		if [ "$NAME" = "" ]; then
-			# remove file extension
-			NAME="$(echo "$1" | sed 's/\.[^.]*//')"
-		fi
-
-		if [ "$WEBSEED" = "" ]; then
-			echo '$WEBSEED not provided' >&2
-			exit 1
-		fi
-		if [ "$ANNOUNCE" = "" ]; then
-			echo '$ANNOUNCE not provided' >&2
-			exit 1
-		fi
-
-		TORRENT_FILE="$NAME.torrent"
-		WEBSEED_URL="$WEBSEED/$1"
-
-		if [ -e "$TORRENT_FILE" ]; then rm "$TORRENT_FILE"; fi
-
-		mktorrent \
-			-n "$NAME" \
-			-a "$ANNOUNCE" \
-			-o "$TORRENT_FILE" \
-			-l 22 \
-			-w "$WEBSEED_URL" \
-			"$1"
-
-		if [ "$user" != "" ] && [ "$group" != "" ]; then
-			chown "$user:$group" "$TORRENT_FILE"
-		fi
-	)
-}
-
-run_packer() {
-	packer build "$NAME.json"
-
-	if [ "$user" != "" ] && [ "$group" != "" ]; then
-		chown $user:$group "/images/${NAME}."*
-	fi
-}
-
-NAME="$1"
-
-if [ "$NAME" = "" ]; then
-	echo 'no name supplied, stopping.' >&2
-	exit 1
-fi
-
-export NAME
-
-if [ -z "$TASK" ] || [ "$TASK" == "packer" ]; then
-	run_packer
-fi
-if [ -z "$TASK" ] || [ "$TASK" == "torrent" ]; then
-	make_torrent "$NAME.squashfs"
-fi
-if [ "$TASK" == "ansible" ]; then
-	cd /ansible
-	/usr/bin/ansible-playbook "$@"
-fi
-

packer/debian-buster.json

@@ -1,124 +0,0 @@
-{
-  "variables": {
-	"name": "{{env `NAME`}}",
-	"squashfs_file": "{{env `NAME`}}.squashfs",
-	"initramfs_file": "{{env `NAME`}}.initramfs",
-	"linux_file": "{{env `NAME`}}.linux",
-	"dpkg_list_file": "{{env `NAME`}}.dpkg-list",
-	"compression_level": "{{env `COMPRESSION_LEVEL`}}",
-	"images": "{{env `IMAGES`}}",
-    "ci_job_id": "{{env `CI_JOB_ID`}}",
-    "ci_commit_sha": "{{env `CI_COMMIT_SHA`}}",
-    "ci_commit_tag": "{{env `CI_COMMIT_TAG`}}",
-    "ci_commit_ref_name": "{{env `CI_COMMIT_REF_NAME`}}",
-    "ci_commit_ref_slug": "{{env `CI_COMMIT_REF_SLUG`}}",
-    "ci_job_name": "{{env `CI_JOB_NAME`}}",
-    "ci_job_stage": "{{env `CI_JOB_STAGE`}}",
-    "ci_project_url": "{{env `CI_PROJECT_URL`}}",
-    "ci_pipeline_triggered": "{{env `PIPELINE_TRIGGERED`}}",
-    "ci_job_manual": "{{env `CI_JOB_MANUAL`}}"
-  },
-  "builders":
-  [
-    {
-      "type": "docker",
-      "image": "debian:buster",
-      "discard": true,
-	  "run_command": [
-		  "-d",
-		  "-i",
-		  "-t",
-		  "-v", "{{user `images`}}:/tmp/images",
-		  "{{.Image}}",
-		  "/bin/bash"
-	  ]
-    }
-  ],
-  "provisioners": [
-    {
-      "type": "shell",
-      "inline": [ "mkdir -p /etc/initramfs-tools/scripts/local-premount/" ]
-    },
-    {
-      "type": "file",
-      "source": "initramfs/labsync",
-      "destination": "/etc/initramfs-tools/scripts/"
-    },
-    {
-      "type": "shell",
-      "inline": [
-        "echo \"LABSYNC_JOB_ID='{{user `ci_job_id`}}'\" >> /etc/environment",
-        "echo \"LABSYNC_COMMIT_SHA='{{user `ci_commit_sha`}}'\" >> /etc/environment",
-        "echo \"LABSYNC_COMMIT_TAG='{{user `ci_commit_tag`}}'\" >> /etc/environment",
-        "echo \"LABSYNC_COMMIT_REF_NAME='{{user `ci_commit_ref_name`}}'\" >> /etc/environment",
-        "echo \"LABSYNC_COMMIT_REF_SLUG='{{user `ci_commit_ref_slug`}}'\" >> /etc/environment",
-        "echo \"LABSYNC_PROJECT_URL='{{user `ci_project_url`}}'\" >> /etc/environment",
-        "sed -i 's#@@PROJECT_URL@@#{{user `ci_project_url`}}#' /etc/initramfs-tools/scripts/labsync"
-      ]
-    },
-    {
-      "type": "file",
-      "source": "initramfs/labsync-prereqs",
-      "destination": "/etc/initramfs-tools/scripts/local-premount/"
-    },
-    {
-      "type": "shell",
-      "inline": [
-        "set -x",
-        "apt-get update",
-        "apt-get -y dist-upgrade",
-        "rmdir /boot && ln -s /usr/local/boot /boot",
-        "apt-get -y install initramfs-tools || true",
-        "echo squashfs >> /etc/initramfs-tools/modules",
-        "echo overlay >> /etc/initramfs-tools/modules",
-        "echo 'RESUME=none' > /etc/initramfs-tools/conf.d/resume",
-        "mkdir /usr/local/boot",
-        "apt-get -f -y install aria2 linux-image-amd64 lvm2 haveged",
-        "cp $(find /boot/ -name 'initrd.img-*' | sort -V | tail -n 1) '/tmp/images/{{user `initramfs_file`}}'",
-        "cp $(find /boot/ -name 'vmlinuz-*' | sort -V | tail -n 1) '/tmp/images/{{user `linux_file`}}'"
-      ]
-    },
-    {
-      "type": "shell",
-      "inline": [ "apt-get -y install openssh-server python lsb-release" ]
-    },
-    {
-      "type": "ansible",
-      "playbook_file": "ansible/playbook.yml",
-      "user": "root"
-    },
-    {
-      "type": "shell",
-      "inline": [ "rm /boot && mkdir /boot" ]
-    },
-    {
-      "type": "shell",
-      "inline": [
-          "set -x",
-		  "apt-get -y install squashfs-tools",
-		  "dpkg -L squashfs-tools liblzo2-2 | while read f; do [ -f \"$f\" ] && echo \"$f\"; done > /tmp/ignore_files",
-		  "dpkg -l > /tmp/images/{{ user `dpkg_list_file` }}",
-		  "echo '/etc/resolv.conf' >> /tmp/ignore_files",
-		  "echo '/etc/hostname' >> /tmp/ignore_files",
-		  "echo '/etc/hosts' >> /tmp/ignore_files",
-		  "echo '/var/lib/docker' >> /tmp/ignore_files",
-		  "echo '/var/cache/apt/archives' >> /tmp/ignore_files",
-		  "echo '/var/lib/apt' >> /tmp/ignore_files",
-		  "mkdir -p /tmp/extra/tmp /tmp/extra/proc /tmp/extra/sys"
-	  ]
-	},
-    {
-      "type": "file",
-      "source": "hosts",
-      "destination": "/etc/hosts.template"
-    },
-	{
-      "type": "shell",
-      "inline": [
-		  "if [ -e '/tmp/images/{{user `squashfs_file`}}' ]; then rm '/tmp/images/{{user `squashfs_file`}}'; fi",
-		  "squashfs_content=\"$(find / -mindepth 1 -maxdepth 1 | grep -vE '^/(proc|sys|tmp|[.]dockerenv|packer-files)$')\"",
-		  "mksquashfs $squashfs_content /tmp/extra/* '/tmp/images/{{user `squashfs_file`}}' -comp lzo -Xcompression-level {{user `compression_level`}} -ef /tmp/ignore_files"
-	  ]
-    }
-  ]
-}

packer/debian-buster.yaml

@@ -0,0 +1,106 @@
+variables:
+  ci_commit_ref_name: '{{env `CI_COMMIT_REF_NAME`}}'
+  ci_commit_ref_slug: '{{env `CI_COMMIT_REF_SLUG`}}'
+  ci_commit_sha: '{{env `CI_COMMIT_SHA`}}'
+  ci_commit_tag: '{{env `CI_COMMIT_TAG`}}'
+  ci_job_id: '{{env `CI_JOB_ID`}}'
+  ci_job_manual: '{{env `CI_JOB_MANUAL`}}'
+  ci_job_name: '{{env `CI_JOB_NAME`}}'
+  ci_job_stage: '{{env `CI_JOB_STAGE`}}'
+  ci_pipeline_triggered: '{{env `PIPELINE_TRIGGERED`}}'
+  ci_project_url: '{{env `CI_PROJECT_URL`}}'
+  compression_level: '{{env `COMPRESSION_LEVEL`}}'
+  dpkg_list_file: '{{env `NAME`}}.dpkg-list'
+  images: '{{env `IMAGES`}}'
+  initramfs_file: '{{env `NAME`}}.initramfs'
+  linux_file: '{{env `NAME`}}.linux'
+  name: '{{env `NAME`}}'
+  squashfs_file: '{{env `NAME`}}.squashfs'
+  torrent_file: '{{env `NAME`}}.torrent'
+
+builders:
+- discard: true
+  image: debian:buster
+  type: docker
+  volumes:
+    '{{user `images`}}': /tmp/images
+
+provisioners:
+- inline:
+  - mkdir -p /etc/initramfs-tools/scripts/local-premount/
+  type: shell
+- destination: /etc/initramfs-tools/scripts/
+  source: initramfs/labsync
+  type: file
+- inline:
+  - echo "LABSYNC_JOB_ID='{{user `ci_job_id`}}'" >> /etc/environment
+  - echo "LABSYNC_COMMIT_SHA='{{user `ci_commit_sha`}}'" >> /etc/environment
+  - echo "LABSYNC_COMMIT_TAG='{{user `ci_commit_tag`}}'" >> /etc/environment
+  - echo "LABSYNC_COMMIT_REF_NAME='{{user `ci_commit_ref_name`}}'" >> /etc/environment
+  - echo "LABSYNC_COMMIT_REF_SLUG='{{user `ci_commit_ref_slug`}}'" >> /etc/environment
+  - echo "LABSYNC_PROJECT_URL='{{user `ci_project_url`}}'" >> /etc/environment
+  - sed -i 's#@@PROJECT_URL@@#{{user `ci_project_url`}}#' /etc/initramfs-tools/scripts/labsync
+  type: shell
+- destination: /etc/initramfs-tools/scripts/local-premount/
+  source: initramfs/labsync-prereqs
+  type: file
+- inline:
+  - set -x
+  - apt-get update
+  - apt-get -y dist-upgrade
+  - rmdir /boot && ln -s /usr/local/boot /boot
+  - apt-get -y install initramfs-tools || true
+  - echo squashfs >> /etc/initramfs-tools/modules
+  - echo overlay >> /etc/initramfs-tools/modules
+  - echo 'RESUME=none' > /etc/initramfs-tools/conf.d/resume
+  - mkdir /usr/local/boot
+  - apt-get -f -y install aria2 linux-image-amd64 lvm2 haveged
+  - mkdir -p /tmp/images
+  - cp $(find /boot/ -name 'initrd.img-*' | sort -V | tail -n 1) '/tmp/images/{{user `initramfs_file`}}'
+  - cp $(find /boot/ -name 'vmlinuz-*' | sort -V | tail -n 1) '/tmp/images/{{user `linux_file`}}'
+  type: shell
+- inline:
+  - apt-get -y install openssh-server python lsb-release
+  type: shell
+- playbook_file: ansible/playbook.yml
+  type: ansible
+  user: root
+- inline:
+  - rm /boot && mkdir /boot
+  type: shell
+- inline:
+  - set -x
+  - apt-get -y install squashfs-tools
+  - dpkg -L squashfs-tools liblzo2-2 | while read f; do [ -f "$f" ] && echo "$f";
+    done > /tmp/ignore_files
+  - dpkg -l > /tmp/images/{{ user `dpkg_list_file` }}
+  - echo '/etc/resolv.conf' >> /tmp/ignore_files
+  - echo '/etc/hostname' >> /tmp/ignore_files
+  - echo '/etc/hosts' >> /tmp/ignore_files
+  - echo '/var/lib/docker' >> /tmp/ignore_files
+  - echo '/var/cache/apt/archives' >> /tmp/ignore_files
+  - echo '/var/lib/apt' >> /tmp/ignore_files
+  - mkdir -p /tmp/extra/tmp /tmp/extra/proc /tmp/extra/sys
+  type: shell
+- destination: /etc/hosts.template
+  source: hosts
+  type: file
+- inline:
+  - if [ -e '/tmp/images/{{user `squashfs_file`}}' ]; then rm '/tmp/images/{{user `squashfs_file`}}'; fi
+  - squashfs_content="$(find / -mindepth 1 -maxdepth 1 | grep -vE '^/(proc|sys|tmp|[.]dockerenv|packer-files)$')"
+  - >
+    mksquashfs $squashfs_content /tmp/extra/* '/tmp/images/{{user `squashfs_file`}}'
+    -comp zstd
+    -Xcompression-level {{user `compression_level`}}
+    -ef /tmp/ignore_files
+  type: shell
+
+post-processors:
+- inline:
+  - >
+    mktorrent
+    -n '{{user `name`}}'
+    -o '{{user `images`}}/{{user `torrent_file`}}'
+    -l 22
+    '{{user `images`}}/{{user `squashfs_file`}}'
+  type: shell-local

scripts/packer.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+set -e
+set -o nounset
+
+export NAME="$1"
+mkdir -p images
+export IMAGES="$(realpath images)"
+
+(
+    cd packer
+
+    python3 -c 'import sys, yaml, json; json.dump(yaml.safe_load(sys.stdin), sys.stdout, indent=4)' < "$NAME.yaml" > "$NAME.json"
+
+    packer build "$NAME.json"
+)
+
+echo "$NAME" >> images.txt