Add security-scanner

.gitignore

@@ -1,5 +1,5 @@
 /**/ansible/**/*.retry
 /**/ansible/inventories/test/
-bin
+/bin
-images
+/images
-tmp
+/tmp

.gitlab-ci.yml

@@ -5,40 +5,62 @@ variables:
   PACKER_VERSION: 1.2.0
   ANNOUNCE: http://labsync.lab.fablab-nea.de:6969/announce
   WEBSEED: http://labsync.lab.fablab-nea.de/labsync/$CI_COMMIT_REF_NAME/$CI_PIPELINE_ID/images
-  DOCKER_IMAGE: ${CI_REGISTRY_IMAGE}/labsync-builder
+  DOCKER_IMAGE_BUILDER: ${CI_REGISTRY_IMAGE}/labsync-builder
+  DOCKER_IMAGE_SECURITY_SCANNER: ${CI_REGISTRY_IMAGE}/security-scanner
 
 stages:
  - prepare
+ - check
  - build
- - torrent
 
 services:
  - docker:dind
 
-before_script:
+dockerimage_builder:
- - docker info
- - apk add --no-cache make openssh-client
-
-dockerimage:
   stage: prepare
+  before_script:
+    - apk add --no-cache make
   script:
+    - docker pull $DOCKER_IMAGE_BUILDER || true
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
-    - make dockerimg
+    - make builderimg
-    - docker push $DOCKER_IMAGE
+    - docker push $DOCKER_IMAGE_BUILDER
   tags:
     - fablab
+  except:
+    - schedules
+
+dockerimage_security_scanner:
+  stage: prepare
+  before_script:
+    - apk add --no-cache make
+  script:
+    - docker pull $DOCKER_IMAGE_SECURITY_SCANNER || true
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+    - make secscanimg
+    - docker push $DOCKER_IMAGE_SECURITY_SCANNER
+  tags:
+    - fablab
+  except:
+    - schedules
+
+security_scanner:
+  stage: check
+  image: $DOCKER_IMAGE_SECURITY_SCANNER
+  script:
+    - set -x
+    - export GITLAB_URL="$(echo "$CI_PROJECT_URL" | grep -Eo '^https?://[^/]*')"
+    - security-scanner stretch
 
 .squashfs_template: &squashfs_template
   stage: build
+  before_script:
+    - apk add --no-cache make
   script:
    - make images/debian-stretch.squashfs
-   - echo "$rsa_labsync_raven" > /dev/shm/id_rsa && chmod 600 /dev/shm/id_rsa
-   - mkdir -p $HOME/.ssh && echo "$hostkeys_raven" >> $HOME/.ssh/known_hosts
-   - ssh -i /dev/shm/id_rsa labsync@raven.lab.fablab-nea.de "cd /opt/docker/tftpgen && make labsync.cfg" || true
   artifacts:
     paths:
      - images
-    expire_in: 2 weeks
   tags:
     - fablab
 
@@ -48,6 +70,7 @@ squashfs_featurebranch:
     COMPRESSION_LEVEL: 5
   except:
     - master
+    - schedules
 
 squashfs_master:
   <<: *squashfs_template
@@ -55,3 +78,7 @@ squashfs_master:
     COMPRESSION_LEVEL: 7
   only:
     - master
+  except:
+    - schedules
+  artifacts:
+    expire_in: 12 weeks

Makefile

@@ -2,7 +2,8 @@ PACKER_VERSION ?= 1.2.0
 ANNOUNCE ?= http://10.2.2.1:6969/announce
 WEBSEED ?= http://10.2.2.1
 
-DOCKER_IMAGE ?= labsync-builder
+DOCKER_IMAGE_BUILDER ?= labsync-builder
+DOCKER_IMAGE_SECURITY_SCANNER ?= security-scanner
 
 CWD=$(abspath $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))))
 
@@ -32,16 +33,20 @@ fi \
 ci_environment=$(shell env | sed -n 's/^\(CI_.*\)=.*/-e \1/p')
 
 .PHONY: default
-default: dockerimg images/debian-stretch.squashfs
+default: builderimg images/debian-stretch.squashfs
 
 .PHONY: clean
 clean:
 	rm -f images/*
 	rm -rf tmp
 
-.PHONY: dockerimg
+.PHONY: builderimg
-dockerimg:
+builderimg:
-	docker build --pull -t "$(DOCKER_IMAGE)" --cache-from "$(DOCKER_IMAGE)" --build-arg "PACKER_VERSION=$(PACKER_VERSION)" docker
+	docker build --pull -t "$(DOCKER_IMAGE_BUILDER)" --cache-from "$(DOCKER_IMAGE_BUILDER)" --build-arg "PACKER_VERSION=$(PACKER_VERSION)" builder
+
+.PHONY: secscanimg
+secscanimg:
+	docker build --pull -t "$(DOCKER_IMAGE_SECURITY_SCANNER)" --cache-from "$(DOCKER_IMAGE_SECURITY_SCANNER)" security-scanner
 
 images:
 	[ ! -d "$@" ] && mkdir "$@"
@@ -61,7 +66,7 @@ images/debian-stretch.squashfs: images
 		-e "WEBSEED=$(WEBSEED)" \
 		-e "COMPRESSION_LEVEL=$(COMPRESSION_LEVEL)" \
 		$(ci_environment) \
-		"$(DOCKER_IMAGE)" \
+		"$(DOCKER_IMAGE_BUILDER)" \
 		debian-stretch
 
 images/debian-stretch.torrent: images
@@ -73,7 +78,7 @@ images/debian-stretch.torrent: images
 		-e "ANNOUNCE=$(ANNOUNCE)" \
 		-e "WEBSEED=$(WEBSEED)" \
 		-e "TASK=torrent" \
-		"$(DOCKER_IMAGE)" \
+		"$(DOCKER_IMAGE_BUILDER)" \
 		debian-stretch
 
 .PHONY: ansible
@@ -88,7 +93,7 @@ ansible:
 		-e "TASK=ansible" \
 		-v "${SSH_AUTH_SOCK}:/var/run/ssh_auth_sock" \
 		-e "SSH_AUTH_SOCK=/var/run/ssh_auth_sock" \
-		"$(DOCKER_IMAGE)" \
+		"$(DOCKER_IMAGE_BUILDER)" \
 			-i inventories \
 			$(if $(ANSIBLE_TAGS),-t $(ANSIBLE_TAGS),) \
 			-l $(ANSIBLE_LIMIT) \

security-scanner/.gitignore

@@ -0,0 +1,7 @@
+*.pyc
+*egg-info
+/venv*
+__pycache__
+/build
+/dist
+/data

security-scanner/Dockerfile

@@ -0,0 +1,19 @@
+FROM debian:stretch-slim
+
+RUN apt-get update \
+	&& apt-get -y install \
+		ca-certificates \
+		curl \
+		python3 \
+		python3-apt \
+		python3-pip \
+		python3-urllib3 \
+	&& rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /tmp/requirements.txt
+
+RUN pip3 install -r /tmp/requirements.txt
+
+ADD . /code
+
+RUN (cd /code && python3 setup.py install)

security-scanner/bin/security-scanner

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+import sys
+
+import security_scanner.main
+
+if __name__ == '__main__':
+    security_scanner.main.main(sys.argv)

security-scanner/requirements.txt

@@ -0,0 +1,2 @@
+python-gitlab==1.4.0
+urllib3==1.22

security-scanner/security_scanner/debian_tracker.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+import json
+import apt_pkg
+import urllib.request
+import os.path
+
+class DebianTracker(object):
+    _debian_security = {}
+    _distro = None
+
+    def __init__(self, distro=None, forceUpdate=False):
+        apt_pkg.init_system()
+
+        self._distro = distro
+        if forceUpdate or (not os.path.exists("debian-security.json")):
+            self.update()
+
+        with open('debian-security.json') as f:
+            self._debian_security = json.load(f)
+
+    def update(self):
+        urllib.request.urlretrieve("https://security-tracker.debian.org/tracker/data/json", "debian-security.json")
+
+    def getLatestVersion(self, package, distro=None):
+        if distro is None:
+            if self._distro is None:
+                raise ValueError('you must pass distro, either to the function or on creating the object')
+            else:
+                distro = self._distro
+
+        latest_version = None
+        if package in self._debian_security:
+            for cve in self._debian_security[package]:
+                if distro in self._debian_security[package][cve]['releases']:
+                    for repository in self._debian_security[package][cve]['releases'][distro]['repositories']:
+                        version = self._debian_security[package][cve]['releases'][distro]['repositories'][repository]
+                        if latest_version is not None:
+                            if apt_pkg.version_compare(version,latest_version) > 0:
+                                latest_version = version
+                        else:
+                             latest_version = version
+                else:
+                    return 'distro not found'
+
+            return latest_version
+        else:
+            return 'package not found'
+
+    def checkPackage(self, package, version, distro=None):
+        result = {
+                'current': version,
+                'latest': None,
+                'comparison': None,
+                'message': '',
+            }
+        latest_version = self.getLatestVersion(package, distro)
+        if latest_version == 'package not found' or latest_version == 'distro not found':
+            result['message'] = latest_version
+        else:
+            result['latest'] = latest_version
+            result['comparison'] = apt_pkg.version_compare(version, latest_version)
+        return result

security-scanner/security_scanner/dpkg_list.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import re
+import os.path
+#from pprint import pprint
+
+
+class DpkgList(object):
+    _distro = None
+
+    def __init__(self, distro):
+        self._distro = distro
+
+    def parseFile(self, path='.', filename=None, encoding="utf-8"):
+        if filename is None:
+            filename = 'debian-' + self._distro + '.dpkg-list'
+
+        dpkg_list = None
+        with open(os.path.join(path, filename), encoding=encoding) as f:
+            dpkg_list = f.readlines()
+
+        installed_packages = {}
+
+        for item in dpkg_list:
+            matches = re.match(r"ii\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$", item)
+            if matches is not None:
+                package = matches.group(1)
+                installed_packages[package] = {}
+                installed_packages[package]['version'] = matches.group(2)
+                installed_packages[package]['arch'] = matches.group(3)
+                installed_packages[package]['description'] = matches.group(4)
+
+        return installed_packages

security-scanner/security_scanner/file_writer.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+class FileWriter(object):
+    def __init__(self, filename):
+        self._fd = open(filename, 'wb')
+
+    def __call__(self, chunk):
+        self._fd.write(chunk)

security-scanner/security_scanner/gitlab.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+import gitlab
+import os
+
+from security_scanner.file_writer import FileWriter
+
+class GitLab:
+    def __init__(self):
+        gitlab_url = os.environ.get('GITLAB_URL')
+        api_token = os.environ.get('PRIVATE_TOKEN')
+        project_id = os.environ.get('CI_PROJECT_ID')
+
+        self._gl = gitlab.Gitlab(gitlab_url, private_token=api_token)
+        self._project = self._gl.projects.get(project_id)
+
+    def getLastSuccessfulJob(self, ref, name):
+        pipelines = self._project.pipelines.list()
+
+        successful_master_jobs = []
+        for pipeline in pipelines:
+            jobs = pipeline.jobs.list()
+            for job in jobs:
+                if job.ref == ref and job.attributes['name'] == name and job.attributes['status'] == 'success':
+                    successful_master_jobs.append(job)
+
+        job = successful_master_jobs[-1]
+
+        return job
+
+    def downloadArtifact(self, job, sourcePath, destPath):
+        job_id = job.attributes['id']
+        print("Downloading artifact {} for job #{}".format(sourcePath, job_id))
+        target = FileWriter(destPath)
+        artifact = self._project.jobs.get(job_id).artifact(sourcePath, streamed=True, action=target)
+        del(target)

security-scanner/security_scanner/main.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import sys
+
+
+import security_scanner
+from security_scanner.debian_tracker import DebianTracker
+from security_scanner.dpkg_list import DpkgList
+from security_scanner.gitlab import GitLab
+
+from pprint import pprint
+
+def stripArch(package):
+    return package.split(":", 2)[0]
+
+def checkDebianDistro(distro):
+    result = 0
+    pkglist = DpkgList(distro)
+    packages = pkglist.parseFile()
+    tracker = DebianTracker(distro)
+
+    for package in packages:
+        state = tracker.checkPackage(stripArch(package), packages[package]['version'])
+        if 'comparison' in state and state['comparison'] is not None and state['comparison'] < 0:
+            print("Package: {}, current: {}, latest: {}, comparison: {}".format(package,state['current'], state['latest'], state['comparison']))
+            result = 1
+    print("checked {} packages; result: {}".format(len(packages), result))
+
+    return result
+
+def main(argv):
+    for distro in argv[1:]:
+    #for distro in ['stretch']:
+        gitlab = GitLab()
+        job = gitlab.getLastSuccessfulJob('master', 'squashfs_master')
+        gitlab.downloadArtifact(job, 'images/debian-' + distro + '.dpkg-list', 'debian-' + distro + '.dpkg-list')
+        if checkDebianDistro(distro) > 0:
+            print("triggering build")
+            pprint(job.attributes)
+            job.play()

security-scanner/setup.py

@@ -0,0 +1,15 @@
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
+
+config = {
+    'name': 'security_scanner',
+    'install_requires': [],
+    'packages': [
+        'security_scanner',
+    ],
+    'scripts': ['bin/security-scanner']
+}
+
+setup(**config)