From 809a5e6012cfcf08c003c874462c19dc0754e4db Mon Sep 17 00:00:00 2001 From: jalr Date: Sat, 9 Jun 2018 14:35:16 +0200 Subject: [PATCH] Add security-scanner --- .gitignore | 6 +- .gitlab-ci.yml | 53 ++++++++++++---- Makefile | 21 ++++--- {docker => builder}/Dockerfile | 0 {docker => builder}/entrypoint.sh | 0 security-scanner/.gitignore | 7 +++ security-scanner/Dockerfile | 19 ++++++ security-scanner/bin/security-scanner | 8 +++ security-scanner/requirements.txt | 2 + security-scanner/security_scanner/__init__.py | 0 .../security_scanner/debian_tracker.py | 63 +++++++++++++++++++ .../security_scanner/dpkg_list.py | 33 ++++++++++ .../security_scanner/file_writer.py | 8 +++ security-scanner/security_scanner/gitlab.py | 36 +++++++++++ security-scanner/security_scanner/main.py | 40 ++++++++++++ security-scanner/setup.py | 15 +++++ 16 files changed, 287 insertions(+), 24 deletions(-) rename {docker => builder}/Dockerfile (100%) rename {docker => builder}/entrypoint.sh (100%) create mode 100644 security-scanner/.gitignore create mode 100644 security-scanner/Dockerfile create mode 100755 security-scanner/bin/security-scanner create mode 100644 security-scanner/requirements.txt create mode 100644 security-scanner/security_scanner/__init__.py create mode 100644 security-scanner/security_scanner/debian_tracker.py create mode 100644 security-scanner/security_scanner/dpkg_list.py create mode 100644 security-scanner/security_scanner/file_writer.py create mode 100644 security-scanner/security_scanner/gitlab.py create mode 100644 security-scanner/security_scanner/main.py create mode 100644 security-scanner/setup.py diff --git a/.gitignore b/.gitignore index 27de7b0..8236110 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ /**/ansible/**/*.retry /**/ansible/inventories/test/ -bin -images -tmp +/bin +/images +/tmp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c685a20..0841ad4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,40 +5,62 @@ variables: PACKER_VERSION: 1.2.0 ANNOUNCE: http://labsync.lab.fablab-nea.de:6969/announce WEBSEED: http://labsync.lab.fablab-nea.de/labsync/$CI_COMMIT_REF_NAME/$CI_PIPELINE_ID/images - DOCKER_IMAGE: ${CI_REGISTRY_IMAGE}/labsync-builder + DOCKER_IMAGE_BUILDER: ${CI_REGISTRY_IMAGE}/labsync-builder + DOCKER_IMAGE_SECURITY_SCANNER: ${CI_REGISTRY_IMAGE}/security-scanner stages: - prepare + - check - build - - torrent services: - docker:dind -before_script: - - docker info - - apk add --no-cache make openssh-client - -dockerimage: +dockerimage_builder: stage: prepare + before_script: + - apk add --no-cache make script: + - docker pull $DOCKER_IMAGE_BUILDER || true - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY - - make dockerimg - - docker push $DOCKER_IMAGE + - make builderimg + - docker push $DOCKER_IMAGE_BUILDER tags: - fablab + except: + - schedules + +dockerimage_security_scanner: + stage: prepare + before_script: + - apk add --no-cache make + script: + - docker pull $DOCKER_IMAGE_SECURITY_SCANNER || true + - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY + - make secscanimg + - docker push $DOCKER_IMAGE_SECURITY_SCANNER + tags: + - fablab + except: + - schedules + +security_scanner: + stage: check + image: $DOCKER_IMAGE_SECURITY_SCANNER + script: + - set -x + - export GITLAB_URL="$(echo "$CI_PROJECT_URL" | grep -Eo '^https?://[^/]*')" + - security-scanner stretch .squashfs_template: &squashfs_template stage: build + before_script: + - apk add --no-cache make script: - make images/debian-stretch.squashfs - - echo "$rsa_labsync_raven" > /dev/shm/id_rsa && chmod 600 /dev/shm/id_rsa - - mkdir -p $HOME/.ssh && echo "$hostkeys_raven" >> $HOME/.ssh/known_hosts - - ssh -i /dev/shm/id_rsa labsync@raven.lab.fablab-nea.de "cd /opt/docker/tftpgen && make labsync.cfg" || true artifacts: paths: - images - expire_in: 2 weeks tags: - fablab @@ -48,6 +70,7 @@ squashfs_featurebranch: COMPRESSION_LEVEL: 5 except: - master + - schedules squashfs_master: <<: *squashfs_template @@ -55,3 +78,7 @@ squashfs_master: COMPRESSION_LEVEL: 7 only: - master + except: + - schedules + artifacts: + expire_in: 12 weeks diff --git a/Makefile b/Makefile index dd903a7..f84b350 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,8 @@ PACKER_VERSION ?= 1.2.0 ANNOUNCE ?= http://10.2.2.1:6969/announce WEBSEED ?= http://10.2.2.1 -DOCKER_IMAGE ?= labsync-builder +DOCKER_IMAGE_BUILDER ?= labsync-builder +DOCKER_IMAGE_SECURITY_SCANNER ?= security-scanner CWD=$(abspath $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))) @@ -32,16 +33,20 @@ fi \ ci_environment=$(shell env | sed -n 's/^\(CI_.*\)=.*/-e \1/p') .PHONY: default -default: dockerimg images/debian-stretch.squashfs +default: builderimg images/debian-stretch.squashfs .PHONY: clean clean: rm -f images/* rm -rf tmp -.PHONY: dockerimg -dockerimg: - docker build --pull -t "$(DOCKER_IMAGE)" --cache-from "$(DOCKER_IMAGE)" --build-arg "PACKER_VERSION=$(PACKER_VERSION)" docker +.PHONY: builderimg +builderimg: + docker build --pull -t "$(DOCKER_IMAGE_BUILDER)" --cache-from "$(DOCKER_IMAGE_BUILDER)" --build-arg "PACKER_VERSION=$(PACKER_VERSION)" builder + +.PHONY: secscanimg +secscanimg: + docker build --pull -t "$(DOCKER_IMAGE_SECURITY_SCANNER)" --cache-from "$(DOCKER_IMAGE_SECURITY_SCANNER)" security-scanner images: [ ! -d "$@" ] && mkdir "$@" @@ -61,7 +66,7 @@ images/debian-stretch.squashfs: images -e "WEBSEED=$(WEBSEED)" \ -e "COMPRESSION_LEVEL=$(COMPRESSION_LEVEL)" \ $(ci_environment) \ - "$(DOCKER_IMAGE)" \ + "$(DOCKER_IMAGE_BUILDER)" \ debian-stretch images/debian-stretch.torrent: images @@ -73,7 +78,7 @@ images/debian-stretch.torrent: images -e "ANNOUNCE=$(ANNOUNCE)" \ -e "WEBSEED=$(WEBSEED)" \ -e "TASK=torrent" \ - "$(DOCKER_IMAGE)" \ + "$(DOCKER_IMAGE_BUILDER)" \ debian-stretch .PHONY: ansible @@ -88,7 +93,7 @@ ansible: -e "TASK=ansible" \ -v "${SSH_AUTH_SOCK}:/var/run/ssh_auth_sock" \ -e "SSH_AUTH_SOCK=/var/run/ssh_auth_sock" \ - "$(DOCKER_IMAGE)" \ + "$(DOCKER_IMAGE_BUILDER)" \ -i inventories \ $(if $(ANSIBLE_TAGS),-t $(ANSIBLE_TAGS),) \ -l $(ANSIBLE_LIMIT) \ diff --git a/docker/Dockerfile b/builder/Dockerfile similarity index 100% rename from docker/Dockerfile rename to builder/Dockerfile diff --git a/docker/entrypoint.sh b/builder/entrypoint.sh similarity index 100% rename from docker/entrypoint.sh rename to builder/entrypoint.sh diff --git a/security-scanner/.gitignore b/security-scanner/.gitignore new file mode 100644 index 0000000..4321473 --- /dev/null +++ b/security-scanner/.gitignore @@ -0,0 +1,7 @@ +*.pyc +*egg-info +/venv* +__pycache__ +/build +/dist +/data diff --git a/security-scanner/Dockerfile b/security-scanner/Dockerfile new file mode 100644 index 0000000..9b732b2 --- /dev/null +++ b/security-scanner/Dockerfile @@ -0,0 +1,19 @@ +FROM debian:stretch-slim + +RUN apt-get update \ + && apt-get -y install \ + ca-certificates \ + curl \ + python3 \ + python3-apt \ + python3-pip \ + python3-urllib3 \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /tmp/requirements.txt + +RUN pip3 install -r /tmp/requirements.txt + +ADD . /code + +RUN (cd /code && python3 setup.py install) diff --git a/security-scanner/bin/security-scanner b/security-scanner/bin/security-scanner new file mode 100755 index 0000000..22c4524 --- /dev/null +++ b/security-scanner/bin/security-scanner @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +import sys + +import security_scanner.main + +if __name__ == '__main__': + security_scanner.main.main(sys.argv) diff --git a/security-scanner/requirements.txt b/security-scanner/requirements.txt new file mode 100644 index 0000000..3324114 --- /dev/null +++ b/security-scanner/requirements.txt @@ -0,0 +1,2 @@ +python-gitlab==1.4.0 +urllib3==1.22 diff --git a/security-scanner/security_scanner/__init__.py b/security-scanner/security_scanner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/security-scanner/security_scanner/debian_tracker.py b/security-scanner/security_scanner/debian_tracker.py new file mode 100644 index 0000000..2a6c07e --- /dev/null +++ b/security-scanner/security_scanner/debian_tracker.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import json +import apt_pkg +import urllib.request +import os.path + +class DebianTracker(object): + _debian_security = {} + _distro = None + + def __init__(self, distro=None, forceUpdate=False): + apt_pkg.init_system() + + self._distro = distro + if forceUpdate or (not os.path.exists("debian-security.json")): + self.update() + + with open('debian-security.json') as f: + self._debian_security = json.load(f) + + def update(self): + urllib.request.urlretrieve("https://security-tracker.debian.org/tracker/data/json", "debian-security.json") + + def getLatestVersion(self, package, distro=None): + if distro is None: + if self._distro is None: + raise ValueError('you must pass distro, either to the function or on creating the object') + else: + distro = self._distro + + latest_version = None + if package in self._debian_security: + for cve in self._debian_security[package]: + if distro in self._debian_security[package][cve]['releases']: + for repository in self._debian_security[package][cve]['releases'][distro]['repositories']: + version = self._debian_security[package][cve]['releases'][distro]['repositories'][repository] + if latest_version is not None: + if apt_pkg.version_compare(version,latest_version) > 0: + latest_version = version + else: + latest_version = version + else: + return 'distro not found' + + return latest_version + else: + return 'package not found' + + def checkPackage(self, package, version, distro=None): + result = { + 'current': version, + 'latest': None, + 'comparison': None, + 'message': '', + } + latest_version = self.getLatestVersion(package, distro) + if latest_version == 'package not found' or latest_version == 'distro not found': + result['message'] = latest_version + else: + result['latest'] = latest_version + result['comparison'] = apt_pkg.version_compare(version, latest_version) + return result diff --git a/security-scanner/security_scanner/dpkg_list.py b/security-scanner/security_scanner/dpkg_list.py new file mode 100644 index 0000000..5e7e42c --- /dev/null +++ b/security-scanner/security_scanner/dpkg_list.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +import re +import os.path +#from pprint import pprint + + +class DpkgList(object): + _distro = None + + def __init__(self, distro): + self._distro = distro + + def parseFile(self, path='.', filename=None, encoding="utf-8"): + if filename is None: + filename = 'debian-' + self._distro + '.dpkg-list' + + dpkg_list = None + with open(os.path.join(path, filename), encoding=encoding) as f: + dpkg_list = f.readlines() + + installed_packages = {} + + for item in dpkg_list: + matches = re.match(r"ii\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$", item) + if matches is not None: + package = matches.group(1) + installed_packages[package] = {} + installed_packages[package]['version'] = matches.group(2) + installed_packages[package]['arch'] = matches.group(3) + installed_packages[package]['description'] = matches.group(4) + + return installed_packages diff --git a/security-scanner/security_scanner/file_writer.py b/security-scanner/security_scanner/file_writer.py new file mode 100644 index 0000000..7c37401 --- /dev/null +++ b/security-scanner/security_scanner/file_writer.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +class FileWriter(object): + def __init__(self, filename): + self._fd = open(filename, 'wb') + + def __call__(self, chunk): + self._fd.write(chunk) diff --git a/security-scanner/security_scanner/gitlab.py b/security-scanner/security_scanner/gitlab.py new file mode 100644 index 0000000..4602428 --- /dev/null +++ b/security-scanner/security_scanner/gitlab.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +import gitlab +import os + +from security_scanner.file_writer import FileWriter + +class GitLab: + def __init__(self): + gitlab_url = os.environ.get('GITLAB_URL') + api_token = os.environ.get('PRIVATE_TOKEN') + project_id = os.environ.get('CI_PROJECT_ID') + + self._gl = gitlab.Gitlab(gitlab_url, private_token=api_token) + self._project = self._gl.projects.get(project_id) + + def getLastSuccessfulJob(self, ref, name): + pipelines = self._project.pipelines.list() + + successful_master_jobs = [] + for pipeline in pipelines: + jobs = pipeline.jobs.list() + for job in jobs: + if job.ref == ref and job.attributes['name'] == name and job.attributes['status'] == 'success': + successful_master_jobs.append(job) + + job = successful_master_jobs[-1] + + return job + + def downloadArtifact(self, job, sourcePath, destPath): + job_id = job.attributes['id'] + print("Downloading artifact {} for job #{}".format(sourcePath, job_id)) + target = FileWriter(destPath) + artifact = self._project.jobs.get(job_id).artifact(sourcePath, streamed=True, action=target) + del(target) diff --git a/security-scanner/security_scanner/main.py b/security-scanner/security_scanner/main.py new file mode 100644 index 0000000..a4a7db2 --- /dev/null +++ b/security-scanner/security_scanner/main.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import sys + + +import security_scanner +from security_scanner.debian_tracker import DebianTracker +from security_scanner.dpkg_list import DpkgList +from security_scanner.gitlab import GitLab + +from pprint import pprint + +def stripArch(package): + return package.split(":", 2)[0] + +def checkDebianDistro(distro): + result = 0 + pkglist = DpkgList(distro) + packages = pkglist.parseFile() + tracker = DebianTracker(distro) + + for package in packages: + state = tracker.checkPackage(stripArch(package), packages[package]['version']) + if 'comparison' in state and state['comparison'] is not None and state['comparison'] < 0: + print("Package: {}, current: {}, latest: {}, comparison: {}".format(package,state['current'], state['latest'], state['comparison'])) + result = 1 + print("checked {} packages; result: {}".format(len(packages), result)) + + return result + +def main(argv): + for distro in argv[1:]: + #for distro in ['stretch']: + gitlab = GitLab() + job = gitlab.getLastSuccessfulJob('master', 'squashfs_master') + gitlab.downloadArtifact(job, 'images/debian-' + distro + '.dpkg-list', 'debian-' + distro + '.dpkg-list') + if checkDebianDistro(distro) > 0: + print("triggering build") + pprint(job.attributes) + job.play() diff --git a/security-scanner/setup.py b/security-scanner/setup.py new file mode 100644 index 0000000..19d4052 --- /dev/null +++ b/security-scanner/setup.py @@ -0,0 +1,15 @@ +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +config = { + 'name': 'security_scanner', + 'install_requires': [], + 'packages': [ + 'security_scanner', + ], + 'scripts': ['bin/security-scanner'] +} + +setup(**config)