Add security-scanner

security-scanner/.gitignore

@@ -0,0 +1,7 @@
+*.pyc
+*egg-info
+/venv*
+__pycache__
+/build
+/dist
+/data

security-scanner/Dockerfile

@@ -0,0 +1,19 @@
+FROM debian:stretch-slim
+
+RUN apt-get update \
+	&& apt-get -y install \
+		ca-certificates \
+		curl \
+		python3 \
+		python3-apt \
+		python3-pip \
+		python3-urllib3 \
+	&& rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /tmp/requirements.txt
+
+RUN pip3 install -r /tmp/requirements.txt
+
+ADD . /code
+
+RUN (cd /code && python3 setup.py install)

security-scanner/bin/security-scanner

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+import sys
+
+import security_scanner.main
+
+if __name__ == '__main__':
+    security_scanner.main.main(sys.argv)

security-scanner/requirements.txt

@@ -0,0 +1,2 @@
+python-gitlab==1.4.0
+urllib3==1.22

security-scanner/security_scanner/debian_tracker.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+import json
+import apt_pkg
+import urllib.request
+import os.path
+
+class DebianTracker(object):
+    _debian_security = {}
+    _distro = None
+
+    def __init__(self, distro=None, forceUpdate=False):
+        apt_pkg.init_system()
+
+        self._distro = distro
+        if forceUpdate or (not os.path.exists("debian-security.json")):
+            self.update()
+
+        with open('debian-security.json') as f:
+            self._debian_security = json.load(f)
+
+    def update(self):
+        urllib.request.urlretrieve("https://security-tracker.debian.org/tracker/data/json", "debian-security.json")
+
+    def getLatestVersion(self, package, distro=None):
+        if distro is None:
+            if self._distro is None:
+                raise ValueError('you must pass distro, either to the function or on creating the object')
+            else:
+                distro = self._distro
+
+        latest_version = None
+        if package in self._debian_security:
+            for cve in self._debian_security[package]:
+                if distro in self._debian_security[package][cve]['releases']:
+                    for repository in self._debian_security[package][cve]['releases'][distro]['repositories']:
+                        version = self._debian_security[package][cve]['releases'][distro]['repositories'][repository]
+                        if latest_version is not None:
+                            if apt_pkg.version_compare(version,latest_version) > 0:
+                                latest_version = version
+                        else:
+                             latest_version = version
+                else:
+                    return 'distro not found'
+
+            return latest_version
+        else:
+            return 'package not found'
+
+    def checkPackage(self, package, version, distro=None):
+        result = {
+                'current': version,
+                'latest': None,
+                'comparison': None,
+                'message': '',
+            }
+        latest_version = self.getLatestVersion(package, distro)
+        if latest_version == 'package not found' or latest_version == 'distro not found':
+            result['message'] = latest_version
+        else:
+            result['latest'] = latest_version
+            result['comparison'] = apt_pkg.version_compare(version, latest_version)
+        return result

security-scanner/security_scanner/dpkg_list.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import re
+import os.path
+#from pprint import pprint
+
+
+class DpkgList(object):
+    _distro = None
+
+    def __init__(self, distro):
+        self._distro = distro
+
+    def parseFile(self, path='.', filename=None, encoding="utf-8"):
+        if filename is None:
+            filename = 'debian-' + self._distro + '.dpkg-list'
+
+        dpkg_list = None
+        with open(os.path.join(path, filename), encoding=encoding) as f:
+            dpkg_list = f.readlines()
+
+        installed_packages = {}
+
+        for item in dpkg_list:
+            matches = re.match(r"ii\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$", item)
+            if matches is not None:
+                package = matches.group(1)
+                installed_packages[package] = {}
+                installed_packages[package]['version'] = matches.group(2)
+                installed_packages[package]['arch'] = matches.group(3)
+                installed_packages[package]['description'] = matches.group(4)
+
+        return installed_packages

security-scanner/security_scanner/file_writer.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+class FileWriter(object):
+    def __init__(self, filename):
+        self._fd = open(filename, 'wb')
+
+    def __call__(self, chunk):
+        self._fd.write(chunk)

security-scanner/security_scanner/gitlab.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+import gitlab
+import os
+
+from security_scanner.file_writer import FileWriter
+
+class GitLab:
+    def __init__(self):
+        gitlab_url = os.environ.get('GITLAB_URL')
+        api_token = os.environ.get('PRIVATE_TOKEN')
+        project_id = os.environ.get('CI_PROJECT_ID')
+
+        self._gl = gitlab.Gitlab(gitlab_url, private_token=api_token)
+        self._project = self._gl.projects.get(project_id)
+
+    def getLastSuccessfulJob(self, ref, name):
+        pipelines = self._project.pipelines.list()
+
+        successful_master_jobs = []
+        for pipeline in pipelines:
+            jobs = pipeline.jobs.list()
+            for job in jobs:
+                if job.ref == ref and job.attributes['name'] == name and job.attributes['status'] == 'success':
+                    successful_master_jobs.append(job)
+
+        job = successful_master_jobs[-1]
+
+        return job
+
+    def downloadArtifact(self, job, sourcePath, destPath):
+        job_id = job.attributes['id']
+        print("Downloading artifact {} for job #{}".format(sourcePath, job_id))
+        target = FileWriter(destPath)
+        artifact = self._project.jobs.get(job_id).artifact(sourcePath, streamed=True, action=target)
+        del(target)

security-scanner/security_scanner/main.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import sys
+
+
+import security_scanner
+from security_scanner.debian_tracker import DebianTracker
+from security_scanner.dpkg_list import DpkgList
+from security_scanner.gitlab import GitLab
+
+from pprint import pprint
+
+def stripArch(package):
+    return package.split(":", 2)[0]
+
+def checkDebianDistro(distro):
+    result = 0
+    pkglist = DpkgList(distro)
+    packages = pkglist.parseFile()
+    tracker = DebianTracker(distro)
+
+    for package in packages:
+        state = tracker.checkPackage(stripArch(package), packages[package]['version'])
+        if 'comparison' in state and state['comparison'] is not None and state['comparison'] < 0:
+            print("Package: {}, current: {}, latest: {}, comparison: {}".format(package,state['current'], state['latest'], state['comparison']))
+            result = 1
+    print("checked {} packages; result: {}".format(len(packages), result))
+
+    return result
+
+def main(argv):
+    for distro in argv[1:]:
+    #for distro in ['stretch']:
+        gitlab = GitLab()
+        job = gitlab.getLastSuccessfulJob('master', 'squashfs_master')
+        gitlab.downloadArtifact(job, 'images/debian-' + distro + '.dpkg-list', 'debian-' + distro + '.dpkg-list')
+        if checkDebianDistro(distro) > 0:
+            print("triggering build")
+            pprint(job.attributes)
+            job.play()

security-scanner/setup.py

@@ -0,0 +1,15 @@
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
+
+config = {
+    'name': 'security_scanner',
+    'install_requires': [],
+    'packages': [
+        'security_scanner',
+    ],
+    'scripts': ['bin/security-scanner']
+}
+
+setup(**config)