From a0e63fdb47067cee8975467310f9405df155f62e Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 25 Feb 2019 15:21:27 +0100
Subject: [PATCH 001/138] Update agent to v0.10

---
 cuckoo/data/agent/agent.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/cuckoo/data/agent/agent.py b/cuckoo/data/agent/agent.py
index 60df37bd42..1be1c170db 100644
--- a/cuckoo/data/agent/agent.py
+++ b/cuckoo/data/agent/agent.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (C) 2015-2017 Cuckoo Foundation.
+# Copyright (C) 2015-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -21,7 +21,7 @@
 import SimpleHTTPServer
 import SocketServer
 
-AGENT_VERSION = "0.9"
+AGENT_VERSION = "0.10"
 AGENT_FEATURES = [
     "execpy", "pinning", "logs", "largefile", "unicodepath",
 ]
@@ -36,9 +36,9 @@ def do_GET(self):
         request.client_ip, request.client_port = self.client_address
         request.form = {}
         request.files = {}
+        request.method = "GET"
 
-        if "client_ip" not in state or request.client_ip == state["client_ip"]:
-            self.httpd.handle(self)
+        self.httpd.handle(self)
 
     def do_POST(self):
         environ = {
@@ -53,6 +53,7 @@ def do_POST(self):
         request.client_ip, request.client_port = self.client_address
         request.form = {}
         request.files = {}
+        request.method = "POST"
 
         # Another pretty fancy workaround. Since we provide backwards
         # compatibility with the Old Agent we will get an xmlrpc request
@@ -68,8 +69,7 @@ def do_POST(self):
                 else:
                     request.form[key] = value.value.decode("utf8")
 
-        if "client_ip" not in state or request.client_ip == state["client_ip"]:
-            self.httpd.handle(self)
+        self.httpd.handle(self)
 
 class MiniHTTPServer(object):
     def __init__(self):
@@ -96,6 +96,12 @@ def register(fn):
         return register
 
     def handle(self, obj):
+        if "client_ip" in state and request.client_ip != state["client_ip"]:
+            if request.client_ip != "127.0.0.1":
+                return
+            if obj.path != "/status" or request.method != "POST":
+                return
+
         for route, fn in self.routes[obj.command]:
             if route.match(obj.path):
                 ret = fn()
@@ -166,6 +172,7 @@ class request(object):
     files = {}
     client_ip = None
     client_port = None
+    method = None
     environ = {
         "werkzeug.server.shutdown": lambda: app.shutdown(),
     }

From b8e859d5c14ced5939748f1fb0697557f8ea41f7 Mon Sep 17 00:00:00 2001
From: Tatsuya-hasegawa <>
Date: Tue, 2 Apr 2019 03:13:45 +0900
Subject: [PATCH 002/138] static analysis view was disable on WebUI if FSG2.0
 packer

---
 cuckoo/web/templates/analysis/pages/static/index.html | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cuckoo/web/templates/analysis/pages/static/index.html b/cuckoo/web/templates/analysis/pages/static/index.html
index 6c68e667dd..e284567a47 100644
--- a/cuckoo/web/templates/analysis/pages/static/index.html
+++ b/cuckoo/web/templates/analysis/pages/static/index.html
@@ -27,6 +27,12 @@ <h1>Static Analysis</h1>
                                 <div class="tab-pane fade in active" id="static_analysis_tab">
                                     {% if "PE32" in report.analysis.target.file.type %}
                                         {% include "analysis/pages/static/_pe32.html" %}
+                                    {% elif "MS-DOS executable" in report.analysis.target.file.type %}
+                    			{% for sig in report.analysis.static.peid_signatures %}
+                        			{% if forloop.first and "FSG" in sig %}
+                                                	{% include "analysis/pages/static/_pe32.html" %}
+                        			{% endif %}
+                    			{% endfor %}
                                     {% elif "ELF" in report.analysis.target.file.type %}
                                         {% include "analysis/pages/static/_elf.html" %}
                                     {% elif "office" in report.analysis.static %}
@@ -66,4 +72,4 @@ <h1>Static Analysis</h1>
         </section>
     </div>
 
-{% endblock %}
\ No newline at end of file
+{% endblock %}

From ab8aafeb4e9af33cf9ae3aceda23a93fa4b7efd7 Mon Sep 17 00:00:00 2001
From: LetMeR00t <alexandre.demeyer31@gmail.com>
Date: Thu, 18 Apr 2019 19:37:16 +0200
Subject: [PATCH 003/138] Fix an API call for recovering data from network
 analysis on Cuckoo web

---
 cuckoo/web/src/scripts/analysis_network.js      | 2 +-
 cuckoo/web/static/js/cuckoo/analysis_network.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cuckoo/web/src/scripts/analysis_network.js b/cuckoo/web/src/scripts/analysis_network.js
index e535a5377a..0bed44bc50 100644
--- a/cuckoo/web/src/scripts/analysis_network.js
+++ b/cuckoo/web/src/scripts/analysis_network.js
@@ -335,7 +335,7 @@ class RequestDisplay {
 
     	// this will later be replaced by the ajax call getting the content
 
-        CuckooWeb.post("/analysis/api/task/network_http_data/", {
+        CuckooWeb.api_post("/analysis/api/task/network_http_data/", {
             "task_id": window.task_id,
             "protocol": _this.protocol,
             "request_body": false,
diff --git a/cuckoo/web/static/js/cuckoo/analysis_network.js b/cuckoo/web/static/js/cuckoo/analysis_network.js
index e6d3d2c4a3..f07668b465 100644
--- a/cuckoo/web/static/js/cuckoo/analysis_network.js
+++ b/cuckoo/web/static/js/cuckoo/analysis_network.js
@@ -357,7 +357,7 @@ var RequestDisplay = function () {
 
             // this will later be replaced by the ajax call getting the content
 
-            CuckooWeb.post("/analysis/api/task/network_http_data/", {
+            CuckooWeb.api_post("/analysis/api/task/network_http_data/", {
                 "task_id": window.task_id,
                 "protocol": _this.protocol,
                 "request_body": false,

From 9a31a996a71cac7bfcdb3117d6692d25977fe573 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 25 Apr 2019 16:45:39 +0200
Subject: [PATCH 004/138] Introduce vulnerable dependency check

---
 cuckoo/common/abstracts.py          |  7 ++-
 cuckoo/common/config.py             |  3 +-
 cuckoo/common/utils.py              | 21 ++++++-
 cuckoo/core/startup.py              | 93 ++++++++++++++++++++++++++++-
 cuckoo/machinery/virtualbox.py      | 28 ++++++++-
 cuckoo/private/cwd/conf/cuckoo.conf |  5 ++
 6 files changed, 148 insertions(+), 9 deletions(-)

diff --git a/cuckoo/common/abstracts.py b/cuckoo/common/abstracts.py
index 1b66461561..d606153c43 100644
--- a/cuckoo/common/abstracts.py
+++ b/cuckoo/common/abstracts.py
@@ -1,5 +1,5 @@
 # Copyright (C) 2012-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2018 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -397,6 +397,11 @@ def _wait_status(self, label, *states):
             waitme += 1
             current = self._status(label)
 
+    @staticmethod
+    def version():
+        """Return the version of the virtualization software"""
+        return None
+
 class LibVirtMachinery(Machinery):
     """Libvirt based machine manager.
 
diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py
index 1ef5333ce0..5862206f80 100644
--- a/cuckoo/common/config.py
+++ b/cuckoo/common/config.py
@@ -1,5 +1,5 @@
 # Copyright (C) 2012-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2018 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -207,6 +207,7 @@ class Config(object):
         "cuckoo": {
             "cuckoo": {
                 "version_check": Boolean(True),
+                "ignore_vulnerabilities": Boolean(False, required=False),
                 "delete_original": Boolean(False),
                 "delete_bin_copy": Boolean(False),
                 "machinery": String("virtualbox"),
diff --git a/cuckoo/common/utils.py b/cuckoo/common/utils.py
index 2453df434d..97a3e98b8f 100644
--- a/cuckoo/common/utils.py
+++ b/cuckoo/common/utils.py
@@ -1,5 +1,5 @@
 # Copyright (C) 2012-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2018 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -11,7 +11,9 @@
 import jsbeautifier
 import json
 import logging
+import operator
 import os
+import pkg_resources
 import platform
 import re
 import string
@@ -20,7 +22,7 @@
 import warnings
 import xmlrpclib
 
-from distutils.version import StrictVersion
+from distutils.version import StrictVersion, LooseVersion
 
 from cuckoo.common.constants import GITHUB_URL, ISSUES_PAGE_URL
 from cuckoo.misc import cwd, version
@@ -262,7 +264,7 @@ def get_os_release():
 
     msg += "Modules: %s\n\n" % " ".join(sorted(
         "%s:%s" % (package.key, package.version)
-        for package in pip.get_installed_distributions()
+        for package in pkg_resources.working_set
     ))
     return msg
 
@@ -356,3 +358,16 @@ def list_of_ints(l):
 
 def list_of_strings(l):
     return list_of(l, basestring)
+
+def cmp_version(first, second, op):
+    op_lookup = {
+        ">": operator.gt,
+        "<": operator.lt,
+        ">=": operator.ge,
+        "<=": operator.le,
+        "!=": operator.ne,
+        "==": operator.eq
+    }
+    op = op_lookup.get(op)
+
+    return op(LooseVersion(first), LooseVersion(second))
diff --git a/cuckoo/core/startup.py b/cuckoo/core/startup.py
index 7cbc0bc338..ad99031204 100644
--- a/cuckoo/core/startup.py
+++ b/cuckoo/core/startup.py
@@ -3,22 +3,26 @@
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
+import errno
 import logging
 import logging.handlers
 import os
+import pkg_resources
 import requests
 import socket
+import sys
 import yara
 
-from distutils.version import StrictVersion
+from distutils.version import StrictVersion, LooseVersion
 
 import cuckoo
 
-from cuckoo.common.colors import red, green, yellow
+from cuckoo.common.colors import red, green, yellow, bold, color
 from cuckoo.common.config import Config, config, config2
 from cuckoo.common.exceptions import CuckooStartupError, CuckooFeedbackError
 from cuckoo.common.files import temppath
 from cuckoo.common.objects import File
+from cuckoo.common.utils import cmp_version
 from cuckoo.core.database import (
     Database, TASK_RUNNING, TASK_FAILED_ANALYSIS, TASK_PENDING
 )
@@ -27,7 +31,7 @@
 from cuckoo.core.log import init_logger
 from cuckoo.core.plugins import RunSignatures
 from cuckoo.core.rooter import rooter
-from cuckoo.misc import cwd, version, getuser, mkdir
+from cuckoo.misc import cwd, version, mkdir
 
 log = logging.getLogger(__name__)
 
@@ -116,6 +120,89 @@ def check_version():
     except ValueError:
         old = True
 
+    warnings = []
+    for deptype, vulns in r.get("vulnerable", {}).iteritems():
+        for dep in vulns:
+            compare = dep.get("highest") or dep.get("lowest")
+
+            # Check if any of the mentioned Python dependencies are installed
+            if deptype == "pydep":
+                try:
+                    v = pkg_resources.get_distribution(
+                        dep["name"]).parsed_version
+                except (pkg_resources.DistributionNotFound, ValueError):
+                    continue
+
+            # See if the mentioned virtualization software is used
+            elif deptype == "machinery":
+                if config("cuckoo:cuckoo:machinery") != dep["name"]:
+                    continue
+
+                # If the version number cannot be determined, raise a warning
+                # to be sure. Virtualization vulnerabilities can potentially
+                # cause a lot of damage
+                v = cuckoo.machinery.plugins[dep["name"]].version()
+                if not v:
+                    warnings.append(
+                        bold(red(
+                            "Potentially vulnerable %s version installed. "
+                            "Failed to retrieve its version. Update if version"
+                            " is: %s" % (dep["name"], compare))))
+                    continue
+
+            else:
+                continue
+
+            warn = False
+            # If a range is specified, check if the current version falls
+            # within the range.
+            if dep.get("highest") and dep.get("lowest"):
+                lv = LooseVersion(str(v))
+                if (lv >= LooseVersion(dep["lowest"]) and
+                        lv <= LooseVersion(dep["highest"])):
+                    warn = True
+
+            # If no range is specified, use the specified operator to see if
+            # the installed version is
+            # 'if <operator> highest/lowest specified'
+            elif cmp_version(str(v), compare, dep["op"]):
+                warn = True
+
+            # Warn the user the dependency must be updated/
+            if warn:
+                info = dep.get("info")
+                message = "Vulnerable version of %s installed (%s). It is " \
+                          "highly recommended to update. Please update and " \
+                          "restart Cuckoo." % (dep["name"], v)
+
+                if deptype == "pydep":
+                    message += " 'pip install %s%s'" % (
+                        dep["name"], dep["recommended"]
+                    )
+
+                else:
+                    message += " Recommended version: %s" % dep["recommended"]
+
+                message = bold(red(message))
+
+                if info:
+                    message += yellow("\nAdditional information: %s" % info)
+
+                warnings.append(message)
+
+    if warnings:
+        print(color(bold(red("Vulnerable dependencies found\n")), 5))
+    for warning in warnings:
+        print("--> %s\n" % color(warning, 4))
+
+    if warnings and not config("cuckoo:cuckoo:ignore_vulnerabilities"):
+        print(
+            "This check can be disabled by enabling "
+            "'ignore_vulnerabilities' in cuckoo.conf under the "
+            "[cuckoo] section"
+        )
+        sys.exit(1)
+
     if old:
         msg = "Cuckoo Sandbox version %s is available now." % r["version"]
         print(red(" Outdated! ") + msg)
diff --git a/cuckoo/machinery/virtualbox.py b/cuckoo/machinery/virtualbox.py
index e2ba53818a..9ed9e5ed93 100644
--- a/cuckoo/machinery/virtualbox.py
+++ b/cuckoo/machinery/virtualbox.py
@@ -1,5 +1,5 @@
 # Copyright (C) 2011-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2018 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -481,3 +481,29 @@ def _set_flag(self, label, key, val):
         )
         _, _ = proc.communicate()
         return proc
+
+    @staticmethod
+    def version():
+        """Get the version for the installed Virtualbox"""
+        try:
+            proc = Popen(
+                [config("virtualbox:virtualbox:path"), "--version"],
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True
+            )
+            output, err = proc.communicate()
+        except OSError:
+            return None
+
+        output = output.strip()
+        version = ""
+        for c in output:
+            if not c.isdigit() and c != ".":
+                break
+            version += c
+
+        # A 3 digit version number is expected. If it has none or fewer, return
+        # None because we are unsure what we have.
+        if len(version.split(".", 2)) < 3:
+            return None
+
+        return version
diff --git a/cuckoo/private/cwd/conf/cuckoo.conf b/cuckoo/private/cwd/conf/cuckoo.conf
index 554efe3d37..2d77890035 100644
--- a/cuckoo/private/cwd/conf/cuckoo.conf
+++ b/cuckoo/private/cwd/conf/cuckoo.conf
@@ -4,6 +4,11 @@
 # one available.
 version_check = {{ cuckoo.cuckoo.version_check }}
 
+# Cuckoo will stop at startup if the version check reports vulnerabilities in
+# one of Cuckoo's dependencies. This setting ignores the vulnerabilities
+# and starts anyway
+ignore_vulnerabilities = {{ cuckoo.cuckoo.ignore_vulnerabilities }}
+
 # The authentication token that is required to access the Cuckoo API, using
 # HTTP Bearer authentication. This will protect the API instance against
 # unauthorized access and CSRF attacks. It is strongly recommended to set this

From e1d2d03bb7bf730dfbda341606a7aa8beb2a46f4 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 29 Apr 2019 16:43:29 +0200
Subject: [PATCH 005/138] Pin dep to unbreak build

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f1f40c7168..91015ac05f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -44,7 +44,7 @@ before_install:
         sudo mysql cuckootestimport < tests/files/sql/11my.sql
         psql -U postgres -c "CREATE DATABASE cuckootestimport"
         psql -U postgres cuckootestimport <tests/files/sql/11pg.sql >/dev/null
-        pip install psycopg2 mysql-python m2crypto==0.24.0 weasyprint
+        pip install psycopg2 mysql-python m2crypto==0.24.0 weasyprint==0.36
     else
         brew update || brew update
         brew install libmagic cairo pango mongodb

From b3d3d37a9377e22babe491c12d196f20f34df336 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 29 Apr 2019 16:57:21 +0200
Subject: [PATCH 006/138] Unbreak signature test

---
 tests/test_signatures.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_signatures.py b/tests/test_signatures.py
index be7f599a83..276facab19 100644
--- a/tests/test_signatures.py
+++ b/tests/test_signatures.py
@@ -181,6 +181,8 @@ class sig3(sig):
         name = "sig3"
         order = 2
 
+    set_cwd(tempfile.mkdtemp())
+    cuckoo_create()
     with mock.patch("cuckoo.core.plugins.cuckoo") as p:
         p.signatures = sig1, sig2, sig3
         RunSignatures.init_once()
@@ -206,6 +208,8 @@ def __init__(self, caller):
         def on_signature(self, sig):
             pass
 
+    set_cwd(tempfile.mkdtemp())
+    cuckoo_create()
     with mock.patch("cuckoo.core.plugins.cuckoo") as p:
         p.signatures = sig,
         RunSignatures.init_once()

From 5744cd16db323c308fa727516502f185aac77d9d Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 29 Apr 2019 17:17:26 +0200
Subject: [PATCH 007/138] Update process_tasks test args

---
 tests/test_apps.py | 2 +-
 tests/test_log.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_apps.py b/tests/test_apps.py
index 0151c19217..f0f90e1cc2 100644
--- a/tests/test_apps.py
+++ b/tests/test_apps.py
@@ -407,7 +407,7 @@ def test_process_many(self, p, q):
             ("--cwd", cwd(), "process", "instance"),
             standalone_mode=False
         )
-        p.assert_called_once_with("instance", 0)
+        p.assert_called_once_with("instance", 0, 0)
         q.assert_called_once()
 
     @mock.patch("cuckoo.apps.apps.Database")
diff --git a/tests/test_log.py b/tests/test_log.py
index 8775194371..bf11672b43 100644
--- a/tests/test_log.py
+++ b/tests/test_log.py
@@ -76,7 +76,7 @@ def test_process_json_logging():
     init_yara()
     init_logfile("process-p0.json")
 
-    def process_tasks(instance, maxcount):
+    def process_tasks(instance, maxcount, timeout):
         logger("foo bar", action="hello.world", status="success")
 
     with mock.patch("cuckoo.main.Database"):

From 954ce3b7bac5296cf201197ac9b5c686e24ca9a2 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 29 Apr 2019 18:02:29 +0200
Subject: [PATCH 008/138] Update guac server tar url

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 91015ac05f..0412437ecc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,7 @@ before_install:
         sudo apt-get install libfreerdp-dev libjpeg-turbo8-dev
 
         # Install guacd.
-        wget https://www.apache.org/dist/guacamole/0.9.14/source/guacamole-server-0.9.14.tar.gz
+        wget http://archive.apache.org/dist/guacamole/0.9.14/source/guacamole-server-0.9.14.tar.gz
         tar xvf guacamole-server-0.9.14.tar.gz && cd guacamole-server-0.9.14
         ./configure --with-init-dir=/etc/init.d && make && sudo make install && sudo ldconfig && cd ../
 

From e5a56addec5c3a0c9b1e62bd85bf8b08e066d141 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 29 Apr 2019 18:12:43 +0200
Subject: [PATCH 009/138] Test sqlalchemy 1.3.3

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 1b0ca2efbe..fdfd5029ac 100755
--- a/setup.py
+++ b/setup.py
@@ -212,7 +212,7 @@ def do_setup(**kwargs):
         "python-magic==0.4.12",
         "roach>=0.1.2, <0.2",
         "sflock>=0.3.8, <0.4",
-        "sqlalchemy==1.0.8",
+        "sqlalchemy==1.3.3",
         "unicorn==1.0.1",
         "wakeonlan==0.2.2",
         "yara-python==3.6.3",

From 380dd984ab713bd0332eacdedf7ef0ea945bfc48 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 2 May 2019 19:04:08 +0200
Subject: [PATCH 010/138] Test travis and tests fix

---
 .travis.yml              |  9 +++-
 cuckoo/common/utils.py   |  3 +-
 cuckoo/reporting/misp.py | 18 +++++---
 tests/test_reporting.py  | 96 ++++++++++++++++++++++++++++++----------
 4 files changed, 93 insertions(+), 33 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0412437ecc..42e5c33057 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -68,7 +68,7 @@ install:
   - pip install -U pip setuptools
   - python setup.py sdist
   - pip install -e .
-  - pip install pytest pytest-cov codecov coveralls distorm3 pycrypto
+  - pip install pytest==4.1.1 pytest-cov codecov coveralls distorm3 pycrypto
   - pip install flask-testing mock pytest-django pytest-pythonpath responses
 
   # Install Volatility.
@@ -97,3 +97,10 @@ script:
 after_success:
   - coveralls
   - codecov
+
+addons:
+  apt:
+    packages:
+    - sqlite3
+    sources:
+    - travis-ci/sqlite3
diff --git a/cuckoo/common/utils.py b/cuckoo/common/utils.py
index fd8e6390a9..b574144e16 100644
--- a/cuckoo/common/utils.py
+++ b/cuckoo/common/utils.py
@@ -3,7 +3,6 @@
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
-import pkg_resources
 import base64
 import bs4
 import chardet
@@ -240,6 +239,8 @@ def get_os_release():
         else:
             return "Unknown"
 
+    import pkg_resources
+
     msg = (
         "Oops! Cuckoo failed in an unhandled exception!\nSometimes bugs are "
         "already fixed in the development release, it is therefore "
diff --git a/cuckoo/reporting/misp.py b/cuckoo/reporting/misp.py
index 7ac6af21f9..d92a5f5779 100644
--- a/cuckoo/reporting/misp.py
+++ b/cuckoo/reporting/misp.py
@@ -41,10 +41,12 @@ def all_urls(self, results, event):
 
     def domain_ipaddr(self, results, event):
         whitelist = [
-            "www.msftncsi.com", "dns.msftncsi.com", "teredo.ipv6.microsoft.com", "time.windows.com",
-            "www.msftconnecttest.com", "v10.vortex-win.data.microsoft.com","settings-win.data.microsoft.com", 
-            "win10.ipv6.microsoft.com", "sls.update.microsoft.com", "13.74.179.117", "40.81.120.221",
-            "40.77.226.249", "8.8.8.8", "fs.microsoft.com", "ctldl.windowsupdate.com"
+            "www.msftncsi.com", "dns.msftncsi.com", "8.8.8.8", "40.77.226.249",
+            "teredo.ipv6.microsoft.com", "time.windows.com",
+            "www.msftconnecttest.com", "v10.vortex-win.data.microsoft.com",
+            "settings-win.data.microsoft.com", "win10.ipv6.microsoft.com",
+            "sls.update.microsoft.com", "13.74.179.117", "40.81.120.221",
+            "fs.microsoft.com", "ctldl.windowsupdate.com"
         ]
 
         domains, ips = {}, set()
@@ -55,7 +57,7 @@ def domain_ipaddr(self, results, event):
 
         ipaddrs = set()
         for ipaddr in results.get("network", {}).get("hosts", []):
-            if ipaddr not in ips:
+            if ipaddr not in whitelist and ipaddr not in ips:
                 ipaddrs.add(ipaddr)
 
         self.misp.add_domains_ips(event, domains)
@@ -69,7 +71,7 @@ def family(self, results, event):
             for cnc in config.get("cnc", []):
                 self.misp.add_url(event, cnc)
             for url in config.get("url", []):
-                self.misp.add_url(event, cnc)
+                self.misp.add_url(event, url)
             for mutex in config.get("mutex", []):
                 self.misp.add_mutex(event, mutex)
             for user_agent in config.get("user_agent", []):
@@ -84,7 +86,9 @@ def signature(self, results, event):
                     log.warning("Description for %s is not found" % (att))
                     continue
 
-                self.misp.add_internal_comment(event, "TTP: %s, short: %s" % (att, description["short"]))
+                self.misp.add_internal_comment(
+                    event, "TTP: %s, short: %s" % (att, description["short"])
+                )
 
     def run(self, results):
         """Submits results to MISP.
diff --git a/tests/test_reporting.py b/tests/test_reporting.py
index e4bc902c70..5c3f592081 100644
--- a/tests/test_reporting.py
+++ b/tests/test_reporting.py
@@ -129,12 +129,12 @@ def test_empty_misp():
 
     with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
         rsps.add(
-            responses.GET, "https://misphost/servers/getVersion.json",
+            responses.GET, "https://misphost/servers/getPyMISPVersion.json",
             json={
-                "version": "2.4.56",
-                "perm_sync": True,
-            },
+                "version": "2.4.103"
+            }
         )
+
         rsps.add(
             responses.GET, "https://misphost/attributes/describeTypes.json",
             json={
@@ -177,31 +177,30 @@ def test_misp_sample_hashes():
         comment="File submitted to Cuckoo"
     )
 
-def test_misp_maldoc():
+def test_misp_signatures():
     r = MISP()
     r.misp = mock.MagicMock()
-    r.misp.add_url.return_value = None
+    r.misp.add_internal_comment.return_value = None
 
-    r.maldoc_network({
-        "signatures": [
-            {
-                "name": "foobar",
-            },
-            {
-                "name": "malicious_document_urls",
-                "marks": [
-                    {
-                        "category": "file",
-                    },
-                    {
-                        "category": "url",
-                        "ioc": "url_ioc",
+    r.signature({
+            "signatures": [
+                {
+                    "description": "Very signature",
+                    "ttp": {
+                        "T1045": {
+                            "short": "Short description",
+                            "long": "A longer description"
+                        }
                     }
-                ],
-            },
-        ],
+                }
+            ]
     }, "event")
-    r.misp.add_url.assert_called_once_with("event", ["url_ioc"])
+
+    assert r.misp.add_internal_comment.call_count == 2
+    r.misp.add_internal_comment.assert_has_calls([
+        mock.call("event", "Very signature - (T1045)"),
+        mock.call("event", "TTP: T1045, short: Short description")
+    ])
 
 def test_misp_all_urls():
     r = MISP()
@@ -252,10 +251,15 @@ def test_misp_domain_ipaddr():
                     "domain": "time.windows.com",
                     "ip": "1.2.3.4",
                 },
+                {
+                    "domain": "www.msftncsi.com",
+                    "ip": "95.101.2.42"
+                }
             ],
             "hosts": [
                 "2.3.4.5",
                 "3.4.5.6",
+                "8.8.8.8"
             ],
         },
     }, "event")
@@ -268,6 +272,50 @@ def test_misp_domain_ipaddr():
         "event", ["2.3.4.5", "3.4.5.6"],
     )
 
+def test_misp_family():
+    r = MISP()
+    r.misp = mock.MagicMock()
+    r.misp.add_detection_name.return_value = None
+    r.misp.add_url.return_value = None
+    r.misp.add_mutex.return_value = None
+    r.misp.add_useragent.return_value = None
+
+    r.family({
+        "metadata": {
+            "cfgextr": [
+                {
+                    "family": "3x4mpl3",
+                    "cnc": ["example.com/gate.php"]
+                },
+                {
+                    "family": "3x4mpl3_2",
+                    "url": ["http://example.org"]
+                },
+                {
+                    "family": "3x4mpl3_3",
+                    "mutex": ["@@@@@@"],
+                    "user_agent": ["M3mebr0wz0r V42"]
+                }
+            ]
+        }
+    }, "event")
+
+    assert r.misp.add_detection_name.call_count == 3
+    r.misp.add_detection_name.assert_has_calls([
+        mock.call("event", "3x4mpl3", "Sandbox detection"),
+        mock.call("event", "3x4mpl3_2", "Sandbox detection"),
+        mock.call("event", "3x4mpl3_3", "Sandbox detection")
+    ])
+
+    assert r.misp.add_url.call_count == 2
+    r.misp.add_url.assert_has_calls([
+        mock.call("event", "example.com/gate.php"),
+        mock.call("event", "http://example.org")
+    ])
+
+    r.misp.add_mutex.assert_called_once_with("event", "@@@@@@")
+    r.misp.add_useragent.assert_called_once_with("event", "M3mebr0wz0r V42")
+
 @mock.patch("cuckoo.reporting.mongodb.mongo")
 def test_mongodb_init_once_new(p):
     p.init.return_value = True

From 7a46b58ba22d0f2da5be7933b54e98f638669684 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 2 May 2019 23:25:53 +0200
Subject: [PATCH 011/138] Test dep upgrade

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index fdfd5029ac..c442d72a71 100755
--- a/setup.py
+++ b/setup.py
@@ -185,7 +185,7 @@ def do_setup(**kwargs):
         ],
     },
     install_requires=[
-        "alembic==0.8.8",
+        "alembic==1.0.10",
         "androguard==3.0.1",
         "beautifulsoup4==4.5.3",
         "chardet==2.3.0",
@@ -196,7 +196,7 @@ def do_setup(**kwargs):
         "egghatch>=0.2.3, <0.3",
         "elasticsearch==5.3.0",
         "flask==0.12.2",
-        "flask-sqlalchemy==2.1",
+        "flask-sqlalchemy==2.4.0",
         "httpreplay>=0.2.4, <0.3",
         "jinja2==2.9.6",
         "jsbeautifier==1.6.2",

From c3fc2cf7eca92a01288be4c4f0bb83bd260945bb Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 3 May 2019 13:42:09 +0200
Subject: [PATCH 012/138] Pin test dep versions and add fix for travis segfault

---
 .travis.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f1f40c7168..42e5c33057 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,7 @@ before_install:
         sudo apt-get install libfreerdp-dev libjpeg-turbo8-dev
 
         # Install guacd.
-        wget https://www.apache.org/dist/guacamole/0.9.14/source/guacamole-server-0.9.14.tar.gz
+        wget http://archive.apache.org/dist/guacamole/0.9.14/source/guacamole-server-0.9.14.tar.gz
         tar xvf guacamole-server-0.9.14.tar.gz && cd guacamole-server-0.9.14
         ./configure --with-init-dir=/etc/init.d && make && sudo make install && sudo ldconfig && cd ../
 
@@ -44,7 +44,7 @@ before_install:
         sudo mysql cuckootestimport < tests/files/sql/11my.sql
         psql -U postgres -c "CREATE DATABASE cuckootestimport"
         psql -U postgres cuckootestimport <tests/files/sql/11pg.sql >/dev/null
-        pip install psycopg2 mysql-python m2crypto==0.24.0 weasyprint
+        pip install psycopg2 mysql-python m2crypto==0.24.0 weasyprint==0.36
     else
         brew update || brew update
         brew install libmagic cairo pango mongodb
@@ -68,7 +68,7 @@ install:
   - pip install -U pip setuptools
   - python setup.py sdist
   - pip install -e .
-  - pip install pytest pytest-cov codecov coveralls distorm3 pycrypto
+  - pip install pytest==4.1.1 pytest-cov codecov coveralls distorm3 pycrypto
   - pip install flask-testing mock pytest-django pytest-pythonpath responses
 
   # Install Volatility.
@@ -97,3 +97,10 @@ script:
 after_success:
   - coveralls
   - codecov
+
+addons:
+  apt:
+    packages:
+    - sqlite3
+    sources:
+    - travis-ci/sqlite3

From 8b3f932ec4e0c72938680c3422cea527c9f04d26 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 3 May 2019 13:44:46 +0200
Subject: [PATCH 013/138] Only load pkg lib when required

---
 cuckoo/common/utils.py | 5 +++--
 cuckoo/core/startup.py | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/cuckoo/common/utils.py b/cuckoo/common/utils.py
index fd8e6390a9..0ac80d8cd3 100644
--- a/cuckoo/common/utils.py
+++ b/cuckoo/common/utils.py
@@ -1,9 +1,8 @@
 # Copyright (C) 2012-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2018 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
-import pkg_resources
 import base64
 import bs4
 import chardet
@@ -240,6 +239,8 @@ def get_os_release():
         else:
             return "Unknown"
 
+    import pkg_resources
+
     msg = (
         "Oops! Cuckoo failed in an unhandled exception!\nSometimes bugs are "
         "already fixed in the development release, it is therefore "
diff --git a/cuckoo/core/startup.py b/cuckoo/core/startup.py
index 7cbc0bc338..e7288bdbaf 100644
--- a/cuckoo/core/startup.py
+++ b/cuckoo/core/startup.py
@@ -3,6 +3,7 @@
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
+import errno
 import logging
 import logging.handlers
 import os

From 18f4a39520103ea70b1c0dc6b350ed386377296c Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 3 May 2019 13:46:19 +0200
Subject: [PATCH 014/138] Use whitelist for IPs and more misp tests

---
 cuckoo/reporting/misp.py | 18 +++++---
 tests/test_reporting.py  | 96 ++++++++++++++++++++++++++++++----------
 2 files changed, 83 insertions(+), 31 deletions(-)

diff --git a/cuckoo/reporting/misp.py b/cuckoo/reporting/misp.py
index 7ac6af21f9..d92a5f5779 100644
--- a/cuckoo/reporting/misp.py
+++ b/cuckoo/reporting/misp.py
@@ -41,10 +41,12 @@ def all_urls(self, results, event):
 
     def domain_ipaddr(self, results, event):
         whitelist = [
-            "www.msftncsi.com", "dns.msftncsi.com", "teredo.ipv6.microsoft.com", "time.windows.com",
-            "www.msftconnecttest.com", "v10.vortex-win.data.microsoft.com","settings-win.data.microsoft.com", 
-            "win10.ipv6.microsoft.com", "sls.update.microsoft.com", "13.74.179.117", "40.81.120.221",
-            "40.77.226.249", "8.8.8.8", "fs.microsoft.com", "ctldl.windowsupdate.com"
+            "www.msftncsi.com", "dns.msftncsi.com", "8.8.8.8", "40.77.226.249",
+            "teredo.ipv6.microsoft.com", "time.windows.com",
+            "www.msftconnecttest.com", "v10.vortex-win.data.microsoft.com",
+            "settings-win.data.microsoft.com", "win10.ipv6.microsoft.com",
+            "sls.update.microsoft.com", "13.74.179.117", "40.81.120.221",
+            "fs.microsoft.com", "ctldl.windowsupdate.com"
         ]
 
         domains, ips = {}, set()
@@ -55,7 +57,7 @@ def domain_ipaddr(self, results, event):
 
         ipaddrs = set()
         for ipaddr in results.get("network", {}).get("hosts", []):
-            if ipaddr not in ips:
+            if ipaddr not in whitelist and ipaddr not in ips:
                 ipaddrs.add(ipaddr)
 
         self.misp.add_domains_ips(event, domains)
@@ -69,7 +71,7 @@ def family(self, results, event):
             for cnc in config.get("cnc", []):
                 self.misp.add_url(event, cnc)
             for url in config.get("url", []):
-                self.misp.add_url(event, cnc)
+                self.misp.add_url(event, url)
             for mutex in config.get("mutex", []):
                 self.misp.add_mutex(event, mutex)
             for user_agent in config.get("user_agent", []):
@@ -84,7 +86,9 @@ def signature(self, results, event):
                     log.warning("Description for %s is not found" % (att))
                     continue
 
-                self.misp.add_internal_comment(event, "TTP: %s, short: %s" % (att, description["short"]))
+                self.misp.add_internal_comment(
+                    event, "TTP: %s, short: %s" % (att, description["short"])
+                )
 
     def run(self, results):
         """Submits results to MISP.
diff --git a/tests/test_reporting.py b/tests/test_reporting.py
index e4bc902c70..5c3f592081 100644
--- a/tests/test_reporting.py
+++ b/tests/test_reporting.py
@@ -129,12 +129,12 @@ def test_empty_misp():
 
     with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
         rsps.add(
-            responses.GET, "https://misphost/servers/getVersion.json",
+            responses.GET, "https://misphost/servers/getPyMISPVersion.json",
             json={
-                "version": "2.4.56",
-                "perm_sync": True,
-            },
+                "version": "2.4.103"
+            }
         )
+
         rsps.add(
             responses.GET, "https://misphost/attributes/describeTypes.json",
             json={
@@ -177,31 +177,30 @@ def test_misp_sample_hashes():
         comment="File submitted to Cuckoo"
     )
 
-def test_misp_maldoc():
+def test_misp_signatures():
     r = MISP()
     r.misp = mock.MagicMock()
-    r.misp.add_url.return_value = None
+    r.misp.add_internal_comment.return_value = None
 
-    r.maldoc_network({
-        "signatures": [
-            {
-                "name": "foobar",
-            },
-            {
-                "name": "malicious_document_urls",
-                "marks": [
-                    {
-                        "category": "file",
-                    },
-                    {
-                        "category": "url",
-                        "ioc": "url_ioc",
+    r.signature({
+            "signatures": [
+                {
+                    "description": "Very signature",
+                    "ttp": {
+                        "T1045": {
+                            "short": "Short description",
+                            "long": "A longer description"
+                        }
                     }
-                ],
-            },
-        ],
+                }
+            ]
     }, "event")
-    r.misp.add_url.assert_called_once_with("event", ["url_ioc"])
+
+    assert r.misp.add_internal_comment.call_count == 2
+    r.misp.add_internal_comment.assert_has_calls([
+        mock.call("event", "Very signature - (T1045)"),
+        mock.call("event", "TTP: T1045, short: Short description")
+    ])
 
 def test_misp_all_urls():
     r = MISP()
@@ -252,10 +251,15 @@ def test_misp_domain_ipaddr():
                     "domain": "time.windows.com",
                     "ip": "1.2.3.4",
                 },
+                {
+                    "domain": "www.msftncsi.com",
+                    "ip": "95.101.2.42"
+                }
             ],
             "hosts": [
                 "2.3.4.5",
                 "3.4.5.6",
+                "8.8.8.8"
             ],
         },
     }, "event")
@@ -268,6 +272,50 @@ def test_misp_domain_ipaddr():
         "event", ["2.3.4.5", "3.4.5.6"],
     )
 
+def test_misp_family():
+    r = MISP()
+    r.misp = mock.MagicMock()
+    r.misp.add_detection_name.return_value = None
+    r.misp.add_url.return_value = None
+    r.misp.add_mutex.return_value = None
+    r.misp.add_useragent.return_value = None
+
+    r.family({
+        "metadata": {
+            "cfgextr": [
+                {
+                    "family": "3x4mpl3",
+                    "cnc": ["example.com/gate.php"]
+                },
+                {
+                    "family": "3x4mpl3_2",
+                    "url": ["http://example.org"]
+                },
+                {
+                    "family": "3x4mpl3_3",
+                    "mutex": ["@@@@@@"],
+                    "user_agent": ["M3mebr0wz0r V42"]
+                }
+            ]
+        }
+    }, "event")
+
+    assert r.misp.add_detection_name.call_count == 3
+    r.misp.add_detection_name.assert_has_calls([
+        mock.call("event", "3x4mpl3", "Sandbox detection"),
+        mock.call("event", "3x4mpl3_2", "Sandbox detection"),
+        mock.call("event", "3x4mpl3_3", "Sandbox detection")
+    ])
+
+    assert r.misp.add_url.call_count == 2
+    r.misp.add_url.assert_has_calls([
+        mock.call("event", "example.com/gate.php"),
+        mock.call("event", "http://example.org")
+    ])
+
+    r.misp.add_mutex.assert_called_once_with("event", "@@@@@@")
+    r.misp.add_useragent.assert_called_once_with("event", "M3mebr0wz0r V42")
+
 @mock.patch("cuckoo.reporting.mongodb.mongo")
 def test_mongodb_init_once_new(p):
     p.init.return_value = True

From f130b47590d506e61356b09cc9dc51f8ab5cc282 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 3 May 2019 13:48:48 +0200
Subject: [PATCH 015/138] Update tests to test merged changes in their coverage

---
 tests/test_log.py        | 2 +-
 tests/test_signatures.py | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/test_log.py b/tests/test_log.py
index 8775194371..bf11672b43 100644
--- a/tests/test_log.py
+++ b/tests/test_log.py
@@ -76,7 +76,7 @@ def test_process_json_logging():
     init_yara()
     init_logfile("process-p0.json")
 
-    def process_tasks(instance, maxcount):
+    def process_tasks(instance, maxcount, timeout):
         logger("foo bar", action="hello.world", status="success")
 
     with mock.patch("cuckoo.main.Database"):
diff --git a/tests/test_signatures.py b/tests/test_signatures.py
index be7f599a83..276facab19 100644
--- a/tests/test_signatures.py
+++ b/tests/test_signatures.py
@@ -181,6 +181,8 @@ class sig3(sig):
         name = "sig3"
         order = 2
 
+    set_cwd(tempfile.mkdtemp())
+    cuckoo_create()
     with mock.patch("cuckoo.core.plugins.cuckoo") as p:
         p.signatures = sig1, sig2, sig3
         RunSignatures.init_once()
@@ -206,6 +208,8 @@ def __init__(self, caller):
         def on_signature(self, sig):
             pass
 
+    set_cwd(tempfile.mkdtemp())
+    cuckoo_create()
     with mock.patch("cuckoo.core.plugins.cuckoo") as p:
         p.signatures = sig,
         RunSignatures.init_once()

From a0a2955a0f8d405ec23c3255e951e3054eceb748 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 3 May 2019 13:49:22 +0200
Subject: [PATCH 016/138] Bump versions of pinned dependencies

---
 setup.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 1b0ca2efbe..c442d72a71 100755
--- a/setup.py
+++ b/setup.py
@@ -185,7 +185,7 @@ def do_setup(**kwargs):
         ],
     },
     install_requires=[
-        "alembic==0.8.8",
+        "alembic==1.0.10",
         "androguard==3.0.1",
         "beautifulsoup4==4.5.3",
         "chardet==2.3.0",
@@ -196,7 +196,7 @@ def do_setup(**kwargs):
         "egghatch>=0.2.3, <0.3",
         "elasticsearch==5.3.0",
         "flask==0.12.2",
-        "flask-sqlalchemy==2.1",
+        "flask-sqlalchemy==2.4.0",
         "httpreplay>=0.2.4, <0.3",
         "jinja2==2.9.6",
         "jsbeautifier==1.6.2",
@@ -212,7 +212,7 @@ def do_setup(**kwargs):
         "python-magic==0.4.12",
         "roach>=0.1.2, <0.2",
         "sflock>=0.3.8, <0.4",
-        "sqlalchemy==1.0.8",
+        "sqlalchemy==1.3.3",
         "unicorn==1.0.1",
         "wakeonlan==0.2.2",
         "yara-python==3.6.3",

From 9eb134ef9f75934e7596f6ebb4e17b41222a114d Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 3 May 2019 16:08:55 +0200
Subject: [PATCH 017/138] Remove unused import

---
 cuckoo/common/utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cuckoo/common/utils.py b/cuckoo/common/utils.py
index 63ab5a1b2e..d39a6a8462 100644
--- a/cuckoo/common/utils.py
+++ b/cuckoo/common/utils.py
@@ -13,7 +13,6 @@
 import logging
 import operator
 import os
-import pkg_resources
 import platform
 import re
 import string

From 2d52b89c8edcb53299aa85b876a0765e82377881 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 3 May 2019 16:09:44 +0200
Subject: [PATCH 018/138] Parse bools as they can no longer be '0' or '1'

---
 cuckoo/core/database.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/cuckoo/core/database.py b/cuckoo/core/database.py
index 07ef8c4524..2cd53c33b0 100644
--- a/cuckoo/core/database.py
+++ b/cuckoo/core/database.py
@@ -16,7 +16,7 @@
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.common.exceptions import CuckooDependencyError
 from cuckoo.common.objects import File, URL, Dictionary
-from cuckoo.common.utils import Singleton, classlock, json_encode
+from cuckoo.common.utils import Singleton, classlock, json_encode, parse_bool
 from cuckoo.misc import cwd, format_command
 
 from sqlalchemy import create_engine, Column, not_, func
@@ -1036,6 +1036,16 @@ def add(self, obj, timeout=0, package="", options="", priority=1,
         if not priority:
             priority = 1
 
+        try:
+            memory = parse_bool(memory)
+        except ValueError:
+            memory = False
+
+        try:
+            enforce_timeout = parse_bool(enforce_timeout)
+        except ValueError:
+            enforce_timeout = False
+
         if isinstance(obj, File):
             sample = Sample(md5=obj.get_md5(),
                             crc32=obj.get_crc32(),

From 1341162ae3ff9af147134809579d4c96514bcc0d Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 3 May 2019 20:00:12 +0200
Subject: [PATCH 019/138] Move import

---
 cuckoo/core/startup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cuckoo/core/startup.py b/cuckoo/core/startup.py
index ad99031204..891c0a2e5c 100644
--- a/cuckoo/core/startup.py
+++ b/cuckoo/core/startup.py
@@ -7,7 +7,6 @@
 import logging
 import logging.handlers
 import os
-import pkg_resources
 import requests
 import socket
 import sys
@@ -102,6 +101,8 @@ def check_version():
     if not config("cuckoo:cuckoo:version_check"):
         return
 
+    import pkg_resources
+
     print(" Checking for updates...")
 
     try:

From 4db3130e1a18d90649e0a61086fb06062bdfe24c Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 13 May 2019 11:09:24 +0200
Subject: [PATCH 020/138] Parse bools in task creation

---
 cuckoo/core/database.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/cuckoo/core/database.py b/cuckoo/core/database.py
index 07ef8c4524..553b8b3605 100644
--- a/cuckoo/core/database.py
+++ b/cuckoo/core/database.py
@@ -1,5 +1,5 @@
 # Copyright (C) 2012-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2018 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -16,7 +16,7 @@
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.common.exceptions import CuckooDependencyError
 from cuckoo.common.objects import File, URL, Dictionary
-from cuckoo.common.utils import Singleton, classlock, json_encode
+from cuckoo.common.utils import Singleton, classlock, json_encode, parse_bool
 from cuckoo.misc import cwd, format_command
 
 from sqlalchemy import create_engine, Column, not_, func
@@ -1036,6 +1036,16 @@ def add(self, obj, timeout=0, package="", options="", priority=1,
         if not priority:
             priority = 1
 
+        try:
+            memory = parse_bool(memory)
+        except ValueError:
+            memory = False
+
+        try:
+            enforce_timeout = parse_bool(enforce_timeout)
+        except ValueError:
+            enforce_timeout = False
+
         if isinstance(obj, File):
             sample = Sample(md5=obj.get_md5(),
                             crc32=obj.get_crc32(),

From d660a8c42338dd345e1b6a49abf6057663ba3ad4 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 13 May 2019 11:54:42 +0200
Subject: [PATCH 021/138] Move import

---
 cuckoo/core/startup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cuckoo/core/startup.py b/cuckoo/core/startup.py
index ad99031204..891c0a2e5c 100644
--- a/cuckoo/core/startup.py
+++ b/cuckoo/core/startup.py
@@ -7,7 +7,6 @@
 import logging
 import logging.handlers
 import os
-import pkg_resources
 import requests
 import socket
 import sys
@@ -102,6 +101,8 @@ def check_version():
     if not config("cuckoo:cuckoo:version_check"):
         return
 
+    import pkg_resources
+
     print(" Checking for updates...")
 
     try:

From d3f94d053bd6a2058c643f5f040f60b8ad672ae5 Mon Sep 17 00:00:00 2001
From: LetMeR00t <alexandre.demeyer31@gmail.com>
Date: Thu, 18 Apr 2019 19:37:16 +0200
Subject: [PATCH 022/138] Fix an API call for recovering data from network
 analysis on Cuckoo web

---
 cuckoo/web/src/scripts/analysis_network.js      | 2 +-
 cuckoo/web/static/js/cuckoo/analysis_network.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cuckoo/web/src/scripts/analysis_network.js b/cuckoo/web/src/scripts/analysis_network.js
index e535a5377a..0bed44bc50 100644
--- a/cuckoo/web/src/scripts/analysis_network.js
+++ b/cuckoo/web/src/scripts/analysis_network.js
@@ -335,7 +335,7 @@ class RequestDisplay {
 
     	// this will later be replaced by the ajax call getting the content
 
-        CuckooWeb.post("/analysis/api/task/network_http_data/", {
+        CuckooWeb.api_post("/analysis/api/task/network_http_data/", {
             "task_id": window.task_id,
             "protocol": _this.protocol,
             "request_body": false,
diff --git a/cuckoo/web/static/js/cuckoo/analysis_network.js b/cuckoo/web/static/js/cuckoo/analysis_network.js
index e6d3d2c4a3..f07668b465 100644
--- a/cuckoo/web/static/js/cuckoo/analysis_network.js
+++ b/cuckoo/web/static/js/cuckoo/analysis_network.js
@@ -357,7 +357,7 @@ var RequestDisplay = function () {
 
             // this will later be replaced by the ajax call getting the content
 
-            CuckooWeb.post("/analysis/api/task/network_http_data/", {
+            CuckooWeb.api_post("/analysis/api/task/network_http_data/", {
                 "task_id": window.task_id,
                 "protocol": _this.protocol,
                 "request_body": false,

From 1ff6d95d3ec7c08d5cb2c6e4dfddac3f1163acfc Mon Sep 17 00:00:00 2001
From: Tatsuya-hasegawa <>
Date: Tue, 2 Apr 2019 03:13:45 +0900
Subject: [PATCH 023/138] static analysis view was disable on WebUI if FSG2.0
 packer

---
 cuckoo/web/templates/analysis/pages/static/index.html | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cuckoo/web/templates/analysis/pages/static/index.html b/cuckoo/web/templates/analysis/pages/static/index.html
index 6c68e667dd..e284567a47 100644
--- a/cuckoo/web/templates/analysis/pages/static/index.html
+++ b/cuckoo/web/templates/analysis/pages/static/index.html
@@ -27,6 +27,12 @@ <h1>Static Analysis</h1>
                                 <div class="tab-pane fade in active" id="static_analysis_tab">
                                     {% if "PE32" in report.analysis.target.file.type %}
                                         {% include "analysis/pages/static/_pe32.html" %}
+                                    {% elif "MS-DOS executable" in report.analysis.target.file.type %}
+                    			{% for sig in report.analysis.static.peid_signatures %}
+                        			{% if forloop.first and "FSG" in sig %}
+                                                	{% include "analysis/pages/static/_pe32.html" %}
+                        			{% endif %}
+                    			{% endfor %}
                                     {% elif "ELF" in report.analysis.target.file.type %}
                                         {% include "analysis/pages/static/_elf.html" %}
                                     {% elif "office" in report.analysis.static %}
@@ -66,4 +72,4 @@ <h1>Static Analysis</h1>
         </section>
     </div>
 
-{% endblock %}
\ No newline at end of file
+{% endblock %}

From d168984fc05c35915563d3839d9d34fbfe0e89c2 Mon Sep 17 00:00:00 2001
From: ihatecsv <me@drakeluce.com>
Date: Wed, 8 May 2019 17:19:05 -0300
Subject: [PATCH 024/138] Fixed valid docstrings to comply with PEP 257

---
 cuckoo/apps/api.py                            |  6 +-
 cuckoo/apps/apps.py                           |  4 +-
 cuckoo/apps/rooter.py                         |  4 +-
 cuckoo/auxiliary/replay.py                    |  2 +-
 cuckoo/auxiliary/services.py                  |  2 +-
 cuckoo/common/abstracts.py                    | 82 +++++++++----------
 cuckoo/common/config.py                       |  6 +-
 cuckoo/common/dns.py                          |  6 +-
 cuckoo/common/files.py                        |  6 +-
 cuckoo/common/netlog.py                       |  8 +-
 cuckoo/common/scripting.py                    |  2 +-
 cuckoo/common/utils.py                        | 16 ++--
 cuckoo/core/database.py                       | 54 ++++++------
 cuckoo/core/feedback.py                       |  4 +-
 cuckoo/core/guest.py                          |  2 +-
 cuckoo/core/init.py                           |  2 +-
 cuckoo/core/plugins.py                        |  4 +-
 cuckoo/core/scheduler.py                      |  2 +-
 cuckoo/core/startup.py                        | 12 +--
 cuckoo/core/submit.py                         |  8 +-
 .../analyzer/darwin/lib/api/screenshot.py     |  2 +-
 .../analyzer/darwin/lib/common/hashing.py     |  2 +-
 .../analyzer/darwin/modules/packages/zip.py   |  2 +-
 .../analyzer/linux/lib/common/abstracts.py    |  2 +-
 .../data/analyzer/linux/lib/common/hashing.py |  2 +-
 .../analyzer/linux/modules/auxiliary/stap.py  |  2 +-
 cuckoo/data/analyzer/windows/analyzer.py      | 10 +--
 .../data/analyzer/windows/lib/api/process.py  |  2 +-
 .../analyzer/windows/lib/common/abstracts.py  |  6 +-
 .../analyzer/windows/lib/common/hashing.py    |  2 +-
 cuckoo/data/analyzer/windows/lib/core/pipe.py | 12 +--
 .../windows/modules/auxiliary/disguise.py     |  2 +-
 .../windows/modules/auxiliary/reboot.py       |  2 +-
 .../windows/modules/auxiliary/recentfiles.py  |  2 +-
 .../windows/modules/auxiliary/zer0m0n.py      |  2 +-
 .../analyzer/windows/modules/packages/zip.py  |  2 +-
 cuckoo/machinery/avd.py                       | 22 ++---
 cuckoo/machinery/esx.py                       |  2 +-
 cuckoo/machinery/physical.py                  | 10 +--
 cuckoo/machinery/qemu.py                      |  6 +-
 cuckoo/machinery/virtualbox.py                | 14 ++--
 cuckoo/machinery/vmware.py                    | 10 +--
 cuckoo/machinery/vsphere.py                   |  2 +-
 cuckoo/machinery/xenserver.py                 |  4 +-
 cuckoo/main.py                                |  8 +-
 cuckoo/misc.py                                | 14 ++--
 cuckoo/processing/apkinfo.py                  |  2 +-
 cuckoo/processing/baseline.py                 |  8 +-
 cuckoo/processing/behavior.py                 |  2 +-
 cuckoo/processing/irma.py                     | 18 ++--
 cuckoo/processing/memory.py                   |  2 +-
 cuckoo/processing/network.py                  | 32 ++++----
 cuckoo/processing/procmemory.py               |  2 +-
 cuckoo/processing/procmon.py                  |  4 +-
 cuckoo/processing/static.py                   | 12 +--
 cuckoo/processing/virustotal.py               |  6 +-
 cuckoo/reporting/elasticsearch.py             |  2 +-
 cuckoo/reporting/feedback.py                  |  2 +-
 cuckoo/reporting/jsondump.py                  |  6 +-
 cuckoo/reporting/misp.py                      |  2 +-
 cuckoo/reporting/mongodb.py                   |  4 +-
 cuckoo/reporting/notification.py              |  2 +-
 cuckoo/reporting/singlefile.py                |  6 +-
 .../analysis/templatetags/analysis_tags.py    |  2 +-
 cuckoo/web/analysis/views.py                  |  2 +-
 cuckoo/web/controllers/analysis/analysis.py   |  2 +-
 cuckoo/web/controllers/analysis/api.py        |  6 +-
 .../web/controllers/analysis/export/export.py |  2 +-
 cuckoo/web/controllers/cuckoo/api.py          |  4 +-
 cuckoo/web/controllers/machines/api.py        |  4 +-
 stuff/vpncheck.py                             |  2 +-
 tests/test_database.py                        |  4 +-
 tests/test_init.py                            |  6 +-
 tests/test_log.py                             |  2 +-
 tests/test_misc.py                            |  2 +-
 tests/test_reporting.py                       |  2 +-
 tests/test_submit.py                          |  6 +-
 tests/test_utils.py                           | 14 ++--
 tests/utils.py                                |  2 +-
 79 files changed, 276 insertions(+), 276 deletions(-)

diff --git a/cuckoo/apps/api.py b/cuckoo/apps/api.py
index 18d66c1f39..3ddb6fb6e0 100644
--- a/cuckoo/apps/api.py
+++ b/cuckoo/apps/api.py
@@ -38,7 +38,7 @@ def json_error(status_code, message):
     return r
 
 def shutdown_server():
-    """Shutdown API werkzeug server"""
+    """Shutdown API werkzeug server."""
     shutdown = request.environ.get("werkzeug.server.shutdown")
     if shutdown:
         shutdown()
@@ -676,8 +676,8 @@ def vpn_status():
 
 @app.route("/exit")
 def exit_api():
-    """Shuts down the server if in debug mode and
-    using the werkzeug server"""
+    """Shut down the server if in debug mode and
+    using the werkzeug server."""
     if not app.debug:
         return json_error(403, "This call can only be used in debug mode")
 
diff --git a/cuckoo/apps/apps.py b/cuckoo/apps/apps.py
index 75aad720a4..467d0df624 100644
--- a/cuckoo/apps/apps.py
+++ b/cuckoo/apps/apps.py
@@ -99,7 +99,7 @@ def fetch_community(branch="master", force=False, filepath=None):
             open(filepath, "wb").write(t.extractfile(member).read())
 
 def enumerate_files(path, pattern):
-    """Yields all filepaths from a directory."""
+    """Yield all filepaths from a directory."""
     if os.path.isfile(path):
         yield path
     elif os.path.isdir(path):
@@ -346,7 +346,7 @@ def process_tasks(instance, maxcount, timeout):
 
 def cuckoo_clean():
     """Clean up cuckoo setup.
-    It deletes logs, all stored data from file system and configured
+    Delete logs, all stored data from file system and configured
     databases (SQL and MongoDB).
     """
     # Init logging (without writing to file).
diff --git a/cuckoo/apps/rooter.py b/cuckoo/apps/rooter.py
index 838c0b7471..a09bb1230b 100644
--- a/cuckoo/apps/rooter.py
+++ b/cuckoo/apps/rooter.py
@@ -59,7 +59,7 @@ def rt_available(rt_table):
         return False
 
 def vpn_status():
-    """Gets current VPN status."""
+    """Get current VPN status."""
     ret = {}
     for line in run(s.service, "openvpn", "status")[0].split("\n"):
         x = re.search("'(?P<vpn>\\w+)'\\ is\\ (?P<running>not)?", line)
@@ -125,7 +125,7 @@ def init_rttable(rt_table, interface):
         run(s.ip, *args)
 
 def flush_rttable(rt_table):
-    """Flushes specified routing table entries."""
+    """Flush specified routing table entries."""
     if rt_table in ["local", "main", "default"]:
         return
 
diff --git a/cuckoo/auxiliary/replay.py b/cuckoo/auxiliary/replay.py
index dc41deea35..984dd11a5e 100644
--- a/cuckoo/auxiliary/replay.py
+++ b/cuckoo/auxiliary/replay.py
@@ -24,7 +24,7 @@ def __init__(self):
         self.port = None
 
     def pcap2mitm(self, pcappath, tlsmaster):
-        """Used to translate a .pcap into a .mitm file."""
+        """Translate a .pcap into a .mitm file."""
         mitmpath = tempfile.mktemp(suffix=".mitm")
         with open(mitmpath, "wb") as f:
             httpreplay.utils.pcap2mitm(pcappath, f, tlsmaster, True)
diff --git a/cuckoo/auxiliary/services.py b/cuckoo/auxiliary/services.py
index e79f5ea2f8..55517e0e84 100644
--- a/cuckoo/auxiliary/services.py
+++ b/cuckoo/auxiliary/services.py
@@ -14,7 +14,7 @@
 db = Database()
 
 class Services(Auxiliary):
-    """Allows one or more additional VMs to be run next to an analysis. Either
+    """Allow one or more additional VMs to be run next to an analysis. Either
     as global services (which are generally never rebooted) or on a
     per-analysis basis."""
 
diff --git a/cuckoo/common/abstracts.py b/cuckoo/common/abstracts.py
index 8c5edc1811..f03be8f834 100644
--- a/cuckoo/common/abstracts.py
+++ b/cuckoo/common/abstracts.py
@@ -166,7 +166,7 @@ def init_once(cls):
         pass
 
     def pcap_path(self, task_id):
-        """Returns the .pcap path for this task id."""
+        """Return the .pcap path for this task id."""
         return cwd("storage", "analyses", "%s" % task_id, "dump.pcap")
 
     def set_options(self, options):
@@ -228,7 +228,7 @@ def _initialize(self, module_name):
             )
 
     def _initialize_check(self):
-        """Runs checks against virtualization software when a machine manager
+        """Run checks against virtualization software when a machine manager
         is initialized.
         @note: in machine manager modules you may override or superclass
                his method.
@@ -271,7 +271,7 @@ def machines(self):
         return self.db.list_machines()
 
     def availables(self):
-        """How many machines are free.
+        """Return how many machines are free.
         @return: free machines count.
         """
         return self.db.count_machines_available()
@@ -297,13 +297,13 @@ def release(self, label=None):
         self.db.unlock_machine(label)
 
     def running(self):
-        """Returns running virtual machines.
+        """Return running virtual machines.
         @return: running virtual machines list.
         """
         return self.db.list_machines(locked=True)
 
     def shutdown(self):
-        """Shutdown the machine manager. Kills all alive machines.
+        """Shutdown the machine manager and kill all alive machines.
         @raise CuckooMachineError: if unable to stop machine.
         """
         if len(self.running()) > 0:
@@ -339,13 +339,13 @@ def stop(self, label=None):
         raise NotImplementedError
 
     def _list(self):
-        """Lists virtual machines configured.
+        """List virtual machines configured.
         @raise NotImplementedError: this method is abstract.
         """
         raise NotImplementedError
 
     def dump_memory(self, label, path):
-        """Takes a memory dump of a machine.
+        """Take a memory dump of a machine.
         @param path: path to where to store the memory dump.
         """
         raise NotImplementedError
@@ -372,7 +372,7 @@ def get_remote_control_params(self, label):
         raise NotImplementedError
 
     def _wait_status(self, label, *states):
-        """Waits for a vm status.
+        """Wait for a vm status.
         @param label: virtual machine name.
         @param state: virtual machine status, accepts multiple states as list.
         @raise CuckooMachineError: if default waiting timeout expire.
@@ -433,7 +433,7 @@ def initialize(self, module):
         super(LibVirtMachinery, self).initialize(module)
 
     def _initialize_check(self):
-        """Runs all checks when a machine manager is initialized.
+        """Run all checks when a machine manager is initialized.
         @raise CuckooMachineError: if libvirt version is not supported.
         """
         # Version checks.
@@ -449,7 +449,7 @@ def _initialize_check(self):
         super(LibVirtMachinery, self)._initialize_check()
 
     def start(self, label, task):
-        """Starts a virtual machine.
+        """Start a virtual machine.
         @param label: virtual machine name.
         @param task: task object.
         @raise CuckooMachineError: if unable to start virtual machine.
@@ -502,7 +502,7 @@ def start(self, label, task):
         self._wait_status(label, self.RUNNING)
 
     def stop(self, label):
-        """Stops a virtual machine. Kill them all.
+        """Stop a virtual machine. Kill them all.
         @param label: virtual machine name.
         @raise CuckooMachineError: if unable to stop virtual machine.
         """
@@ -536,7 +536,7 @@ def shutdown(self):
         self.vms = None
 
     def dump_memory(self, label, path):
-        """Takes a memory dump.
+        """Take a memory dump.
         @param path: path to where to store the memory dump.
         """
         log.debug("Dumping memory for machine %s", label)
@@ -555,7 +555,7 @@ def dump_memory(self, label, path):
             self._disconnect(conn)
 
     def _status(self, label):
-        """Gets current status of a vm.
+        """Get current status of a vm.
         @param label: virtual machine name.
         @return: status string.
         """
@@ -600,7 +600,7 @@ def _status(self, label):
                                      "{0}".format(label))
 
     def _connect(self):
-        """Connects to libvirt subsystem.
+        """Connect to libvirt subsystem.
         @raise CuckooMachineError: when unable to connect to libvirt.
         """
         # Check if a connection string is available.
@@ -614,7 +614,7 @@ def _connect(self):
             raise CuckooMachineError("Cannot connect to libvirt")
 
     def _disconnect(self, conn):
-        """Disconnects to libvirt subsystem.
+        """Disconnect from libvirt subsystem.
         @raise CuckooMachineError: if cannot disconnect from libvirt.
         """
         try:
@@ -870,7 +870,7 @@ def init_once(cls):
         pass
 
     def _check_value(self, pattern, subject, regex=False, all=False):
-        """Checks a pattern against a given subject.
+        """Check a pattern against a given subject.
         @param pattern: string or expression to check for.
         @param subject: target of the check.
         @param regex: boolean representing if the pattern is a regular
@@ -995,7 +995,7 @@ def get_keys(self, pid=None, actions=None):
 
     def check_file(self, pattern, regex=False, actions=None, pid=None,
                    all=False):
-        """Checks for a file being opened.
+        """Check for a file being opened.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1018,7 +1018,7 @@ def check_file(self, pattern, regex=False, actions=None, pid=None,
 
     def check_dll_loaded(self, pattern, regex=False, actions=None, pid=None,
                          all=False):
-        """Checks for DLLs being loaded.
+        """Check for DLLs being loaded.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1032,7 +1032,7 @@ def check_dll_loaded(self, pattern, regex=False, actions=None, pid=None,
                                  all=all)
 
     def check_command_line(self, pattern, regex=False, all=False):
-        """Checks for a command line being opened.
+        """Check for a command line being opened.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1045,7 +1045,7 @@ def check_command_line(self, pattern, regex=False, all=False):
 
     def check_key(self, pattern, regex=False, actions=None, pid=None,
                   all=False):
-        """Checks for a registry key being accessed.
+        """Check for a registry key being accessed.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1073,7 +1073,7 @@ def get_mutexes(self, pid=None):
         return self.get_summary_generic(pid, ["mutex"])
 
     def check_mutex(self, pattern, regex=False, all=False):
-        """Checks for a mutex being opened.
+        """Check for a mutex being opened.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1085,11 +1085,11 @@ def check_mutex(self, pattern, regex=False, all=False):
                                  all=all)
 
     def get_command_lines(self):
-        """Retrieves all command lines used."""
+        """Retrieve all command lines used."""
         return self.get_summary("command_line")
 
     def get_wmi_queries(self):
-        """Retrieves all executed WMI queries."""
+        """Retrieve all executed WMI queries."""
         return self.get_summary("wmi_query")
 
     def get_net_generic(self, subtype):
@@ -1100,68 +1100,68 @@ def get_net_generic(self, subtype):
         return self.get_results("network", {}).get(subtype, [])
 
     def get_net_hosts(self):
-        """Returns a list of all hosts."""
+        """Return a list of all hosts."""
         return self.get_net_generic("hosts")
 
     def get_net_domains(self):
-        """Returns a list of all domains."""
+        """Return a list of all domains."""
         return self.get_net_generic("domains")
 
     def get_net_http(self):
-        """Returns a list of all http data."""
+        """Return a list of all http data."""
         return self.get_net_generic("http")
 
     def get_net_http_ex(self):
-        """Returns a list of all http data."""
+        """Return a list of all http data."""
         return \
             self.get_net_generic("http_ex") + self.get_net_generic("https_ex")
 
     def get_net_udp(self):
-        """Returns a list of all udp data."""
+        """Return a list of all udp data."""
         return self.get_net_generic("udp")
 
     def get_net_icmp(self):
-        """Returns a list of all icmp data."""
+        """Return a list of all icmp data."""
         return self.get_net_generic("icmp")
 
     def get_net_irc(self):
-        """Returns a list of all irc data."""
+        """Return a list of all irc data."""
         return self.get_net_generic("irc")
 
     def get_net_smtp(self):
-        """Returns a list of all smtp data."""
+        """Return a list of all smtp data."""
         return self.get_net_generic("smtp")
 
     def get_net_smtp_ex(self):
-        """"Returns a list of all smtp data"""
+        """"Return a list of all smtp data"""
         return self.get_net_generic("smtp_ex")
 
     def get_virustotal(self):
-        """Returns the information retrieved from virustotal."""
+        """Return the information retrieved from virustotal."""
         return self.get_results("virustotal", {})
 
     def get_volatility(self, module=None):
-        """Returns the data that belongs to the given module."""
+        """Return the data that belongs to the given module."""
         volatility = self.get_results("memory", {})
         return volatility if module is None else volatility.get(module, {})
 
     def get_apkinfo(self, section=None, default={}):
-        """Returns the apkinfo results for this analysis."""
+        """Return the apkinfo results for this analysis."""
         apkinfo = self.get_results("apkinfo", {})
         return apkinfo if section is None else apkinfo.get(section, default)
 
     def get_droidmon(self, section=None, default={}):
-        """Returns the droidmon results for this analysis."""
+        """Return the droidmon results for this analysis."""
         droidmon = self.get_results("droidmon", {})
         return droidmon if section is None else droidmon.get(section, default)
 
     def get_googleplay(self, section=None, default={}):
-        """Returns the Google Play results for this analysis."""
+        """Return the Google Play results for this analysis."""
         googleplay = self.get_results("googleplay", {})
         return googleplay if section is None else googleplay.get(section, default)
 
     def check_ip(self, pattern, regex=False, all=False):
-        """Checks for an IP address being contacted.
+        """Check for an IP address being contacted.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1173,7 +1173,7 @@ def check_ip(self, pattern, regex=False, all=False):
                                  all=all)
 
     def check_domain(self, pattern, regex=False, all=False):
-        """Checks for a domain being contacted.
+        """Check for a domain being contacted.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1189,7 +1189,7 @@ def check_domain(self, pattern, regex=False, all=False):
                                  all=all)
 
     def check_url(self, pattern, regex=False, all=False):
-        """Checks for a URL being contacted.
+        """Check for a URL being contacted.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1279,7 +1279,7 @@ def mark(self, **kwargs):
         self.marks.append(mark)
 
     def has_marks(self, count=None):
-        """Returns true if this signature has one or more marks."""
+        """Return true if this signature has one or more marks."""
         if count is not None:
             return len(self.marks) >= count
         return not not self.marks
diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py
index 92cfa31376..31e711c475 100644
--- a/cuckoo/common/config.py
+++ b/cuckoo/common/config.py
@@ -148,7 +148,7 @@ def parse(self, value):
             log.error("Incorrect UUID %s", value)
 
     def check(self, value):
-        """Checks if the value is of type UUID."""
+        """Check if the value is of type UUID."""
         try:
             click.UUID(value)
             return True
@@ -1042,7 +1042,7 @@ def get(self, section):
 
     @staticmethod
     def from_confdir(dirpath, loose=False, sanitize=False):
-        """Reads all the configuration from a configuration directory. If
+        """Read all the configuration from a configuration directory. If
         `sanitize` is set, then black out sensitive fields."""
         ret = {}
         for filename in os.listdir(dirpath):
@@ -1183,7 +1183,7 @@ def cast(s, value):
     return type_.parse(value)
 
 def read_kv_conf(filepath):
-    """Reads a flat Cuckoo key/value configuration file."""
+    """Read a flat Cuckoo key/value configuration file."""
     ret = {}
     for line in open(filepath, "rb"):
         line = line.strip()
diff --git a/cuckoo/common/dns.py b/cuckoo/common/dns.py
index 56f6009b78..a13bce6081 100644
--- a/cuckoo/common/dns.py
+++ b/cuckoo/common/dns.py
@@ -36,9 +36,9 @@ def set_timeout_value(value):
 # standard gethostbyname in thread
 # http://code.activestate.com/recipes/473878/
 def with_timeout(func, args=(), kwargs={}):
-    """This function will spawn a thread and run the given function
-    using the args, kwargs and return the given default value if the
-    timeout_duration is exceeded.
+    """Spawn a thread and run the given function using the args,
+    kwargs and return the given default value if the timeout_duration
+    is exceeded.
     """
     class ResultThread(threading.Thread):
         daemon = True
diff --git a/cuckoo/common/files.py b/cuckoo/common/files.py
index 525382b818..5ae8e24811 100644
--- a/cuckoo/common/files.py
+++ b/cuckoo/common/files.py
@@ -13,7 +13,7 @@
 from cuckoo.misc import getuser
 
 def temppath():
-    """Returns the true temporary directory."""
+    """Return the true temporary directory."""
     tmppath = config("cuckoo:cuckoo:tmppath")
 
     # Backwards compatibility with older configuration.
@@ -37,7 +37,7 @@ def get_filename_from_path(path):
 class Folders(Storage):
     @staticmethod
     def create(root=".", folders=None):
-        """Creates a directory or multiple directories.
+        """Create a directory or multiple directories.
         @param root: root path.
         @param folders: folders list to be created.
         @raise CuckooOperationalError: if fails to create folder.
@@ -149,7 +149,7 @@ def copy(path_target, path_dest):
 
     @staticmethod
     def hash_file(method, filepath):
-        """Calculates an hash on a file by path.
+        """Calculate a hash on a file by path.
         @param method: callable hashing method
         @param path: file path
         @return: computed hash string
diff --git a/cuckoo/common/netlog.py b/cuckoo/common/netlog.py
index 54ff29a74c..9fb65ae7c4 100644
--- a/cuckoo/common/netlog.py
+++ b/cuckoo/common/netlog.py
@@ -55,7 +55,7 @@ def default_converter_64bit(v):
     return v
 
 class BsonParser(ProtocolHandler):
-    """Receives and interprets .bson logs from the monitor.
+    """Receive and interpret .bson logs from the monitor.
 
     The monitor provides us with "info" messages that explain how the function
     arguments will come through later on. This class remembers these info
@@ -117,9 +117,9 @@ def resolve_flags(self, apiname, argdict, flags):
             flags[argument] = "|".join(flags[argument])
 
     def determine_unserializers(self, arginfo):
-        """Determines which unserializers (or converters) have to be used in
-        order to parse the various arguments for this function call. Keeps in
-        mind whether the current bson is 32-bit or 64-bit."""
+        """Determine which unserializers (or converters) have to be used in
+        order to parse the various arguments for this function call. Maintains
+        whether the current bson is 32-bit or 64-bit."""
         argnames, converters = [], []
 
         for argument in arginfo:
diff --git a/cuckoo/common/scripting.py b/cuckoo/common/scripting.py
index ae1a73f96b..0f12c3dd0b 100644
--- a/cuckoo/common/scripting.py
+++ b/cuckoo/common/scripting.py
@@ -105,7 +105,7 @@ def get_script(self):
         return " ".join(self.args.get("command", []))
 
 def ps1_cmdarg(s, minimum=1):
-    """Creates an exactly matching PowerShell command line argument regex,
+    """Create an exactly matching PowerShell command line argument regex,
     instead of a regex that matches anything with the same characters."""
     return "".join(
         "([%s%s^]" % (ch.lower(), ch.upper()) for ch in s
diff --git a/cuckoo/common/utils.py b/cuckoo/common/utils.py
index d39a6a8462..e587712998 100644
--- a/cuckoo/common/utils.py
+++ b/cuckoo/common/utils.py
@@ -35,7 +35,7 @@
 )
 
 def convert_char(c):
-    """Escapes characters.
+    """Escape characters.
     @param c: dirty char.
     @return: sanitized char.
     """
@@ -75,14 +75,14 @@ def constant_time_compare(a, b):
     return result == 0
 
 def validate_hash(h):
-    """Validates a hash by length and contents."""
+    """Validate a hash by length and contents."""
     if len(h) not in (32, 40, 64, 128):
         return False
 
     return bool(re.match("[0-9a-fA-F]*$", h))
 
 def validate_url(url, allow_invalid=False):
-    """Validates an URL using Django's built-in URL validator"""
+    """Validate an URL using Django's built-in URL validator"""
     from django.core.validators import URLValidator
     val = URLValidator(schemes=["http", "https"])
 
@@ -230,7 +230,7 @@ def guid_name(guid):
     return GUIDS.get(guid)
 
 def exception_message():
-    """Creates a message describing an unhandled exception."""
+    """Create a message describing an unhandled exception."""
     def get_os_release():
         """Returns detailed OS release."""
         if platform.linux_distribution()[0]:
@@ -271,7 +271,7 @@ def get_os_release():
 _jsbeautify_lock = threading.Lock()
 
 def jsbeautify(javascript):
-    """Beautifies Javascript through jsbeautifier and ignore some messages."""
+    """Beautify Javascript through jsbeautifier and ignore some messages."""
     with _jsbeautify_lock:
         origout, sys.stdout = sys.stdout, io.StringIO()
 
@@ -289,7 +289,7 @@ def jsbeautify(javascript):
     return javascript
 
 def htmlprettify(html):
-    """Beautifies HTML through BeautifulSoup4."""
+    """Beautify HTML through BeautifulSoup4."""
     # The following ignores the following bs4 warning:
     # UserWarning: "." looks like a filename, not markup.
     with warnings.catch_warnings():
@@ -297,7 +297,7 @@ def htmlprettify(html):
         return bs4.BeautifulSoup(html, "html.parser").prettify()
 
 def json_default(obj):
-    """JSON serializer for objects not serializable by default json code"""
+    """JSON serialize objects not serializable by default json code"""
     if hasattr(obj, "to_dict"):
         return obj.to_dict()
 
@@ -330,7 +330,7 @@ def parse_bool(value):
     return bool(int(value))
 
 def supported_version(version, minimum, maximum):
-    """Checks if a version number is supported as per the minimum and maximum
+    """Check if a version number is supported as per the minimum and maximum
     version numbers."""
     if minimum and StrictVersion(version) < StrictVersion(minimum):
         return False
diff --git a/cuckoo/core/database.py b/cuckoo/core/database.py
index 2cd53c33b0..98c1a90fd5 100644
--- a/cuckoo/core/database.py
+++ b/cuckoo/core/database.py
@@ -120,7 +120,7 @@ def rcparams(self, value):
             self._rcparams = value
 
     def to_dict(self):
-        """Converts object to dict.
+        """Convert object to dict.
         @return: dict
         """
         d = {}
@@ -136,7 +136,7 @@ def to_dict(self):
         return d
 
     def to_json(self):
-        """Converts object to JSON.
+        """Convert object to JSON.
         @return: JSON data
         """
         return json.dumps(self.to_dict())
@@ -198,7 +198,7 @@ def __repr__(self):
         return "<Guest('{0}','{1}')>".format(self.id, self.name)
 
     def to_dict(self):
-        """Converts object to dict.
+        """Convert object to dict.
         @return: dict
         """
         d = {}
@@ -211,7 +211,7 @@ def to_dict(self):
         return d
 
     def to_json(self):
-        """Converts object to JSON.
+        """Convert object to JSON.
         @return: JSON data
         """
         return json.dumps(self.to_dict())
@@ -256,7 +256,7 @@ def __repr__(self):
         return "<Sample('{0}','{1}')>".format(self.id, self.sha256)
 
     def to_dict(self):
-        """Converts object to dict.
+        """Convert object to dict.
         @return: dict
         """
         d = {}
@@ -265,7 +265,7 @@ def to_dict(self):
         return d
 
     def to_json(self):
-        """Converts object to JSON.
+        """Convert object to JSON.
         @return: JSON data
         """
         return json.dumps(self.to_dict())
@@ -291,7 +291,7 @@ class Error(Base):
     task_id = Column(Integer, ForeignKey("tasks.id"), nullable=False)
 
     def to_dict(self):
-        """Converts object to dict.
+        """Convert object to dict.
         @return: dict
         """
         d = {}
@@ -300,7 +300,7 @@ def to_dict(self):
         return d
 
     def to_json(self):
-        """Converts object to JSON.
+        """Convert object to JSON.
         @return: JSON data
         """
         return json.dumps(self.to_dict())
@@ -371,7 +371,7 @@ def options(self, value):
             self._options = value
 
     def to_dict(self, dt=False):
-        """Converts object to dict.
+        """Convert object to dict.
         @param dt: encode datetime objects
         @return: dict
         """
@@ -398,7 +398,7 @@ def to_dict(self, dt=False):
         return d
 
     def to_json(self):
-        """Converts object to JSON.
+        """Convert object to JSON.
         @return: JSON data
         """
         return json_encode(self.to_dict())
@@ -465,7 +465,7 @@ def connect(self, schema_check=None, dsn=None, create=True):
             self._create_tables()
 
     def _create_tables(self):
-        """Creates all the database tables etc."""
+        """Create all database tables etc."""
         try:
             Base.metadata.create_all(self.engine)
         except SQLAlchemyError as e:
@@ -509,7 +509,7 @@ def __del__(self):
         self.engine.dispose()
 
     def _connect_database(self, connection_string):
-        """Connect to a Database.
+        """Connect to a database.
         @param connection_string: Connection string specifying the database
         """
         try:
@@ -677,7 +677,7 @@ def set_route(self, task_id, route):
 
     @classlock
     def fetch(self, machine=None, service=True):
-        """Fetches a task waiting to be processed and locks it for running.
+        """Fetch a task waiting to be processed and lock it for running.
         @return: None or task
         """
         session = self.Session()
@@ -704,7 +704,7 @@ def fetch(self, machine=None, service=True):
 
     @classlock
     def guest_start(self, task_id, name, label, manager):
-        """Logs guest start.
+        """Log guest start.
         @param task_id: task identifier
         @param name: vm name
         @param label: vm label
@@ -728,7 +728,7 @@ def guest_start(self, task_id, name, label, manager):
 
     @classlock
     def guest_get_status(self, task_id):
-        """Logs guest start.
+        """Log guest start.
         @param task_id: task id
         @return: guest status
         """
@@ -745,7 +745,7 @@ def guest_get_status(self, task_id):
 
     @classlock
     def guest_set_status(self, task_id, status):
-        """Logs guest start.
+        """Log guest start.
         @param task_id: task identifier
         @param status: status
         """
@@ -764,7 +764,7 @@ def guest_set_status(self, task_id, status):
 
     @classlock
     def guest_remove(self, guest_id):
-        """Removes a guest start entry."""
+        """Remove a guest start entry."""
         session = self.Session()
         try:
             guest = session.query(Guest).get(guest_id)
@@ -779,7 +779,7 @@ def guest_remove(self, guest_id):
 
     @classlock
     def guest_stop(self, guest_id):
-        """Logs guest stop.
+        """Log guest stop.
         @param guest_id: guest log entry id
         """
         session = self.Session()
@@ -799,7 +799,7 @@ def guest_stop(self, guest_id):
 
     @classlock
     def list_machines(self, locked=False):
-        """Lists virtual machines.
+        """List virtual machines.
         @return: list of virtual machines
         """
         session = self.Session()
@@ -817,7 +817,7 @@ def list_machines(self, locked=False):
 
     @classlock
     def lock_machine(self, label=None, platform=None, tags=None):
-        """Places a lock on a free virtual machine.
+        """Place a lock on a free virtual machine.
         @param label: optional virtual machine label
         @param platform: optional virtual machine platform
         @param tags: optional tags required (list)
@@ -876,7 +876,7 @@ def lock_machine(self, label=None, platform=None, tags=None):
 
     @classlock
     def unlock_machine(self, label):
-        """Remove lock form a virtual machine.
+        """Remove a lock from a virtual machine.
         @param label: virtual machine label
         @return: unlocked machine
         """
@@ -905,7 +905,7 @@ def unlock_machine(self, label):
 
     @classlock
     def count_machines_available(self):
-        """How many virtual machines are ready for analysis.
+        """Return number of virtual machines ready for analysis.
         @return: free virtual machines count
         """
         session = self.Session()
@@ -920,7 +920,7 @@ def count_machines_available(self):
 
     @classlock
     def get_available_machines(self):
-        """  Which machines are available
+        """Return machines that are available.
         @return: free virtual machines
         """
         session = self.Session()
@@ -1406,7 +1406,7 @@ def list_tasks(self, limit=None, details=True, category=None, owner=None,
             session.close()
 
     def minmax_tasks(self):
-        """Find tasks minimum and maximum
+        """Find tasks minimum and maximum.
         @return: unix timestamps of minimum and maximum
         """
         session = self.Session()
@@ -1426,7 +1426,7 @@ def minmax_tasks(self):
 
     @classlock
     def count_tasks(self, status=None):
-        """Count tasks in the database
+        """Count tasks in the database.
         @param status: apply a filter according to the task status
         @return: number of tasks found
         """
@@ -1513,7 +1513,7 @@ def delete_task(self, task_id):
 
     @classlock
     def view_sample(self, sample_id):
-        """Retrieve information on a sample given a sample id.
+        """Retrieve information on a sample given a sample ID.
         @param sample_id: ID of the sample to query.
         @return: details on the sample used in sample: sample_id.
         """
@@ -1557,7 +1557,7 @@ def find_sample(self, md5=None, sha256=None):
 
     @classlock
     def count_samples(self):
-        """Counts the amount of samples in the database."""
+        """Count number of samples in the database."""
         session = self.Session()
         try:
             sample_count = session.query(Sample).count()
diff --git a/cuckoo/core/feedback.py b/cuckoo/core/feedback.py
index 2735fc7d19..2e7ec64c75 100644
--- a/cuckoo/core/feedback.py
+++ b/cuckoo/core/feedback.py
@@ -18,7 +18,7 @@
 log = logging.getLogger(__name__)
 
 class CuckooFeedback(object):
-    """Contacts Cuckoo HQ with feedback & optional analysis dump."""
+    """Contact Cuckoo HQ with feedback & optional analysis dump."""
     endpoint = "https://feedback.cuckoosandbox.org/api/submit/"
     exc_whitelist = (
         CuckooFeedbackError,
@@ -200,7 +200,7 @@ def include_report_web(self, task_id):
         return self.include_report(report)
 
     def gather_export_files(self, dirpath):
-        """Returns a list of all files of interest from an analysis."""
+        """Return a list of all files of interest from an analysis."""
         ret = []
         for name in self.export_files:
             if isinstance(name, basestring):
diff --git a/cuckoo/core/guest.py b/cuckoo/core/guest.py
index 2803f93188..5f148d58f1 100644
--- a/cuckoo/core/guest.py
+++ b/cuckoo/core/guest.py
@@ -30,7 +30,7 @@
 db = Database()
 
 def analyzer_zipfile(platform, monitor):
-    """Creates the Zip file that is sent to the Guest."""
+    """Create the zip file that is sent to the Guest."""
     t = time.time()
 
     zip_data = io.BytesIO()
diff --git a/cuckoo/core/init.py b/cuckoo/core/init.py
index 53ba3b8c8b..0fdb1b9da0 100644
--- a/cuckoo/core/init.py
+++ b/cuckoo/core/init.py
@@ -11,7 +11,7 @@
 from cuckoo.misc import cwd
 
 def write_supervisor_conf(username):
-    """Writes supervisord.conf configuration file if it does not exist yet."""
+    """Write supervisord.conf configuration file if it does not exist yet."""
     # TODO Handle updates?
     if os.path.exists(cwd("supervisord.conf")):
         return
diff --git a/cuckoo/core/plugins.py b/cuckoo/core/plugins.py
index 0869947c76..e5880b0b17 100644
--- a/cuckoo/core/plugins.py
+++ b/cuckoo/core/plugins.py
@@ -466,7 +466,7 @@ def yield_calls(self, proc):
                     self.api_sigs[call["api"]].remove(sig)
 
     def process_yara_matches(self):
-        """Yields any Yara matches to each signature."""
+        """Yield any Yara matches to each signature."""
         def loop_yara(category, filepath, matches):
             for match in matches:
                 match = YaraMatch(match, category)
@@ -650,7 +650,7 @@ def process(self, module):
             )
 
     def run(self):
-        """Generates all reports.
+        """Generate all reports.
         @raise CuckooReportError: if a report module fails.
         """
         # In every reporting module you can specify a numeric value that
diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py
index c13c02e542..695dd23715 100644
--- a/cuckoo/core/scheduler.py
+++ b/cuckoo/core/scheduler.py
@@ -138,7 +138,7 @@ def init(self):
         return True
 
     def store_task_info(self):
-        """grab latest task from db (if available) and update self.task"""
+        """Grab latest task from db (if available) and update self.task"""
         dbtask = self.db.view_task(self.task.id)
         self.task = dbtask.to_dict()
 
diff --git a/cuckoo/core/startup.py b/cuckoo/core/startup.py
index 891c0a2e5c..4f121b9a59 100644
--- a/cuckoo/core/startup.py
+++ b/cuckoo/core/startup.py
@@ -51,7 +51,7 @@ def check_specific_config(filename):
             )
 
 def check_configs():
-    """Checks if config files exist.
+    """Check if config files exist.
     @raise CuckooStartupError: if config files do not exist.
     """
     configs = (
@@ -97,7 +97,7 @@ def check_configs():
     return True
 
 def check_version():
-    """Checks version of Cuckoo."""
+    """Check version of Cuckoo."""
     if not config("cuckoo:cuckoo:version_check"):
         return
 
@@ -219,14 +219,14 @@ def check_version():
     return r
 
 def init_logging(level):
-    """Initializes logging."""
+    """Initialize logging."""
     logging.getLogger().setLevel(logging.DEBUG)
     init_logger("cuckoo.log", level)
     init_logger("cuckoo.json")
     init_logger("task")
 
 def init_console_logging(level=logging.INFO):
-    """Initializes logging only to console and database."""
+    """Initialize logging only to console and database."""
     logging.getLogger().setLevel(logging.DEBUG)
     init_logger("console", level)
     init_logger("database")
@@ -258,7 +258,7 @@ def init_tasks():
         db.set_status(task.id, TASK_FAILED_ANALYSIS)
 
 def init_modules():
-    """Initializes plugins."""
+    """Initialize plugins."""
     log.debug("Imported modules...")
 
     categories = (
@@ -503,7 +503,7 @@ def init_routing():
             rooter("init_rttable", rt_table, interface)
 
 def ensure_tmpdir():
-    """Verifies if the current user can read and create files in the
+    """Verify if the current user can read and create files in the
     cuckoo temporary directory (and creates it, if needed)."""
     try:
         if not os.path.isdir(temppath()):
diff --git a/cuckoo/core/submit.py b/cuckoo/core/submit.py
index 22657bf86c..1ecaa9dc85 100644
--- a/cuckoo/core/submit.py
+++ b/cuckoo/core/submit.py
@@ -58,7 +58,7 @@ def _handle_string(self, submit, tmppath, line):
         )
 
     def translate_options_from(self, entry, options):
-        """Translates from Web Interface options to Cuckoo database options."""
+        """Translate from Web Interface options to Cuckoo database options."""
         ret = {}
 
         if not options.get("simulated-human-interaction", True):
@@ -88,7 +88,7 @@ def translate_options_from(self, entry, options):
         return ret
 
     def translate_options_to(self, options):
-        """Translates from Cuckoo database options to Web Interface options."""
+        """Translate from Cuckoo database options to Web Interface options."""
         ret = {}
 
         if not int(options.get("human", "1")):
@@ -147,7 +147,7 @@ def pre(self, submit_type, data, options=None):
 
     def get_files(self, submit_id, password=None, astree=False):
         """
-        Returns files or URLs from a submitted analysis.
+        Return files or URLs from a submitted analysis.
         @param password: The password to unlock container archives with
         @param astree: sflock option; determines the format in which the files are returned
         @return: A tree of files
@@ -195,7 +195,7 @@ def get_files(self, submit_id, password=None, astree=False):
         return files, submit.data["errors"], submit.data["options"]
 
     def submit(self, submit_id, config):
-        """Reads, interprets, and converts the JSON configuration provided by
+        """Read, interpret, and convert the JSON configuration provided by
         the Web Interface into something we insert into the database."""
         ret = []
         submit = db.view_submit(submit_id)
diff --git a/cuckoo/data/analyzer/darwin/lib/api/screenshot.py b/cuckoo/data/analyzer/darwin/lib/api/screenshot.py
index ea22a8ab94..613551fd07 100644
--- a/cuckoo/data/analyzer/darwin/lib/api/screenshot.py
+++ b/cuckoo/data/analyzer/darwin/lib/api/screenshot.py
@@ -38,7 +38,7 @@ def have_pil(self):
         return HAVE_PIL
 
     def equal(self, img1, img2, skip_area=None):
-        """Compares two screenshots using Root-Mean-Square Difference (RMS).
+        """Compare two screenshots using Root-Mean-Square Difference (RMS).
         @param img1: screenshot to compare.
         @param img2: screenshot to compare.
         @return: equal status.
diff --git a/cuckoo/data/analyzer/darwin/lib/common/hashing.py b/cuckoo/data/analyzer/darwin/lib/common/hashing.py
index fac551f2fd..010d4fa11c 100644
--- a/cuckoo/data/analyzer/darwin/lib/common/hashing.py
+++ b/cuckoo/data/analyzer/darwin/lib/common/hashing.py
@@ -6,7 +6,7 @@
 
 
 def hash_file(method, path):
-    """Calculates an hash on a file by path.
+    """Calculate a hash on a file by path.
     @param method: callable hashing method
     @param path: file path
     @return: computed hash string
diff --git a/cuckoo/data/analyzer/darwin/modules/packages/zip.py b/cuckoo/data/analyzer/darwin/modules/packages/zip.py
index eaf475c6da..18388c8a03 100644
--- a/cuckoo/data/analyzer/darwin/modules/packages/zip.py
+++ b/cuckoo/data/analyzer/darwin/modules/packages/zip.py
@@ -90,7 +90,7 @@ def _extract_nested_archives(self, archive, where, password):
 
 
 def _prepare_archive_at_path(filename):
-    """ Verifies that there's a readable zip archive at the given path.
+    """ Verify that there's a readable zip archive at the given path.
 
     This function returns a new name for the archive (for most cases it's
     the same as the original one; but if an archive named "foo.zip" contains
diff --git a/cuckoo/data/analyzer/linux/lib/common/abstracts.py b/cuckoo/data/analyzer/linux/lib/common/abstracts.py
index 015e035dab..3bf8fd0c96 100644
--- a/cuckoo/data/analyzer/linux/lib/common/abstracts.py
+++ b/cuckoo/data/analyzer/linux/lib/common/abstracts.py
@@ -31,7 +31,7 @@ def check(self):
         return True
 
     def execute(self, cmd):
-        """Starts an executable for analysis.
+        """Start an executable for analysis.
         @param path: executable path
         @param args: executable arguments
         @return: process pid
diff --git a/cuckoo/data/analyzer/linux/lib/common/hashing.py b/cuckoo/data/analyzer/linux/lib/common/hashing.py
index 78d1d1936e..1ffa54190d 100644
--- a/cuckoo/data/analyzer/linux/lib/common/hashing.py
+++ b/cuckoo/data/analyzer/linux/lib/common/hashing.py
@@ -10,7 +10,7 @@ def sha256_file(path):
     return hash_file(hashlib.sha256, path)
 
 def hash_file(method, path):
-    """Calculates an hash on a file by path.
+    """Calculate a hash on a file by path.
     @param method: callable hashing method
     @param path: file path
     @return: computed hash string
diff --git a/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py b/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py
index ae2b20b390..3ddbdfeec0 100644
--- a/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py
+++ b/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py
@@ -14,7 +14,7 @@
 log = logging.getLogger(__name__)
 
 class STAP(Auxiliary):
-    """system-wide syscall trace with stap."""
+    """System-wide syscall trace with stap."""
     priority = -10  # low prio to wrap tightly around the analysis
 
     def __init__(self):
diff --git a/cuckoo/data/analyzer/windows/analyzer.py b/cuckoo/data/analyzer/windows/analyzer.py
index 66d0a6bdb4..0a3868ae71 100644
--- a/cuckoo/data/analyzer/windows/analyzer.py
+++ b/cuckoo/data/analyzer/windows/analyzer.py
@@ -46,11 +46,11 @@ def __init__(self):
         self.dumped = []
 
     def is_protected_filename(self, file_name):
-        """Do we want to inject into a process with this name?"""
+        """Return whether or not to inject into a process with this name."""
         return file_name.lower() in self.PROTECTED_NAMES
 
     def add_pid(self, filepath, pid, verbose=True):
-        """Tracks a process identifier for this file."""
+        """Track a process identifier for this file."""
         if not pid or filepath.lower() not in self.files:
             return
 
@@ -150,7 +150,7 @@ def add_pids(self, pids):
             self.add_pid(pids)
 
     def has_pid(self, pid, notrack=True):
-        """Is this process identifier being tracked?"""
+        """Return whether or not this process identifier being tracked."""
         if int(pid) in self.pids:
             return True
 
@@ -435,7 +435,7 @@ def __init__(self):
         self.reboot = []
 
     def get_pipe_path(self, name):
-        """Returns \\\\.\\PIPE on Windows XP and \\??\\PIPE elsewhere."""
+        """Return \\\\.\\PIPE on Windows XP and \\??\\PIPE elsewhere."""
         version = sys.getwindowsversion()
         if version.major == 5 and version.minor == 1:
             return "\\\\.\\PIPE\\%s" % name
@@ -509,7 +509,7 @@ def prepare(self):
             self.target = self.config.target
 
     def stop(self):
-        """Allows an auxiliary module to stop the analysis."""
+        """Allow an auxiliary module to stop the analysis."""
         self.do_run = False
 
     def complete(self):
diff --git a/cuckoo/data/analyzer/windows/lib/api/process.py b/cuckoo/data/analyzer/windows/lib/api/process.py
index ba8770174d..ae7359795a 100644
--- a/cuckoo/data/analyzer/windows/lib/api/process.py
+++ b/cuckoo/data/analyzer/windows/lib/api/process.py
@@ -127,7 +127,7 @@ def __init__(self, pid=None, tid=None, process_name=None):
 
     @staticmethod
     def set_config(config):
-        """Sets the analyzer configuration once."""
+        """Set the analyzer configuration once."""
         Process.config = config
 
     def get_system_info(self):
diff --git a/cuckoo/data/analyzer/windows/lib/common/abstracts.py b/cuckoo/data/analyzer/windows/lib/common/abstracts.py
index c9ae91a119..e47109f3d6 100644
--- a/cuckoo/data/analyzer/windows/lib/common/abstracts.py
+++ b/cuckoo/data/analyzer/windows/lib/common/abstracts.py
@@ -107,7 +107,7 @@ def move_curdir(self, filepath):
         return outpath
 
     def init_regkeys(self, regkeys):
-        """Initializes the registry to avoid annoying popups, configure
+        """Initialize the registry to avoid annoying popups, configure
         settings, etc.
         @param regkeys: the root keys, subkeys, and key/value pairs.
         """
@@ -130,7 +130,7 @@ def init_regkeys(self, regkeys):
 
     def execute(self, path, args, mode=None, maximize=False, env=None,
                 source=None, trigger=None):
-        """Starts an executable for analysis.
+        """Start an executable for analysis.
         @param path: executable path
         @param args: executable arguments
         @param mode: monitor mode - which functions to instrument
@@ -169,7 +169,7 @@ def execute(self, path, args, mode=None, maximize=False, env=None,
         return p.pid
 
     def package_files(self):
-        """A list of files to upload to host.
+        """Return a list of files to upload to host.
         The list should be a list of tuples (<path on guest>, <name of file in package_files folder>).
         (package_files is a folder that will be created in analysis folder).
         """
diff --git a/cuckoo/data/analyzer/windows/lib/common/hashing.py b/cuckoo/data/analyzer/windows/lib/common/hashing.py
index bbea930962..adbd321349 100644
--- a/cuckoo/data/analyzer/windows/lib/common/hashing.py
+++ b/cuckoo/data/analyzer/windows/lib/common/hashing.py
@@ -7,7 +7,7 @@
 
 
 def hash_file(method, path):
-    """Calculates an hash on a file by path.
+    """Calculate a hash on a file by path.
     @param method: callable hashing method
     @param path: file path
     @return: computed hash string
diff --git a/cuckoo/data/analyzer/windows/lib/core/pipe.py b/cuckoo/data/analyzer/windows/lib/core/pipe.py
index 88598b2be5..26594266ee 100644
--- a/cuckoo/data/analyzer/windows/lib/core/pipe.py
+++ b/cuckoo/data/analyzer/windows/lib/core/pipe.py
@@ -21,8 +21,8 @@
 BUFSIZE = 0x10000
 
 class PipeForwarder(threading.Thread):
-    """The Pipe Forwarder forwards all data received from a local pipe to
-    the Cuckoo server through a socket."""
+    """Forward all data received from a local pipe to the Cuckoo
+    server through a socket."""
     sockets = {}
     active = {}
 
@@ -99,8 +99,8 @@ def run(self):
             self.active[pid.value] = False
 
 class PipeDispatcher(threading.Thread):
-    """Receives commands through a local pipe, forwards them to the
-    dispatcher, and returns the response."""
+    """Receive commands through a local pipe, forward them to the
+    dispatcher, and return the response."""
 
     def __init__(self, pipe_handle, dispatcher):
         threading.Thread.__init__(self)
@@ -146,8 +146,8 @@ def run(self):
         KERNEL32.CloseHandle(self.pipe_handle)
 
 class PipeServer(threading.Thread):
-    """The Pipe Server accepts incoming pipe handlers and initializes
-    them in a new thread."""
+    """Accept incoming pipe handlers and initialize them in
+    a new thread."""
 
     def __init__(self, pipe_handler, pipe_name, message=False, **kwargs):
         threading.Thread.__init__(self)
diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py b/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py
index fa74c71de9..f5fa69a7e8 100644
--- a/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py
+++ b/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py
@@ -66,7 +66,7 @@ class Disguise(Auxiliary):
     ]
 
     def change_productid(self):
-        """Randomizes Windows ProductId.
+        """Randomize Windows ProductId.
         The Windows ProductId is occasionally used by malware
         to detect public setups of Cuckoo, e.g., Malwr.com.
         """
diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py b/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py
index 144f36ac51..54c421d278 100644
--- a/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py
+++ b/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py
@@ -12,7 +12,7 @@
 log = logging.getLogger(__name__)
 
 class Reboot(Auxiliary):
-    """Prepares the environment to behave as if the VM has been rebooted."""
+    """Prepare the environment to behave as if the VM has been rebooted."""
 
     def start(self):
         if self.analyzer.config.package != "reboot":
diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py b/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py
index 25497dd032..50eed421fc 100644
--- a/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py
+++ b/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py
@@ -17,7 +17,7 @@
 log = logging.getLogger(__name__)
 
 class RecentFiles(Auxiliary):
-    """Populates the Desktop with recent files in order to combat recent
+    """Populate the Desktop with recent files in order to combat recent
     anti-sandbox measures."""
 
     extensions = [
diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py b/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py
index e5b656e75c..3d8e479922 100644
--- a/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py
+++ b/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py
@@ -13,7 +13,7 @@
 log = logging.getLogger(__name__)
 
 class LoadZer0m0n(Auxiliary):
-    """Loads the zer0m0n kernel driver."""
+    """Load the zer0m0n kernel driver."""
 
     def start(self):
         if self.options.get("analysis") not in ("both", "kernel"):
diff --git a/cuckoo/data/analyzer/windows/modules/packages/zip.py b/cuckoo/data/analyzer/windows/modules/packages/zip.py
index 339cf41833..de33fc355a 100644
--- a/cuckoo/data/analyzer/windows/modules/packages/zip.py
+++ b/cuckoo/data/analyzer/windows/modules/packages/zip.py
@@ -51,7 +51,7 @@ def extract_zip(self, zip_path, extract_path, password):
                         self.extract_zip(os.path.join(extract_path, name), extract_path, password)
 
     def is_overwritten(self, zip_path):
-        """Checks if the ZIP file contains another file with the same name, so it is going to be overwritten.
+        """Check if the ZIP file contains another file with the same name, so it is going to be overwritten.
         @param zip_path: zip file path
         @return: comparison boolean
         """
diff --git a/cuckoo/machinery/avd.py b/cuckoo/machinery/avd.py
index bfa789464d..387800f630 100644
--- a/cuckoo/machinery/avd.py
+++ b/cuckoo/machinery/avd.py
@@ -19,7 +19,7 @@ class Avd(Machinery):
     """Virtualization layer for Android Emulator."""
 
     def _initialize_check(self):
-        """Runs all checks when a machine manager is initialized.
+        """Run all checks when a machine manager is initialized.
         @raise CuckooMachineError: if the android emulator is not found.
         """
         self.emulator_processes = {}
@@ -76,7 +76,7 @@ def start(self, label, task):
         self.start_agent(label)
 
     def stop(self, label):
-        """Stops a virtual machine.
+        """Stop a virtual machine.
         @param label: virtual machine name.
         @raise CuckooMachineError: if unable to stop.
         """
@@ -84,20 +84,20 @@ def stop(self, label):
         self.stop_emulator(label)
 
     def _list(self):
-        """Lists virtual machines installed.
+        """List virtual machines installed.
         @return: virtual machine names list.
         """
         return self.options.avd.machines
 
     def _status(self, label):
-        """Gets current status of a vm.
+        """Get current status of a vm.
         @param label: virtual machine name.
         @return: status string.
         """
         log.debug("Getting status for %s" % label)
 
     def duplicate_reference_machine(self, label):
-        """Creates a new emulator based on a reference one."""
+        """Create a new emulator based on a reference one."""
         reference_machine = self.options.avd.reference_machine
         log.debug("Duplicate Reference Machine '{0}'.".format(reference_machine))
 
@@ -127,7 +127,7 @@ def duplicate_reference_machine(self, label):
         # todo:will see
 
     def delete_old_emulator(self, label):
-        """Deletes any trace of an emulator that would have the same name as
+        """Delete any trace of an emulator that would have the same name as
         the one of the current emulator."""
         old_emulator_config_file = os.path.join(self.options.avd.avd_path,
                                                 "%s.ini" % label)
@@ -142,7 +142,7 @@ def delete_old_emulator(self, label):
             shutil.rmtree(old_emulator_path)
 
     def replace_content_in_file(self, fileName, contentToReplace, replacementContent):
-        """Replaces the specified motif by a specified value in the specified
+        """Replace the specified motif by a specified value in the specified
         file.
         """
 
@@ -157,7 +157,7 @@ def replace_content_in_file(self, fileName, contentToReplace, replacementContent
             fd.writelines(newLines)
 
     def start_emulator(self, label, task):
-        """Starts the emulator."""
+        """Start the emulator."""
         emulator_port = self.options.get(label)["emulator_port"]
 
         cmd = [
@@ -216,7 +216,7 @@ def stop_emulator(self, label):
             del self.emulator_processes[label]
 
     def wait_for_device_ready(self, label):
-        """Analyzes the emulator and returns when it's ready."""
+        """Analyze the emulator and return when it's ready."""
 
         emulator_port = str(self.options.get(label)["emulator_port"])
         adb = self.options.avd.adb_path
@@ -294,7 +294,7 @@ def start_agent(self, label):
         time.sleep(10)
 
     def check_adb_recognize_emulator(self, label):
-        """Checks that ADB recognizes the emulator. Returns True if device is
+        """Check that ADB recognizes the emulator. Return True if device is
         recognized by ADB, False otherwise.
         """
         log.debug("Checking if ADB recognizes emulator...")
@@ -311,7 +311,7 @@ def check_adb_recognize_emulator(self, label):
         return False
 
     def restart_adb_server(self):
-        """Restarts ADB server. This function is not used because we have to
+        """Restart ADB server. This function is not used because we have to
         verify we don't have multiple devices.
         """
         log.debug("Restarting ADB server...")
diff --git a/cuckoo/machinery/esx.py b/cuckoo/machinery/esx.py
index 68caa84fb3..3ae4c307af 100644
--- a/cuckoo/machinery/esx.py
+++ b/cuckoo/machinery/esx.py
@@ -18,7 +18,7 @@ class ESX(LibVirtMachinery):
     """Virtualization layer for ESXi/ESX based on python-libvirt."""
 
     def _initialize_check(self):
-        """Runs all checks when a machine manager is initialized.
+        """Run all checks when a machine manager is initialized.
         @raise CuckooMachineError: if configuration is invalid
         """
         if not self.options.esx.dsn:
diff --git a/cuckoo/machinery/physical.py b/cuckoo/machinery/physical.py
index ff56937ab6..1fa40c75e1 100644
--- a/cuckoo/machinery/physical.py
+++ b/cuckoo/machinery/physical.py
@@ -30,7 +30,7 @@ class Physical(Machinery):
     ERROR = "error"
 
     def _initialize_check(self):
-        """Ensures that credentials have been entered into the config file.
+        """Ensure that credentials have been entered into the config file.
         @raise CuckooCriticalError: if no credentials were provided or if
             one or more physical machines are offline.
         """
@@ -85,7 +85,7 @@ def start(self, label, task):
                                      "%s (STATUS=%s)" % (label, status))
 
     def stop(self, label):
-        """Stops a physical machine.
+        """Stop a physical machine.
         @param label: physical machine name.
         @raise CuckooMachineError: if unable to stop.
         """
@@ -119,7 +119,7 @@ def stop(self, label):
                 continue
 
     def _list(self):
-        """Lists physical machines installed.
+        """List physical machines installed.
         @return: physical machine names list.
         """
         active_machines = []
@@ -130,7 +130,7 @@ def _list(self):
         return active_machines
 
     def _status(self, label):
-        """Gets current status of a physical machine.
+        """Get current status of a physical machine.
         @param label: physical machine name.
         @return: status string.
         """
@@ -239,7 +239,7 @@ def fog_init(self):
                 )
 
     def fog_queue_task(self, hostname):
-        """Queues a task with FOG to deploy the given machine after reboot."""
+        """Queue a task with FOG to deploy the given machine after reboot."""
         if hostname in self.fog_machines:
             macaddr, download = self.fog_machines[hostname]
             self.fog_query(download)
diff --git a/cuckoo/machinery/qemu.py b/cuckoo/machinery/qemu.py
index 5efdb853c0..712f37b725 100644
--- a/cuckoo/machinery/qemu.py
+++ b/cuckoo/machinery/qemu.py
@@ -116,7 +116,7 @@ def __init__(self):
         self.state = {}
 
     def _initialize_check(self):
-        """Runs all checks when a machine manager is initialized.
+        """Run all checks when a machine manager is initialized.
         @raise CuckooMachineError: if QEMU binary is not found.
         """
         # VirtualBox specific checks.
@@ -205,7 +205,7 @@ def start(self, label, task):
             raise CuckooMachineError("QEMU failed starting the machine: %s" % e)
 
     def stop(self, label):
-        """Stops a virtual machine.
+        """Stop a virtual machine.
         @param label: virtual machine label.
         @raise CuckooMachineError: if unable to stop.
         """
@@ -235,7 +235,7 @@ def stop(self, label):
         self.state[vm_info.name] = None
 
     def _status(self, name):
-        """Gets current status of a vm.
+        """Get current status of a vm.
         @param name: virtual machine name.
         @return: status string.
         """
diff --git a/cuckoo/machinery/virtualbox.py b/cuckoo/machinery/virtualbox.py
index 1df84a51cd..a4255abf6f 100644
--- a/cuckoo/machinery/virtualbox.py
+++ b/cuckoo/machinery/virtualbox.py
@@ -30,7 +30,7 @@ class VirtualBox(Machinery):
     ERROR = "machete"
 
     def _initialize_check(self):
-        """Runs all checks when a machine manager is initialized.
+        """Run all checks when a machine manager is initialized.
         @raise CuckooMachineError: if VBoxManage is not found.
         """
         if not self.options.virtualbox.path:
@@ -182,7 +182,7 @@ def dump_pcap(self, label, task):
             return
 
     def stop(self, label):
-        """Stops a virtual machine.
+        """Stop a virtual machine.
         @param label: virtual machine name.
         @raise CuckooMachineError: if unable to stop.
         """
@@ -235,7 +235,7 @@ def stop(self, label):
         self._wait_status(label, self.POWEROFF, self.ABORTED, self.SAVED)
 
     def _list(self):
-        """Lists virtual machines installed.
+        """List virtual machines installed.
         @return: virtual machine names list.
         """
         try:
@@ -268,8 +268,8 @@ def _list(self):
         return machines
 
     def vminfo(self, label, field):
-        """Returns False if invoking vboxmanage fails. Otherwise the VM
-        information value, if any."""
+        """Return False if invoking vboxmanage fails. Otherwise return the
+        VM information value, if any."""
         try:
             args = [
                 self.options.virtualbox.path,
@@ -314,7 +314,7 @@ def vminfo(self, label, field):
                 return line.split("=", 1)[1]
 
     def _status(self, label):
-        """Gets current status of a vm.
+        """Get current status of a vm.
         @param label: virtual machine name.
         @return: status string.
         """
@@ -332,7 +332,7 @@ def _status(self, label):
         )
 
     def dump_memory(self, label, path):
-        """Takes a memory dump.
+        """Take a memory dump.
         @param path: path to where to store the memory dump.
         """
 
diff --git a/cuckoo/machinery/vmware.py b/cuckoo/machinery/vmware.py
index 498be2a351..a19b2eb521 100644
--- a/cuckoo/machinery/vmware.py
+++ b/cuckoo/machinery/vmware.py
@@ -44,7 +44,7 @@ def _initialize_check(self):
         super(VMware, self)._initialize_check()
 
     def _check_vmx(self, vmx_path):
-        """Checks whether a vmx file exists and is valid.
+        """Check whether a vmx file exists and is valid.
         @param vmx_path: path to vmx file
         @raise CuckooMachineError: if file not found or not ending with .vmx
         """
@@ -56,7 +56,7 @@ def _check_vmx(self, vmx_path):
             raise CuckooMachineError("Vm file %s not found" % vmx_path)
 
     def _check_snapshot(self, vmx_path, snapshot):
-        """Checks snapshot existance.
+        """Check snapshot existence.
         @param vmx_path: path to vmx file
         @param snapshot: snapshot name
         @raise CuckooMachineError: if snapshot not found
@@ -114,7 +114,7 @@ def start(self, vmx_path, task):
                                      "mode: %s" % (vmx_path, mode, e))
 
     def stop(self, vmx_path):
-        """Stops a virtual machine.
+        """Stop a virtual machine.
         @param vmx_path: path to vmx file
         @raise CuckooMachineError: if unable to stop.
         """
@@ -135,7 +135,7 @@ def stop(self, vmx_path):
                         vmx_path)
 
     def _revert(self, vmx_path, snapshot):
-        """Revets machine to snapshot.
+        """Revert machine to snapshot.
         @param vmx_path: path to vmx file
         @param snapshot: snapshot name
         @raise CuckooMachineError: if unable to revert
@@ -154,7 +154,7 @@ def _revert(self, vmx_path, snapshot):
                                      "machine %s: %s" % (vmx_path, e))
 
     def _is_running(self, vmx_path):
-        """Checks if virtual machine is running.
+        """Check if virtual machine is running.
         @param vmx_path: path to vmx file
         @return: running status
         """
diff --git a/cuckoo/machinery/vsphere.py b/cuckoo/machinery/vsphere.py
index 4f947937c4..2054f395c9 100644
--- a/cuckoo/machinery/vsphere.py
+++ b/cuckoo/machinery/vsphere.py
@@ -55,7 +55,7 @@ def _initialize(self, module_name):
         random.seed()
 
     def _initialize_check(self):
-        """Runs checks against virtualization software when a machine manager
+        """Run checks against virtualization software when a machine manager
         is initialized.
         @raise CuckooCriticalError: if a misconfiguration or unsupported state
                                     is found.
diff --git a/cuckoo/machinery/xenserver.py b/cuckoo/machinery/xenserver.py
index 8023ba901c..3a5ba2b291 100644
--- a/cuckoo/machinery/xenserver.py
+++ b/cuckoo/machinery/xenserver.py
@@ -189,7 +189,7 @@ def _snapshot_from_vm_uuid(self, uuid):
         return machine.snapshot
 
     def _is_halted(self, vm):
-        """Checks if the virtual machine is running.
+        """Check if the virtual machine is running.
         @param uuid: vm uuid
         """
         return vm["power_state"] == "Halted"
@@ -262,7 +262,7 @@ def _list(self):
             return vm_list
 
     def _status(self, label):
-        """Gets current status of a vm.
+        """Get current status of a vm.
         @param label: virtual machine uuid
         @return: status string.
         """
diff --git a/cuckoo/main.py b/cuckoo/main.py
index 292a8c5dd7..03958f3e4d 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -194,7 +194,7 @@ def cuckoo_main(max_analysis_count=0):
 @click.option("--cwd", help="Cuckoo Working Directory")
 @click.pass_context
 def main(ctx, debug, quiet, nolog, maxcount, user, cwd):
-    """Invokes the Cuckoo daemon or one of its subcommands.
+    """Invoke the Cuckoo daemon or one of its subcommands.
 
     To be able to use different Cuckoo configurations on the same machine with
     the same Cuckoo installation, we use the so-called Cuckoo Working
@@ -247,7 +247,7 @@ def main(ctx, debug, quiet, nolog, maxcount, user, cwd):
 @click.pass_context
 @click.option("--conf", type=click.Path(exists=True, file_okay=True, readable=True), help="Flat key/value configuration file")
 def init(ctx, conf):
-    """Initializes Cuckoo and its configuration."""
+    """Initialize Cuckoo and its configuration."""
     if conf and os.path.exists(conf):
         cfg = read_kv_conf(conf)
     else:
@@ -398,7 +398,7 @@ def process(ctx, instance, report, maxcount, timeout):
 @click.option("--sudo", is_flag=True, help="Request superuser privileges")
 @click.pass_context
 def rooter(ctx, socket, group, service, iptables, ip, sudo):
-    """Instantiates the Cuckoo Rooter."""
+    """Instantiate the Cuckoo Rooter."""
     init_console_logging(level=ctx.parent.level)
 
     if sudo:
@@ -626,7 +626,7 @@ def migrate(revision):
 @click.argument("path", type=click.Path(file_okay=False, exists=True))
 @click.pass_context
 def import_(ctx, mode, path):
-    """Imports an older Cuckoo setup into a new CWD. The old setup should be
+    """Import an older Cuckoo setup into a new CWD. The old setup should be
     identified by PATH and the new CWD may be specified with the --cwd
     parameter, e.g., "cuckoo --cwd /tmp/cwd import old-cuckoo"."""
     if os.path.exists(os.path.join(path, ".cwd")):
diff --git a/cuckoo/misc.py b/cuckoo/misc.py
index a137abc1b7..6e82f9f1c3 100644
--- a/cuckoo/misc.py
+++ b/cuckoo/misc.py
@@ -41,7 +41,7 @@ def set_cwd(path, raw=None):
     _raw = raw
 
 def cwd(*args, **kwargs):
-    """Returns absolute path to this file in the Cuckoo Working Directory or
+    """Return absolute path to this file in the Cuckoo Working Directory or
     optionally - when private=True has been passed along - to our private
     Cuckoo Working Directory which is not configurable."""
     if kwargs.get("private"):
@@ -62,7 +62,7 @@ def cwd(*args, **kwargs):
         return os.path.join(_root, *args)
 
 def decide_cwd(cwd=None, exists=False):
-    """Decides and sets the CWD, optionally checks if it's a valid CWD."""
+    """Decide and set the CWD, optionally check if it's a valid CWD."""
     if not cwd:
         cwd = os.environ.get("CUCKOO_CWD")
 
@@ -104,7 +104,7 @@ def getuser():
     return ""
 
 def load_signatures():
-    """Loads additional Signatures from the Cuckoo Working Directory.
+    """Load additional Signatures from the Cuckoo Working Directory.
 
     This method is quite hacky in the sense that it magically imports
     Signatures from an arbitrary directory - one that doesn't belong to the
@@ -188,7 +188,7 @@ def is_macosx():
     return sys.platform == "darwin"
 
 def Popen(*args, **kwargs):
-    """Drops the close_fds argument on Windows platforms in certain situations
+    """Drop the close_fds argument on Windows platforms in certain situations
     where it'd otherwise cause an exception from the subprocess module."""
     if is_windows() and "close_fds" in kwargs:
         if "stdin" in kwargs or "stdout" in kwargs or "stderr" in kwargs:
@@ -197,7 +197,7 @@ def Popen(*args, **kwargs):
     return subprocess.Popen(*args, **kwargs)
 
 def drop_privileges(username):
-    """Drops privileges to selected user.
+    """Drop privileges to selected user.
     @param username: drop privileges to this username
     """
     if not HAVE_PWD:
@@ -225,7 +225,7 @@ def __init__(self, name):
         self.pid = None
 
     def create(self):
-        """Creates pidfile for the current process."""
+        """Create pidfile for the current process."""
         with open(self.filepath, "wb") as f:
             f.write(str(os.getpid()))
 
@@ -249,7 +249,7 @@ def read(self):
         return self.pid
 
     def proc_exists(self, pid):
-        """Returns boolean if the process exists or None when unsupported."""
+        """Return boolean of process existence, or None when unsupported."""
         if not pid:
             return False
 
diff --git a/cuckoo/processing/apkinfo.py b/cuckoo/processing/apkinfo.py
index 7e64cffec4..a5b7f3fd6f 100644
--- a/cuckoo/processing/apkinfo.py
+++ b/cuckoo/processing/apkinfo.py
@@ -30,7 +30,7 @@ def check_size(self, file_list):
         return False
 
     def _apk_files(self, apk):
-        """Returns a list of files in the APK."""
+        """Return a list of files in the APK."""
         ret = []
         for fname, filetype in apk.get_files_types().items():
             buf = apk.zip.read(fname)
diff --git a/cuckoo/processing/baseline.py b/cuckoo/processing/baseline.py
index 3f7dc83812..58f0ca9f9a 100644
--- a/cuckoo/processing/baseline.py
+++ b/cuckoo/processing/baseline.py
@@ -12,7 +12,7 @@
 log = logging.getLogger(__name__)
 
 class Baseline(Processing):
-    """Reduces Baseline results from gathered information."""
+    """Reduce Baseline results from gathered information."""
     order = 2
 
     def deep_tuple(self, o, bl=None):
@@ -39,9 +39,9 @@ def normalize(self, plugin, o):
         return self.deep_tuple(o, plugins.get(plugin))
 
     def memory(self, baseline, report):
-        """Finds the differences between the analysis report and the baseline
-        report. Puts the differences into the baseline part of the report and
-        also marks the existing rows with a `class_` attribute."""
+        """Find the differences between the analysis report and the baseline
+        report. Put the differences into the baseline part of the report and
+        mark the existing rows with a `class_` attribute."""
         results = {}
 
         for plugin in baseline.keys() + report.keys():
diff --git a/cuckoo/processing/behavior.py b/cuckoo/processing/behavior.py
index d711cead30..ab24898060 100644
--- a/cuckoo/processing/behavior.py
+++ b/cuckoo/processing/behavior.py
@@ -19,7 +19,7 @@
 log = logging.getLogger(__name__)
 
 class Summary(BehaviorHandler):
-    """Generates overview summary information (not split by process)."""
+    """Generate overview summary information (not split by process)."""
 
     key = "summary"
     event_types = ["generic"]
diff --git a/cuckoo/processing/irma.py b/cuckoo/processing/irma.py
index 927cbe1470..013f1b1b91 100644
--- a/cuckoo/processing/irma.py
+++ b/cuckoo/processing/irma.py
@@ -14,7 +14,7 @@
 log = logging.getLogger(__name__)
 
 class Irma(Processing):
-    """Gets antivirus signatures from IRMA for various results.
+    """Get antivirus signatures from IRMA for various results.
 
     Currently obtains IRMA results for the target sample.
     """
@@ -97,7 +97,7 @@ def _get_results(self, sha256):
         )
 
     def run(self):
-        """Runs IRMA processing
+        """Run IRMA processing
         @return: full IRMA report.
         """
         self.key = "irma"
@@ -123,13 +123,13 @@ def run(self):
             self._scan_file(self.file_path, self.force)
             results = self._get_results(sha256) or {}
 
-        """ FIXME! could use a proper fix here
-        that probably needs changes on IRMA side aswell
-        --
-        related to  https://github.com/elastic/elasticsearch/issues/15377
-        entropy value is sometimes 0 and sometimes like  0.10191042566270775
-        other issue is that results type changes between string and object :/
-        """
+        # FIXME! could use a proper fix here
+        # that probably needs changes on IRMA side aswell
+        # --
+        # related to  https://github.com/elastic/elasticsearch/issues/15377
+        # entropy value is sometimes 0 and sometimes like  0.10191042566270775
+        # other issue is that results type changes between string and object :/
+        
         for idx, result in enumerate(results["probe_results"]):
             if result["name"] == "PE Static Analyzer":
                 log.debug("Ignoring PE results at index {0}".format(idx))
diff --git a/cuckoo/processing/memory.py b/cuckoo/processing/memory.py
index 26873ae291..b30853749a 100644
--- a/cuckoo/processing/memory.py
+++ b/cuckoo/processing/memory.py
@@ -90,7 +90,7 @@ def get_dtb(self):
         return False
 
     def init_config(self):
-        """Creates a volatility configuration."""
+        """Create a volatility configuration."""
         if self.config is not None and self.addr_space is not None:
             return
 
diff --git a/cuckoo/processing/network.py b/cuckoo/processing/network.py
index 5dc39fe962..e5c13adca7 100644
--- a/cuckoo/processing/network.py
+++ b/cuckoo/processing/network.py
@@ -42,7 +42,7 @@ class Pcap(object):
     ssl_ports = 443,
 
     def __init__(self, filepath, options):
-        """Creates a new instance.
+        """Create a new instance.
         @param filepath: path to PCAP file
         @param options: config options
         """
@@ -93,7 +93,7 @@ def __init__(self, filepath, options):
         self.dns_servers = []
 
     def _is_whitelisted(self, conn, hostname):
-        """Checks if whitelisting conditions are met"""
+        """Check if whitelisting conditions are met"""
         # Is whitelistng enabled?
         if not self.whitelist_enabled:
             return False
@@ -217,7 +217,7 @@ def _add_hosts(self, connection):
             pass
 
     def _tcp_dissect(self, conn, data):
-        """Runs all TCP dissectors.
+        """Run all TCP dissectors.
         @param conn: connection.
         @param data: payload data.
         """
@@ -237,7 +237,7 @@ def _tcp_dissect(self, conn, data):
             self._https_identify(conn, data)
 
     def _udp_dissect(self, conn, data):
-        """Runs all UDP dissectors.
+        """Run all UDP dissectors.
         @param conn: connection.
         @param data: payload data.
         """
@@ -247,7 +247,7 @@ def _udp_dissect(self, conn, data):
                 self._add_dns(conn, data)
 
     def _check_icmp(self, icmp_data):
-        """Checks for ICMP traffic.
+        """Check for ICMP traffic.
         @param icmp_data: ICMP data flow.
         """
         try:
@@ -257,7 +257,7 @@ def _check_icmp(self, icmp_data):
             return False
 
     def _icmp_dissect(self, conn, data):
-        """Runs all ICMP dissectors.
+        """Run all ICMP dissectors.
         @param conn: connection.
         @param data: payload data.
         """
@@ -282,7 +282,7 @@ def _icmp_dissect(self, conn, data):
             self.icmp_requests.append(entry)
 
     def _check_dns(self, udpdata):
-        """Checks for DNS traffic.
+        """Check for DNS traffic.
         @param udpdata: UDP data flow.
         """
         try:
@@ -293,7 +293,7 @@ def _check_dns(self, udpdata):
         return True
 
     def _add_dns(self, conn, udpdata):
-        """Adds a DNS data flow.
+        """Add a DNS data flow.
         @param udpdata: UDP data flow.
         """
         dns = dpkt.dns.DNS(udpdata)
@@ -443,7 +443,7 @@ def _add_domain(self, domain):
                                     "ip": self._dns_gethostbyname(domain)})
 
     def _check_http(self, tcpdata):
-        """Checks for HTTP traffic.
+        """Check for HTTP traffic.
         @param tcpdata: TCP data flow.
         """
         try:
@@ -459,7 +459,7 @@ def _check_http(self, tcpdata):
         return True
 
     def _add_http(self, tcpdata, dport):
-        """Adds an HTTP flow.
+        """Add an HTTP flow.
         @param tcpdata: TCP data flow.
         @param dport: destination port.
         """
@@ -562,7 +562,7 @@ def _process_smtp(self):
 
     def _check_irc(self, tcpdata):
         """
-        Checks for IRC traffic.
+        Check for IRC traffic.
         @param tcpdata: tcp data flow
         """
         try:
@@ -574,7 +574,7 @@ def _check_irc(self, tcpdata):
 
     def _add_irc(self, tcpdata):
         """
-        Adds an IRC communication.
+        Add an IRC communication.
         @param tcpdata: TCP data in flow
         @param dport: destination port
         """
@@ -732,7 +732,7 @@ def run(self):
         return self.results
 
 class Pcap2(object):
-    """Interprets the PCAP file through the httpreplay library which parses
+    """Interpret the PCAP file through the httpreplay library which parses
     the various protocols, decrypts and decodes them, and then provides us
     with the high level representation of it."""
 
@@ -936,7 +936,7 @@ def conn_from_flowtuple(ft):
 # it for the temp files
 # this code is mostly taken from some SO post, can't remember the url though
 def batch_sort(input_iterator, output_path, output_class):
-    """batch sort helper with temporary files, supports sorting large stuff"""
+    """Batch sort helper with temporary files, supports sorting large stuff."""
     chunks = []
     try:
         while True:
@@ -1015,7 +1015,7 @@ def next(self):
         return Keyed((flowtuple, ts, self.ctr), rpkt)
 
 def sort_pcap(inpath, outpath):
-    """Use SortCap class together with batch_sort to sort a pcap"""
+    """Use SortCap class together with batch_sort to sort a pcap."""
     inc = SortCap(inpath)
     batch_sort(
         inc, outpath, lambda path: SortCap(path, linktype=inc.linktype)
@@ -1023,7 +1023,7 @@ def sort_pcap(inpath, outpath):
     return 0
 
 def flowtuple_from_raw(raw, linktype=1):
-    """Parse a packet from a pcap just enough to gain a flow description tuple"""
+    """Parse a packet from a pcap just enough to gain a flow description tuple."""
     ip = iplayer_from_raw(raw, linktype)
 
     if isinstance(ip, dpkt.ip.IP):
diff --git a/cuckoo/processing/procmemory.py b/cuckoo/processing/procmemory.py
index 7f13418b84..e4f0e45afc 100644
--- a/cuckoo/processing/procmemory.py
+++ b/cuckoo/processing/procmemory.py
@@ -52,7 +52,7 @@ def create_idapy(self, process):
                 print>>o, "autoMark(%s, AU_CODE)" % region["addr"]
 
     def _fixup_pe_header(self, pe):
-        """Fixes the PE header from an in-memory representation to an
+        """Fix the PE header from an in-memory representation to an
         on-disk representation."""
         for section in pe.sections:
             section.PointerToRawData = section.VirtualAddress
diff --git a/cuckoo/processing/procmon.py b/cuckoo/processing/procmon.py
index ac7cfe36f0..494f835182 100644
--- a/cuckoo/processing/procmon.py
+++ b/cuckoo/processing/procmon.py
@@ -8,7 +8,7 @@
 from cuckoo.common.abstracts import Processing
 
 class ProcmonLog(list):
-    """Yields each API call event to the parent handler."""
+    """Yield each API call event to the parent handler."""
 
     def __init__(self, filepath):
         list.__init__(self)
@@ -32,7 +32,7 @@ def __nonzero__(self):
         return True
 
 class Procmon(Processing):
-    """Extracts events from procmon.exe output."""
+    """Extract events from procmon.exe output."""
 
     key = "procmon"
 
diff --git a/cuckoo/processing/static.py b/cuckoo/processing/static.py
index 46b5a2c95f..e5ac96d2bb 100644
--- a/cuckoo/processing/static.py
+++ b/cuckoo/processing/static.py
@@ -63,14 +63,14 @@ def __init__(self, file_path):
         self.pe = None
 
     def _get_filetype(self, data):
-        """Gets filetype, uses libmagic if available.
+        """Get filetype, use libmagic if available.
         @param data: data to be analyzed.
         @return: file type or None.
         """
         return sflock.magic.from_buffer(data)
 
     def _get_peid_signatures(self):
-        """Gets PEID signatures.
+        """Get PEID signatures.
         @return: matched signatures or None.
         """
         try:
@@ -81,7 +81,7 @@ def _get_peid_signatures(self):
             return None
 
     def _get_imported_symbols(self):
-        """Gets imported symbols.
+        """Get imported symbols.
         @return: imported symbols dict or None.
         """
         imports = []
@@ -105,7 +105,7 @@ def _get_imported_symbols(self):
         return imports
 
     def _get_exported_symbols(self):
-        """Gets exported symbols.
+        """Get exported symbols.
         @return: exported symbols dict or None.
         """
         exports = []
@@ -122,7 +122,7 @@ def _get_exported_symbols(self):
         return exports
 
     def _get_sections(self):
-        """Gets sections.
+        """Get sections.
         @return: sections dict or None.
         """
         sections = []
@@ -207,7 +207,7 @@ def _get_versioninfo(self):
         return infos
 
     def _get_imphash(self):
-        """Gets imphash.
+        """Get imphash.
         @return: imphash string or None.
         """
         try:
diff --git a/cuckoo/processing/virustotal.py b/cuckoo/processing/virustotal.py
index 7652a2b0a0..c903a9a866 100644
--- a/cuckoo/processing/virustotal.py
+++ b/cuckoo/processing/virustotal.py
@@ -18,7 +18,7 @@
 log = logging.getLogger(__name__)
 
 class VirusTotal(Processing):
-    """Gets antivirus signatures from VirusTotal.com for various results.
+    """Get antivirus signatures from VirusTotal.com for various results.
 
     Currently obtains VirusTotal results for the target sample or URL and the
     dropped files.
@@ -26,7 +26,7 @@ class VirusTotal(Processing):
     order = 2
 
     def run(self):
-        """Runs VirusTotal processing
+        """Run VirusTotal processing
         @return: full VirusTotal report.
         """
         self.key = "virustotal"
@@ -94,7 +94,7 @@ def scan_url(self, url, summary=False):
                         "\"%s\": %s", url, e.message)
 
     def should_scan_file(self, filetype):
-        """Determines whether a certain filetype should be scanned on
+        """Determine whether a certain filetype should be scanned on
         VirusTotal. For example, we're not interested in scanning text
         files.
         @param filetype: file type
diff --git a/cuckoo/reporting/elasticsearch.py b/cuckoo/reporting/elasticsearch.py
index b6701db999..9c2e810b5f 100644
--- a/cuckoo/reporting/elasticsearch.py
+++ b/cuckoo/reporting/elasticsearch.py
@@ -21,7 +21,7 @@
 log = logging.getLogger(__name__)
 
 class ElasticSearch(Report):
-    """Stores report in Elasticsearch."""
+    """Store report in Elasticsearch."""
 
     @classmethod
     def init_once(cls):
diff --git a/cuckoo/reporting/feedback.py b/cuckoo/reporting/feedback.py
index 6ca53d3f19..44fda322f8 100644
--- a/cuckoo/reporting/feedback.py
+++ b/cuckoo/reporting/feedback.py
@@ -6,7 +6,7 @@
 from cuckoo.core.feedback import CuckooFeedbackObject, CuckooFeedback
 
 class Feedback(Report):
-    """Reports feedback to the Cuckoo Feedback backend if required."""
+    """Report feedback to the Cuckoo Feedback backend if required."""
 
     def run(self, results):
         # Nothing to see here.
diff --git a/cuckoo/reporting/jsondump.py b/cuckoo/reporting/jsondump.py
index 54f86a8fba..73041e7253 100644
--- a/cuckoo/reporting/jsondump.py
+++ b/cuckoo/reporting/jsondump.py
@@ -19,10 +19,10 @@ def default(obj):
     raise TypeError("%r is not JSON serializable" % obj)
 
 class JsonDump(Report):
-    """Saves analysis results in JSON format."""
+    """Save analysis results in JSON format."""
 
     def erase_calls(self, results):
-        """Temporarily removes calls from the report by replacing them with
+        """Temporarily remove calls from the report by replacing them with
         empty lists."""
         if self.calls:
             self.calls = None
@@ -34,7 +34,7 @@ def erase_calls(self, results):
             process["calls"] = []
 
     def restore_calls(self, results):
-        """Restores calls that were temporarily removed in the report by
+        """Restore calls that were temporarily removed in the report by
         replacing the calls with the original values."""
         if not self.calls:
             return
diff --git a/cuckoo/reporting/misp.py b/cuckoo/reporting/misp.py
index d92a5f5779..14508d2f71 100644
--- a/cuckoo/reporting/misp.py
+++ b/cuckoo/reporting/misp.py
@@ -91,7 +91,7 @@ def signature(self, results, event):
                 )
 
     def run(self, results):
-        """Submits results to MISP.
+        """Submit results to MISP.
         @param results: Cuckoo results dict.
         """
         url = self.options.get("url")
diff --git a/cuckoo/reporting/mongodb.py b/cuckoo/reporting/mongodb.py
index c6df12f385..b170cdc010 100644
--- a/cuckoo/reporting/mongodb.py
+++ b/cuckoo/reporting/mongodb.py
@@ -12,7 +12,7 @@
 from cuckoo.common.objects import File
 
 class MongoDB(Report):
-    """Stores report in MongoDB."""
+    """Store report in MongoDB."""
     order = 2
 
     # Mongo schema version, used for data migration.
@@ -77,7 +77,7 @@ def store_file(self, file_obj, filename=""):
             return self.db.fs.files.find_one(to_find)["_id"]
 
     def run(self, results):
-        """Writes report.
+        """Write report.
         @param results: analysis results dictionary.
         @raise CuckooReportError: if fails to connect or write to MongoDB.
         """
diff --git a/cuckoo/reporting/notification.py b/cuckoo/reporting/notification.py
index 7a56105d4b..2518d9f882 100644
--- a/cuckoo/reporting/notification.py
+++ b/cuckoo/reporting/notification.py
@@ -18,7 +18,7 @@ def default(obj):
     raise TypeError("%r is not JSON serializable" % obj)
 
 class Notification(Report):
-    """Notifies external service about finished analysis via URL."""
+    """Notify external service about finished analysis via URL."""
     order = 3
 
     def run(self, results):
diff --git a/cuckoo/reporting/singlefile.py b/cuckoo/reporting/singlefile.py
index 148b7c8272..202b5a7eb9 100644
--- a/cuckoo/reporting/singlefile.py
+++ b/cuckoo/reporting/singlefile.py
@@ -20,7 +20,7 @@
 logging.getLogger("weasyprint").setLevel(logging.ERROR)
 
 class SingleFile(Report):
-    """Stores report in a single-file HTML and/or PDF format."""
+    """Store report in a single-file HTML and/or PDF format."""
 
     fonts = [{
         "family": "Roboto",
@@ -120,14 +120,14 @@ def generate_jinja2_template(self, results):
         )
 
     def combine_css(self):
-        """Scans the static/css/ directory and concatenates stylesheets"""
+        """Scan the static/css/ directory and concatenate stylesheets"""
         css_includes = []
         for filepath in glob.glob("%s/static/css/*.css" % self.path_base):
             css_includes.append(open(filepath, "rb").read().decode("utf8"))
         return "\n".join(css_includes)
 
     def combine_js(self):
-        """Scans the static/js/ directory and concatenates js files"""
+        """Scan the static/js/ directory and concatenate js files"""
         js_includes = []
         # Note: jquery-2.2.4.min.js must be the first file.
         filepaths = sorted(glob.glob("%s/static/js/*.js" % self.path_base))
diff --git a/cuckoo/web/analysis/templatetags/analysis_tags.py b/cuckoo/web/analysis/templatetags/analysis_tags.py
index f5e18ba51e..e34d251a6e 100644
--- a/cuckoo/web/analysis/templatetags/analysis_tags.py
+++ b/cuckoo/web/analysis/templatetags/analysis_tags.py
@@ -19,7 +19,7 @@ def mongo_id(value):
 
 @register.filter
 def is_dict(value):
-    """Checks if value is an instance of dict"""
+    """Check if value is an instance of dict"""
     return isinstance(value, dict)
 
 @register.filter
diff --git a/cuckoo/web/analysis/views.py b/cuckoo/web/analysis/views.py
index 7b4fcbb1f3..d9d60d9145 100644
--- a/cuckoo/web/analysis/views.py
+++ b/cuckoo/web/analysis/views.py
@@ -84,7 +84,7 @@ def chunk(request, task_id, pid, pagenum):
 
 @require_safe
 def filtered_chunk(request, task_id, pid, category):
-    """Filters calls for call category.
+    """Filter calls for call category.
     @param task_id: cuckoo task id
     @param pid: pid you want calls
     @param category: call category type
diff --git a/cuckoo/web/controllers/analysis/analysis.py b/cuckoo/web/controllers/analysis/analysis.py
index 0c71128054..5d7a445e76 100644
--- a/cuckoo/web/controllers/analysis/analysis.py
+++ b/cuckoo/web/controllers/analysis/analysis.py
@@ -34,7 +34,7 @@ def _get_report(task_id):
 
     @staticmethod
     def _get_dnsinfo(report):
-        """Create DNS information dicts by domain and ip"""
+        """Create DNS information dicts by domain and ip."""
 
         if "network" in report and "domains" in report["network"]:
             domainlookups = dict((i["domain"], i["ip"]) for i in report["network"]["domains"])
diff --git a/cuckoo/web/controllers/analysis/api.py b/cuckoo/web/controllers/analysis/api.py
index 278f2a805b..829e91c71f 100644
--- a/cuckoo/web/controllers/analysis/api.py
+++ b/cuckoo/web/controllers/analysis/api.py
@@ -102,7 +102,7 @@ def tasks_info(request, body):
     @api_get
     def task_delete(request, task_id):
         """
-        Deletes a task
+        Delete a task.
         :param body: required: task_id
         :return:
         """
@@ -126,7 +126,7 @@ def task_delete(request, task_id):
     @api_get
     def tasks_reschedule(request, task_id, priority=None):
         """
-        Reschedules a task
+        Reschedule a task.
         :param body: required: task_id, priority
         :return: new task_id
         """
@@ -363,7 +363,7 @@ def tasks_recent(request, body):
     @api_post
     def tasks_stats(request, body):
         """
-        Fetches the number of analysis over a
+        Fetch the number of analysis over a
         given period for the "failed" and
         "successful" states. Values are
         returned in months.
diff --git a/cuckoo/web/controllers/analysis/export/export.py b/cuckoo/web/controllers/analysis/export/export.py
index 83ebf0f7fc..e12eebd28f 100644
--- a/cuckoo/web/controllers/analysis/export/export.py
+++ b/cuckoo/web/controllers/analysis/export/export.py
@@ -44,7 +44,7 @@ def estimate_size(task_id, taken_dirs, taken_files):
     @staticmethod
     def create(task_id, taken_dirs, taken_files, report=None):
         """
-        Returns a zip file as a file like object.
+        Return a zip file as a file like object.
         :param task_id: task id
         :param taken_dirs: directories to include
         :param taken_files: files to include
diff --git a/cuckoo/web/controllers/cuckoo/api.py b/cuckoo/web/controllers/cuckoo/api.py
index 5b36ab17bd..8748cdf1e1 100644
--- a/cuckoo/web/controllers/cuckoo/api.py
+++ b/cuckoo/web/controllers/cuckoo/api.py
@@ -21,7 +21,7 @@
 updates = {}
 
 def latest_updates():
-    """Updates the latest Cuckoo version & blogposts at maximum once a day."""
+    """Update the latest Cuckoo version & blogposts at maximum once a day."""
     next_check = datetime.datetime.now() - datetime.timedelta(days=1)
     if updates and updates["timestamp"] > next_check:
         return updates
@@ -36,7 +36,7 @@ class CuckooApi(object):
     @api_get
     def status(request):
         """
-        Returns a variety of information about both
+        Return a variety of information about both
         Cuckoo and the operating system.
         :return: Dictionary
         """
diff --git a/cuckoo/web/controllers/machines/api.py b/cuckoo/web/controllers/machines/api.py
index 50a2b57d17..f1eb67bc2b 100644
--- a/cuckoo/web/controllers/machines/api.py
+++ b/cuckoo/web/controllers/machines/api.py
@@ -13,7 +13,7 @@ class MachinesApi:
     @api_get
     def list(request):
         """
-        Returns a list of all machines currently registered in Cuckoo
+        Return a list of all machines currently registered in Cuckoo
         :return:
         """
         data = {}
@@ -29,7 +29,7 @@ def list(request):
     @api_get
     def view(request, name=None):
         """
-        Returns information about a machine
+        Return information about a machine
         :param name: machine name
         :return: Machine information as a dictionary
         """
diff --git a/stuff/vpncheck.py b/stuff/vpncheck.py
index 31b464b432..92670186dd 100755
--- a/stuff/vpncheck.py
+++ b/stuff/vpncheck.py
@@ -16,7 +16,7 @@
 SIOCGIFADDR = 0x8915
 
 def get_ip_address(interface):
-    """Retrieves the local IP address of a network interface."""
+    """Retrieve the local IP address of a network interface."""
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     buf = fcntl.ioctl(s.fileno(), SIOCGIFADDR, struct.pack("256s", interface))
     return socket.inet_ntoa(buf[20:24])
diff --git a/tests/test_database.py b/tests/test_database.py
index b91a8be73c..03803ce7cf 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -17,7 +17,7 @@
 from cuckoo.misc import set_cwd, cwd, mkdir
 
 class DatabaseEngine(object):
-    """Tests database stuff."""
+    """Test database stuff."""
     URI = None
 
     def setup_class(self):
@@ -275,7 +275,7 @@ class TestMySQL(DatabaseEngine):
 
 @pytest.mark.skipif("sys.platform != 'linux2'")
 class DatabaseMigrationEngine(object):
-    """Tests database migration(s)."""
+    """Test database migration(s)."""
     URI = None
     SRC = None
 
diff --git a/tests/test_init.py b/tests/test_init.py
index a374b59ee1..ca7dd8988f 100644
--- a/tests/test_init.py
+++ b/tests/test_init.py
@@ -60,7 +60,7 @@ def test_venv_new_unicode(self):
         write_supervisor_conf(None)
 
     def test_cuckoo_init(self):
-        """Tests that 'cuckoo init' works with a new CWD."""
+        """Test that 'cuckoo init' works with a new CWD."""
         with pytest.raises(SystemExit):
             main.main(
                 ("--cwd", cwd(), "--nolog", "init"),
@@ -77,7 +77,7 @@ def test_cuckoo_init(self):
         assert os.path.exists(os.path.join(cwd(), "storage", "baseline"))
 
     def test_cuckoo_init_main(self):
-        """Tests that 'cuckoo' works with a new CWD."""
+        """Test that 'cuckoo' works with a new CWD."""
         main.main(
             ("--cwd", cwd(), "--nolog"),
             standalone_mode=False
@@ -95,7 +95,7 @@ def test_cuckoo_init_main_nosigs(self, p):
         p.assert_not_called()
 
     def test_cuckoo_init_no_resultserver(self):
-        """Tests that 'cuckoo init' doesn't launch the ResultServer."""
+        """Test that 'cuckoo init' doesn't launch the ResultServer."""
         with pytest.raises(SystemExit):
             main.main(
                 ("--cwd", cwd(), "--nolog", "init"),
diff --git a/tests/test_log.py b/tests/test_log.py
index bf11672b43..4072665382 100644
--- a/tests/test_log.py
+++ b/tests/test_log.py
@@ -18,7 +18,7 @@
 db = Database()
 
 def reset_logging():
-    """Resets the logging module to its initial state so that we can
+    """Reset the logging module to its initial state so that we can
     re-register all kinds of logging logic for unit testing purposes."""
     logging.root = logging.RootLogger(logging.WARNING)
     logging.Logger.root = logging.root
diff --git a/tests/test_misc.py b/tests/test_misc.py
index 1c3a78b6bf..4d1ef2efa6 100644
--- a/tests/test_misc.py
+++ b/tests/test_misc.py
@@ -111,7 +111,7 @@ def test_platforms():
     assert sys.platform in ("win32", "linux2", "darwin")
 
 def test_popen():
-    """Ensures that Popen is working properly."""
+    """Ensure that Popen is working properly."""
     with mock.patch("subprocess.Popen") as p:
         p.return_value = None
         Popen(["foo", "bar"])
diff --git a/tests/test_reporting.py b/tests/test_reporting.py
index 5c3f592081..c25b4a9434 100644
--- a/tests/test_reporting.py
+++ b/tests/test_reporting.py
@@ -116,7 +116,7 @@ def test_empty_mattermost():
 
 @responses.activate
 def test_empty_misp():
-    """Merely connects to MISP and creates the new event."""
+    """Merely connect to MISP and create the new event."""
     set_cwd(tempfile.mkdtemp())
     conf = {
         "misp": {
diff --git a/tests/test_submit.py b/tests/test_submit.py
index 10d6517176..e27db869f8 100644
--- a/tests/test_submit.py
+++ b/tests/test_submit.py
@@ -38,7 +38,7 @@ def setup(self):
         self.submit_manager = SubmitManager()
 
     def test_pre_file(self):
-        """Tests the submission of a plaintext file"""
+        """Test the submission of a plaintext file"""
         assert self.submit_manager.pre(submit_type="files", data=[{
             "name": "foo.txt",
             "data": open("tests/files/foo.txt", "rb").read()
@@ -54,7 +54,7 @@ def test_pre_file(self):
         assert filedata == open("tests/files/foo.txt", "rb").read()
 
     def test_pre_url(self):
-        """Tests the submission of URLs (http/https)"""
+        """Test the submission of URLs (http/https)"""
         assert self.submit_manager.pre(submit_type="strings", data=[
             "http://theguardian.com/",
             "https://news.ycombinator.com/",
@@ -83,7 +83,7 @@ def test_invalid_strings(self):
 
     @responses.activate
     def test_pre_hash(self):
-        """Tests the submission of a VirusTotal hash."""
+        """Test the submission of a VirusTotal hash."""
         with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
             rsps.add(
                 responses.GET, VirusTotalAPI.HASH_DOWNLOAD, body="A"*1024*1024
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 01383b3a83..4bb2263451 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -26,19 +26,19 @@ def setup(self):
         self.tmp_dir = tempfile.gettempdir()
 
     def test_root_folder(self):
-        """Tests a single folder creation based on the root parameter."""
+        """Test single folder creation based on the root parameter."""
         Folders.create(os.path.join(self.tmp_dir, "foo"))
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         os.rmdir(os.path.join(self.tmp_dir, "foo"))
 
     def test_single_folder(self):
-        """Tests a single folder creation."""
+        """Test single folder creation."""
         Folders.create(self.tmp_dir, "foo")
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         os.rmdir(os.path.join(self.tmp_dir, "foo"))
 
     def test_multiple_folders(self):
-        """Tests multiple folders creation."""
+        """Test multiple folder creation."""
         Folders.create(self.tmp_dir, ["foo", "bar"])
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         assert os.path.exists(os.path.join(self.tmp_dir, "bar"))
@@ -46,7 +46,7 @@ def test_multiple_folders(self):
         os.rmdir(os.path.join(self.tmp_dir, "bar"))
 
     def test_copy_folder(self):
-        """Tests recursive folder copy"""
+        """Test recursive folder copy."""
         dirpath = tempfile.mkdtemp()
         set_cwd(dirpath)
 
@@ -54,21 +54,21 @@ def test_copy_folder(self):
         assert os.path.isfile("%s/reports/report.json" % dirpath)
 
     def test_duplicate_folder(self):
-        """Tests a duplicate folder creation."""
+        """Test a duplicate folder creation."""
         Folders.create(self.tmp_dir, "foo")
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         Folders.create(self.tmp_dir, "foo")
         os.rmdir(os.path.join(self.tmp_dir, "foo"))
 
     def test_delete_folder(self):
-        """Tests folder deletion #1."""
+        """Test folder deletion #1."""
         Folders.create(self.tmp_dir, "foo")
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         Folders.delete(os.path.join(self.tmp_dir, "foo"))
         assert not os.path.exists(os.path.join(self.tmp_dir, "foo"))
 
     def test_delete_folder2(self):
-        """Tests folder deletion #2."""
+        """Test folder deletion #2."""
         Folders.create(self.tmp_dir, "foo")
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         Folders.delete(self.tmp_dir, "foo")
diff --git a/tests/utils.py b/tests/utils.py
index 323861f3cd..7d32016f34 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -21,7 +21,7 @@ def __exit__(self, type_, value, traceback):
         os.chdir(self.origpath)
 
 def init_analysis(task_id, package, *filename):
-    """Initializes an analysis with an "encrypted" binary from tests/files/."""
+    """Initialize an analysis with an "encrypted" binary from tests/files/."""
     mkdir(cwd(analysis=task_id))
     content = open(os.path.join("tests", "files", *filename), "rb").read()
     open(cwd("binary", analysis=task_id), "wb").write(content[::-1])

From e12ca9b3310315b8eb13cf1ab9c47eb24afdd51a Mon Sep 17 00:00:00 2001
From: Prockwin <prockwin@yandex.ru>
Date: Mon, 11 Mar 2019 13:27:06 +0500
Subject: [PATCH 025/138] Horisontal scroll fix

Fix fullscreen horisontal scroll on some pages
---
 cuckoo/web/static/css/main.css | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cuckoo/web/static/css/main.css b/cuckoo/web/static/css/main.css
index c013d94c2c..de6bd26365 100644
--- a/cuckoo/web/static/css/main.css
+++ b/cuckoo/web/static/css/main.css
@@ -12430,6 +12430,7 @@ pre
 [id='dashboard.views.index'] .app
 {
     padding: 0;
+    overflow: hidden;
 }
 
 .content

From cb3c2fc0a9a70c2c6855e34448255c3b4a500f04 Mon Sep 17 00:00:00 2001
From: Prockwin <prockwin@yandex.ru>
Date: Mon, 11 Mar 2019 13:27:06 +0500
Subject: [PATCH 026/138] Horisontal scroll fix

Fix fullscreen horisontal scroll on some pages
---
 cuckoo/web/static/css/main.css | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cuckoo/web/static/css/main.css b/cuckoo/web/static/css/main.css
index c013d94c2c..de6bd26365 100644
--- a/cuckoo/web/static/css/main.css
+++ b/cuckoo/web/static/css/main.css
@@ -12430,6 +12430,7 @@ pre
 [id='dashboard.views.index'] .app
 {
     padding: 0;
+    overflow: hidden;
 }
 
 .content

From 52dc77569cab8f0b5186df207753777915b595cc Mon Sep 17 00:00:00 2001
From: ihatecsv <me@drakeluce.com>
Date: Wed, 8 May 2019 17:19:05 -0300
Subject: [PATCH 027/138] Fixed valid docstrings to comply with PEP 257

---
 cuckoo/apps/api.py                            |  6 +-
 cuckoo/apps/apps.py                           |  4 +-
 cuckoo/apps/rooter.py                         |  4 +-
 cuckoo/auxiliary/replay.py                    |  2 +-
 cuckoo/auxiliary/services.py                  |  2 +-
 cuckoo/common/abstracts.py                    | 82 +++++++++----------
 cuckoo/common/config.py                       |  6 +-
 cuckoo/common/dns.py                          |  6 +-
 cuckoo/common/files.py                        |  6 +-
 cuckoo/common/netlog.py                       |  8 +-
 cuckoo/common/scripting.py                    |  2 +-
 cuckoo/common/utils.py                        | 16 ++--
 cuckoo/core/database.py                       | 54 ++++++------
 cuckoo/core/feedback.py                       |  4 +-
 cuckoo/core/guest.py                          |  2 +-
 cuckoo/core/init.py                           |  2 +-
 cuckoo/core/plugins.py                        |  4 +-
 cuckoo/core/scheduler.py                      |  2 +-
 cuckoo/core/startup.py                        | 12 +--
 cuckoo/core/submit.py                         |  8 +-
 .../analyzer/darwin/lib/api/screenshot.py     |  2 +-
 .../analyzer/darwin/lib/common/hashing.py     |  2 +-
 .../analyzer/darwin/modules/packages/zip.py   |  2 +-
 .../analyzer/linux/lib/common/abstracts.py    |  2 +-
 .../data/analyzer/linux/lib/common/hashing.py |  2 +-
 .../analyzer/linux/modules/auxiliary/stap.py  |  2 +-
 cuckoo/data/analyzer/windows/analyzer.py      | 10 +--
 .../data/analyzer/windows/lib/api/process.py  |  2 +-
 .../analyzer/windows/lib/common/abstracts.py  |  6 +-
 .../analyzer/windows/lib/common/hashing.py    |  2 +-
 cuckoo/data/analyzer/windows/lib/core/pipe.py | 12 +--
 .../windows/modules/auxiliary/disguise.py     |  2 +-
 .../windows/modules/auxiliary/reboot.py       |  2 +-
 .../windows/modules/auxiliary/recentfiles.py  |  2 +-
 .../windows/modules/auxiliary/zer0m0n.py      |  2 +-
 .../analyzer/windows/modules/packages/zip.py  |  2 +-
 cuckoo/machinery/avd.py                       | 22 ++---
 cuckoo/machinery/esx.py                       |  2 +-
 cuckoo/machinery/physical.py                  | 10 +--
 cuckoo/machinery/qemu.py                      |  6 +-
 cuckoo/machinery/virtualbox.py                | 14 ++--
 cuckoo/machinery/vmware.py                    | 10 +--
 cuckoo/machinery/vsphere.py                   |  2 +-
 cuckoo/machinery/xenserver.py                 |  4 +-
 cuckoo/main.py                                |  8 +-
 cuckoo/misc.py                                | 14 ++--
 cuckoo/processing/apkinfo.py                  |  2 +-
 cuckoo/processing/baseline.py                 |  8 +-
 cuckoo/processing/behavior.py                 |  2 +-
 cuckoo/processing/irma.py                     | 18 ++--
 cuckoo/processing/memory.py                   |  2 +-
 cuckoo/processing/network.py                  | 32 ++++----
 cuckoo/processing/procmemory.py               |  2 +-
 cuckoo/processing/procmon.py                  |  4 +-
 cuckoo/processing/static.py                   | 12 +--
 cuckoo/processing/virustotal.py               |  6 +-
 cuckoo/reporting/elasticsearch.py             |  2 +-
 cuckoo/reporting/feedback.py                  |  2 +-
 cuckoo/reporting/jsondump.py                  |  6 +-
 cuckoo/reporting/misp.py                      |  2 +-
 cuckoo/reporting/mongodb.py                   |  4 +-
 cuckoo/reporting/notification.py              |  2 +-
 cuckoo/reporting/singlefile.py                |  6 +-
 .../analysis/templatetags/analysis_tags.py    |  2 +-
 cuckoo/web/analysis/views.py                  |  2 +-
 cuckoo/web/controllers/analysis/analysis.py   |  2 +-
 cuckoo/web/controllers/analysis/api.py        |  6 +-
 .../web/controllers/analysis/export/export.py |  2 +-
 cuckoo/web/controllers/cuckoo/api.py          |  4 +-
 cuckoo/web/controllers/machines/api.py        |  4 +-
 stuff/vpncheck.py                             |  2 +-
 tests/test_database.py                        |  4 +-
 tests/test_init.py                            |  6 +-
 tests/test_log.py                             |  2 +-
 tests/test_misc.py                            |  2 +-
 tests/test_reporting.py                       |  2 +-
 tests/test_submit.py                          |  6 +-
 tests/test_utils.py                           | 14 ++--
 tests/utils.py                                |  2 +-
 79 files changed, 276 insertions(+), 276 deletions(-)

diff --git a/cuckoo/apps/api.py b/cuckoo/apps/api.py
index 18d66c1f39..3ddb6fb6e0 100644
--- a/cuckoo/apps/api.py
+++ b/cuckoo/apps/api.py
@@ -38,7 +38,7 @@ def json_error(status_code, message):
     return r
 
 def shutdown_server():
-    """Shutdown API werkzeug server"""
+    """Shutdown API werkzeug server."""
     shutdown = request.environ.get("werkzeug.server.shutdown")
     if shutdown:
         shutdown()
@@ -676,8 +676,8 @@ def vpn_status():
 
 @app.route("/exit")
 def exit_api():
-    """Shuts down the server if in debug mode and
-    using the werkzeug server"""
+    """Shut down the server if in debug mode and
+    using the werkzeug server."""
     if not app.debug:
         return json_error(403, "This call can only be used in debug mode")
 
diff --git a/cuckoo/apps/apps.py b/cuckoo/apps/apps.py
index 75aad720a4..467d0df624 100644
--- a/cuckoo/apps/apps.py
+++ b/cuckoo/apps/apps.py
@@ -99,7 +99,7 @@ def fetch_community(branch="master", force=False, filepath=None):
             open(filepath, "wb").write(t.extractfile(member).read())
 
 def enumerate_files(path, pattern):
-    """Yields all filepaths from a directory."""
+    """Yield all filepaths from a directory."""
     if os.path.isfile(path):
         yield path
     elif os.path.isdir(path):
@@ -346,7 +346,7 @@ def process_tasks(instance, maxcount, timeout):
 
 def cuckoo_clean():
     """Clean up cuckoo setup.
-    It deletes logs, all stored data from file system and configured
+    Delete logs, all stored data from file system and configured
     databases (SQL and MongoDB).
     """
     # Init logging (without writing to file).
diff --git a/cuckoo/apps/rooter.py b/cuckoo/apps/rooter.py
index 838c0b7471..a09bb1230b 100644
--- a/cuckoo/apps/rooter.py
+++ b/cuckoo/apps/rooter.py
@@ -59,7 +59,7 @@ def rt_available(rt_table):
         return False
 
 def vpn_status():
-    """Gets current VPN status."""
+    """Get current VPN status."""
     ret = {}
     for line in run(s.service, "openvpn", "status")[0].split("\n"):
         x = re.search("'(?P<vpn>\\w+)'\\ is\\ (?P<running>not)?", line)
@@ -125,7 +125,7 @@ def init_rttable(rt_table, interface):
         run(s.ip, *args)
 
 def flush_rttable(rt_table):
-    """Flushes specified routing table entries."""
+    """Flush specified routing table entries."""
     if rt_table in ["local", "main", "default"]:
         return
 
diff --git a/cuckoo/auxiliary/replay.py b/cuckoo/auxiliary/replay.py
index dc41deea35..984dd11a5e 100644
--- a/cuckoo/auxiliary/replay.py
+++ b/cuckoo/auxiliary/replay.py
@@ -24,7 +24,7 @@ def __init__(self):
         self.port = None
 
     def pcap2mitm(self, pcappath, tlsmaster):
-        """Used to translate a .pcap into a .mitm file."""
+        """Translate a .pcap into a .mitm file."""
         mitmpath = tempfile.mktemp(suffix=".mitm")
         with open(mitmpath, "wb") as f:
             httpreplay.utils.pcap2mitm(pcappath, f, tlsmaster, True)
diff --git a/cuckoo/auxiliary/services.py b/cuckoo/auxiliary/services.py
index e79f5ea2f8..55517e0e84 100644
--- a/cuckoo/auxiliary/services.py
+++ b/cuckoo/auxiliary/services.py
@@ -14,7 +14,7 @@
 db = Database()
 
 class Services(Auxiliary):
-    """Allows one or more additional VMs to be run next to an analysis. Either
+    """Allow one or more additional VMs to be run next to an analysis. Either
     as global services (which are generally never rebooted) or on a
     per-analysis basis."""
 
diff --git a/cuckoo/common/abstracts.py b/cuckoo/common/abstracts.py
index 8c5edc1811..f03be8f834 100644
--- a/cuckoo/common/abstracts.py
+++ b/cuckoo/common/abstracts.py
@@ -166,7 +166,7 @@ def init_once(cls):
         pass
 
     def pcap_path(self, task_id):
-        """Returns the .pcap path for this task id."""
+        """Return the .pcap path for this task id."""
         return cwd("storage", "analyses", "%s" % task_id, "dump.pcap")
 
     def set_options(self, options):
@@ -228,7 +228,7 @@ def _initialize(self, module_name):
             )
 
     def _initialize_check(self):
-        """Runs checks against virtualization software when a machine manager
+        """Run checks against virtualization software when a machine manager
         is initialized.
         @note: in machine manager modules you may override or superclass
                his method.
@@ -271,7 +271,7 @@ def machines(self):
         return self.db.list_machines()
 
     def availables(self):
-        """How many machines are free.
+        """Return how many machines are free.
         @return: free machines count.
         """
         return self.db.count_machines_available()
@@ -297,13 +297,13 @@ def release(self, label=None):
         self.db.unlock_machine(label)
 
     def running(self):
-        """Returns running virtual machines.
+        """Return running virtual machines.
         @return: running virtual machines list.
         """
         return self.db.list_machines(locked=True)
 
     def shutdown(self):
-        """Shutdown the machine manager. Kills all alive machines.
+        """Shutdown the machine manager and kill all alive machines.
         @raise CuckooMachineError: if unable to stop machine.
         """
         if len(self.running()) > 0:
@@ -339,13 +339,13 @@ def stop(self, label=None):
         raise NotImplementedError
 
     def _list(self):
-        """Lists virtual machines configured.
+        """List virtual machines configured.
         @raise NotImplementedError: this method is abstract.
         """
         raise NotImplementedError
 
     def dump_memory(self, label, path):
-        """Takes a memory dump of a machine.
+        """Take a memory dump of a machine.
         @param path: path to where to store the memory dump.
         """
         raise NotImplementedError
@@ -372,7 +372,7 @@ def get_remote_control_params(self, label):
         raise NotImplementedError
 
     def _wait_status(self, label, *states):
-        """Waits for a vm status.
+        """Wait for a vm status.
         @param label: virtual machine name.
         @param state: virtual machine status, accepts multiple states as list.
         @raise CuckooMachineError: if default waiting timeout expire.
@@ -433,7 +433,7 @@ def initialize(self, module):
         super(LibVirtMachinery, self).initialize(module)
 
     def _initialize_check(self):
-        """Runs all checks when a machine manager is initialized.
+        """Run all checks when a machine manager is initialized.
         @raise CuckooMachineError: if libvirt version is not supported.
         """
         # Version checks.
@@ -449,7 +449,7 @@ def _initialize_check(self):
         super(LibVirtMachinery, self)._initialize_check()
 
     def start(self, label, task):
-        """Starts a virtual machine.
+        """Start a virtual machine.
         @param label: virtual machine name.
         @param task: task object.
         @raise CuckooMachineError: if unable to start virtual machine.
@@ -502,7 +502,7 @@ def start(self, label, task):
         self._wait_status(label, self.RUNNING)
 
     def stop(self, label):
-        """Stops a virtual machine. Kill them all.
+        """Stop a virtual machine. Kill them all.
         @param label: virtual machine name.
         @raise CuckooMachineError: if unable to stop virtual machine.
         """
@@ -536,7 +536,7 @@ def shutdown(self):
         self.vms = None
 
     def dump_memory(self, label, path):
-        """Takes a memory dump.
+        """Take a memory dump.
         @param path: path to where to store the memory dump.
         """
         log.debug("Dumping memory for machine %s", label)
@@ -555,7 +555,7 @@ def dump_memory(self, label, path):
             self._disconnect(conn)
 
     def _status(self, label):
-        """Gets current status of a vm.
+        """Get current status of a vm.
         @param label: virtual machine name.
         @return: status string.
         """
@@ -600,7 +600,7 @@ def _status(self, label):
                                      "{0}".format(label))
 
     def _connect(self):
-        """Connects to libvirt subsystem.
+        """Connect to libvirt subsystem.
         @raise CuckooMachineError: when unable to connect to libvirt.
         """
         # Check if a connection string is available.
@@ -614,7 +614,7 @@ def _connect(self):
             raise CuckooMachineError("Cannot connect to libvirt")
 
     def _disconnect(self, conn):
-        """Disconnects to libvirt subsystem.
+        """Disconnect from libvirt subsystem.
         @raise CuckooMachineError: if cannot disconnect from libvirt.
         """
         try:
@@ -870,7 +870,7 @@ def init_once(cls):
         pass
 
     def _check_value(self, pattern, subject, regex=False, all=False):
-        """Checks a pattern against a given subject.
+        """Check a pattern against a given subject.
         @param pattern: string or expression to check for.
         @param subject: target of the check.
         @param regex: boolean representing if the pattern is a regular
@@ -995,7 +995,7 @@ def get_keys(self, pid=None, actions=None):
 
     def check_file(self, pattern, regex=False, actions=None, pid=None,
                    all=False):
-        """Checks for a file being opened.
+        """Check for a file being opened.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1018,7 +1018,7 @@ def check_file(self, pattern, regex=False, actions=None, pid=None,
 
     def check_dll_loaded(self, pattern, regex=False, actions=None, pid=None,
                          all=False):
-        """Checks for DLLs being loaded.
+        """Check for DLLs being loaded.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1032,7 +1032,7 @@ def check_dll_loaded(self, pattern, regex=False, actions=None, pid=None,
                                  all=all)
 
     def check_command_line(self, pattern, regex=False, all=False):
-        """Checks for a command line being opened.
+        """Check for a command line being opened.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1045,7 +1045,7 @@ def check_command_line(self, pattern, regex=False, all=False):
 
     def check_key(self, pattern, regex=False, actions=None, pid=None,
                   all=False):
-        """Checks for a registry key being accessed.
+        """Check for a registry key being accessed.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1073,7 +1073,7 @@ def get_mutexes(self, pid=None):
         return self.get_summary_generic(pid, ["mutex"])
 
     def check_mutex(self, pattern, regex=False, all=False):
-        """Checks for a mutex being opened.
+        """Check for a mutex being opened.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1085,11 +1085,11 @@ def check_mutex(self, pattern, regex=False, all=False):
                                  all=all)
 
     def get_command_lines(self):
-        """Retrieves all command lines used."""
+        """Retrieve all command lines used."""
         return self.get_summary("command_line")
 
     def get_wmi_queries(self):
-        """Retrieves all executed WMI queries."""
+        """Retrieve all executed WMI queries."""
         return self.get_summary("wmi_query")
 
     def get_net_generic(self, subtype):
@@ -1100,68 +1100,68 @@ def get_net_generic(self, subtype):
         return self.get_results("network", {}).get(subtype, [])
 
     def get_net_hosts(self):
-        """Returns a list of all hosts."""
+        """Return a list of all hosts."""
         return self.get_net_generic("hosts")
 
     def get_net_domains(self):
-        """Returns a list of all domains."""
+        """Return a list of all domains."""
         return self.get_net_generic("domains")
 
     def get_net_http(self):
-        """Returns a list of all http data."""
+        """Return a list of all http data."""
         return self.get_net_generic("http")
 
     def get_net_http_ex(self):
-        """Returns a list of all http data."""
+        """Return a list of all http data."""
         return \
             self.get_net_generic("http_ex") + self.get_net_generic("https_ex")
 
     def get_net_udp(self):
-        """Returns a list of all udp data."""
+        """Return a list of all udp data."""
         return self.get_net_generic("udp")
 
     def get_net_icmp(self):
-        """Returns a list of all icmp data."""
+        """Return a list of all icmp data."""
         return self.get_net_generic("icmp")
 
     def get_net_irc(self):
-        """Returns a list of all irc data."""
+        """Return a list of all irc data."""
         return self.get_net_generic("irc")
 
     def get_net_smtp(self):
-        """Returns a list of all smtp data."""
+        """Return a list of all smtp data."""
         return self.get_net_generic("smtp")
 
     def get_net_smtp_ex(self):
-        """"Returns a list of all smtp data"""
+        """"Return a list of all smtp data"""
         return self.get_net_generic("smtp_ex")
 
     def get_virustotal(self):
-        """Returns the information retrieved from virustotal."""
+        """Return the information retrieved from virustotal."""
         return self.get_results("virustotal", {})
 
     def get_volatility(self, module=None):
-        """Returns the data that belongs to the given module."""
+        """Return the data that belongs to the given module."""
         volatility = self.get_results("memory", {})
         return volatility if module is None else volatility.get(module, {})
 
     def get_apkinfo(self, section=None, default={}):
-        """Returns the apkinfo results for this analysis."""
+        """Return the apkinfo results for this analysis."""
         apkinfo = self.get_results("apkinfo", {})
         return apkinfo if section is None else apkinfo.get(section, default)
 
     def get_droidmon(self, section=None, default={}):
-        """Returns the droidmon results for this analysis."""
+        """Return the droidmon results for this analysis."""
         droidmon = self.get_results("droidmon", {})
         return droidmon if section is None else droidmon.get(section, default)
 
     def get_googleplay(self, section=None, default={}):
-        """Returns the Google Play results for this analysis."""
+        """Return the Google Play results for this analysis."""
         googleplay = self.get_results("googleplay", {})
         return googleplay if section is None else googleplay.get(section, default)
 
     def check_ip(self, pattern, regex=False, all=False):
-        """Checks for an IP address being contacted.
+        """Check for an IP address being contacted.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1173,7 +1173,7 @@ def check_ip(self, pattern, regex=False, all=False):
                                  all=all)
 
     def check_domain(self, pattern, regex=False, all=False):
-        """Checks for a domain being contacted.
+        """Check for a domain being contacted.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1189,7 +1189,7 @@ def check_domain(self, pattern, regex=False, all=False):
                                  all=all)
 
     def check_url(self, pattern, regex=False, all=False):
-        """Checks for a URL being contacted.
+        """Check for a URL being contacted.
         @param pattern: string or expression to check for.
         @param regex: boolean representing if the pattern is a regular
                       expression or not and therefore should be compiled.
@@ -1279,7 +1279,7 @@ def mark(self, **kwargs):
         self.marks.append(mark)
 
     def has_marks(self, count=None):
-        """Returns true if this signature has one or more marks."""
+        """Return true if this signature has one or more marks."""
         if count is not None:
             return len(self.marks) >= count
         return not not self.marks
diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py
index 92cfa31376..31e711c475 100644
--- a/cuckoo/common/config.py
+++ b/cuckoo/common/config.py
@@ -148,7 +148,7 @@ def parse(self, value):
             log.error("Incorrect UUID %s", value)
 
     def check(self, value):
-        """Checks if the value is of type UUID."""
+        """Check if the value is of type UUID."""
         try:
             click.UUID(value)
             return True
@@ -1042,7 +1042,7 @@ def get(self, section):
 
     @staticmethod
     def from_confdir(dirpath, loose=False, sanitize=False):
-        """Reads all the configuration from a configuration directory. If
+        """Read all the configuration from a configuration directory. If
         `sanitize` is set, then black out sensitive fields."""
         ret = {}
         for filename in os.listdir(dirpath):
@@ -1183,7 +1183,7 @@ def cast(s, value):
     return type_.parse(value)
 
 def read_kv_conf(filepath):
-    """Reads a flat Cuckoo key/value configuration file."""
+    """Read a flat Cuckoo key/value configuration file."""
     ret = {}
     for line in open(filepath, "rb"):
         line = line.strip()
diff --git a/cuckoo/common/dns.py b/cuckoo/common/dns.py
index 56f6009b78..a13bce6081 100644
--- a/cuckoo/common/dns.py
+++ b/cuckoo/common/dns.py
@@ -36,9 +36,9 @@ def set_timeout_value(value):
 # standard gethostbyname in thread
 # http://code.activestate.com/recipes/473878/
 def with_timeout(func, args=(), kwargs={}):
-    """This function will spawn a thread and run the given function
-    using the args, kwargs and return the given default value if the
-    timeout_duration is exceeded.
+    """Spawn a thread and run the given function using the args,
+    kwargs and return the given default value if the timeout_duration
+    is exceeded.
     """
     class ResultThread(threading.Thread):
         daemon = True
diff --git a/cuckoo/common/files.py b/cuckoo/common/files.py
index 525382b818..5ae8e24811 100644
--- a/cuckoo/common/files.py
+++ b/cuckoo/common/files.py
@@ -13,7 +13,7 @@
 from cuckoo.misc import getuser
 
 def temppath():
-    """Returns the true temporary directory."""
+    """Return the true temporary directory."""
     tmppath = config("cuckoo:cuckoo:tmppath")
 
     # Backwards compatibility with older configuration.
@@ -37,7 +37,7 @@ def get_filename_from_path(path):
 class Folders(Storage):
     @staticmethod
     def create(root=".", folders=None):
-        """Creates a directory or multiple directories.
+        """Create a directory or multiple directories.
         @param root: root path.
         @param folders: folders list to be created.
         @raise CuckooOperationalError: if fails to create folder.
@@ -149,7 +149,7 @@ def copy(path_target, path_dest):
 
     @staticmethod
     def hash_file(method, filepath):
-        """Calculates an hash on a file by path.
+        """Calculate a hash on a file by path.
         @param method: callable hashing method
         @param path: file path
         @return: computed hash string
diff --git a/cuckoo/common/netlog.py b/cuckoo/common/netlog.py
index 54ff29a74c..9fb65ae7c4 100644
--- a/cuckoo/common/netlog.py
+++ b/cuckoo/common/netlog.py
@@ -55,7 +55,7 @@ def default_converter_64bit(v):
     return v
 
 class BsonParser(ProtocolHandler):
-    """Receives and interprets .bson logs from the monitor.
+    """Receive and interpret .bson logs from the monitor.
 
     The monitor provides us with "info" messages that explain how the function
     arguments will come through later on. This class remembers these info
@@ -117,9 +117,9 @@ def resolve_flags(self, apiname, argdict, flags):
             flags[argument] = "|".join(flags[argument])
 
     def determine_unserializers(self, arginfo):
-        """Determines which unserializers (or converters) have to be used in
-        order to parse the various arguments for this function call. Keeps in
-        mind whether the current bson is 32-bit or 64-bit."""
+        """Determine which unserializers (or converters) have to be used in
+        order to parse the various arguments for this function call. Maintains
+        whether the current bson is 32-bit or 64-bit."""
         argnames, converters = [], []
 
         for argument in arginfo:
diff --git a/cuckoo/common/scripting.py b/cuckoo/common/scripting.py
index ae1a73f96b..0f12c3dd0b 100644
--- a/cuckoo/common/scripting.py
+++ b/cuckoo/common/scripting.py
@@ -105,7 +105,7 @@ def get_script(self):
         return " ".join(self.args.get("command", []))
 
 def ps1_cmdarg(s, minimum=1):
-    """Creates an exactly matching PowerShell command line argument regex,
+    """Create an exactly matching PowerShell command line argument regex,
     instead of a regex that matches anything with the same characters."""
     return "".join(
         "([%s%s^]" % (ch.lower(), ch.upper()) for ch in s
diff --git a/cuckoo/common/utils.py b/cuckoo/common/utils.py
index 63ab5a1b2e..2246745591 100644
--- a/cuckoo/common/utils.py
+++ b/cuckoo/common/utils.py
@@ -36,7 +36,7 @@
 )
 
 def convert_char(c):
-    """Escapes characters.
+    """Escape characters.
     @param c: dirty char.
     @return: sanitized char.
     """
@@ -76,14 +76,14 @@ def constant_time_compare(a, b):
     return result == 0
 
 def validate_hash(h):
-    """Validates a hash by length and contents."""
+    """Validate a hash by length and contents."""
     if len(h) not in (32, 40, 64, 128):
         return False
 
     return bool(re.match("[0-9a-fA-F]*$", h))
 
 def validate_url(url, allow_invalid=False):
-    """Validates an URL using Django's built-in URL validator"""
+    """Validate an URL using Django's built-in URL validator"""
     from django.core.validators import URLValidator
     val = URLValidator(schemes=["http", "https"])
 
@@ -231,7 +231,7 @@ def guid_name(guid):
     return GUIDS.get(guid)
 
 def exception_message():
-    """Creates a message describing an unhandled exception."""
+    """Create a message describing an unhandled exception."""
     def get_os_release():
         """Returns detailed OS release."""
         if platform.linux_distribution()[0]:
@@ -272,7 +272,7 @@ def get_os_release():
 _jsbeautify_lock = threading.Lock()
 
 def jsbeautify(javascript):
-    """Beautifies Javascript through jsbeautifier and ignore some messages."""
+    """Beautify Javascript through jsbeautifier and ignore some messages."""
     with _jsbeautify_lock:
         origout, sys.stdout = sys.stdout, io.StringIO()
 
@@ -290,7 +290,7 @@ def jsbeautify(javascript):
     return javascript
 
 def htmlprettify(html):
-    """Beautifies HTML through BeautifulSoup4."""
+    """Beautify HTML through BeautifulSoup4."""
     # The following ignores the following bs4 warning:
     # UserWarning: "." looks like a filename, not markup.
     with warnings.catch_warnings():
@@ -298,7 +298,7 @@ def htmlprettify(html):
         return bs4.BeautifulSoup(html, "html.parser").prettify()
 
 def json_default(obj):
-    """JSON serializer for objects not serializable by default json code"""
+    """JSON serialize objects not serializable by default json code"""
     if hasattr(obj, "to_dict"):
         return obj.to_dict()
 
@@ -331,7 +331,7 @@ def parse_bool(value):
     return bool(int(value))
 
 def supported_version(version, minimum, maximum):
-    """Checks if a version number is supported as per the minimum and maximum
+    """Check if a version number is supported as per the minimum and maximum
     version numbers."""
     if minimum and StrictVersion(version) < StrictVersion(minimum):
         return False
diff --git a/cuckoo/core/database.py b/cuckoo/core/database.py
index 553b8b3605..364e8973d3 100644
--- a/cuckoo/core/database.py
+++ b/cuckoo/core/database.py
@@ -120,7 +120,7 @@ def rcparams(self, value):
             self._rcparams = value
 
     def to_dict(self):
-        """Converts object to dict.
+        """Convert object to dict.
         @return: dict
         """
         d = {}
@@ -136,7 +136,7 @@ def to_dict(self):
         return d
 
     def to_json(self):
-        """Converts object to JSON.
+        """Convert object to JSON.
         @return: JSON data
         """
         return json.dumps(self.to_dict())
@@ -198,7 +198,7 @@ def __repr__(self):
         return "<Guest('{0}','{1}')>".format(self.id, self.name)
 
     def to_dict(self):
-        """Converts object to dict.
+        """Convert object to dict.
         @return: dict
         """
         d = {}
@@ -211,7 +211,7 @@ def to_dict(self):
         return d
 
     def to_json(self):
-        """Converts object to JSON.
+        """Convert object to JSON.
         @return: JSON data
         """
         return json.dumps(self.to_dict())
@@ -256,7 +256,7 @@ def __repr__(self):
         return "<Sample('{0}','{1}')>".format(self.id, self.sha256)
 
     def to_dict(self):
-        """Converts object to dict.
+        """Convert object to dict.
         @return: dict
         """
         d = {}
@@ -265,7 +265,7 @@ def to_dict(self):
         return d
 
     def to_json(self):
-        """Converts object to JSON.
+        """Convert object to JSON.
         @return: JSON data
         """
         return json.dumps(self.to_dict())
@@ -291,7 +291,7 @@ class Error(Base):
     task_id = Column(Integer, ForeignKey("tasks.id"), nullable=False)
 
     def to_dict(self):
-        """Converts object to dict.
+        """Convert object to dict.
         @return: dict
         """
         d = {}
@@ -300,7 +300,7 @@ def to_dict(self):
         return d
 
     def to_json(self):
-        """Converts object to JSON.
+        """Convert object to JSON.
         @return: JSON data
         """
         return json.dumps(self.to_dict())
@@ -371,7 +371,7 @@ def options(self, value):
             self._options = value
 
     def to_dict(self, dt=False):
-        """Converts object to dict.
+        """Convert object to dict.
         @param dt: encode datetime objects
         @return: dict
         """
@@ -398,7 +398,7 @@ def to_dict(self, dt=False):
         return d
 
     def to_json(self):
-        """Converts object to JSON.
+        """Convert object to JSON.
         @return: JSON data
         """
         return json_encode(self.to_dict())
@@ -465,7 +465,7 @@ def connect(self, schema_check=None, dsn=None, create=True):
             self._create_tables()
 
     def _create_tables(self):
-        """Creates all the database tables etc."""
+        """Create all database tables etc."""
         try:
             Base.metadata.create_all(self.engine)
         except SQLAlchemyError as e:
@@ -509,7 +509,7 @@ def __del__(self):
         self.engine.dispose()
 
     def _connect_database(self, connection_string):
-        """Connect to a Database.
+        """Connect to a database.
         @param connection_string: Connection string specifying the database
         """
         try:
@@ -677,7 +677,7 @@ def set_route(self, task_id, route):
 
     @classlock
     def fetch(self, machine=None, service=True):
-        """Fetches a task waiting to be processed and locks it for running.
+        """Fetch a task waiting to be processed and lock it for running.
         @return: None or task
         """
         session = self.Session()
@@ -704,7 +704,7 @@ def fetch(self, machine=None, service=True):
 
     @classlock
     def guest_start(self, task_id, name, label, manager):
-        """Logs guest start.
+        """Log guest start.
         @param task_id: task identifier
         @param name: vm name
         @param label: vm label
@@ -728,7 +728,7 @@ def guest_start(self, task_id, name, label, manager):
 
     @classlock
     def guest_get_status(self, task_id):
-        """Logs guest start.
+        """Log guest start.
         @param task_id: task id
         @return: guest status
         """
@@ -745,7 +745,7 @@ def guest_get_status(self, task_id):
 
     @classlock
     def guest_set_status(self, task_id, status):
-        """Logs guest start.
+        """Log guest start.
         @param task_id: task identifier
         @param status: status
         """
@@ -764,7 +764,7 @@ def guest_set_status(self, task_id, status):
 
     @classlock
     def guest_remove(self, guest_id):
-        """Removes a guest start entry."""
+        """Remove a guest start entry."""
         session = self.Session()
         try:
             guest = session.query(Guest).get(guest_id)
@@ -779,7 +779,7 @@ def guest_remove(self, guest_id):
 
     @classlock
     def guest_stop(self, guest_id):
-        """Logs guest stop.
+        """Log guest stop.
         @param guest_id: guest log entry id
         """
         session = self.Session()
@@ -799,7 +799,7 @@ def guest_stop(self, guest_id):
 
     @classlock
     def list_machines(self, locked=False):
-        """Lists virtual machines.
+        """List virtual machines.
         @return: list of virtual machines
         """
         session = self.Session()
@@ -817,7 +817,7 @@ def list_machines(self, locked=False):
 
     @classlock
     def lock_machine(self, label=None, platform=None, tags=None):
-        """Places a lock on a free virtual machine.
+        """Place a lock on a free virtual machine.
         @param label: optional virtual machine label
         @param platform: optional virtual machine platform
         @param tags: optional tags required (list)
@@ -876,7 +876,7 @@ def lock_machine(self, label=None, platform=None, tags=None):
 
     @classlock
     def unlock_machine(self, label):
-        """Remove lock form a virtual machine.
+        """Remove a lock from a virtual machine.
         @param label: virtual machine label
         @return: unlocked machine
         """
@@ -905,7 +905,7 @@ def unlock_machine(self, label):
 
     @classlock
     def count_machines_available(self):
-        """How many virtual machines are ready for analysis.
+        """Return number of virtual machines ready for analysis.
         @return: free virtual machines count
         """
         session = self.Session()
@@ -920,7 +920,7 @@ def count_machines_available(self):
 
     @classlock
     def get_available_machines(self):
-        """  Which machines are available
+        """Return machines that are available.
         @return: free virtual machines
         """
         session = self.Session()
@@ -1406,7 +1406,7 @@ def list_tasks(self, limit=None, details=True, category=None, owner=None,
             session.close()
 
     def minmax_tasks(self):
-        """Find tasks minimum and maximum
+        """Find tasks minimum and maximum.
         @return: unix timestamps of minimum and maximum
         """
         session = self.Session()
@@ -1426,7 +1426,7 @@ def minmax_tasks(self):
 
     @classlock
     def count_tasks(self, status=None):
-        """Count tasks in the database
+        """Count tasks in the database.
         @param status: apply a filter according to the task status
         @return: number of tasks found
         """
@@ -1513,7 +1513,7 @@ def delete_task(self, task_id):
 
     @classlock
     def view_sample(self, sample_id):
-        """Retrieve information on a sample given a sample id.
+        """Retrieve information on a sample given a sample ID.
         @param sample_id: ID of the sample to query.
         @return: details on the sample used in sample: sample_id.
         """
@@ -1557,7 +1557,7 @@ def find_sample(self, md5=None, sha256=None):
 
     @classlock
     def count_samples(self):
-        """Counts the amount of samples in the database."""
+        """Count number of samples in the database."""
         session = self.Session()
         try:
             sample_count = session.query(Sample).count()
diff --git a/cuckoo/core/feedback.py b/cuckoo/core/feedback.py
index 2735fc7d19..2e7ec64c75 100644
--- a/cuckoo/core/feedback.py
+++ b/cuckoo/core/feedback.py
@@ -18,7 +18,7 @@
 log = logging.getLogger(__name__)
 
 class CuckooFeedback(object):
-    """Contacts Cuckoo HQ with feedback & optional analysis dump."""
+    """Contact Cuckoo HQ with feedback & optional analysis dump."""
     endpoint = "https://feedback.cuckoosandbox.org/api/submit/"
     exc_whitelist = (
         CuckooFeedbackError,
@@ -200,7 +200,7 @@ def include_report_web(self, task_id):
         return self.include_report(report)
 
     def gather_export_files(self, dirpath):
-        """Returns a list of all files of interest from an analysis."""
+        """Return a list of all files of interest from an analysis."""
         ret = []
         for name in self.export_files:
             if isinstance(name, basestring):
diff --git a/cuckoo/core/guest.py b/cuckoo/core/guest.py
index 2803f93188..5f148d58f1 100644
--- a/cuckoo/core/guest.py
+++ b/cuckoo/core/guest.py
@@ -30,7 +30,7 @@
 db = Database()
 
 def analyzer_zipfile(platform, monitor):
-    """Creates the Zip file that is sent to the Guest."""
+    """Create the zip file that is sent to the Guest."""
     t = time.time()
 
     zip_data = io.BytesIO()
diff --git a/cuckoo/core/init.py b/cuckoo/core/init.py
index 53ba3b8c8b..0fdb1b9da0 100644
--- a/cuckoo/core/init.py
+++ b/cuckoo/core/init.py
@@ -11,7 +11,7 @@
 from cuckoo.misc import cwd
 
 def write_supervisor_conf(username):
-    """Writes supervisord.conf configuration file if it does not exist yet."""
+    """Write supervisord.conf configuration file if it does not exist yet."""
     # TODO Handle updates?
     if os.path.exists(cwd("supervisord.conf")):
         return
diff --git a/cuckoo/core/plugins.py b/cuckoo/core/plugins.py
index 0869947c76..e5880b0b17 100644
--- a/cuckoo/core/plugins.py
+++ b/cuckoo/core/plugins.py
@@ -466,7 +466,7 @@ def yield_calls(self, proc):
                     self.api_sigs[call["api"]].remove(sig)
 
     def process_yara_matches(self):
-        """Yields any Yara matches to each signature."""
+        """Yield any Yara matches to each signature."""
         def loop_yara(category, filepath, matches):
             for match in matches:
                 match = YaraMatch(match, category)
@@ -650,7 +650,7 @@ def process(self, module):
             )
 
     def run(self):
-        """Generates all reports.
+        """Generate all reports.
         @raise CuckooReportError: if a report module fails.
         """
         # In every reporting module you can specify a numeric value that
diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py
index c13c02e542..695dd23715 100644
--- a/cuckoo/core/scheduler.py
+++ b/cuckoo/core/scheduler.py
@@ -138,7 +138,7 @@ def init(self):
         return True
 
     def store_task_info(self):
-        """grab latest task from db (if available) and update self.task"""
+        """Grab latest task from db (if available) and update self.task"""
         dbtask = self.db.view_task(self.task.id)
         self.task = dbtask.to_dict()
 
diff --git a/cuckoo/core/startup.py b/cuckoo/core/startup.py
index 891c0a2e5c..4f121b9a59 100644
--- a/cuckoo/core/startup.py
+++ b/cuckoo/core/startup.py
@@ -51,7 +51,7 @@ def check_specific_config(filename):
             )
 
 def check_configs():
-    """Checks if config files exist.
+    """Check if config files exist.
     @raise CuckooStartupError: if config files do not exist.
     """
     configs = (
@@ -97,7 +97,7 @@ def check_configs():
     return True
 
 def check_version():
-    """Checks version of Cuckoo."""
+    """Check version of Cuckoo."""
     if not config("cuckoo:cuckoo:version_check"):
         return
 
@@ -219,14 +219,14 @@ def check_version():
     return r
 
 def init_logging(level):
-    """Initializes logging."""
+    """Initialize logging."""
     logging.getLogger().setLevel(logging.DEBUG)
     init_logger("cuckoo.log", level)
     init_logger("cuckoo.json")
     init_logger("task")
 
 def init_console_logging(level=logging.INFO):
-    """Initializes logging only to console and database."""
+    """Initialize logging only to console and database."""
     logging.getLogger().setLevel(logging.DEBUG)
     init_logger("console", level)
     init_logger("database")
@@ -258,7 +258,7 @@ def init_tasks():
         db.set_status(task.id, TASK_FAILED_ANALYSIS)
 
 def init_modules():
-    """Initializes plugins."""
+    """Initialize plugins."""
     log.debug("Imported modules...")
 
     categories = (
@@ -503,7 +503,7 @@ def init_routing():
             rooter("init_rttable", rt_table, interface)
 
 def ensure_tmpdir():
-    """Verifies if the current user can read and create files in the
+    """Verify if the current user can read and create files in the
     cuckoo temporary directory (and creates it, if needed)."""
     try:
         if not os.path.isdir(temppath()):
diff --git a/cuckoo/core/submit.py b/cuckoo/core/submit.py
index 22657bf86c..1ecaa9dc85 100644
--- a/cuckoo/core/submit.py
+++ b/cuckoo/core/submit.py
@@ -58,7 +58,7 @@ def _handle_string(self, submit, tmppath, line):
         )
 
     def translate_options_from(self, entry, options):
-        """Translates from Web Interface options to Cuckoo database options."""
+        """Translate from Web Interface options to Cuckoo database options."""
         ret = {}
 
         if not options.get("simulated-human-interaction", True):
@@ -88,7 +88,7 @@ def translate_options_from(self, entry, options):
         return ret
 
     def translate_options_to(self, options):
-        """Translates from Cuckoo database options to Web Interface options."""
+        """Translate from Cuckoo database options to Web Interface options."""
         ret = {}
 
         if not int(options.get("human", "1")):
@@ -147,7 +147,7 @@ def pre(self, submit_type, data, options=None):
 
     def get_files(self, submit_id, password=None, astree=False):
         """
-        Returns files or URLs from a submitted analysis.
+        Return files or URLs from a submitted analysis.
         @param password: The password to unlock container archives with
         @param astree: sflock option; determines the format in which the files are returned
         @return: A tree of files
@@ -195,7 +195,7 @@ def get_files(self, submit_id, password=None, astree=False):
         return files, submit.data["errors"], submit.data["options"]
 
     def submit(self, submit_id, config):
-        """Reads, interprets, and converts the JSON configuration provided by
+        """Read, interpret, and convert the JSON configuration provided by
         the Web Interface into something we insert into the database."""
         ret = []
         submit = db.view_submit(submit_id)
diff --git a/cuckoo/data/analyzer/darwin/lib/api/screenshot.py b/cuckoo/data/analyzer/darwin/lib/api/screenshot.py
index ea22a8ab94..613551fd07 100644
--- a/cuckoo/data/analyzer/darwin/lib/api/screenshot.py
+++ b/cuckoo/data/analyzer/darwin/lib/api/screenshot.py
@@ -38,7 +38,7 @@ def have_pil(self):
         return HAVE_PIL
 
     def equal(self, img1, img2, skip_area=None):
-        """Compares two screenshots using Root-Mean-Square Difference (RMS).
+        """Compare two screenshots using Root-Mean-Square Difference (RMS).
         @param img1: screenshot to compare.
         @param img2: screenshot to compare.
         @return: equal status.
diff --git a/cuckoo/data/analyzer/darwin/lib/common/hashing.py b/cuckoo/data/analyzer/darwin/lib/common/hashing.py
index fac551f2fd..010d4fa11c 100644
--- a/cuckoo/data/analyzer/darwin/lib/common/hashing.py
+++ b/cuckoo/data/analyzer/darwin/lib/common/hashing.py
@@ -6,7 +6,7 @@
 
 
 def hash_file(method, path):
-    """Calculates an hash on a file by path.
+    """Calculate a hash on a file by path.
     @param method: callable hashing method
     @param path: file path
     @return: computed hash string
diff --git a/cuckoo/data/analyzer/darwin/modules/packages/zip.py b/cuckoo/data/analyzer/darwin/modules/packages/zip.py
index eaf475c6da..18388c8a03 100644
--- a/cuckoo/data/analyzer/darwin/modules/packages/zip.py
+++ b/cuckoo/data/analyzer/darwin/modules/packages/zip.py
@@ -90,7 +90,7 @@ def _extract_nested_archives(self, archive, where, password):
 
 
 def _prepare_archive_at_path(filename):
-    """ Verifies that there's a readable zip archive at the given path.
+    """ Verify that there's a readable zip archive at the given path.
 
     This function returns a new name for the archive (for most cases it's
     the same as the original one; but if an archive named "foo.zip" contains
diff --git a/cuckoo/data/analyzer/linux/lib/common/abstracts.py b/cuckoo/data/analyzer/linux/lib/common/abstracts.py
index 015e035dab..3bf8fd0c96 100644
--- a/cuckoo/data/analyzer/linux/lib/common/abstracts.py
+++ b/cuckoo/data/analyzer/linux/lib/common/abstracts.py
@@ -31,7 +31,7 @@ def check(self):
         return True
 
     def execute(self, cmd):
-        """Starts an executable for analysis.
+        """Start an executable for analysis.
         @param path: executable path
         @param args: executable arguments
         @return: process pid
diff --git a/cuckoo/data/analyzer/linux/lib/common/hashing.py b/cuckoo/data/analyzer/linux/lib/common/hashing.py
index 78d1d1936e..1ffa54190d 100644
--- a/cuckoo/data/analyzer/linux/lib/common/hashing.py
+++ b/cuckoo/data/analyzer/linux/lib/common/hashing.py
@@ -10,7 +10,7 @@ def sha256_file(path):
     return hash_file(hashlib.sha256, path)
 
 def hash_file(method, path):
-    """Calculates an hash on a file by path.
+    """Calculate a hash on a file by path.
     @param method: callable hashing method
     @param path: file path
     @return: computed hash string
diff --git a/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py b/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py
index ae2b20b390..3ddbdfeec0 100644
--- a/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py
+++ b/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py
@@ -14,7 +14,7 @@
 log = logging.getLogger(__name__)
 
 class STAP(Auxiliary):
-    """system-wide syscall trace with stap."""
+    """System-wide syscall trace with stap."""
     priority = -10  # low prio to wrap tightly around the analysis
 
     def __init__(self):
diff --git a/cuckoo/data/analyzer/windows/analyzer.py b/cuckoo/data/analyzer/windows/analyzer.py
index 66d0a6bdb4..0a3868ae71 100644
--- a/cuckoo/data/analyzer/windows/analyzer.py
+++ b/cuckoo/data/analyzer/windows/analyzer.py
@@ -46,11 +46,11 @@ def __init__(self):
         self.dumped = []
 
     def is_protected_filename(self, file_name):
-        """Do we want to inject into a process with this name?"""
+        """Return whether or not to inject into a process with this name."""
         return file_name.lower() in self.PROTECTED_NAMES
 
     def add_pid(self, filepath, pid, verbose=True):
-        """Tracks a process identifier for this file."""
+        """Track a process identifier for this file."""
         if not pid or filepath.lower() not in self.files:
             return
 
@@ -150,7 +150,7 @@ def add_pids(self, pids):
             self.add_pid(pids)
 
     def has_pid(self, pid, notrack=True):
-        """Is this process identifier being tracked?"""
+        """Return whether or not this process identifier being tracked."""
         if int(pid) in self.pids:
             return True
 
@@ -435,7 +435,7 @@ def __init__(self):
         self.reboot = []
 
     def get_pipe_path(self, name):
-        """Returns \\\\.\\PIPE on Windows XP and \\??\\PIPE elsewhere."""
+        """Return \\\\.\\PIPE on Windows XP and \\??\\PIPE elsewhere."""
         version = sys.getwindowsversion()
         if version.major == 5 and version.minor == 1:
             return "\\\\.\\PIPE\\%s" % name
@@ -509,7 +509,7 @@ def prepare(self):
             self.target = self.config.target
 
     def stop(self):
-        """Allows an auxiliary module to stop the analysis."""
+        """Allow an auxiliary module to stop the analysis."""
         self.do_run = False
 
     def complete(self):
diff --git a/cuckoo/data/analyzer/windows/lib/api/process.py b/cuckoo/data/analyzer/windows/lib/api/process.py
index ba8770174d..ae7359795a 100644
--- a/cuckoo/data/analyzer/windows/lib/api/process.py
+++ b/cuckoo/data/analyzer/windows/lib/api/process.py
@@ -127,7 +127,7 @@ def __init__(self, pid=None, tid=None, process_name=None):
 
     @staticmethod
     def set_config(config):
-        """Sets the analyzer configuration once."""
+        """Set the analyzer configuration once."""
         Process.config = config
 
     def get_system_info(self):
diff --git a/cuckoo/data/analyzer/windows/lib/common/abstracts.py b/cuckoo/data/analyzer/windows/lib/common/abstracts.py
index c9ae91a119..e47109f3d6 100644
--- a/cuckoo/data/analyzer/windows/lib/common/abstracts.py
+++ b/cuckoo/data/analyzer/windows/lib/common/abstracts.py
@@ -107,7 +107,7 @@ def move_curdir(self, filepath):
         return outpath
 
     def init_regkeys(self, regkeys):
-        """Initializes the registry to avoid annoying popups, configure
+        """Initialize the registry to avoid annoying popups, configure
         settings, etc.
         @param regkeys: the root keys, subkeys, and key/value pairs.
         """
@@ -130,7 +130,7 @@ def init_regkeys(self, regkeys):
 
     def execute(self, path, args, mode=None, maximize=False, env=None,
                 source=None, trigger=None):
-        """Starts an executable for analysis.
+        """Start an executable for analysis.
         @param path: executable path
         @param args: executable arguments
         @param mode: monitor mode - which functions to instrument
@@ -169,7 +169,7 @@ def execute(self, path, args, mode=None, maximize=False, env=None,
         return p.pid
 
     def package_files(self):
-        """A list of files to upload to host.
+        """Return a list of files to upload to host.
         The list should be a list of tuples (<path on guest>, <name of file in package_files folder>).
         (package_files is a folder that will be created in analysis folder).
         """
diff --git a/cuckoo/data/analyzer/windows/lib/common/hashing.py b/cuckoo/data/analyzer/windows/lib/common/hashing.py
index bbea930962..adbd321349 100644
--- a/cuckoo/data/analyzer/windows/lib/common/hashing.py
+++ b/cuckoo/data/analyzer/windows/lib/common/hashing.py
@@ -7,7 +7,7 @@
 
 
 def hash_file(method, path):
-    """Calculates an hash on a file by path.
+    """Calculate a hash on a file by path.
     @param method: callable hashing method
     @param path: file path
     @return: computed hash string
diff --git a/cuckoo/data/analyzer/windows/lib/core/pipe.py b/cuckoo/data/analyzer/windows/lib/core/pipe.py
index 88598b2be5..26594266ee 100644
--- a/cuckoo/data/analyzer/windows/lib/core/pipe.py
+++ b/cuckoo/data/analyzer/windows/lib/core/pipe.py
@@ -21,8 +21,8 @@
 BUFSIZE = 0x10000
 
 class PipeForwarder(threading.Thread):
-    """The Pipe Forwarder forwards all data received from a local pipe to
-    the Cuckoo server through a socket."""
+    """Forward all data received from a local pipe to the Cuckoo
+    server through a socket."""
     sockets = {}
     active = {}
 
@@ -99,8 +99,8 @@ def run(self):
             self.active[pid.value] = False
 
 class PipeDispatcher(threading.Thread):
-    """Receives commands through a local pipe, forwards them to the
-    dispatcher, and returns the response."""
+    """Receive commands through a local pipe, forward them to the
+    dispatcher, and return the response."""
 
     def __init__(self, pipe_handle, dispatcher):
         threading.Thread.__init__(self)
@@ -146,8 +146,8 @@ def run(self):
         KERNEL32.CloseHandle(self.pipe_handle)
 
 class PipeServer(threading.Thread):
-    """The Pipe Server accepts incoming pipe handlers and initializes
-    them in a new thread."""
+    """Accept incoming pipe handlers and initialize them in
+    a new thread."""
 
     def __init__(self, pipe_handler, pipe_name, message=False, **kwargs):
         threading.Thread.__init__(self)
diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py b/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py
index fa74c71de9..f5fa69a7e8 100644
--- a/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py
+++ b/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py
@@ -66,7 +66,7 @@ class Disguise(Auxiliary):
     ]
 
     def change_productid(self):
-        """Randomizes Windows ProductId.
+        """Randomize Windows ProductId.
         The Windows ProductId is occasionally used by malware
         to detect public setups of Cuckoo, e.g., Malwr.com.
         """
diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py b/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py
index 144f36ac51..54c421d278 100644
--- a/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py
+++ b/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py
@@ -12,7 +12,7 @@
 log = logging.getLogger(__name__)
 
 class Reboot(Auxiliary):
-    """Prepares the environment to behave as if the VM has been rebooted."""
+    """Prepare the environment to behave as if the VM has been rebooted."""
 
     def start(self):
         if self.analyzer.config.package != "reboot":
diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py b/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py
index 25497dd032..50eed421fc 100644
--- a/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py
+++ b/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py
@@ -17,7 +17,7 @@
 log = logging.getLogger(__name__)
 
 class RecentFiles(Auxiliary):
-    """Populates the Desktop with recent files in order to combat recent
+    """Populate the Desktop with recent files in order to combat recent
     anti-sandbox measures."""
 
     extensions = [
diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py b/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py
index e5b656e75c..3d8e479922 100644
--- a/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py
+++ b/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py
@@ -13,7 +13,7 @@
 log = logging.getLogger(__name__)
 
 class LoadZer0m0n(Auxiliary):
-    """Loads the zer0m0n kernel driver."""
+    """Load the zer0m0n kernel driver."""
 
     def start(self):
         if self.options.get("analysis") not in ("both", "kernel"):
diff --git a/cuckoo/data/analyzer/windows/modules/packages/zip.py b/cuckoo/data/analyzer/windows/modules/packages/zip.py
index 339cf41833..de33fc355a 100644
--- a/cuckoo/data/analyzer/windows/modules/packages/zip.py
+++ b/cuckoo/data/analyzer/windows/modules/packages/zip.py
@@ -51,7 +51,7 @@ def extract_zip(self, zip_path, extract_path, password):
                         self.extract_zip(os.path.join(extract_path, name), extract_path, password)
 
     def is_overwritten(self, zip_path):
-        """Checks if the ZIP file contains another file with the same name, so it is going to be overwritten.
+        """Check if the ZIP file contains another file with the same name, so it is going to be overwritten.
         @param zip_path: zip file path
         @return: comparison boolean
         """
diff --git a/cuckoo/machinery/avd.py b/cuckoo/machinery/avd.py
index bfa789464d..387800f630 100644
--- a/cuckoo/machinery/avd.py
+++ b/cuckoo/machinery/avd.py
@@ -19,7 +19,7 @@ class Avd(Machinery):
     """Virtualization layer for Android Emulator."""
 
     def _initialize_check(self):
-        """Runs all checks when a machine manager is initialized.
+        """Run all checks when a machine manager is initialized.
         @raise CuckooMachineError: if the android emulator is not found.
         """
         self.emulator_processes = {}
@@ -76,7 +76,7 @@ def start(self, label, task):
         self.start_agent(label)
 
     def stop(self, label):
-        """Stops a virtual machine.
+        """Stop a virtual machine.
         @param label: virtual machine name.
         @raise CuckooMachineError: if unable to stop.
         """
@@ -84,20 +84,20 @@ def stop(self, label):
         self.stop_emulator(label)
 
     def _list(self):
-        """Lists virtual machines installed.
+        """List virtual machines installed.
         @return: virtual machine names list.
         """
         return self.options.avd.machines
 
     def _status(self, label):
-        """Gets current status of a vm.
+        """Get current status of a vm.
         @param label: virtual machine name.
         @return: status string.
         """
         log.debug("Getting status for %s" % label)
 
     def duplicate_reference_machine(self, label):
-        """Creates a new emulator based on a reference one."""
+        """Create a new emulator based on a reference one."""
         reference_machine = self.options.avd.reference_machine
         log.debug("Duplicate Reference Machine '{0}'.".format(reference_machine))
 
@@ -127,7 +127,7 @@ def duplicate_reference_machine(self, label):
         # todo:will see
 
     def delete_old_emulator(self, label):
-        """Deletes any trace of an emulator that would have the same name as
+        """Delete any trace of an emulator that would have the same name as
         the one of the current emulator."""
         old_emulator_config_file = os.path.join(self.options.avd.avd_path,
                                                 "%s.ini" % label)
@@ -142,7 +142,7 @@ def delete_old_emulator(self, label):
             shutil.rmtree(old_emulator_path)
 
     def replace_content_in_file(self, fileName, contentToReplace, replacementContent):
-        """Replaces the specified motif by a specified value in the specified
+        """Replace the specified motif by a specified value in the specified
         file.
         """
 
@@ -157,7 +157,7 @@ def replace_content_in_file(self, fileName, contentToReplace, replacementContent
             fd.writelines(newLines)
 
     def start_emulator(self, label, task):
-        """Starts the emulator."""
+        """Start the emulator."""
         emulator_port = self.options.get(label)["emulator_port"]
 
         cmd = [
@@ -216,7 +216,7 @@ def stop_emulator(self, label):
             del self.emulator_processes[label]
 
     def wait_for_device_ready(self, label):
-        """Analyzes the emulator and returns when it's ready."""
+        """Analyze the emulator and return when it's ready."""
 
         emulator_port = str(self.options.get(label)["emulator_port"])
         adb = self.options.avd.adb_path
@@ -294,7 +294,7 @@ def start_agent(self, label):
         time.sleep(10)
 
     def check_adb_recognize_emulator(self, label):
-        """Checks that ADB recognizes the emulator. Returns True if device is
+        """Check that ADB recognizes the emulator. Return True if device is
         recognized by ADB, False otherwise.
         """
         log.debug("Checking if ADB recognizes emulator...")
@@ -311,7 +311,7 @@ def check_adb_recognize_emulator(self, label):
         return False
 
     def restart_adb_server(self):
-        """Restarts ADB server. This function is not used because we have to
+        """Restart ADB server. This function is not used because we have to
         verify we don't have multiple devices.
         """
         log.debug("Restarting ADB server...")
diff --git a/cuckoo/machinery/esx.py b/cuckoo/machinery/esx.py
index 68caa84fb3..3ae4c307af 100644
--- a/cuckoo/machinery/esx.py
+++ b/cuckoo/machinery/esx.py
@@ -18,7 +18,7 @@ class ESX(LibVirtMachinery):
     """Virtualization layer for ESXi/ESX based on python-libvirt."""
 
     def _initialize_check(self):
-        """Runs all checks when a machine manager is initialized.
+        """Run all checks when a machine manager is initialized.
         @raise CuckooMachineError: if configuration is invalid
         """
         if not self.options.esx.dsn:
diff --git a/cuckoo/machinery/physical.py b/cuckoo/machinery/physical.py
index ff56937ab6..1fa40c75e1 100644
--- a/cuckoo/machinery/physical.py
+++ b/cuckoo/machinery/physical.py
@@ -30,7 +30,7 @@ class Physical(Machinery):
     ERROR = "error"
 
     def _initialize_check(self):
-        """Ensures that credentials have been entered into the config file.
+        """Ensure that credentials have been entered into the config file.
         @raise CuckooCriticalError: if no credentials were provided or if
             one or more physical machines are offline.
         """
@@ -85,7 +85,7 @@ def start(self, label, task):
                                      "%s (STATUS=%s)" % (label, status))
 
     def stop(self, label):
-        """Stops a physical machine.
+        """Stop a physical machine.
         @param label: physical machine name.
         @raise CuckooMachineError: if unable to stop.
         """
@@ -119,7 +119,7 @@ def stop(self, label):
                 continue
 
     def _list(self):
-        """Lists physical machines installed.
+        """List physical machines installed.
         @return: physical machine names list.
         """
         active_machines = []
@@ -130,7 +130,7 @@ def _list(self):
         return active_machines
 
     def _status(self, label):
-        """Gets current status of a physical machine.
+        """Get current status of a physical machine.
         @param label: physical machine name.
         @return: status string.
         """
@@ -239,7 +239,7 @@ def fog_init(self):
                 )
 
     def fog_queue_task(self, hostname):
-        """Queues a task with FOG to deploy the given machine after reboot."""
+        """Queue a task with FOG to deploy the given machine after reboot."""
         if hostname in self.fog_machines:
             macaddr, download = self.fog_machines[hostname]
             self.fog_query(download)
diff --git a/cuckoo/machinery/qemu.py b/cuckoo/machinery/qemu.py
index 5efdb853c0..712f37b725 100644
--- a/cuckoo/machinery/qemu.py
+++ b/cuckoo/machinery/qemu.py
@@ -116,7 +116,7 @@ def __init__(self):
         self.state = {}
 
     def _initialize_check(self):
-        """Runs all checks when a machine manager is initialized.
+        """Run all checks when a machine manager is initialized.
         @raise CuckooMachineError: if QEMU binary is not found.
         """
         # VirtualBox specific checks.
@@ -205,7 +205,7 @@ def start(self, label, task):
             raise CuckooMachineError("QEMU failed starting the machine: %s" % e)
 
     def stop(self, label):
-        """Stops a virtual machine.
+        """Stop a virtual machine.
         @param label: virtual machine label.
         @raise CuckooMachineError: if unable to stop.
         """
@@ -235,7 +235,7 @@ def stop(self, label):
         self.state[vm_info.name] = None
 
     def _status(self, name):
-        """Gets current status of a vm.
+        """Get current status of a vm.
         @param name: virtual machine name.
         @return: status string.
         """
diff --git a/cuckoo/machinery/virtualbox.py b/cuckoo/machinery/virtualbox.py
index 1df84a51cd..a4255abf6f 100644
--- a/cuckoo/machinery/virtualbox.py
+++ b/cuckoo/machinery/virtualbox.py
@@ -30,7 +30,7 @@ class VirtualBox(Machinery):
     ERROR = "machete"
 
     def _initialize_check(self):
-        """Runs all checks when a machine manager is initialized.
+        """Run all checks when a machine manager is initialized.
         @raise CuckooMachineError: if VBoxManage is not found.
         """
         if not self.options.virtualbox.path:
@@ -182,7 +182,7 @@ def dump_pcap(self, label, task):
             return
 
     def stop(self, label):
-        """Stops a virtual machine.
+        """Stop a virtual machine.
         @param label: virtual machine name.
         @raise CuckooMachineError: if unable to stop.
         """
@@ -235,7 +235,7 @@ def stop(self, label):
         self._wait_status(label, self.POWEROFF, self.ABORTED, self.SAVED)
 
     def _list(self):
-        """Lists virtual machines installed.
+        """List virtual machines installed.
         @return: virtual machine names list.
         """
         try:
@@ -268,8 +268,8 @@ def _list(self):
         return machines
 
     def vminfo(self, label, field):
-        """Returns False if invoking vboxmanage fails. Otherwise the VM
-        information value, if any."""
+        """Return False if invoking vboxmanage fails. Otherwise return the
+        VM information value, if any."""
         try:
             args = [
                 self.options.virtualbox.path,
@@ -314,7 +314,7 @@ def vminfo(self, label, field):
                 return line.split("=", 1)[1]
 
     def _status(self, label):
-        """Gets current status of a vm.
+        """Get current status of a vm.
         @param label: virtual machine name.
         @return: status string.
         """
@@ -332,7 +332,7 @@ def _status(self, label):
         )
 
     def dump_memory(self, label, path):
-        """Takes a memory dump.
+        """Take a memory dump.
         @param path: path to where to store the memory dump.
         """
 
diff --git a/cuckoo/machinery/vmware.py b/cuckoo/machinery/vmware.py
index 498be2a351..a19b2eb521 100644
--- a/cuckoo/machinery/vmware.py
+++ b/cuckoo/machinery/vmware.py
@@ -44,7 +44,7 @@ def _initialize_check(self):
         super(VMware, self)._initialize_check()
 
     def _check_vmx(self, vmx_path):
-        """Checks whether a vmx file exists and is valid.
+        """Check whether a vmx file exists and is valid.
         @param vmx_path: path to vmx file
         @raise CuckooMachineError: if file not found or not ending with .vmx
         """
@@ -56,7 +56,7 @@ def _check_vmx(self, vmx_path):
             raise CuckooMachineError("Vm file %s not found" % vmx_path)
 
     def _check_snapshot(self, vmx_path, snapshot):
-        """Checks snapshot existance.
+        """Check snapshot existence.
         @param vmx_path: path to vmx file
         @param snapshot: snapshot name
         @raise CuckooMachineError: if snapshot not found
@@ -114,7 +114,7 @@ def start(self, vmx_path, task):
                                      "mode: %s" % (vmx_path, mode, e))
 
     def stop(self, vmx_path):
-        """Stops a virtual machine.
+        """Stop a virtual machine.
         @param vmx_path: path to vmx file
         @raise CuckooMachineError: if unable to stop.
         """
@@ -135,7 +135,7 @@ def stop(self, vmx_path):
                         vmx_path)
 
     def _revert(self, vmx_path, snapshot):
-        """Revets machine to snapshot.
+        """Revert machine to snapshot.
         @param vmx_path: path to vmx file
         @param snapshot: snapshot name
         @raise CuckooMachineError: if unable to revert
@@ -154,7 +154,7 @@ def _revert(self, vmx_path, snapshot):
                                      "machine %s: %s" % (vmx_path, e))
 
     def _is_running(self, vmx_path):
-        """Checks if virtual machine is running.
+        """Check if virtual machine is running.
         @param vmx_path: path to vmx file
         @return: running status
         """
diff --git a/cuckoo/machinery/vsphere.py b/cuckoo/machinery/vsphere.py
index 4f947937c4..2054f395c9 100644
--- a/cuckoo/machinery/vsphere.py
+++ b/cuckoo/machinery/vsphere.py
@@ -55,7 +55,7 @@ def _initialize(self, module_name):
         random.seed()
 
     def _initialize_check(self):
-        """Runs checks against virtualization software when a machine manager
+        """Run checks against virtualization software when a machine manager
         is initialized.
         @raise CuckooCriticalError: if a misconfiguration or unsupported state
                                     is found.
diff --git a/cuckoo/machinery/xenserver.py b/cuckoo/machinery/xenserver.py
index 8023ba901c..3a5ba2b291 100644
--- a/cuckoo/machinery/xenserver.py
+++ b/cuckoo/machinery/xenserver.py
@@ -189,7 +189,7 @@ def _snapshot_from_vm_uuid(self, uuid):
         return machine.snapshot
 
     def _is_halted(self, vm):
-        """Checks if the virtual machine is running.
+        """Check if the virtual machine is running.
         @param uuid: vm uuid
         """
         return vm["power_state"] == "Halted"
@@ -262,7 +262,7 @@ def _list(self):
             return vm_list
 
     def _status(self, label):
-        """Gets current status of a vm.
+        """Get current status of a vm.
         @param label: virtual machine uuid
         @return: status string.
         """
diff --git a/cuckoo/main.py b/cuckoo/main.py
index 292a8c5dd7..03958f3e4d 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -194,7 +194,7 @@ def cuckoo_main(max_analysis_count=0):
 @click.option("--cwd", help="Cuckoo Working Directory")
 @click.pass_context
 def main(ctx, debug, quiet, nolog, maxcount, user, cwd):
-    """Invokes the Cuckoo daemon or one of its subcommands.
+    """Invoke the Cuckoo daemon or one of its subcommands.
 
     To be able to use different Cuckoo configurations on the same machine with
     the same Cuckoo installation, we use the so-called Cuckoo Working
@@ -247,7 +247,7 @@ def main(ctx, debug, quiet, nolog, maxcount, user, cwd):
 @click.pass_context
 @click.option("--conf", type=click.Path(exists=True, file_okay=True, readable=True), help="Flat key/value configuration file")
 def init(ctx, conf):
-    """Initializes Cuckoo and its configuration."""
+    """Initialize Cuckoo and its configuration."""
     if conf and os.path.exists(conf):
         cfg = read_kv_conf(conf)
     else:
@@ -398,7 +398,7 @@ def process(ctx, instance, report, maxcount, timeout):
 @click.option("--sudo", is_flag=True, help="Request superuser privileges")
 @click.pass_context
 def rooter(ctx, socket, group, service, iptables, ip, sudo):
-    """Instantiates the Cuckoo Rooter."""
+    """Instantiate the Cuckoo Rooter."""
     init_console_logging(level=ctx.parent.level)
 
     if sudo:
@@ -626,7 +626,7 @@ def migrate(revision):
 @click.argument("path", type=click.Path(file_okay=False, exists=True))
 @click.pass_context
 def import_(ctx, mode, path):
-    """Imports an older Cuckoo setup into a new CWD. The old setup should be
+    """Import an older Cuckoo setup into a new CWD. The old setup should be
     identified by PATH and the new CWD may be specified with the --cwd
     parameter, e.g., "cuckoo --cwd /tmp/cwd import old-cuckoo"."""
     if os.path.exists(os.path.join(path, ".cwd")):
diff --git a/cuckoo/misc.py b/cuckoo/misc.py
index a137abc1b7..6e82f9f1c3 100644
--- a/cuckoo/misc.py
+++ b/cuckoo/misc.py
@@ -41,7 +41,7 @@ def set_cwd(path, raw=None):
     _raw = raw
 
 def cwd(*args, **kwargs):
-    """Returns absolute path to this file in the Cuckoo Working Directory or
+    """Return absolute path to this file in the Cuckoo Working Directory or
     optionally - when private=True has been passed along - to our private
     Cuckoo Working Directory which is not configurable."""
     if kwargs.get("private"):
@@ -62,7 +62,7 @@ def cwd(*args, **kwargs):
         return os.path.join(_root, *args)
 
 def decide_cwd(cwd=None, exists=False):
-    """Decides and sets the CWD, optionally checks if it's a valid CWD."""
+    """Decide and set the CWD, optionally check if it's a valid CWD."""
     if not cwd:
         cwd = os.environ.get("CUCKOO_CWD")
 
@@ -104,7 +104,7 @@ def getuser():
     return ""
 
 def load_signatures():
-    """Loads additional Signatures from the Cuckoo Working Directory.
+    """Load additional Signatures from the Cuckoo Working Directory.
 
     This method is quite hacky in the sense that it magically imports
     Signatures from an arbitrary directory - one that doesn't belong to the
@@ -188,7 +188,7 @@ def is_macosx():
     return sys.platform == "darwin"
 
 def Popen(*args, **kwargs):
-    """Drops the close_fds argument on Windows platforms in certain situations
+    """Drop the close_fds argument on Windows platforms in certain situations
     where it'd otherwise cause an exception from the subprocess module."""
     if is_windows() and "close_fds" in kwargs:
         if "stdin" in kwargs or "stdout" in kwargs or "stderr" in kwargs:
@@ -197,7 +197,7 @@ def Popen(*args, **kwargs):
     return subprocess.Popen(*args, **kwargs)
 
 def drop_privileges(username):
-    """Drops privileges to selected user.
+    """Drop privileges to selected user.
     @param username: drop privileges to this username
     """
     if not HAVE_PWD:
@@ -225,7 +225,7 @@ def __init__(self, name):
         self.pid = None
 
     def create(self):
-        """Creates pidfile for the current process."""
+        """Create pidfile for the current process."""
         with open(self.filepath, "wb") as f:
             f.write(str(os.getpid()))
 
@@ -249,7 +249,7 @@ def read(self):
         return self.pid
 
     def proc_exists(self, pid):
-        """Returns boolean if the process exists or None when unsupported."""
+        """Return boolean of process existence, or None when unsupported."""
         if not pid:
             return False
 
diff --git a/cuckoo/processing/apkinfo.py b/cuckoo/processing/apkinfo.py
index 7e64cffec4..a5b7f3fd6f 100644
--- a/cuckoo/processing/apkinfo.py
+++ b/cuckoo/processing/apkinfo.py
@@ -30,7 +30,7 @@ def check_size(self, file_list):
         return False
 
     def _apk_files(self, apk):
-        """Returns a list of files in the APK."""
+        """Return a list of files in the APK."""
         ret = []
         for fname, filetype in apk.get_files_types().items():
             buf = apk.zip.read(fname)
diff --git a/cuckoo/processing/baseline.py b/cuckoo/processing/baseline.py
index 3f7dc83812..58f0ca9f9a 100644
--- a/cuckoo/processing/baseline.py
+++ b/cuckoo/processing/baseline.py
@@ -12,7 +12,7 @@
 log = logging.getLogger(__name__)
 
 class Baseline(Processing):
-    """Reduces Baseline results from gathered information."""
+    """Reduce Baseline results from gathered information."""
     order = 2
 
     def deep_tuple(self, o, bl=None):
@@ -39,9 +39,9 @@ def normalize(self, plugin, o):
         return self.deep_tuple(o, plugins.get(plugin))
 
     def memory(self, baseline, report):
-        """Finds the differences between the analysis report and the baseline
-        report. Puts the differences into the baseline part of the report and
-        also marks the existing rows with a `class_` attribute."""
+        """Find the differences between the analysis report and the baseline
+        report. Put the differences into the baseline part of the report and
+        mark the existing rows with a `class_` attribute."""
         results = {}
 
         for plugin in baseline.keys() + report.keys():
diff --git a/cuckoo/processing/behavior.py b/cuckoo/processing/behavior.py
index d711cead30..ab24898060 100644
--- a/cuckoo/processing/behavior.py
+++ b/cuckoo/processing/behavior.py
@@ -19,7 +19,7 @@
 log = logging.getLogger(__name__)
 
 class Summary(BehaviorHandler):
-    """Generates overview summary information (not split by process)."""
+    """Generate overview summary information (not split by process)."""
 
     key = "summary"
     event_types = ["generic"]
diff --git a/cuckoo/processing/irma.py b/cuckoo/processing/irma.py
index 927cbe1470..013f1b1b91 100644
--- a/cuckoo/processing/irma.py
+++ b/cuckoo/processing/irma.py
@@ -14,7 +14,7 @@
 log = logging.getLogger(__name__)
 
 class Irma(Processing):
-    """Gets antivirus signatures from IRMA for various results.
+    """Get antivirus signatures from IRMA for various results.
 
     Currently obtains IRMA results for the target sample.
     """
@@ -97,7 +97,7 @@ def _get_results(self, sha256):
         )
 
     def run(self):
-        """Runs IRMA processing
+        """Run IRMA processing
         @return: full IRMA report.
         """
         self.key = "irma"
@@ -123,13 +123,13 @@ def run(self):
             self._scan_file(self.file_path, self.force)
             results = self._get_results(sha256) or {}
 
-        """ FIXME! could use a proper fix here
-        that probably needs changes on IRMA side aswell
-        --
-        related to  https://github.com/elastic/elasticsearch/issues/15377
-        entropy value is sometimes 0 and sometimes like  0.10191042566270775
-        other issue is that results type changes between string and object :/
-        """
+        # FIXME! could use a proper fix here
+        # that probably needs changes on IRMA side aswell
+        # --
+        # related to  https://github.com/elastic/elasticsearch/issues/15377
+        # entropy value is sometimes 0 and sometimes like  0.10191042566270775
+        # other issue is that results type changes between string and object :/
+        
         for idx, result in enumerate(results["probe_results"]):
             if result["name"] == "PE Static Analyzer":
                 log.debug("Ignoring PE results at index {0}".format(idx))
diff --git a/cuckoo/processing/memory.py b/cuckoo/processing/memory.py
index 26873ae291..b30853749a 100644
--- a/cuckoo/processing/memory.py
+++ b/cuckoo/processing/memory.py
@@ -90,7 +90,7 @@ def get_dtb(self):
         return False
 
     def init_config(self):
-        """Creates a volatility configuration."""
+        """Create a volatility configuration."""
         if self.config is not None and self.addr_space is not None:
             return
 
diff --git a/cuckoo/processing/network.py b/cuckoo/processing/network.py
index 5dc39fe962..e5c13adca7 100644
--- a/cuckoo/processing/network.py
+++ b/cuckoo/processing/network.py
@@ -42,7 +42,7 @@ class Pcap(object):
     ssl_ports = 443,
 
     def __init__(self, filepath, options):
-        """Creates a new instance.
+        """Create a new instance.
         @param filepath: path to PCAP file
         @param options: config options
         """
@@ -93,7 +93,7 @@ def __init__(self, filepath, options):
         self.dns_servers = []
 
     def _is_whitelisted(self, conn, hostname):
-        """Checks if whitelisting conditions are met"""
+        """Check if whitelisting conditions are met"""
         # Is whitelistng enabled?
         if not self.whitelist_enabled:
             return False
@@ -217,7 +217,7 @@ def _add_hosts(self, connection):
             pass
 
     def _tcp_dissect(self, conn, data):
-        """Runs all TCP dissectors.
+        """Run all TCP dissectors.
         @param conn: connection.
         @param data: payload data.
         """
@@ -237,7 +237,7 @@ def _tcp_dissect(self, conn, data):
             self._https_identify(conn, data)
 
     def _udp_dissect(self, conn, data):
-        """Runs all UDP dissectors.
+        """Run all UDP dissectors.
         @param conn: connection.
         @param data: payload data.
         """
@@ -247,7 +247,7 @@ def _udp_dissect(self, conn, data):
                 self._add_dns(conn, data)
 
     def _check_icmp(self, icmp_data):
-        """Checks for ICMP traffic.
+        """Check for ICMP traffic.
         @param icmp_data: ICMP data flow.
         """
         try:
@@ -257,7 +257,7 @@ def _check_icmp(self, icmp_data):
             return False
 
     def _icmp_dissect(self, conn, data):
-        """Runs all ICMP dissectors.
+        """Run all ICMP dissectors.
         @param conn: connection.
         @param data: payload data.
         """
@@ -282,7 +282,7 @@ def _icmp_dissect(self, conn, data):
             self.icmp_requests.append(entry)
 
     def _check_dns(self, udpdata):
-        """Checks for DNS traffic.
+        """Check for DNS traffic.
         @param udpdata: UDP data flow.
         """
         try:
@@ -293,7 +293,7 @@ def _check_dns(self, udpdata):
         return True
 
     def _add_dns(self, conn, udpdata):
-        """Adds a DNS data flow.
+        """Add a DNS data flow.
         @param udpdata: UDP data flow.
         """
         dns = dpkt.dns.DNS(udpdata)
@@ -443,7 +443,7 @@ def _add_domain(self, domain):
                                     "ip": self._dns_gethostbyname(domain)})
 
     def _check_http(self, tcpdata):
-        """Checks for HTTP traffic.
+        """Check for HTTP traffic.
         @param tcpdata: TCP data flow.
         """
         try:
@@ -459,7 +459,7 @@ def _check_http(self, tcpdata):
         return True
 
     def _add_http(self, tcpdata, dport):
-        """Adds an HTTP flow.
+        """Add an HTTP flow.
         @param tcpdata: TCP data flow.
         @param dport: destination port.
         """
@@ -562,7 +562,7 @@ def _process_smtp(self):
 
     def _check_irc(self, tcpdata):
         """
-        Checks for IRC traffic.
+        Check for IRC traffic.
         @param tcpdata: tcp data flow
         """
         try:
@@ -574,7 +574,7 @@ def _check_irc(self, tcpdata):
 
     def _add_irc(self, tcpdata):
         """
-        Adds an IRC communication.
+        Add an IRC communication.
         @param tcpdata: TCP data in flow
         @param dport: destination port
         """
@@ -732,7 +732,7 @@ def run(self):
         return self.results
 
 class Pcap2(object):
-    """Interprets the PCAP file through the httpreplay library which parses
+    """Interpret the PCAP file through the httpreplay library which parses
     the various protocols, decrypts and decodes them, and then provides us
     with the high level representation of it."""
 
@@ -936,7 +936,7 @@ def conn_from_flowtuple(ft):
 # it for the temp files
 # this code is mostly taken from some SO post, can't remember the url though
 def batch_sort(input_iterator, output_path, output_class):
-    """batch sort helper with temporary files, supports sorting large stuff"""
+    """Batch sort helper with temporary files, supports sorting large stuff."""
     chunks = []
     try:
         while True:
@@ -1015,7 +1015,7 @@ def next(self):
         return Keyed((flowtuple, ts, self.ctr), rpkt)
 
 def sort_pcap(inpath, outpath):
-    """Use SortCap class together with batch_sort to sort a pcap"""
+    """Use SortCap class together with batch_sort to sort a pcap."""
     inc = SortCap(inpath)
     batch_sort(
         inc, outpath, lambda path: SortCap(path, linktype=inc.linktype)
@@ -1023,7 +1023,7 @@ def sort_pcap(inpath, outpath):
     return 0
 
 def flowtuple_from_raw(raw, linktype=1):
-    """Parse a packet from a pcap just enough to gain a flow description tuple"""
+    """Parse a packet from a pcap just enough to gain a flow description tuple."""
     ip = iplayer_from_raw(raw, linktype)
 
     if isinstance(ip, dpkt.ip.IP):
diff --git a/cuckoo/processing/procmemory.py b/cuckoo/processing/procmemory.py
index 7f13418b84..e4f0e45afc 100644
--- a/cuckoo/processing/procmemory.py
+++ b/cuckoo/processing/procmemory.py
@@ -52,7 +52,7 @@ def create_idapy(self, process):
                 print>>o, "autoMark(%s, AU_CODE)" % region["addr"]
 
     def _fixup_pe_header(self, pe):
-        """Fixes the PE header from an in-memory representation to an
+        """Fix the PE header from an in-memory representation to an
         on-disk representation."""
         for section in pe.sections:
             section.PointerToRawData = section.VirtualAddress
diff --git a/cuckoo/processing/procmon.py b/cuckoo/processing/procmon.py
index ac7cfe36f0..494f835182 100644
--- a/cuckoo/processing/procmon.py
+++ b/cuckoo/processing/procmon.py
@@ -8,7 +8,7 @@
 from cuckoo.common.abstracts import Processing
 
 class ProcmonLog(list):
-    """Yields each API call event to the parent handler."""
+    """Yield each API call event to the parent handler."""
 
     def __init__(self, filepath):
         list.__init__(self)
@@ -32,7 +32,7 @@ def __nonzero__(self):
         return True
 
 class Procmon(Processing):
-    """Extracts events from procmon.exe output."""
+    """Extract events from procmon.exe output."""
 
     key = "procmon"
 
diff --git a/cuckoo/processing/static.py b/cuckoo/processing/static.py
index 46b5a2c95f..e5ac96d2bb 100644
--- a/cuckoo/processing/static.py
+++ b/cuckoo/processing/static.py
@@ -63,14 +63,14 @@ def __init__(self, file_path):
         self.pe = None
 
     def _get_filetype(self, data):
-        """Gets filetype, uses libmagic if available.
+        """Get filetype, use libmagic if available.
         @param data: data to be analyzed.
         @return: file type or None.
         """
         return sflock.magic.from_buffer(data)
 
     def _get_peid_signatures(self):
-        """Gets PEID signatures.
+        """Get PEID signatures.
         @return: matched signatures or None.
         """
         try:
@@ -81,7 +81,7 @@ def _get_peid_signatures(self):
             return None
 
     def _get_imported_symbols(self):
-        """Gets imported symbols.
+        """Get imported symbols.
         @return: imported symbols dict or None.
         """
         imports = []
@@ -105,7 +105,7 @@ def _get_imported_symbols(self):
         return imports
 
     def _get_exported_symbols(self):
-        """Gets exported symbols.
+        """Get exported symbols.
         @return: exported symbols dict or None.
         """
         exports = []
@@ -122,7 +122,7 @@ def _get_exported_symbols(self):
         return exports
 
     def _get_sections(self):
-        """Gets sections.
+        """Get sections.
         @return: sections dict or None.
         """
         sections = []
@@ -207,7 +207,7 @@ def _get_versioninfo(self):
         return infos
 
     def _get_imphash(self):
-        """Gets imphash.
+        """Get imphash.
         @return: imphash string or None.
         """
         try:
diff --git a/cuckoo/processing/virustotal.py b/cuckoo/processing/virustotal.py
index 7652a2b0a0..c903a9a866 100644
--- a/cuckoo/processing/virustotal.py
+++ b/cuckoo/processing/virustotal.py
@@ -18,7 +18,7 @@
 log = logging.getLogger(__name__)
 
 class VirusTotal(Processing):
-    """Gets antivirus signatures from VirusTotal.com for various results.
+    """Get antivirus signatures from VirusTotal.com for various results.
 
     Currently obtains VirusTotal results for the target sample or URL and the
     dropped files.
@@ -26,7 +26,7 @@ class VirusTotal(Processing):
     order = 2
 
     def run(self):
-        """Runs VirusTotal processing
+        """Run VirusTotal processing
         @return: full VirusTotal report.
         """
         self.key = "virustotal"
@@ -94,7 +94,7 @@ def scan_url(self, url, summary=False):
                         "\"%s\": %s", url, e.message)
 
     def should_scan_file(self, filetype):
-        """Determines whether a certain filetype should be scanned on
+        """Determine whether a certain filetype should be scanned on
         VirusTotal. For example, we're not interested in scanning text
         files.
         @param filetype: file type
diff --git a/cuckoo/reporting/elasticsearch.py b/cuckoo/reporting/elasticsearch.py
index b6701db999..9c2e810b5f 100644
--- a/cuckoo/reporting/elasticsearch.py
+++ b/cuckoo/reporting/elasticsearch.py
@@ -21,7 +21,7 @@
 log = logging.getLogger(__name__)
 
 class ElasticSearch(Report):
-    """Stores report in Elasticsearch."""
+    """Store report in Elasticsearch."""
 
     @classmethod
     def init_once(cls):
diff --git a/cuckoo/reporting/feedback.py b/cuckoo/reporting/feedback.py
index 6ca53d3f19..44fda322f8 100644
--- a/cuckoo/reporting/feedback.py
+++ b/cuckoo/reporting/feedback.py
@@ -6,7 +6,7 @@
 from cuckoo.core.feedback import CuckooFeedbackObject, CuckooFeedback
 
 class Feedback(Report):
-    """Reports feedback to the Cuckoo Feedback backend if required."""
+    """Report feedback to the Cuckoo Feedback backend if required."""
 
     def run(self, results):
         # Nothing to see here.
diff --git a/cuckoo/reporting/jsondump.py b/cuckoo/reporting/jsondump.py
index 54f86a8fba..73041e7253 100644
--- a/cuckoo/reporting/jsondump.py
+++ b/cuckoo/reporting/jsondump.py
@@ -19,10 +19,10 @@ def default(obj):
     raise TypeError("%r is not JSON serializable" % obj)
 
 class JsonDump(Report):
-    """Saves analysis results in JSON format."""
+    """Save analysis results in JSON format."""
 
     def erase_calls(self, results):
-        """Temporarily removes calls from the report by replacing them with
+        """Temporarily remove calls from the report by replacing them with
         empty lists."""
         if self.calls:
             self.calls = None
@@ -34,7 +34,7 @@ def erase_calls(self, results):
             process["calls"] = []
 
     def restore_calls(self, results):
-        """Restores calls that were temporarily removed in the report by
+        """Restore calls that were temporarily removed in the report by
         replacing the calls with the original values."""
         if not self.calls:
             return
diff --git a/cuckoo/reporting/misp.py b/cuckoo/reporting/misp.py
index d92a5f5779..14508d2f71 100644
--- a/cuckoo/reporting/misp.py
+++ b/cuckoo/reporting/misp.py
@@ -91,7 +91,7 @@ def signature(self, results, event):
                 )
 
     def run(self, results):
-        """Submits results to MISP.
+        """Submit results to MISP.
         @param results: Cuckoo results dict.
         """
         url = self.options.get("url")
diff --git a/cuckoo/reporting/mongodb.py b/cuckoo/reporting/mongodb.py
index c6df12f385..b170cdc010 100644
--- a/cuckoo/reporting/mongodb.py
+++ b/cuckoo/reporting/mongodb.py
@@ -12,7 +12,7 @@
 from cuckoo.common.objects import File
 
 class MongoDB(Report):
-    """Stores report in MongoDB."""
+    """Store report in MongoDB."""
     order = 2
 
     # Mongo schema version, used for data migration.
@@ -77,7 +77,7 @@ def store_file(self, file_obj, filename=""):
             return self.db.fs.files.find_one(to_find)["_id"]
 
     def run(self, results):
-        """Writes report.
+        """Write report.
         @param results: analysis results dictionary.
         @raise CuckooReportError: if fails to connect or write to MongoDB.
         """
diff --git a/cuckoo/reporting/notification.py b/cuckoo/reporting/notification.py
index 7a56105d4b..2518d9f882 100644
--- a/cuckoo/reporting/notification.py
+++ b/cuckoo/reporting/notification.py
@@ -18,7 +18,7 @@ def default(obj):
     raise TypeError("%r is not JSON serializable" % obj)
 
 class Notification(Report):
-    """Notifies external service about finished analysis via URL."""
+    """Notify external service about finished analysis via URL."""
     order = 3
 
     def run(self, results):
diff --git a/cuckoo/reporting/singlefile.py b/cuckoo/reporting/singlefile.py
index 148b7c8272..202b5a7eb9 100644
--- a/cuckoo/reporting/singlefile.py
+++ b/cuckoo/reporting/singlefile.py
@@ -20,7 +20,7 @@
 logging.getLogger("weasyprint").setLevel(logging.ERROR)
 
 class SingleFile(Report):
-    """Stores report in a single-file HTML and/or PDF format."""
+    """Store report in a single-file HTML and/or PDF format."""
 
     fonts = [{
         "family": "Roboto",
@@ -120,14 +120,14 @@ def generate_jinja2_template(self, results):
         )
 
     def combine_css(self):
-        """Scans the static/css/ directory and concatenates stylesheets"""
+        """Scan the static/css/ directory and concatenate stylesheets"""
         css_includes = []
         for filepath in glob.glob("%s/static/css/*.css" % self.path_base):
             css_includes.append(open(filepath, "rb").read().decode("utf8"))
         return "\n".join(css_includes)
 
     def combine_js(self):
-        """Scans the static/js/ directory and concatenates js files"""
+        """Scan the static/js/ directory and concatenate js files"""
         js_includes = []
         # Note: jquery-2.2.4.min.js must be the first file.
         filepaths = sorted(glob.glob("%s/static/js/*.js" % self.path_base))
diff --git a/cuckoo/web/analysis/templatetags/analysis_tags.py b/cuckoo/web/analysis/templatetags/analysis_tags.py
index f5e18ba51e..e34d251a6e 100644
--- a/cuckoo/web/analysis/templatetags/analysis_tags.py
+++ b/cuckoo/web/analysis/templatetags/analysis_tags.py
@@ -19,7 +19,7 @@ def mongo_id(value):
 
 @register.filter
 def is_dict(value):
-    """Checks if value is an instance of dict"""
+    """Check if value is an instance of dict"""
     return isinstance(value, dict)
 
 @register.filter
diff --git a/cuckoo/web/analysis/views.py b/cuckoo/web/analysis/views.py
index 7b4fcbb1f3..d9d60d9145 100644
--- a/cuckoo/web/analysis/views.py
+++ b/cuckoo/web/analysis/views.py
@@ -84,7 +84,7 @@ def chunk(request, task_id, pid, pagenum):
 
 @require_safe
 def filtered_chunk(request, task_id, pid, category):
-    """Filters calls for call category.
+    """Filter calls for call category.
     @param task_id: cuckoo task id
     @param pid: pid you want calls
     @param category: call category type
diff --git a/cuckoo/web/controllers/analysis/analysis.py b/cuckoo/web/controllers/analysis/analysis.py
index 0c71128054..5d7a445e76 100644
--- a/cuckoo/web/controllers/analysis/analysis.py
+++ b/cuckoo/web/controllers/analysis/analysis.py
@@ -34,7 +34,7 @@ def _get_report(task_id):
 
     @staticmethod
     def _get_dnsinfo(report):
-        """Create DNS information dicts by domain and ip"""
+        """Create DNS information dicts by domain and ip."""
 
         if "network" in report and "domains" in report["network"]:
             domainlookups = dict((i["domain"], i["ip"]) for i in report["network"]["domains"])
diff --git a/cuckoo/web/controllers/analysis/api.py b/cuckoo/web/controllers/analysis/api.py
index 278f2a805b..829e91c71f 100644
--- a/cuckoo/web/controllers/analysis/api.py
+++ b/cuckoo/web/controllers/analysis/api.py
@@ -102,7 +102,7 @@ def tasks_info(request, body):
     @api_get
     def task_delete(request, task_id):
         """
-        Deletes a task
+        Delete a task.
         :param body: required: task_id
         :return:
         """
@@ -126,7 +126,7 @@ def task_delete(request, task_id):
     @api_get
     def tasks_reschedule(request, task_id, priority=None):
         """
-        Reschedules a task
+        Reschedule a task.
         :param body: required: task_id, priority
         :return: new task_id
         """
@@ -363,7 +363,7 @@ def tasks_recent(request, body):
     @api_post
     def tasks_stats(request, body):
         """
-        Fetches the number of analysis over a
+        Fetch the number of analysis over a
         given period for the "failed" and
         "successful" states. Values are
         returned in months.
diff --git a/cuckoo/web/controllers/analysis/export/export.py b/cuckoo/web/controllers/analysis/export/export.py
index 83ebf0f7fc..e12eebd28f 100644
--- a/cuckoo/web/controllers/analysis/export/export.py
+++ b/cuckoo/web/controllers/analysis/export/export.py
@@ -44,7 +44,7 @@ def estimate_size(task_id, taken_dirs, taken_files):
     @staticmethod
     def create(task_id, taken_dirs, taken_files, report=None):
         """
-        Returns a zip file as a file like object.
+        Return a zip file as a file like object.
         :param task_id: task id
         :param taken_dirs: directories to include
         :param taken_files: files to include
diff --git a/cuckoo/web/controllers/cuckoo/api.py b/cuckoo/web/controllers/cuckoo/api.py
index 5b36ab17bd..8748cdf1e1 100644
--- a/cuckoo/web/controllers/cuckoo/api.py
+++ b/cuckoo/web/controllers/cuckoo/api.py
@@ -21,7 +21,7 @@
 updates = {}
 
 def latest_updates():
-    """Updates the latest Cuckoo version & blogposts at maximum once a day."""
+    """Update the latest Cuckoo version & blogposts at maximum once a day."""
     next_check = datetime.datetime.now() - datetime.timedelta(days=1)
     if updates and updates["timestamp"] > next_check:
         return updates
@@ -36,7 +36,7 @@ class CuckooApi(object):
     @api_get
     def status(request):
         """
-        Returns a variety of information about both
+        Return a variety of information about both
         Cuckoo and the operating system.
         :return: Dictionary
         """
diff --git a/cuckoo/web/controllers/machines/api.py b/cuckoo/web/controllers/machines/api.py
index 50a2b57d17..f1eb67bc2b 100644
--- a/cuckoo/web/controllers/machines/api.py
+++ b/cuckoo/web/controllers/machines/api.py
@@ -13,7 +13,7 @@ class MachinesApi:
     @api_get
     def list(request):
         """
-        Returns a list of all machines currently registered in Cuckoo
+        Return a list of all machines currently registered in Cuckoo
         :return:
         """
         data = {}
@@ -29,7 +29,7 @@ def list(request):
     @api_get
     def view(request, name=None):
         """
-        Returns information about a machine
+        Return information about a machine
         :param name: machine name
         :return: Machine information as a dictionary
         """
diff --git a/stuff/vpncheck.py b/stuff/vpncheck.py
index 31b464b432..92670186dd 100755
--- a/stuff/vpncheck.py
+++ b/stuff/vpncheck.py
@@ -16,7 +16,7 @@
 SIOCGIFADDR = 0x8915
 
 def get_ip_address(interface):
-    """Retrieves the local IP address of a network interface."""
+    """Retrieve the local IP address of a network interface."""
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     buf = fcntl.ioctl(s.fileno(), SIOCGIFADDR, struct.pack("256s", interface))
     return socket.inet_ntoa(buf[20:24])
diff --git a/tests/test_database.py b/tests/test_database.py
index b91a8be73c..03803ce7cf 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -17,7 +17,7 @@
 from cuckoo.misc import set_cwd, cwd, mkdir
 
 class DatabaseEngine(object):
-    """Tests database stuff."""
+    """Test database stuff."""
     URI = None
 
     def setup_class(self):
@@ -275,7 +275,7 @@ class TestMySQL(DatabaseEngine):
 
 @pytest.mark.skipif("sys.platform != 'linux2'")
 class DatabaseMigrationEngine(object):
-    """Tests database migration(s)."""
+    """Test database migration(s)."""
     URI = None
     SRC = None
 
diff --git a/tests/test_init.py b/tests/test_init.py
index a374b59ee1..ca7dd8988f 100644
--- a/tests/test_init.py
+++ b/tests/test_init.py
@@ -60,7 +60,7 @@ def test_venv_new_unicode(self):
         write_supervisor_conf(None)
 
     def test_cuckoo_init(self):
-        """Tests that 'cuckoo init' works with a new CWD."""
+        """Test that 'cuckoo init' works with a new CWD."""
         with pytest.raises(SystemExit):
             main.main(
                 ("--cwd", cwd(), "--nolog", "init"),
@@ -77,7 +77,7 @@ def test_cuckoo_init(self):
         assert os.path.exists(os.path.join(cwd(), "storage", "baseline"))
 
     def test_cuckoo_init_main(self):
-        """Tests that 'cuckoo' works with a new CWD."""
+        """Test that 'cuckoo' works with a new CWD."""
         main.main(
             ("--cwd", cwd(), "--nolog"),
             standalone_mode=False
@@ -95,7 +95,7 @@ def test_cuckoo_init_main_nosigs(self, p):
         p.assert_not_called()
 
     def test_cuckoo_init_no_resultserver(self):
-        """Tests that 'cuckoo init' doesn't launch the ResultServer."""
+        """Test that 'cuckoo init' doesn't launch the ResultServer."""
         with pytest.raises(SystemExit):
             main.main(
                 ("--cwd", cwd(), "--nolog", "init"),
diff --git a/tests/test_log.py b/tests/test_log.py
index bf11672b43..4072665382 100644
--- a/tests/test_log.py
+++ b/tests/test_log.py
@@ -18,7 +18,7 @@
 db = Database()
 
 def reset_logging():
-    """Resets the logging module to its initial state so that we can
+    """Reset the logging module to its initial state so that we can
     re-register all kinds of logging logic for unit testing purposes."""
     logging.root = logging.RootLogger(logging.WARNING)
     logging.Logger.root = logging.root
diff --git a/tests/test_misc.py b/tests/test_misc.py
index 1c3a78b6bf..4d1ef2efa6 100644
--- a/tests/test_misc.py
+++ b/tests/test_misc.py
@@ -111,7 +111,7 @@ def test_platforms():
     assert sys.platform in ("win32", "linux2", "darwin")
 
 def test_popen():
-    """Ensures that Popen is working properly."""
+    """Ensure that Popen is working properly."""
     with mock.patch("subprocess.Popen") as p:
         p.return_value = None
         Popen(["foo", "bar"])
diff --git a/tests/test_reporting.py b/tests/test_reporting.py
index 5c3f592081..c25b4a9434 100644
--- a/tests/test_reporting.py
+++ b/tests/test_reporting.py
@@ -116,7 +116,7 @@ def test_empty_mattermost():
 
 @responses.activate
 def test_empty_misp():
-    """Merely connects to MISP and creates the new event."""
+    """Merely connect to MISP and create the new event."""
     set_cwd(tempfile.mkdtemp())
     conf = {
         "misp": {
diff --git a/tests/test_submit.py b/tests/test_submit.py
index 10d6517176..e27db869f8 100644
--- a/tests/test_submit.py
+++ b/tests/test_submit.py
@@ -38,7 +38,7 @@ def setup(self):
         self.submit_manager = SubmitManager()
 
     def test_pre_file(self):
-        """Tests the submission of a plaintext file"""
+        """Test the submission of a plaintext file"""
         assert self.submit_manager.pre(submit_type="files", data=[{
             "name": "foo.txt",
             "data": open("tests/files/foo.txt", "rb").read()
@@ -54,7 +54,7 @@ def test_pre_file(self):
         assert filedata == open("tests/files/foo.txt", "rb").read()
 
     def test_pre_url(self):
-        """Tests the submission of URLs (http/https)"""
+        """Test the submission of URLs (http/https)"""
         assert self.submit_manager.pre(submit_type="strings", data=[
             "http://theguardian.com/",
             "https://news.ycombinator.com/",
@@ -83,7 +83,7 @@ def test_invalid_strings(self):
 
     @responses.activate
     def test_pre_hash(self):
-        """Tests the submission of a VirusTotal hash."""
+        """Test the submission of a VirusTotal hash."""
         with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
             rsps.add(
                 responses.GET, VirusTotalAPI.HASH_DOWNLOAD, body="A"*1024*1024
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 01383b3a83..4bb2263451 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -26,19 +26,19 @@ def setup(self):
         self.tmp_dir = tempfile.gettempdir()
 
     def test_root_folder(self):
-        """Tests a single folder creation based on the root parameter."""
+        """Test single folder creation based on the root parameter."""
         Folders.create(os.path.join(self.tmp_dir, "foo"))
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         os.rmdir(os.path.join(self.tmp_dir, "foo"))
 
     def test_single_folder(self):
-        """Tests a single folder creation."""
+        """Test single folder creation."""
         Folders.create(self.tmp_dir, "foo")
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         os.rmdir(os.path.join(self.tmp_dir, "foo"))
 
     def test_multiple_folders(self):
-        """Tests multiple folders creation."""
+        """Test multiple folder creation."""
         Folders.create(self.tmp_dir, ["foo", "bar"])
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         assert os.path.exists(os.path.join(self.tmp_dir, "bar"))
@@ -46,7 +46,7 @@ def test_multiple_folders(self):
         os.rmdir(os.path.join(self.tmp_dir, "bar"))
 
     def test_copy_folder(self):
-        """Tests recursive folder copy"""
+        """Test recursive folder copy."""
         dirpath = tempfile.mkdtemp()
         set_cwd(dirpath)
 
@@ -54,21 +54,21 @@ def test_copy_folder(self):
         assert os.path.isfile("%s/reports/report.json" % dirpath)
 
     def test_duplicate_folder(self):
-        """Tests a duplicate folder creation."""
+        """Test a duplicate folder creation."""
         Folders.create(self.tmp_dir, "foo")
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         Folders.create(self.tmp_dir, "foo")
         os.rmdir(os.path.join(self.tmp_dir, "foo"))
 
     def test_delete_folder(self):
-        """Tests folder deletion #1."""
+        """Test folder deletion #1."""
         Folders.create(self.tmp_dir, "foo")
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         Folders.delete(os.path.join(self.tmp_dir, "foo"))
         assert not os.path.exists(os.path.join(self.tmp_dir, "foo"))
 
     def test_delete_folder2(self):
-        """Tests folder deletion #2."""
+        """Test folder deletion #2."""
         Folders.create(self.tmp_dir, "foo")
         assert os.path.exists(os.path.join(self.tmp_dir, "foo"))
         Folders.delete(self.tmp_dir, "foo")
diff --git a/tests/utils.py b/tests/utils.py
index 323861f3cd..7d32016f34 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -21,7 +21,7 @@ def __exit__(self, type_, value, traceback):
         os.chdir(self.origpath)
 
 def init_analysis(task_id, package, *filename):
-    """Initializes an analysis with an "encrypted" binary from tests/files/."""
+    """Initialize an analysis with an "encrypted" binary from tests/files/."""
     mkdir(cwd(analysis=task_id))
     content = open(os.path.join("tests", "files", *filename), "rb").read()
     open(cwd("binary", analysis=task_id), "wb").write(content[::-1])

From 722bc165e2819495721126b31ba69a57a633e37e Mon Sep 17 00:00:00 2001
From: Lilly Chalupowski <lchalupowski@gosecure.ca>
Date: Fri, 16 Mar 2018 11:26:21 -0300
Subject: [PATCH 028/138] Disable SmartScreen in IE

New registry keys added to disable smart screen filter.

Allows for more accurate URL analysis.

:wink:
---
 .../data/analyzer/windows/modules/packages/ie.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/cuckoo/data/analyzer/windows/modules/packages/ie.py b/cuckoo/data/analyzer/windows/modules/packages/ie.py
index 8252072a0b..9084c8c588 100644
--- a/cuckoo/data/analyzer/windows/modules/packages/ie.py
+++ b/cuckoo/data/analyzer/windows/modules/packages/ie.py
@@ -94,6 +94,22 @@ class IE(Package):
                 "CheckExeSignatures": "no",
             },
         ],
+        [
+            HKEY_LOCAL_MACHINE,
+            "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer",
+            {
+                # Disable SmartScreen Windows 8
+                "SmartScreenEnabled": "Off"
+            }
+        ],
+        [
+            HKEY_CURRENT_USER,
+            "Software\\Microsoft\\Internet Explorer\\PhishingFilter",
+            {
+                # Disable SmartScreen Filter Windows 7
+                "EnabledV9": 0
+            }
+        ],
     ]
 
     def setup_proxy(self, proxy_host):

From 25592a5ac02c764b651463491065653702313d7a Mon Sep 17 00:00:00 2001
From: Lilly Chalupowski <lchalupowski@gosecure.ca>
Date: Fri, 16 Mar 2018 11:26:21 -0300
Subject: [PATCH 029/138] Disable SmartScreen in IE

New registry keys added to disable smart screen filter.

Allows for more accurate URL analysis.

:wink:
---
 .../data/analyzer/windows/modules/packages/ie.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/cuckoo/data/analyzer/windows/modules/packages/ie.py b/cuckoo/data/analyzer/windows/modules/packages/ie.py
index 8252072a0b..9084c8c588 100644
--- a/cuckoo/data/analyzer/windows/modules/packages/ie.py
+++ b/cuckoo/data/analyzer/windows/modules/packages/ie.py
@@ -94,6 +94,22 @@ class IE(Package):
                 "CheckExeSignatures": "no",
             },
         ],
+        [
+            HKEY_LOCAL_MACHINE,
+            "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer",
+            {
+                # Disable SmartScreen Windows 8
+                "SmartScreenEnabled": "Off"
+            }
+        ],
+        [
+            HKEY_CURRENT_USER,
+            "Software\\Microsoft\\Internet Explorer\\PhishingFilter",
+            {
+                # Disable SmartScreen Filter Windows 7
+                "EnabledV9": 0
+            }
+        ],
     ]
 
     def setup_proxy(self, proxy_host):

From cf9746f478084e8474f0da48eb27dda23e443330 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Tue, 14 May 2019 16:17:39 +0200
Subject: [PATCH 030/138] Update appveyor conf

---
 appveyor.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 50a420393c..21e919aa07 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -25,8 +25,8 @@ install:
   - "python.exe setup.py sdist"
   - "pip.exe install -e ."
   - "pip.exe install psycopg2 mysqlclient==1.3.9 codecov flask-testing"
-  - "pip.exe install pytest pytest-cov pytest-django pytest-pythonpath"
-  - "pip.exe install flask-sqlalchemy==2.1 mock==2.0.0 responses==0.5.1"
+  - "pip.exe install pytest==4.1.1 pytest-cov pytest-django pytest-pythonpath"
+  - "pip.exe install flask-sqlalchemy==2.4.0 mock==2.0.0 responses==0.5.1"
 
 build: false
 

From 2161447c8dda17f97faf26ab53472eb8cff9c8db Mon Sep 17 00:00:00 2001
From: in2etv <in2etv@in2e.tv>
Date: Sat, 4 Aug 2018 23:17:09 +0900
Subject: [PATCH 031/138] Bugfix : String extraction does not work on Windows
 host

---
 cuckoo/processing/strings.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cuckoo/processing/strings.py b/cuckoo/processing/strings.py
index baa1454f6c..e255d0414d 100644
--- a/cuckoo/processing/strings.py
+++ b/cuckoo/processing/strings.py
@@ -29,12 +29,14 @@ def run(self):
                 )
 
             try:
-                data = open(self.file_path, "r").read(self.MAX_FILESIZE)
+                data = open(self.file_path, "rb").read(self.MAX_FILESIZE)
             except (IOError, OSError) as e:
                 raise CuckooProcessingError("Error opening file %s" % e)
 
-            strings = re.findall("[\x1f-\x7e]{6,}", data)
-            for s in re.findall("(?:[\x1f-\x7e][\x00]){6,}", data):
+            strings = []
+            for s in re.findall(b"[\x1f-\x7e]{6,}", data):
+                strings.append(s.decode("utf-8"))
+            for s in re.findall(b"(?:[\x1f-\x7e][\x00]){6,}", data):
                 strings.append(s.decode("utf-16le"))
 
         # Now limit the amount & length of the strings.
@@ -42,4 +44,4 @@ def run(self):
         for idx, s in enumerate(strings):
             strings[idx] = s[:self.MAX_STRINGLEN]
 
-        return strings
+        return strings
\ No newline at end of file

From 8abf3c711317bedaf228199ef2a8786b79083f18 Mon Sep 17 00:00:00 2001
From: Calogero Lo Leggio <kalos@nerdrug.org>
Date: Wed, 11 Jul 2018 15:29:29 +0200
Subject: [PATCH 032/138] avoid SSL/TLS problem with recent vSphere 6.7 release

TLSv1 is unsupported in recent vSphere.
Using '_create_unverified_context' for create correct context.
---
 cuckoo/machinery/vsphere.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cuckoo/machinery/vsphere.py b/cuckoo/machinery/vsphere.py
index 2054f395c9..7ada17d6c1 100644
--- a/cuckoo/machinery/vsphere.py
+++ b/cuckoo/machinery/vsphere.py
@@ -96,8 +96,7 @@ def _initialize_check(self):
 
         # Workaround for PEP-0476 issues in recent Python versions
         if self.options.vsphere.unverified_ssl:
-            sslContext = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
-            sslContext.verify_mode = ssl.CERT_NONE
+            sslContext = ssl._create_unverified_context()
             self.connect_opts["sslContext"] = sslContext
             log.warn("Turning off SSL certificate verification!")
 

From 0bbb49f2c291a235212d3a898a38f18160b34239 Mon Sep 17 00:00:00 2001
From: sebdg <sebdg@binarycompany.com>
Date: Wed, 6 Dec 2017 12:53:31 +0000
Subject: [PATCH 033/138] path not null check

---
 cuckoo/web/controllers/submission/routes.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cuckoo/web/controllers/submission/routes.py b/cuckoo/web/controllers/submission/routes.py
index f689b00371..eb0d835e98 100644
--- a/cuckoo/web/controllers/submission/routes.py
+++ b/cuckoo/web/controllers/submission/routes.py
@@ -77,7 +77,8 @@ def resubmit(request, task_id):
                 task.target,
             ], submit_manager.translate_options_to(task.options))
         else:
-            if not os.path.exists(task.target):
+            file_path = binary_filepath(task_id) if not os.path.exists(task.target) else task.target
+            if not file_path or not os.path.exists(file_path):
                 return view_error(
                     request, "The file you're trying to resubmit "
                     "no longer exists. Please resubmit it altogether!"

From 3fb0cf0517e627cb0659eaaea5e098461738fe7d Mon Sep 17 00:00:00 2001
From: sebdg <sebdg@binarycompany.com>
Date: Wed, 6 Dec 2017 12:08:40 +0000
Subject: [PATCH 034/138] path could be missing

---
 cuckoo/web/controllers/submission/routes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cuckoo/web/controllers/submission/routes.py b/cuckoo/web/controllers/submission/routes.py
index eb0d835e98..aa8238999c 100644
--- a/cuckoo/web/controllers/submission/routes.py
+++ b/cuckoo/web/controllers/submission/routes.py
@@ -78,7 +78,7 @@ def resubmit(request, task_id):
             ], submit_manager.translate_options_to(task.options))
         else:
             file_path = binary_filepath(task_id) if not os.path.exists(task.target) else task.target
-            if not file_path or not os.path.exists(file_path):
+            if file_path and not os.path.exists(file_path):
                 return view_error(
                     request, "The file you're trying to resubmit "
                     "no longer exists. Please resubmit it altogether!"

From 33b53e3edfbb226afe1224add8c65e34bbb0bda5 Mon Sep 17 00:00:00 2001
From: sebdg <sebdg@binarycompany.com>
Date: Tue, 5 Dec 2017 22:54:57 +0000
Subject: [PATCH 035/138] fix issues locating file to resubmit

---
 cuckoo/web/controllers/submission/routes.py |  6 +++---
 cuckoo/web/utils.py                         | 10 ++++++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/cuckoo/web/controllers/submission/routes.py b/cuckoo/web/controllers/submission/routes.py
index aa8238999c..38af0ff1b1 100644
--- a/cuckoo/web/controllers/submission/routes.py
+++ b/cuckoo/web/controllers/submission/routes.py
@@ -10,7 +10,7 @@
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.core.database import Database
 from cuckoo.core.submit import SubmitManager
-from cuckoo.web.utils import view_error, render_template, dropped_filepath
+from cuckoo.web.utils import view_error, render_template, dropped_filepath, binary_filepath
 
 log = logging.getLogger(__name__)
 submit_manager = SubmitManager()
@@ -78,7 +78,7 @@ def resubmit(request, task_id):
             ], submit_manager.translate_options_to(task.options))
         else:
             file_path = binary_filepath(task_id) if not os.path.exists(task.target) else task.target
-            if file_path and not os.path.exists(file_path):
+            if not os.path.exists(file_path):
                 return view_error(
                     request, "The file you're trying to resubmit "
                     "no longer exists. Please resubmit it altogether!"
@@ -88,7 +88,7 @@ def resubmit(request, task_id):
             # analyses of type "archive".
             submit_id = submit_manager.pre("files", [{
                 "name": os.path.basename(task.target),
-                "data": open(task.target, "rb"),
+                "data": open(file_path, "rb"),
             }], submit_manager.translate_options_to(task.options))
 
         return redirect("submission/pre", submit_id=submit_id)
diff --git a/cuckoo/web/utils.py b/cuckoo/web/utils.py
index 2656844a7b..46ba0adc51 100644
--- a/cuckoo/web/utils.py
+++ b/cuckoo/web/utils.py
@@ -120,6 +120,16 @@ def dropped_filepath(task_id, sha1):
         if dropped["sha1"] == sha1:
             return dropped["path"]
 
+def binary_filepath(task_id):
+    record = mongo.db.analysis.find_one({
+        "info.id": int(task_id)
+    })
+
+    if not record or not record["target"]["file"]:
+        return
+
+    return record["target"]["file"]["path"]
+
 def normalize_task(task):
     if task["category"] == "file":
         task["target"] = os.path.basename(task["target"])

From 58af31ddb7ba27dd761811ad096000884b7d4715 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 15 May 2019 20:26:39 +0200
Subject: [PATCH 036/138] Handle in path is None case

---
 cuckoo/web/controllers/submission/routes.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cuckoo/web/controllers/submission/routes.py b/cuckoo/web/controllers/submission/routes.py
index 38af0ff1b1..69322bea09 100644
--- a/cuckoo/web/controllers/submission/routes.py
+++ b/cuckoo/web/controllers/submission/routes.py
@@ -77,8 +77,11 @@ def resubmit(request, task_id):
                 task.target,
             ], submit_manager.translate_options_to(task.options))
         else:
-            file_path = binary_filepath(task_id) if not os.path.exists(task.target) else task.target
-            if not os.path.exists(file_path):
+            file_path = binary_filepath(task_id)
+            if not file_path or not os.path.exists(file_path):
+                file_path = task.target
+
+            if not file_path or not os.path.exists(file_path):
                 return view_error(
                     request, "The file you're trying to resubmit "
                     "no longer exists. Please resubmit it altogether!"

From ae42390eda102c19be6657483be090a602f04dd3 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 15 May 2019 20:28:11 +0200
Subject: [PATCH 037/138] Cleanup network rules in case of analysis manager
 crash

---
 cuckoo/core/scheduler.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py
index 695dd23715..dad8e07ec2 100644
--- a/cuckoo/core/scheduler.py
+++ b/cuckoo/core/scheduler.py
@@ -62,6 +62,7 @@ def __init__(self, task_id, error_queue):
         self.route = None
         self.interface = None
         self.rt_table = None
+        self.unrouted_network = False
 
     def init(self):
         """Initialize the analysis."""
@@ -370,6 +371,8 @@ def unroute_network(self):
                 str(config("routing:tor:proxyport"))
             )
 
+        self.unrouted_network = True
+
     def wait_finish(self):
         """Some VMs don't have an actual agent. Mainly those that are used as
         assistance for an analysis through the services auxiliary module. This
@@ -786,6 +789,11 @@ def run(self):
                 "status": "error",
             })
         finally:
+            # In case the analysis manager crashes, the network cleanup
+            # should still be performed.
+            if not self.unrouted_network:
+                self.unroute_network()
+
             if self.cfg.cuckoo.process_results:
                 self.db.set_status(self.task.id, TASK_REPORTED)
             else:

From 3d716487eb7f90ff0aa728de477f505d7ea13143 Mon Sep 17 00:00:00 2001
From: in2etv <in2etv@in2e.tv>
Date: Sat, 4 Aug 2018 23:17:09 +0900
Subject: [PATCH 038/138] Bugfix : String extraction does not work on Windows
 host

---
 cuckoo/processing/strings.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cuckoo/processing/strings.py b/cuckoo/processing/strings.py
index baa1454f6c..e255d0414d 100644
--- a/cuckoo/processing/strings.py
+++ b/cuckoo/processing/strings.py
@@ -29,12 +29,14 @@ def run(self):
                 )
 
             try:
-                data = open(self.file_path, "r").read(self.MAX_FILESIZE)
+                data = open(self.file_path, "rb").read(self.MAX_FILESIZE)
             except (IOError, OSError) as e:
                 raise CuckooProcessingError("Error opening file %s" % e)
 
-            strings = re.findall("[\x1f-\x7e]{6,}", data)
-            for s in re.findall("(?:[\x1f-\x7e][\x00]){6,}", data):
+            strings = []
+            for s in re.findall(b"[\x1f-\x7e]{6,}", data):
+                strings.append(s.decode("utf-8"))
+            for s in re.findall(b"(?:[\x1f-\x7e][\x00]){6,}", data):
                 strings.append(s.decode("utf-16le"))
 
         # Now limit the amount & length of the strings.
@@ -42,4 +44,4 @@ def run(self):
         for idx, s in enumerate(strings):
             strings[idx] = s[:self.MAX_STRINGLEN]
 
-        return strings
+        return strings
\ No newline at end of file

From eda242bd840c3377a1897b154e01db09d99d38a7 Mon Sep 17 00:00:00 2001
From: Calogero Lo Leggio <kalos@nerdrug.org>
Date: Wed, 11 Jul 2018 15:29:29 +0200
Subject: [PATCH 039/138] avoid SSL/TLS problem with recent vSphere 6.7 release

TLSv1 is unsupported in recent vSphere.
Using '_create_unverified_context' for create correct context.
---
 cuckoo/machinery/vsphere.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cuckoo/machinery/vsphere.py b/cuckoo/machinery/vsphere.py
index 2054f395c9..7ada17d6c1 100644
--- a/cuckoo/machinery/vsphere.py
+++ b/cuckoo/machinery/vsphere.py
@@ -96,8 +96,7 @@ def _initialize_check(self):
 
         # Workaround for PEP-0476 issues in recent Python versions
         if self.options.vsphere.unverified_ssl:
-            sslContext = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
-            sslContext.verify_mode = ssl.CERT_NONE
+            sslContext = ssl._create_unverified_context()
             self.connect_opts["sslContext"] = sslContext
             log.warn("Turning off SSL certificate verification!")
 

From 3be3c6979ec4801506eb32ad5360b2e2a9ed9570 Mon Sep 17 00:00:00 2001
From: sebdg <sebdg@binarycompany.com>
Date: Tue, 5 Dec 2017 22:54:57 +0000
Subject: [PATCH 040/138] fix issues locating file to resubmit

---
 cuckoo/web/controllers/submission/routes.py |  7 ++++---
 cuckoo/web/utils.py                         | 10 ++++++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/cuckoo/web/controllers/submission/routes.py b/cuckoo/web/controllers/submission/routes.py
index f689b00371..38af0ff1b1 100644
--- a/cuckoo/web/controllers/submission/routes.py
+++ b/cuckoo/web/controllers/submission/routes.py
@@ -10,7 +10,7 @@
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.core.database import Database
 from cuckoo.core.submit import SubmitManager
-from cuckoo.web.utils import view_error, render_template, dropped_filepath
+from cuckoo.web.utils import view_error, render_template, dropped_filepath, binary_filepath
 
 log = logging.getLogger(__name__)
 submit_manager = SubmitManager()
@@ -77,7 +77,8 @@ def resubmit(request, task_id):
                 task.target,
             ], submit_manager.translate_options_to(task.options))
         else:
-            if not os.path.exists(task.target):
+            file_path = binary_filepath(task_id) if not os.path.exists(task.target) else task.target
+            if not os.path.exists(file_path):
                 return view_error(
                     request, "The file you're trying to resubmit "
                     "no longer exists. Please resubmit it altogether!"
@@ -87,7 +88,7 @@ def resubmit(request, task_id):
             # analyses of type "archive".
             submit_id = submit_manager.pre("files", [{
                 "name": os.path.basename(task.target),
-                "data": open(task.target, "rb"),
+                "data": open(file_path, "rb"),
             }], submit_manager.translate_options_to(task.options))
 
         return redirect("submission/pre", submit_id=submit_id)
diff --git a/cuckoo/web/utils.py b/cuckoo/web/utils.py
index 2656844a7b..46ba0adc51 100644
--- a/cuckoo/web/utils.py
+++ b/cuckoo/web/utils.py
@@ -120,6 +120,16 @@ def dropped_filepath(task_id, sha1):
         if dropped["sha1"] == sha1:
             return dropped["path"]
 
+def binary_filepath(task_id):
+    record = mongo.db.analysis.find_one({
+        "info.id": int(task_id)
+    })
+
+    if not record or not record["target"]["file"]:
+        return
+
+    return record["target"]["file"]["path"]
+
 def normalize_task(task):
     if task["category"] == "file":
         task["target"] = os.path.basename(task["target"])

From 577b6acc5036daf7d0020464ea320edab23f1242 Mon Sep 17 00:00:00 2001
From: sebdg <sebdg@binarycompany.com>
Date: Wed, 6 Dec 2017 12:08:40 +0000
Subject: [PATCH 041/138] path could be missing

---
 cuckoo/web/controllers/submission/routes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cuckoo/web/controllers/submission/routes.py b/cuckoo/web/controllers/submission/routes.py
index 38af0ff1b1..90beaf0626 100644
--- a/cuckoo/web/controllers/submission/routes.py
+++ b/cuckoo/web/controllers/submission/routes.py
@@ -78,7 +78,7 @@ def resubmit(request, task_id):
             ], submit_manager.translate_options_to(task.options))
         else:
             file_path = binary_filepath(task_id) if not os.path.exists(task.target) else task.target
-            if not os.path.exists(file_path):
+            if file_path and not os.path.exists(file_path):
                 return view_error(
                     request, "The file you're trying to resubmit "
                     "no longer exists. Please resubmit it altogether!"

From 909c7cca6ff9efa6168af4a0d6256b1d61d6e53e Mon Sep 17 00:00:00 2001
From: sebdg <sebdg@binarycompany.com>
Date: Wed, 6 Dec 2017 12:53:31 +0000
Subject: [PATCH 042/138] path not null check

---
 cuckoo/web/controllers/submission/routes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cuckoo/web/controllers/submission/routes.py b/cuckoo/web/controllers/submission/routes.py
index 90beaf0626..c9c901242d 100644
--- a/cuckoo/web/controllers/submission/routes.py
+++ b/cuckoo/web/controllers/submission/routes.py
@@ -78,7 +78,7 @@ def resubmit(request, task_id):
             ], submit_manager.translate_options_to(task.options))
         else:
             file_path = binary_filepath(task_id) if not os.path.exists(task.target) else task.target
-            if file_path and not os.path.exists(file_path):
+            if not file_path or not os.path.exists(file_path):
                 return view_error(
                     request, "The file you're trying to resubmit "
                     "no longer exists. Please resubmit it altogether!"

From 7984c14174de1ab28158b6708545b9e131930ee9 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 15 May 2019 23:19:48 +0200
Subject: [PATCH 043/138] Unroute network in case of analysis manager fail

---
 cuckoo/core/scheduler.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py
index 695dd23715..dad8e07ec2 100644
--- a/cuckoo/core/scheduler.py
+++ b/cuckoo/core/scheduler.py
@@ -62,6 +62,7 @@ def __init__(self, task_id, error_queue):
         self.route = None
         self.interface = None
         self.rt_table = None
+        self.unrouted_network = False
 
     def init(self):
         """Initialize the analysis."""
@@ -370,6 +371,8 @@ def unroute_network(self):
                 str(config("routing:tor:proxyport"))
             )
 
+        self.unrouted_network = True
+
     def wait_finish(self):
         """Some VMs don't have an actual agent. Mainly those that are used as
         assistance for an analysis through the services auxiliary module. This
@@ -786,6 +789,11 @@ def run(self):
                 "status": "error",
             })
         finally:
+            # In case the analysis manager crashes, the network cleanup
+            # should still be performed.
+            if not self.unrouted_network:
+                self.unroute_network()
+
             if self.cfg.cuckoo.process_results:
                 self.db.set_status(self.task.id, TASK_REPORTED)
             else:

From ef28f06557196961bae26d8c87d1a290bccdea65 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 15 May 2019 23:20:04 +0200
Subject: [PATCH 044/138] Code cleanup/style

---
 cuckoo/web/controllers/submission/routes.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cuckoo/web/controllers/submission/routes.py b/cuckoo/web/controllers/submission/routes.py
index c9c901242d..69322bea09 100644
--- a/cuckoo/web/controllers/submission/routes.py
+++ b/cuckoo/web/controllers/submission/routes.py
@@ -77,7 +77,10 @@ def resubmit(request, task_id):
                 task.target,
             ], submit_manager.translate_options_to(task.options))
         else:
-            file_path = binary_filepath(task_id) if not os.path.exists(task.target) else task.target
+            file_path = binary_filepath(task_id)
+            if not file_path or not os.path.exists(file_path):
+                file_path = task.target
+
             if not file_path or not os.path.exists(file_path):
                 return view_error(
                     request, "The file you're trying to resubmit "

From dbe2a0c42cbc4c330e7192960e461d80e5b23d5c Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 16 May 2019 13:55:45 +0200
Subject: [PATCH 045/138] Update latest monitor binary commit hash

---
 cuckoo/data/monitor/latest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cuckoo/data/monitor/latest b/cuckoo/data/monitor/latest
index 85b52a2642..8879054a99 100644
--- a/cuckoo/data/monitor/latest
+++ b/cuckoo/data/monitor/latest
@@ -1 +1 @@
-e071e63a66e831163a40abc45109fdf71fee829e
\ No newline at end of file
+2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b

From b47b8f8b0a2251394fe22788098dbbac35cbe3e5 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 16 May 2019 13:56:22 +0200
Subject: [PATCH 046/138] Update data directory hashes list

---
 cuckoo/private/cwd/hashes.txt | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/cuckoo/private/cwd/hashes.txt b/cuckoo/private/cwd/hashes.txt
index 0c5ee3432a..8f36b32968 100644
--- a/cuckoo/private/cwd/hashes.txt
+++ b/cuckoo/private/cwd/hashes.txt
@@ -292,7 +292,6 @@ c8492c74db400e6300194c1bafd3088a102bdc8e analyzer/windows/modules/auxiliary/huma
 e86627abeb5ecc0112438ad179e9d0487870785a analyzer/windows/modules/packages/ie.py
 
 # TBD
-b327de7ae427d9e39f43d11f15b4754fc99ed98b agent/agent.py
 cb3a77d8dd7edf46de54545ca7b0c5b201f85917 analyzer/windows/bin/execsc.exe
 93727e778dadc13d83cea61a9ea88bf6b5906686 analyzer/windows/modules/auxiliary/human.py
 24cbd18428df8dbc6b9ccd7896d066c492f6d381 analyzer/windows/modules/packages/ps1.py
@@ -300,3 +299,22 @@ cb3a77d8dd7edf46de54545ca7b0c5b201f85917 analyzer/windows/bin/execsc.exe
 d8fce614d615f6bdb3117e92bfa6e4ae2b48ea52 analyzer/windows/modules/packages/vbs.py
 f81e71f788149039f31c9b0b5e2891733e51b5d7 whitelist/ip.txt
 4575c7d09b316cad3a71059a27b78cbcb9c1f6b4 stuff/ttp_descriptions.json
+74c4c577a61f96571ac47e86c83fc0ece9d5f0ad agent/agent.py
+4d567f35bd79192d8f279f474816b9686e71896b analyzer/darwin/lib/api/screenshot.py
+633ab6bd08eb393ca630b59ce0ee5374862c6558 analyzer/darwin/lib/common/hashing.py
+d5e3410184765dba6af2895b0dd528558f51ef8d analyzer/darwin/modules/packages/zip.py
+6db386e3ebea277637397a42616b2232e2d6b771 analyzer/linux/lib/common/abstracts.py
+0c77d682544f214e3314350fe7767e40ecf5b174 analyzer/linux/lib/common/hashing.py
+11d0d726c6c17e8abce33b07df9cb498e021da1c analyzer/linux/modules/auxiliary/stap.py
+faf94dddbe6fc6a262c56735e7c437f326fffe59 analyzer/windows/analyzer.py
+a3847083dc4ee78e186359fb03071489ebfd5932 analyzer/windows/lib/api/process.py
+ba6b59b09ef3a157f6081cd1e0f12168cd20538d analyzer/windows/lib/common/abstracts.py
+105aac03a5a5ddf1eaf9262389a593d1aaebd0fe analyzer/windows/lib/common/hashing.py
+9b0df7467fa48ea6451475c93c27682ae492c33c analyzer/windows/lib/core/pipe.py
+8f531c7997a8e36e16f1511322adb718630476cd analyzer/windows/modules/auxiliary/disguise.py
+90bfc348b008e717b5a44cb4ae91b8260682bf84 analyzer/windows/modules/auxiliary/reboot.py
+44187be47fd7ddb3b7bed7ab596510168eec1294 analyzer/windows/modules/auxiliary/recentfiles.py
+348e720075790c862be1e67894ed4ab30ed4f7ce analyzer/windows/modules/auxiliary/zer0m0n.py
+e13518903a2fcaec3d0b140d3164c426f07ab647 analyzer/windows/modules/packages/ie.py
+09702bc15041a80f399f0c143cc3ec29196e4962 analyzer/windows/modules/packages/zip.py
+c278d582c26003467794ee4d0ab7ae337034c2ea monitor/latest

From 1d4c755cc152c75a504733e9f7f70f515d5456bb Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Thu, 12 Apr 2018 17:29:04 +0200
Subject: [PATCH 047/138] Improve performance of signature on_call

---
 cuckoo/common/abstracts.py |  9 ----
 cuckoo/core/plugins.py     | 77 +++++++++++++++++++++-------------
 tests/test_signatures.py   | 85 ++++++++++++++++++++++++++++++++++++--
 3 files changed, 130 insertions(+), 41 deletions(-)

diff --git a/cuckoo/common/abstracts.py b/cuckoo/common/abstracts.py
index f03be8f834..408cd3cac6 100644
--- a/cuckoo/common/abstracts.py
+++ b/cuckoo/common/abstracts.py
@@ -846,11 +846,6 @@ class Signature(object):
     filter_apinames = []
     filter_categories = []
 
-    # If no on_call() handler is present and this field has been set, then
-    # dispatch on a per-API basis to the accompanying API. That is, rather
-    # than calling the generic on_call(), call, e.g., on_call_CreateFile().
-    on_call_dispatch = False
-
     def __init__(self, caller):
         """
         @param caller: calling object. Stores results in caller.results
@@ -1293,10 +1288,6 @@ def on_call(self, call, process):
         @param call: logged API call.
         @param process: proc object.
         """
-        # Dispatch this call to a per-API specific handler.
-        if self.on_call_dispatch:
-            return getattr(self, "on_call_%s" % call["api"])(call, process)
-
         raise NotImplementedError
 
     def on_signature(self, signature):
diff --git a/cuckoo/core/plugins.py b/cuckoo/core/plugins.py
index e5880b0b17..36c16fd89c 100644
--- a/cuckoo/core/plugins.py
+++ b/cuckoo/core/plugins.py
@@ -12,7 +12,7 @@
 
 import cuckoo
 
-from cuckoo.common.abstracts import Configuration
+from cuckoo.common.abstracts import Configuration, Signature
 from cuckoo.common.config import config2
 from cuckoo.common.exceptions import (
     CuckooConfigurationError, CuckooProcessingError, CuckooReportError,
@@ -337,9 +337,42 @@ def __init__(self, results):
             if self.should_enable_signature(signature):
                 self.signatures.append(signature(self))
 
-        # Signatures to call per API name.
+        # Cache of signatures to call per API name.
         self.api_sigs = {}
 
+        # Prebuild a list of signatures that *may* be interested
+        self.call_always = set()
+        self.call_for_api = {}
+        self.call_for_cat = {}
+        for sig in self.signatures:
+            # Direct dispatch per API call
+            for n in dir(sig):
+                if n.startswith("on_call_"):
+                    self.call_for_api.setdefault(n[8:], set()).add(sig)
+            if not self._on_call_defined(sig):
+                # Not implemented...
+                continue
+            if not sig.filter_apinames and not sig.filter_categories:
+                self.call_always.add(sig)
+                continue
+            for api in sig.filter_apinames:
+                self.call_for_api.setdefault(api, set()).add(sig)
+            for cat in sig.filter_categories:
+                self.call_for_cat.setdefault(cat, set()).add(sig)
+
+    def _on_call_defined(self, sig):
+        """Test if on_call is defined.  This is not pretty, but it allows
+        on_call to be defined in `abstracts` for documentation purposes.
+
+        NB: In Python 3, we can just use `sig.on_call is Signature.on_call`."""
+        try:
+            sig.on_call(None, None)
+        except NotImplementedError:
+            return False
+        except:
+            pass
+        return True
+
     @classmethod
     def init_once(cls):
         cls.available_signatures = []
@@ -425,8 +458,6 @@ def call_signature(self, signature, handler, *args, **kwargs):
                 signature.matched = True
                 for sig in self.signatures:
                     self.call_signature(sig, sig.on_signature, signature)
-        except NotImplementedError:
-            return False
         except:
             task_id = self.results.get("info", {}).get("id")
             log.exception(
@@ -436,34 +467,24 @@ def call_signature(self, signature, handler, *args, **kwargs):
             )
         return True
 
-    def init_api_sigs(self, apiname, category):
-        """Initialize a list of signatures for which we should trigger its
-        on_call method for this particular API name and category."""
-        self.api_sigs[apiname] = []
-
-        for sig in self.signatures:
-            if sig.filter_apinames and apiname not in sig.filter_apinames:
-                continue
-
-            if sig.filter_categories and category not in sig.filter_categories:
-                continue
-
-            self.api_sigs[apiname].append(sig)
-
     def yield_calls(self, proc):
         """Yield calls of interest to each interested signature."""
         for idx, call in enumerate(proc.get("calls", [])):
-
-            # Initialize a list of signatures to call for this API call.
-            if call["api"] not in self.api_sigs:
-                self.init_api_sigs(call["api"], call.get("category"))
-
-            # See the following SO answer on why we're using reversed() here.
-            # http://stackoverflow.com/a/10665800
-            for sig in reversed(self.api_sigs[call["api"]]):
+            api = call.get("api")
+            sigs = self.api_sigs.get(api)
+            if sigs is None:
+                # Build interested signatures
+                cat = call.get("category")
+                sigs = self.call_always.union(
+                    self.call_for_api.get(api, set()),
+                    self.call_for_cat.get(cat, set())
+                )
+                self.api_sigs[api] = sigs
+            name = "on_call_" + api
+            for sig in sigs:
                 sig.cid, sig.call = idx, call
-                if self.call_signature(sig, sig.on_call, call, proc) is False:
-                    self.api_sigs[call["api"]].remove(sig)
+                func = getattr(sig, name, sig.on_call)
+                self.call_signature(sig, func, call, proc)
 
     def process_yara_matches(self):
         """Yield any Yara matches to each signature."""
diff --git a/tests/test_signatures.py b/tests/test_signatures.py
index 276facab19..e574153d03 100644
--- a/tests/test_signatures.py
+++ b/tests/test_signatures.py
@@ -159,7 +159,7 @@ class sig_windows_platform(object):
     assert rs.should_enable_signature(sig_windows_platform())
 
 def test_signature_order():
-    class sig(object):
+    class sig(Signature):
         enabled = True
         minimum = "2.0.0"
         maximum = None
@@ -193,7 +193,7 @@ class sig3(sig):
     assert isinstance(rs.signatures[2], sig1)
 
 class test_call_signature():
-    class sig(object):
+    class sig(Signature):
         enabled = True
         name = "sig"
         minimum = "2.0.0"
@@ -354,7 +354,7 @@ def test_on_yara():
         "vmware1": [(0, 0)],
     }
 
-    class sig1(object):
+    class sig1(Signature):
         name = "sig1"
 
         @property
@@ -381,7 +381,7 @@ def on_extract(self, match):
 
     rs = RunSignatures(results)
 
-    rs.signatures = sig1(),
+    rs.signatures = sig1(rs),
     rs.run()
 
     assert sig1.on_yara.call_count == 3
@@ -485,3 +485,80 @@ def test_check_command_line(self):
         })
         r.check_command_line("foo") == "foo"
         r.check_command_line("ar$", regex=True) == "bar"
+
+def classes(objects):
+    return [s.__class__ for s in objects]
+
+def lazy_compare(unsorted, sorted_lst):
+    items = sorted(unsorted, key=lambda v: repr(v))
+    assert items == sorted_lst
+
+class TestRunSignatures(object):
+    _fake_results = {
+        "behavior": {
+            "processes": [{
+                "process_path": "C:\\Temp\\malware.exe",
+                "pid": 123,
+                "calls": [
+                    {"category": "system", "api": "LdrLoadDll"},
+                    {"category": "registry", "api": "GetSystemTimeAsFileTime"},
+                    {"category": "xundefined", "api": "XUndefined"}
+                ]
+            }]
+        }
+    }
+
+    def _runner(self, signatures):
+        class TestRunSignatures(RunSignatures):
+            available_signatures = signatures
+        return TestRunSignatures(self._fake_results)
+
+    def test_yield_calls(self):
+        pass
+
+    def test_dispatch_building(self):
+        class SignatureAll(Signature):
+            def on_call(self, call, proc):
+                pass
+        class SignatureDispatch(Signature):
+            def on_call_LdrLoadDll(self, call, proc):
+                pass
+        class SignatureAPI(Signature):
+            filter_apinames = ["GetSystemTimeAsFileTime"]
+            def on_call(self, call, proc):
+                pass
+        class SignatureCategory(Signature):
+            filter_categories = ["system"]
+            def on_call(self, call, proc):
+                pass
+        class SignatureDummy(Signature):
+            pass
+
+        r = self._runner([SignatureAPI, SignatureAll, SignatureCategory,
+            SignatureDispatch, SignatureDummy])
+        assert len(r.signatures) == 5
+        lazy_compare(
+            r.call_for_api.keys(),
+            ["GetSystemTimeAsFileTime", "LdrLoadDll"]
+        )
+        assert classes(r.call_for_api["LdrLoadDll"]) == [SignatureDispatch]
+        assert classes(r.call_for_cat["system"]) == [SignatureCategory]
+        assert SignatureDummy not in classes(r.call_always)
+        for v in r.call_for_api.values():
+            assert SignatureDummy not in classes(v)
+        for v in r.call_for_cat.values():
+            assert SignatureDummy not in classes(v)
+        assert not r.api_sigs
+        r.run()
+        lazy_compare(
+            r.api_sigs.keys(),
+            ["GetSystemTimeAsFileTime", "LdrLoadDll", "XUndefined"]
+        )
+        lazy_compare(
+            classes(r.api_sigs["GetSystemTimeAsFileTime"]),
+            [SignatureAPI, SignatureAll]
+        )
+        lazy_compare(
+            classes(r.api_sigs["LdrLoadDll"]),
+            [SignatureAll, SignatureCategory, SignatureDispatch]
+        )

From b5cbf2b51f8b15e6392591570a6537a24c9c68c8 Mon Sep 17 00:00:00 2001
From: Jurriaan Bremer <jbr@cuckoo.sh>
Date: Mon, 19 Feb 2018 23:58:18 +0100
Subject: [PATCH 048/138] work in progress on the new gevent-based resultserver

The "legacy" ResultServer consumes huge amounts of CPU & RAM resources
due to the way it operates, namely by creating a thread for each
incoming connection. Needless to say, in case of analyzing, e.g.,
Ransomware, with potentially thousands of dropped files this results
in thousands of connections and as such, Python threads.
The switch to Gevent will provide a huge performance boost for Cuckoo.
---
 cuckoo/common/abstracts.py  |   9 ++
 cuckoo/core/resultserver.py | 178 ++++++++++++++++++++++++++++++++++--
 2 files changed, 179 insertions(+), 8 deletions(-)

diff --git a/cuckoo/common/abstracts.py b/cuckoo/common/abstracts.py
index 408cd3cac6..64b723acfb 100644
--- a/cuckoo/common/abstracts.py
+++ b/cuckoo/common/abstracts.py
@@ -1432,13 +1432,22 @@ class ProtocolHandler(object):
     def __init__(self, handler, version=None):
         self.handler = handler
         self.version = version
+        self.sock = None
+        self.task_id = None
+        self.running = True
 
     def init(self):
         pass
 
+    def handle(self):
+        pass
+
     def close(self):
         pass
 
+    def read(self, length):
+        return self.handler.read(self, length)
+
 class Extractor(object):
     """One piece in a series of recursive extractors & unpackers."""
     yara_rules = []
diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index a76533a205..bc7aafaef7 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -4,13 +4,15 @@
 # See the file 'docs/LICENSE' for copying permission.
 
 import errno
+import datetime
+import gevent.server
 import json
+import logging
 import os
-import socket
 import select
-import logging
-import datetime
+import socket
 import SocketServer
+import struct
 import threading
 
 from cuckoo.common.abstracts import ProtocolHandler
@@ -31,7 +33,7 @@
 class Disconnect(Exception):
     pass
 
-class ResultServer(SocketServer.ThreadingTCPServer, object):
+class OldResultServer(SocketServer.ThreadingTCPServer, object):
     """Result server. Singleton!
 
     This class handles results coming back from the analysis machines.
@@ -71,8 +73,8 @@ def __init__(self, *args, **kwargs):
                         "usually happens when you start Cuckoo without "
                         "bringing up the virtual interface associated with "
                         "the ResultServer IP address. Please refer to "
-                        "https://cuckoo.sh/docs/faq/#troubles-problem"
-                        " for more information." % (self.ip, self.port, e)
+                        "https://cuckoo.sh/docs/faq/#troubles-problem "
+                        "for more information." % (self.ip, self.port, e)
                     )
                 else:
                     raise CuckooCriticalError(
@@ -90,7 +92,7 @@ def __init__(self, *args, **kwargs):
 
     def serve_forever(self, poll_interval=0.5):
         try:
-            super(ResultServer, self).serve_forever(poll_interval)
+            super(OldResultServer, self).serve_forever(poll_interval)
         except AttributeError as e:
             if "NoneType" not in e.message or "select" not in e.message:
                 raise
@@ -134,7 +136,7 @@ def build_storage_path(self, ip):
         if not task or not machine:
             return
 
-        return cwd("storage", "analyses", "%s" % task.id)
+        return cwd(analysis=task.id)
 
 class ResultHandler(SocketServer.BaseRequestHandler):
     """Result handler.
@@ -458,3 +460,163 @@ def _open(self):
         )
 
         return fd
+
+class BsonStore(ProtocolHandler):
+    def init(self):
+        # We cheat a little bit through the "version" variable, but that's
+        # acceptable and backwards compatible (for now). Backwards compatible
+        # in the sense that newer Cuckoo Monitor binaries work with older
+        # versions of Cuckoo, the other way around doesn't apply here.
+        self.f = open(cwd(
+            os.path.join("logs", "%d.bson" % self.version),
+            analysis=self.task_id
+        ), "wb")
+
+    def handle(self):
+        while self.running:
+            lenbuf = self.read(4)
+            if len(lenbuf) != 4:
+                break
+
+            length = struct.unpack("I", lenbuf)[0]
+            buf = self.read(length)
+            if len(buf) != length:
+                break
+
+            # TODO Handle out of disk space.
+            self.f.write(lenbuf + buf)
+
+    def close(self):
+        self.f.close()
+
+class NewResultServerWorker(gevent.server.StreamServer):
+    """The new ResultServer, providing a huge performance boost as well as
+    implementing a new dropped file storage format avoiding small fd limits.
+
+    The old ResultServer would start a new thread per socket, greatly impacting
+    the overall performance of Cuckoo Sandbox. The new ResultServer uses
+    so-called Greenlets, low overhead green-threads by Gevent, imposing much
+    less kernel overhead.
+
+    Furthermore, instead of writing each dropped file to its own location (in
+    $CWD/storage/analyses/<task_id>/files/<partial_hash>_filename.ext) it's
+    capable of storing all dropped files in a streamable container format. This
+    is one of various steps to start being able to use less fd's in Cuckoo.
+    """
+    commands = {
+        "BSON": BsonStore,
+        "FILE": FileUpload,
+        "LOG": LogHandler,
+    }
+
+    def init(self):
+        self.tasks = {}
+        self.handlers = {}
+        self.buf = bytearray(BUFSIZE)
+
+    def add_task(self, task_id, ipaddr):
+        self.tasks[ipaddr] = task_id
+        self.handlers[ipaddr] = self.handlers.get(ipaddr, {})
+
+    def del_task(self, task_id, ipaddr):
+        self.tasks.pop(ipaddr)
+
+        for handler in self.handlers[ipaddr].values():
+            handler.running = False
+
+        self.handlers.pop(ipaddr, None)
+
+    def read_newline(self, sock):
+        buf = ""
+        while "\n" not in buf:
+            buf += sock.recv(1)
+        return buf
+
+    def read(self, protocol, length):
+        ret, offset = [], 0
+        while protocol.running and offset != length:
+            buf = protocol.sock.recv(length - offset)
+            if not buf:
+                # TODO Do we need to do something here?
+                continue
+
+            ret.append(buf)
+            offset += len(buf)
+        return "".join(ret)
+
+    def handle(self, sock, (ipaddr, port)):
+        if ipaddr not in self.tasks:
+            return
+
+        task_log_start(self.tasks[ipaddr])
+
+        protocol = self.read_newline(sock).strip()
+
+        if " " in protocol:
+            command, version = protocol.split()
+            version = int(version)
+        else:
+            command, version = protocol, None
+
+        if command not in self.commands:
+            log.warning(
+                "Unknown netlog protocol requested ('%s'), "
+                "terminating connection.", command
+            )
+            task_log_stop(self.tasks[ipaddr])
+            return
+
+        protocol = self.commands[command](self, version)
+        protocol.sock = sock
+        protocol.task_id = self.tasks[ipaddr]
+
+        # Registering the protocol allows for the handler getting its "running"
+        # field set to False (among other use-cases in the future).
+        self.handlers[ipaddr][port] = protocol
+
+        protocol.init()
+        protocol.handle()
+        protocol.close()
+
+        task_log_stop(self.tasks[ipaddr])
+
+class NewResultServer(object):
+    """Wrapper around the NewResultServerWorker."""
+    __metaclass__ = Singleton
+
+    def __init__(self):
+        self.instance = None
+        self.thread = threading.Thread(target=self.do_run)
+        self.thread.daemon = True
+        self.thread.start()
+
+    def do_run(self):
+        self.instance = NewResultServerWorker((
+            config("cuckoo:resultserver:ip"),
+            config("cuckoo:resultserver:port")
+        ))
+        self.instance.init()
+
+        try:
+            self.instance.serve_forever()
+        except socket.error as e:
+            if e.errno == errno.EACCES:
+                pass
+            if e.errno == errno.EADDRINUSE:
+                pass
+            if e.errno == errno.EADDRNOTAVAIL:
+                pass
+            raise
+
+    def add_task(self, task, machine):
+        self.instance.add_task(task.id, machine.ip)
+
+    def del_task(self, task, machine):
+        self.instance.del_task(task.id, machine.ip)
+
+    @property
+    def port(self):
+        return self.instance.server_port
+
+# TODO Should this be configurable?
+ResultServer = NewResultServer

From fda1a773c595b32e84b20d0d842f95ff5a96b362 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Fri, 2 Mar 2018 13:03:41 +0100
Subject: [PATCH 049/138] Functional gevent-based ResultServer

However, this implementation still needs to properly implement task
management, some timeout functionality, and path security features. Also
need to run a benchmark to determine a proper pool size.
---
 cuckoo/core/resultserver.py | 537 ++++++++++--------------------------
 tests/test_resultserver.py  | 109 ++++++--
 2 files changed, 226 insertions(+), 420 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index bc7aafaef7..693569429c 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -3,15 +3,23 @@
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
+# TODO:
+# * NTFS ADS / windows fn protection
+# * gevent doesn't bind until serve_forever()
+# * replace threading with gevent locks (needed?)
+# * lock all filesystem operations?
+# * lock in ProtocolHandler for .running?
+# * fd leaking / no timeout
+# * consider creating a folder whitelist
+from __future__ import print_function
+
 import errno
 import datetime
 import gevent.server
+import gevent.pool
 import json
 import logging
 import os
-import select
-import socket
-import SocketServer
 import struct
 import threading
 
@@ -21,325 +29,66 @@
 from cuckoo.common.exceptions import CuckooCriticalError
 from cuckoo.common.exceptions import CuckooResultError
 from cuckoo.common.files import Folders
-from cuckoo.common.netlog import BsonParser
 from cuckoo.common.utils import Singleton
 from cuckoo.core.log import task_log_start, task_log_stop
 from cuckoo.misc import cwd
 
 log = logging.getLogger(__name__)
 
-BUFSIZE = 1024 * 1024
-
-class Disconnect(Exception):
-    pass
-
-class OldResultServer(SocketServer.ThreadingTCPServer, object):
-    """Result server. Singleton!
-
-    This class handles results coming back from the analysis machines.
-    """
+# Maximum line length to read for netlog messages, to avoid memory exhaustion
+MAX_NETLOG_LINE = 4 * 1024
 
-    __metaclass__ = Singleton
+BUFSIZE = 16 * 1024
 
-    allow_reuse_address = True
-    daemon_threads = True
+NETLOG_RECV_TIMEOUT = 60
 
-    def __init__(self, *args, **kwargs):
-        self.analysistasks = {}
-        self.analysishandlers = {}
 
-        self.ip = config("cuckoo:resultserver:ip")
-        self.port = config("cuckoo:resultserver:port")
-        while True:
-            try:
-                server_addr = self.ip, self.port
-                SocketServer.ThreadingTCPServer.__init__(
-                    self, server_addr, ResultHandler, *args, **kwargs
-                )
-            except Exception as e:
-                if e.errno == errno.EADDRINUSE:
-                    if config("cuckoo:resultserver:force_port"):
-                        raise CuckooCriticalError(
-                            "Cannot bind ResultServer on port %d, "
-                            "bailing." % self.port
-                        )
-                    else:
-                        log.warning("Cannot bind ResultServer on port %s, "
-                                    "trying another port.", self.port)
-                        self.port += 1
-                elif e.errno == errno.EADDRNOTAVAIL:
-                    raise CuckooCriticalError(
-                        "Unable to bind ResultServer on %s:%s %s. This "
-                        "usually happens when you start Cuckoo without "
-                        "bringing up the virtual interface associated with "
-                        "the ResultServer IP address. Please refer to "
-                        "https://cuckoo.sh/docs/faq/#troubles-problem "
-                        "for more information." % (self.ip, self.port, e)
-                    )
-                else:
-                    raise CuckooCriticalError(
-                        "Unable to bind ResultServer on %s:%s: %s" %
-                        (self.ip, self.port, e)
-                    )
-            else:
-                log.debug(
-                    "ResultServer running on %s:%s.", self.ip, self.port
-                )
-                self.servethread = threading.Thread(target=self.serve_forever)
-                self.servethread.setDaemon(True)
-                self.servethread.start()
-                break
-
-    def serve_forever(self, poll_interval=0.5):
-        try:
-            super(OldResultServer, self).serve_forever(poll_interval)
-        except AttributeError as e:
-            if "NoneType" not in e.message or "select" not in e.message:
-                raise
-
-    def add_task(self, task, machine):
-        """Register a task/machine with the ResultServer."""
-        self.analysistasks[machine.ip] = task, machine
-        self.analysishandlers[task.id] = []
-
-    def del_task(self, task, machine):
-        """Delete ResultServer state and wait for pending RequestHandlers."""
-        x = self.analysistasks.pop(machine.ip, None)
-        if not x:
-            log.warning("ResultServer did not have %s in its task info.",
-                        machine.ip)
-        handlers = self.analysishandlers.pop(task.id, None)
-        for h in handlers:
-            h.end_request.set()
-            h.done_event.wait()
-
-    def register_handler(self, handler):
-        """Register a RequestHandler so that we can later wait for it."""
-        task, machine = self.get_ctx_for_ip(handler.client_address[0])
-        if not task or not machine:
-            return False
-
-        self.analysishandlers[task.id].append(handler)
-
-    def get_ctx_for_ip(self, ip):
-        """Return state for this IP's task."""
-        x = self.analysistasks.get(ip)
-        if not x:
-            log.debug("ResultServer unable to map ip to context: %s.", ip)
-            return None, None
-
-        return x
-
-    def build_storage_path(self, ip):
-        """Initialize analysis storage folder."""
-        task, machine = self.get_ctx_for_ip(ip)
-        if not task or not machine:
-            return
-
-        return cwd(analysis=task.id)
-
-class ResultHandler(SocketServer.BaseRequestHandler):
-    """Result handler.
-
-    This handler speaks our analysis log network protocol.
-    """
-
-    def setup(self):
-        self.rawlogfd = None
-        self.protocol = None
-        self.startbuf = ""
-        self.end_request = threading.Event()
-        self.done_event = threading.Event()
-        self.server.register_handler(self)
-
-        if hasattr(select, "poll"):
-            self.poll = select.poll()
-            self.poll.register(self.request, select.POLLIN)
-        else:
-            self.poll = None
-
-    def finish(self):
-        self.done_event.set()
-
-        if self.protocol:
-            self.protocol.close()
-        if self.rawlogfd:
-            self.rawlogfd.close()
-
-    def wait_sock_or_end(self):
-        while True:
-            if self.end_request.isSet():
-                return False
-
-            if self.poll:
-                if self.poll.poll(1000):
-                    return True
-            else:
-                rs, _, _ = select.select([self.request], [], [], 1)
-                if rs:
-                    return True
-
-    def seek(self, pos):
-        pass
-
-    def read(self, length):
-        buf = ""
-        while len(buf) < length:
-            if not self.wait_sock_or_end():
-                raise Disconnect()
-            tmp = self.request.recv(length-len(buf))
-            if not tmp:
-                raise Disconnect()
-            buf += tmp
-
-        if isinstance(self.protocol, BsonParser):
-            if self.rawlogfd:
-                self.rawlogfd.write(buf)
-            else:
-                self.startbuf += buf
-
-                if len(self.startbuf) > 0x10000:
-                    raise CuckooResultError(
-                        "Somebody is knowingly overflowing the startbuf "
-                        "buffer, possibly to use excessive amounts of memory."
-                    )
+class HandlerContext:
+    """Holds context for protocol handlers"""
+    def __init__(self, storagepath, sock):
+        # The part where artifacts will be stored
+        self.storagepath = storagepath
+        self.sock = sock.makefile(mode='rb')
 
+    def read(self, size):
+        buf = self.sock.read(size)
+        if not buf:
+            raise EOFError
         return buf
 
-    def read_any(self):
-        if not self.wait_sock_or_end():
-            raise Disconnect()
-        tmp = self.request.recv(BUFSIZE)
-        if not tmp:
-            raise Disconnect()
-        return tmp
-
-    def read_newline(self, strip=False):
-        buf = ""
-        while "\n" not in buf:
-            buf += self.read(1)
-
-        if strip:
-            buf = buf.strip()
+    def read_newline(self):
+        line = self.sock.readline(MAX_NETLOG_LINE)
+        if not line:
+            raise EOFError
+        elif not line.endswith('\n'):
+            raise CuckooOperationalError('Received overly long line')
+        return line[:-1]
 
+    def read_any(self):
+        buf = self.sock.read(BUFSIZE)
+        if not buf:
+            raise EOFError
         return buf
 
-    def negotiate_protocol(self):
-        protocol = self.read_newline(strip=True)
-
-        # Command with version number.
-        if " " in protocol:
-            command, version = protocol.split()
-            version = int(version)
-        else:
-            command, version = protocol, None
-
-        if command == "BSON":
-            self.protocol = BsonParser(self, version)
-        elif command == "FILE":
-            self.protocol = FileUpload(self, version)
-        elif command == "LOG":
-            self.protocol = LogHandler(self, version)
-        else:
-            raise CuckooOperationalError(
-                "Netlog failure, unknown protocol requested."
-            )
-
-        self.protocol.init()
-
-    def handle(self):
-        ip, port = self.client_address
-
-        self.storagepath = self.server.build_storage_path(ip)
-        if not self.storagepath:
-            return
-
-        task, _ = self.server.get_ctx_for_ip(ip)
-        task_log_start(task.id)
-
-        # Create all missing folders for this analysis.
-        self.create_folders()
-
-        try:
-            # Initialize the protocol handler class for this connection.
-            self.negotiate_protocol()
-
-            for event in self.protocol:
-                if isinstance(self.protocol, BsonParser) and event["type"] == "process":
-                    self.open_process_log(event)
-        except CuckooResultError as e:
-            log.warning(
-                "ResultServer connection stopping because of "
-                "CuckooResultError: %s.", e
-            )
-        except (Disconnect, socket.error):
-            pass
-        except:
-            log.exception("FIXME - exception in resultserver connection %s",
-                          self.client_address)
-
-        task_log_stop(task.id)
-
-    def open_process_log(self, event):
-        pid = event["pid"]
-        ppid = event["ppid"]
-        procname = event["process_name"]
-
-        if self.rawlogfd:
-            log.debug(
-                "ResultServer got a new process message but already "
-                "has pid %d ppid %s procname %s.", pid, ppid, procname
-            )
-            raise CuckooResultError(
-                "ResultServer connection state inconsistent."
-            )
-
-        if not isinstance(pid, (int, long)):
-            raise CuckooResultError(
-                "An invalid process identifier has been provided, this "
-                "could be a potential security hazard."
-            )
-
-        # Only report this process when we're tracking it.
-        if event["track"]:
-            log.debug(
-                "New process (pid=%s, ppid=%s, name=%s)",
-                pid, ppid, procname.encode("utf8")
-            )
-
-        filepath = os.path.join(self.storagepath, "logs", "%s.bson" % pid)
-        self.rawlogfd = open(filepath, "wb")
-        self.rawlogfd.write(self.startbuf)
-
-    def create_folders(self):
-        folders = "shots", "files", "logs", "buffer", "extracted"
-
-        try:
-            Folders.create(self.storagepath, folders)
-        except CuckooOperationalError as e:
-            log.error("Issue creating analyses folders: %s", e)
-            return False
 
 class FileUpload(ProtocolHandler):
     RESTRICTED_DIRECTORIES = "reports/",
-    lock = threading.Lock()
 
     def init(self):
         self.upload_max_size = config("cuckoo:resultserver:upload_max_size")
         self.storagepath = self.handler.storagepath
         self.fd = None
-
         self.filelog = os.path.join(self.handler.storagepath, "files.json")
 
-    def __iter__(self):
+    def handle(self):
         # Read until newline for file path, e.g.,
         # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin
 
-        dump_path = self.handler.read_newline(strip=True).replace("\\", "/")
+        dump_path = self.handler.read_newline().replace("\\", "/")
 
         if self.version >= 2:
-            filepath = self.handler.read_newline(strip=True)
-            pids = map(int, self.handler.read_newline(strip=True).split())
+            filepath = self.handler.read_newline()
+            pids = map(int, self.handler.read_newline().split())
         else:
             filepath, pids = None, []
 
@@ -396,8 +145,6 @@ def __iter__(self):
             except:
                 break
 
-        self.lock.acquire()
-
         with open(self.filelog, "a+b") as f:
             f.write("%s\n" % json.dumps({
                 "path": dump_path,
@@ -405,40 +152,35 @@ def __iter__(self):
                 "pids": pids,
             }))
 
-        self.lock.release()
-
         log.debug("Uploaded file length: %s", self.fd.tell())
-        return
-        yield
+        self.fd.close()
 
     def close(self):
         if self.fd:
             self.fd.close()
 
 class LogHandler(ProtocolHandler):
+    # TODO: not protected against opening multiple times
     def init(self):
         self.logpath = os.path.join(self.handler.storagepath, "analysis.log")
         self.fd = self._open()
         log.debug("LogHandler for live analysis.log initialized.")
 
-    def __iter__(self):
+    def handle(self):
         if not self.fd:
             return
 
         while True:
             try:
                 buf = self.handler.read_any()
-            except Disconnect:
+            except EOFError:
                 break
 
             if not buf:
                 break
 
             self.fd.write(buf)
-            self.fd.flush()
-
-        return
-        yield
+            self.fd.flush()  # Expensive...
 
     def close(self):
         if self.fd:
@@ -446,6 +188,7 @@ def close(self):
 
     def _open(self):
         if not os.path.exists(self.logpath):
+            # (Race condition)
             return open(self.logpath, "wb")
 
         log.debug("Log analysis.log already existing, appending data.")
@@ -455,10 +198,10 @@ def _open(self):
         #  use the same format as the default logger, in case anyone wants to parse this
         #  2015-02-23 12:05:05,092 [lib.api.process] DEBUG: Using QueueUserAPC injection.
         now = datetime.datetime.now()
-        print >>fd, "\n%s,%03.0f [lib.core.resultserver] WARNING: This log file was re-opened, log entries will be appended." % (
-            now.strftime("%Y-%m-%d %H:%M:%S"), now.microsecond / 1000.0
-        )
-
+        print("\n", now.strftime("%Y-%m-%d %H:%M:%S",),
+              now.microsecond / 1000.0,
+              " [lib.core.resultserver] WARNING: This log file was re-opened, log entries will be appended.",
+              sep='', file=fd)
         return fd
 
 class BsonStore(ProtocolHandler):
@@ -467,20 +210,24 @@ def init(self):
         # acceptable and backwards compatible (for now). Backwards compatible
         # in the sense that newer Cuckoo Monitor binaries work with older
         # versions of Cuckoo, the other way around doesn't apply here.
-        self.f = open(cwd(
-            os.path.join("logs", "%d.bson" % self.version),
-            analysis=self.task_id
-        ), "wb")
+        self.f = open(os.path.join(self.handler.storagepath,
+                                   "logs", "%d.bson" % self.version), "wb")
 
     def handle(self):
         while self.running:
-            lenbuf = self.read(4)
-            if len(lenbuf) != 4:
+            # TODO: just loop read_any
+            try:
+                lenbuf = self.handler.read(4)
+                if len(lenbuf) != 4:
+                    log.warning("BsonStore short read")
+                    break
+            except EOFError:
                 break
 
             length = struct.unpack("I", lenbuf)[0]
-            buf = self.read(length)
+            buf = self.handler.read(length)
             if len(buf) != length:
+                log.warning("BsonStore short read")
                 break
 
             # TODO Handle out of disk space.
@@ -489,7 +236,8 @@ def handle(self):
     def close(self):
         self.f.close()
 
-class NewResultServerWorker(gevent.server.StreamServer):
+
+class GeventResultServerWorker(gevent.server.StreamServer):
     """The new ResultServer, providing a huge performance boost as well as
     implementing a new dropped file storage format avoiding small fd limits.
 
@@ -509,114 +257,123 @@ class NewResultServerWorker(gevent.server.StreamServer):
         "LOG": LogHandler,
     }
 
-    def init(self):
+    def __init__(self, *args, **kwargs):
+        super(GeventResultServerWorker, self).__init__(*args, **kwargs)
         self.tasks = {}
         self.handlers = {}
-        self.buf = bytearray(BUFSIZE)
+
+    def do_run(self):
+        self.serve_forever()
 
     def add_task(self, task_id, ipaddr):
         self.tasks[ipaddr] = task_id
         self.handlers[ipaddr] = self.handlers.get(ipaddr, {})
 
     def del_task(self, task_id, ipaddr):
-        self.tasks.pop(ipaddr)
+        """Delete ResultServer state and wait for pending RequestHandlers."""
+        if self.tasks.pop(ipaddr, None) is None:
+            log.warning("ResultServer did not have a task with ID %s",
+                        task_id)
 
-        for handler in self.handlers[ipaddr].values():
+        handlers = self.handlers.pop(ipaddr, {})
+        for handler in handlers.values():
             handler.running = False
-
-        self.handlers.pop(ipaddr, None)
-
-    def read_newline(self, sock):
-        buf = ""
-        while "\n" not in buf:
-            buf += sock.recv(1)
-        return buf
-
-    def read(self, protocol, length):
-        ret, offset = [], 0
-        while protocol.running and offset != length:
-            buf = protocol.sock.recv(length - offset)
-            if not buf:
-                # TODO Do we need to do something here?
-                continue
-
-            ret.append(buf)
-            offset += len(buf)
-        return "".join(ret)
-
-    def handle(self, sock, (ipaddr, port)):
-        if ipaddr not in self.tasks:
+            # <WAIT>
+            #h.end_request.set()
+            #h.done_event.wait()
+
+    def handle(self, sock, addr):
+        ipaddr, port = addr
+        task_id = self.tasks.get(ipaddr)
+        if not task_id:
+            log.warning("ResultServer did not have a task for IP %s", ipaddr)
             return
 
-        task_log_start(self.tasks[ipaddr])
+        storagepath = cwd(analysis=task_id)
+        ctx = HandlerContext(storagepath, sock)
+        task_log_start(task_id)
+        try:
+            protocol = self.negotiate_protocol(ctx)
 
-        protocol = self.read_newline(sock).strip()
+            # Registering the protocol allows for the handler getting its "running"
+            # field set to False (among other use-cases in the future).
+            self.handlers[ipaddr][port] = protocol
 
-        if " " in protocol:
-            command, version = protocol.split()
+            protocol.task_id = task_id
+            try:
+                protocol.init()
+                protocol.handle()
+            finally:
+                protocol.close()
+
+        finally:
+            task_log_stop(task_id)
+
+    def negotiate_protocol(self, ctx):
+        header = ctx.read_newline()
+        if " " in header:
+            command, version = header.split()
             version = int(version)
         else:
-            command, version = protocol, None
-
+            command, version = header, None
         if command not in self.commands:
             log.warning(
-                "Unknown netlog protocol requested ('%s'), "
+                "Unknown netlog protocol requested (%r), "
                 "terminating connection.", command
             )
-            task_log_stop(self.tasks[ipaddr])
             return
+        return self.commands[command](ctx, version)
 
-        protocol = self.commands[command](self, version)
-        protocol.sock = sock
-        protocol.task_id = self.tasks[ipaddr]
-
-        # Registering the protocol allows for the handler getting its "running"
-        # field set to False (among other use-cases in the future).
-        self.handlers[ipaddr][port] = protocol
 
-        protocol.init()
-        protocol.handle()
-        protocol.close()
-
-        task_log_stop(self.tasks[ipaddr])
-
-class NewResultServer(object):
-    """Wrapper around the NewResultServerWorker."""
+class ResultServer(object):
+    """Manager for the ResultServer worker and task state."""
     __metaclass__ = Singleton
 
     def __init__(self):
-        self.instance = None
-        self.thread = threading.Thread(target=self.do_run)
+        self.thread = threading.Thread(target=self.create_bg_server)
         self.thread.daemon = True
         self.thread.start()
 
-    def do_run(self):
-        self.instance = NewResultServerWorker((
-            config("cuckoo:resultserver:ip"),
-            config("cuckoo:resultserver:port")
-        ))
-        self.instance.init()
-
-        try:
-            self.instance.serve_forever()
-        except socket.error as e:
-            if e.errno == errno.EACCES:
-                pass
-            if e.errno == errno.EADDRINUSE:
-                pass
-            if e.errno == errno.EADDRNOTAVAIL:
-                pass
-            raise
-
     def add_task(self, task, machine):
+        """Register a task/machine with the ResultServer."""
         self.instance.add_task(task.id, machine.ip)
 
     def del_task(self, task, machine):
+        """Delete ResultServer state and wait for pending RequestHandlers."""
         self.instance.del_task(task.id, machine.ip)
 
-    @property
-    def port(self):
-        return self.instance.server_port
+    def create_bg_server(self):
+        ip = config("cuckoo:resultserver:ip")
+        port = self.port = config("cuckoo:resultserver:port")
+        pool_size = config('cuckoo:resultserver:poolsize')
+        if pool_size:
+            pool_size = int(pool_size)
+        else:
+            pool_size = 32
 
-# TODO Should this be configurable?
-ResultServer = NewResultServer
+        pool = gevent.pool.Pool(pool_size)
+        try:
+            # TODO: support binding to port 0 for random port
+            self.instance = GeventResultServerWorker((ip, port),
+                                                     spawn=pool)
+        except OSError as e:
+            if e.errno == errno.EADDRINUSE:
+                    raise CuckooCriticalError(
+                        "Cannot bind ResultServer on port %d "
+                        "because it was in use, bailing." % port
+                        )
+            elif e.errno == errno.EADDRNOTAVAIL:
+                raise CuckooCriticalError(
+                    "Unable to bind ResultServer on %s:%s %s. This "
+                    "usually happens when you start Cuckoo without "
+                    "bringing up the virtual interface associated with "
+                    "the ResultServer IP address. Please refer to "
+                    "https://cuckoo.sh/docs/faq/#troubles-problem "
+                    "for more information." % (ip, port, e)
+                )
+            else:
+                raise CuckooCriticalError(
+                    "Unable to bind ResultServer on %s:%s: %s" %
+                    (ip, port, e)
+                )
+        self.instance.do_run()
diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py
index b79ced476f..3b44b3fcc7 100644
--- a/tests/test_resultserver.py
+++ b/tests/test_resultserver.py
@@ -1,19 +1,30 @@
-# Copyright (C) 2017 Cuckoo Foundation.
+# Copyright (C) 2017-2018 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
+from __future__ import print_function
+
+# Testing TODO:
+# - Socket timeout, cleanup
+# - Task cleanup
+# - Invalid path tests
+# - Double LOG command
+
 import logging
 import mock
 import pytest
 import tempfile
+import shutil
 
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.core.log import task_log_start, task_log_stop
-from cuckoo.core.resultserver import ResultHandler, FileUpload
+from cuckoo.core.resultserver import FileUpload, LogHandler, BsonStore
 from cuckoo.core.startup import init_logging
 from cuckoo.main import cuckoo_create
 from cuckoo.misc import mkdir, set_cwd, cwd
 
+# TODO: restore this test
+'''
 @mock.patch("cuckoo.core.resultserver.select")
 def test_open_process_log_unicode(p):
     set_cwd(tempfile.mkdtemp())
@@ -38,48 +49,86 @@ def handle(self):
         })
     finally:
         task_log_stop(1)
+'''
+
+@pytest.fixture(scope='module')
+def cuckoo_cwd():
+    """Create a temporary Cuckoo working directory"""
+    path = tempfile.mkdtemp()
+    print('Temporary path:', path)
+    set_cwd(path)
+    cuckoo_create()
+    mkdir(cwd(analysis=1))
+    yield path
+    shutil.rmtree(path)
 
-class TestFileUpload(object):
-    def fileupload(self, handler):
-        set_cwd(tempfile.mkdtemp())
-        cuckoo_create()
-        mkdir(cwd(analysis=1))
-        mkdir(cwd("logs", analysis=1))
-
-        handler.storagepath = cwd(analysis=1)
-        fu = FileUpload(handler, None)
-        fu.init()
-        for x in fu:
-            pass
-        fu.close()
 
-    def test_success(self):
-        class Handler(object):
-            reads = [
-                "this", "is", "a", "test", None
-            ]
+def mock_handler_context(klass, path, lines, data, version=None):
+    class FakeContext:
+        storagepath = path
+
+        def read_newline(self):
+            if not lines:
+                raise EOFError
+            return lines.pop(0)
 
-            def read_newline(self, strip):
-                return "logs/1.log"
+        def read_any(self):
+            if not data:
+                raise EOFError
+            return data.pop(0)
 
-            def read_any(self):
-                return self.reads.pop(0)
+        def read(self, size):
+            # TODO: we can test expected sizes here
+            return self.read_any()
 
-        self.fileupload(Handler())
+    h = klass(FakeContext(), version)
+    h.init()
+    h.handle()
+    h.close()
+    return h
+
+@pytest.mark.usefixtures('cuckoo_cwd')
+class TestFileUpload(object):
+    def test_success(self):
+        mock_handler_context(FileUpload,
+                             cwd(analysis=1),
+                             ['logs/1.log'],
+                             ['this', 'is', 'a', 'test'])
 
         with open(cwd("logs", "1.log", analysis=1), "rb") as f:
             assert f.read() == "thisisatest"
 
     def invalid_path(self, path):
-        class Handler(object):
-            def read_newline(self, strip):
-                return path
-
         with pytest.raises(CuckooOperationalError) as e:
-            self.fileupload(Handler())
+            mock_handler_context(FileUpload, cwd(analysis=1), [path], [])
         e.match("banned path")
 
     def test_invalid_paths(self):
         self.invalid_path("/tmp/foobar")
         self.invalid_path("../hello")
         self.invalid_path("../../foobar")
+
+
+@pytest.mark.usefixtures('cuckoo_cwd')
+class TestLogHandler(object):
+    def test_success(self):
+        mock_handler_context(LogHandler,
+                             cwd(analysis=1),
+                             [],
+                             ['first\n', 'second\n'])
+
+        with open(cwd("analysis.log", analysis=1), "rb") as f:
+            assert f.read() == "first\nsecond\n"
+
+
+@pytest.mark.usefixtures('cuckoo_cwd')
+class TestBsonStore(object):
+    def test_success(self):
+        mock_handler_context(BsonStore,
+                             cwd(analysis=1),
+                             [],
+                             ['\x01\x00\x00\x00', 'a'],
+                             1)
+
+        with open(cwd("logs/1.bson", analysis=1), "rb") as f:
+            assert f.read() == "\x01\x00\x00\x00a"

From 5765bdfd56b2d5bb824d559a3330bb473975cf82 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Fri, 2 Mar 2018 16:00:20 +0100
Subject: [PATCH 050/138] Forcefully close sockets on `del_task`

This should prevent sockets remaining open when a task is deleted.
Also clean up the list of open sockets as soon as a connection is
closed.
---
 cuckoo/core/resultserver.py | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 693569429c..6d31021550 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -50,6 +50,9 @@ def __init__(self, storagepath, sock):
         self.storagepath = storagepath
         self.sock = sock.makefile(mode='rb')
 
+    def __del__(self):
+        self.sock.close()
+
     def read(self, size):
         buf = self.sock.read(size)
         if not buf:
@@ -256,10 +259,15 @@ class GeventResultServerWorker(gevent.server.StreamServer):
         "FILE": FileUpload,
         "LOG": LogHandler,
     }
+    handler_lock = threading.Lock()
 
     def __init__(self, *args, **kwargs):
         super(GeventResultServerWorker, self).__init__(*args, **kwargs)
+
+        # Store IP address to task_id mapping
         self.tasks = {}
+
+        # Store running handlers for task_id
         self.handlers = {}
 
     def do_run(self):
@@ -267,7 +275,6 @@ def do_run(self):
 
     def add_task(self, task_id, ipaddr):
         self.tasks[ipaddr] = task_id
-        self.handlers[ipaddr] = self.handlers.get(ipaddr, {})
 
     def del_task(self, task_id, ipaddr):
         """Delete ResultServer state and wait for pending RequestHandlers."""
@@ -275,15 +282,13 @@ def del_task(self, task_id, ipaddr):
             log.warning("ResultServer did not have a task with ID %s",
                         task_id)
 
-        handlers = self.handlers.pop(ipaddr, {})
-        for handler in handlers.values():
-            handler.running = False
-            # <WAIT>
-            #h.end_request.set()
-            #h.done_event.wait()
+        with self.handler_lock:
+            socks = self.handlers.pop(task_id, set())
+            for sock in socks:
+                sock.close()
 
     def handle(self, sock, addr):
-        ipaddr, port = addr
+        ipaddr = addr[0]
         task_id = self.tasks.get(ipaddr)
         if not task_id:
             log.warning("ResultServer did not have a task for IP %s", ipaddr)
@@ -297,16 +302,21 @@ def handle(self, sock, addr):
 
             # Registering the protocol allows for the handler getting its "running"
             # field set to False (among other use-cases in the future).
-            self.handlers[ipaddr][port] = protocol
+            with self.handler_lock:
+                s = self.handlers.setdefault(ipaddr, set())
+                s.add(sock)
 
-            protocol.task_id = task_id
             try:
+                protocol.task_id = task_id  # TODO
                 protocol.init()
                 protocol.handle()
             finally:
                 protocol.close()
+                with self.handler_lock:
+                    s.discard(sock)
 
         finally:
+            sock.close()
             task_log_stop(task_id)
 
     def negotiate_protocol(self, ctx):
@@ -339,7 +349,7 @@ def add_task(self, task, machine):
         self.instance.add_task(task.id, machine.ip)
 
     def del_task(self, task, machine):
-        """Delete ResultServer state and wait for pending RequestHandlers."""
+        """Delete running task and cancel existing handlers."""
         self.instance.del_task(task.id, machine.ip)
 
     def create_bg_server(self):

From b22e065f953a936b8783b26047e71ede60faf7db Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Fri, 2 Mar 2018 16:04:52 +0100
Subject: [PATCH 051/138] Handlers are mapped based on task ID

---
 cuckoo/core/resultserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 6d31021550..1e62b30c95 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -303,7 +303,7 @@ def handle(self, sock, addr):
             # Registering the protocol allows for the handler getting its "running"
             # field set to False (among other use-cases in the future).
             with self.handler_lock:
-                s = self.handlers.setdefault(ipaddr, set())
+                s = self.handlers.setdefault(task_id, set())
                 s.add(sock)
 
             try:

From 8ef0dbe3a8a8627392f431180ffe59c669dc86c7 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Sat, 3 Mar 2018 17:10:00 +0100
Subject: [PATCH 052/138] Improve BSON and filename security

Avoid potentially dangerous filenames. Also limit potential memory usage
when sending large BSON document parts.
---
 cuckoo/core/resultserver.py | 85 +++++++++++++++++++++++++------------
 1 file changed, 57 insertions(+), 28 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 1e62b30c95..ec6c98f915 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -4,13 +4,11 @@
 # See the file 'docs/LICENSE' for copying permission.
 
 # TODO:
-# * NTFS ADS / windows fn protection
 # * gevent doesn't bind until serve_forever()
-# * replace threading with gevent locks (needed?)
-# * lock all filesystem operations?
-# * lock in ProtocolHandler for .running?
-# * fd leaking / no timeout
+# * document filesystem operations locking concern
+# * test fd leaking / timeout
 # * consider creating a folder whitelist
+# * update setup.py dep
 from __future__ import print_function
 
 import errno
@@ -42,6 +40,21 @@
 
 NETLOG_RECV_TIMEOUT = 60
 
+# Prevent malicious clients from using potentially dangerious filenames
+# E.g. C API confusion by using null, or using the colon on NTFS (Alternate
+# Data Streams); XXX: just replace illegal chars?
+BANNED_PATH_CHARS = b'\x00:'
+
+
+def netlog_sanitize_fname(fname):
+    """Validate agent-provided path for result files"""
+    fname = fname.replace("\\", "/")
+    if (fname.startswith('/') or './' in fname or
+            any(c in BANNED_PATH_CHARS for c in fname)):
+        raise CuckooOperationalError("Netlog client supplied banned path: %s"
+                                     % fname)
+    return fname
+
 
 class HandlerContext:
     """Holds context for protocol handlers"""
@@ -87,9 +100,10 @@ def handle(self):
         # Read until newline for file path, e.g.,
         # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin
 
-        dump_path = self.handler.read_newline().replace("\\", "/")
+        dump_path = netlog_sanitize_fname(self.handler.read_newline())
 
         if self.version >= 2:
+            # NB: filepath is only used as metadata
             filepath = self.handler.read_newline()
             pids = map(int, self.handler.read_newline().split())
         else:
@@ -99,7 +113,7 @@ def handle(self):
 
         dir_part, filename = os.path.split(dump_path)
 
-        if "./" in dump_path or not dir_part or dump_path.startswith("/"):
+        if not dir_part:
             raise CuckooOperationalError(
                 "FileUpload failure, banned path: %s" % dump_path
             )
@@ -162,6 +176,7 @@ def close(self):
         if self.fd:
             self.fd.close()
 
+
 class LogHandler(ProtocolHandler):
     # TODO: not protected against opening multiple times
     def init(self):
@@ -197,16 +212,19 @@ def _open(self):
         log.debug("Log analysis.log already existing, appending data.")
         fd = open(self.logpath, "ab")
 
-        # add a fake log entry, saying this had to be re-opened
-        #  use the same format as the default logger, in case anyone wants to parse this
-        #  2015-02-23 12:05:05,092 [lib.api.process] DEBUG: Using QueueUserAPC injection.
+        # Add a fake log entry, saying this had to be re-opened. Use the same
+        # format as the default logger, in case anyone wants to parse this:
+        #  2015-02-23 12:05:05,092 [lib.api.process] DEBUG: Using QueueUserAPC
+        #   injection.
         now = datetime.datetime.now()
         print("\n", now.strftime("%Y-%m-%d %H:%M:%S",),
               now.microsecond / 1000.0,
-              " [lib.core.resultserver] WARNING: This log file was re-opened, log entries will be appended.",
+              " [lib.core.resultserver] WARNING: This log file was re-opened,"
+              " log entries will be appended.",
               sep='', file=fd)
         return fd
 
+
 class BsonStore(ProtocolHandler):
     def init(self):
         # We cheat a little bit through the "version" variable, but that's
@@ -227,14 +245,21 @@ def handle(self):
             except EOFError:
                 break
 
-            length = struct.unpack("I", lenbuf)[0]
-            buf = self.handler.read(length)
-            if len(buf) != length:
-                log.warning("BsonStore short read")
-                break
+            self.f.write(lenbuf)
+
+            length = struct.unpack("<I", lenbuf)[0]
+            remain = length
+            while remain > 0:
+                size = min(BUFSIZE, remain)
+                buf = self.handler.read(size)
+                remain -= len(buf)
+
+                # TODO Handle out of disk space.
+                self.f.write(buf)
 
-            # TODO Handle out of disk space.
-            self.f.write(lenbuf + buf)
+            log.debug("Task %s: uploaded BSON part for PID %s length: %s",
+                      self.task_id,
+                      self.version, length)
 
     def close(self):
         self.f.close()
@@ -274,6 +299,7 @@ def do_run(self):
         self.serve_forever()
 
     def add_task(self, task_id, ipaddr):
+        Folders.create(cwd(analysis=task_id), 'logs')
         self.tasks[ipaddr] = task_id
 
     def del_task(self, task_id, ipaddr):
@@ -284,6 +310,9 @@ def del_task(self, task_id, ipaddr):
 
         with self.handler_lock:
             socks = self.handlers.pop(task_id, set())
+            if socks:
+                log.debug("Cancel %s socket(s) for task %r", len(socks),
+                          task_id)
             for sock in socks:
                 sock.close()
 
@@ -300,15 +329,15 @@ def handle(self, sock, addr):
         try:
             protocol = self.negotiate_protocol(ctx)
 
-            # Registering the protocol allows for the handler getting its "running"
-            # field set to False (among other use-cases in the future).
+            # Registering the socket allows us to cancel the handler by
+            # closing the socket when the task is deleted
             with self.handler_lock:
                 s = self.handlers.setdefault(task_id, set())
                 s.add(sock)
 
+            protocol.task_id = task_id  # TODO
+            protocol.init()
             try:
-                protocol.task_id = task_id  # TODO
-                protocol.init()
                 protocol.handle()
             finally:
                 protocol.close()
@@ -359,19 +388,20 @@ def create_bg_server(self):
         if pool_size:
             pool_size = int(pool_size)
         else:
-            pool_size = 32
+            pool_size = 128
 
         pool = gevent.pool.Pool(pool_size)
         try:
             # TODO: support binding to port 0 for random port
             self.instance = GeventResultServerWorker((ip, port),
                                                      spawn=pool)
+            self.instance.do_run()
         except OSError as e:
             if e.errno == errno.EADDRINUSE:
-                    raise CuckooCriticalError(
-                        "Cannot bind ResultServer on port %d "
-                        "because it was in use, bailing." % port
-                        )
+                raise CuckooCriticalError(
+                    "Cannot bind ResultServer on port %d "
+                    "because it was in use, bailing." % port
+                )
             elif e.errno == errno.EADDRNOTAVAIL:
                 raise CuckooCriticalError(
                     "Unable to bind ResultServer on %s:%s %s. This "
@@ -386,4 +416,3 @@ def create_bg_server(self):
                     "Unable to bind ResultServer on %s:%s: %s" %
                     (ip, port, e)
                 )
-        self.instance.do_run()

From 3201314f9de1d0b07e3b016528695711572a3626 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Tue, 6 Mar 2018 10:43:34 +0100
Subject: [PATCH 053/138] Work on ResultServer tests/stability

---
 cuckoo/common/files.py      | 18 ++++++-
 cuckoo/core/resultserver.py | 87 +++++++++++++++++++--------------
 setup.py                    |  2 +-
 tests/test_resultserver.py  | 95 +++++++++++++++++++++++--------------
 4 files changed, 130 insertions(+), 72 deletions(-)

diff --git a/cuckoo/common/files.py b/cuckoo/common/files.py
index 5ae8e24811..ddaf8d57e7 100644
--- a/cuckoo/common/files.py
+++ b/cuckoo/common/files.py
@@ -7,6 +7,7 @@
 import tempfile
 import ntpath
 import shutil
+import errno
 
 from cuckoo.common.config import config
 from cuckoo.common.exceptions import CuckooOperationalError
@@ -24,6 +25,18 @@ def temppath():
 
     return tmppath
 
+
+def open_exclusive(path, mode='wb'):
+    """Open a file with O_EXCL, failing if it already exists
+    [In Python 3, use open with x]"""
+    fd = os.open(path, os.O_CREAT|os.O_EXCL|os.O_WRONLY)
+    try:
+        return os.fdopen(fd, mode)
+    except:
+        os.close(fd)
+        raise
+
+
 class Storage(object):
     @staticmethod
     def get_filename_from_path(path):
@@ -56,7 +69,10 @@ def create(root=".", folders=None):
             if not os.path.isdir(folder_path):
                 try:
                     os.makedirs(folder_path)
-                except OSError:
+                except OSError as e:
+                    if e.errno == errno.EEXIST:
+                        # Race condition, ignore
+                        continue
                     raise CuckooOperationalError(
                         "Unable to create folder: %s" % folder_path
                     )
diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index ec6c98f915..009f67c7d0 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -3,14 +3,9 @@
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
-# TODO:
-# * gevent doesn't bind until serve_forever()
-# * document filesystem operations locking concern
-# * test fd leaking / timeout
-# * consider creating a folder whitelist
-# * update setup.py dep
 from __future__ import print_function
 
+import socket
 import errno
 import datetime
 import gevent.server
@@ -26,7 +21,7 @@
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.common.exceptions import CuckooCriticalError
 from cuckoo.common.exceptions import CuckooResultError
-from cuckoo.common.files import Folders
+from cuckoo.common.files import Folders, open_exclusive
 from cuckoo.common.utils import Singleton
 from cuckoo.core.log import task_log_start, task_log_stop
 from cuckoo.misc import cwd
@@ -67,7 +62,12 @@ def __del__(self):
         self.sock.close()
 
     def read(self, size):
-        buf = self.sock.read(size)
+        try:
+            buf = self.sock.read(size)
+        except socket.error as e:
+            if e.errno == errno.ECONNRESET:
+                raise EOFError
+            raise
         if not buf:
             raise EOFError
         return buf
@@ -81,13 +81,11 @@ def read_newline(self):
         return line[:-1]
 
     def read_any(self):
-        buf = self.sock.read(BUFSIZE)
-        if not buf:
-            raise EOFError
-        return buf
+        return self.read(BUFSIZE)
 
 
 class FileUpload(ProtocolHandler):
+    # TODO: this should be a whitelist
     RESTRICTED_DIRECTORIES = "reports/",
 
     def init(self):
@@ -110,7 +108,6 @@ def handle(self):
             filepath, pids = None, []
 
         log.debug("File upload request for %s", dump_path)
-
         dir_part, filename = os.path.split(dump_path)
 
         if not dir_part:
@@ -118,10 +115,12 @@ def handle(self):
                 "FileUpload failure, banned path: %s" % dump_path
             )
 
+        dir_part += "/"
+
         for restricted in self.RESTRICTED_DIRECTORIES:
-            if restricted in dir_part:
+            if dir_part.startswith(restricted):
                 raise CuckooOperationalError(
-                    "FileUpload failure, banned path."
+                    "FileUpload failure, banned path: %s" % dump_path
                 )
 
         try:
@@ -137,14 +136,14 @@ def handle(self):
                 "FileUpload failure, path sanitization failed."
             )
 
-        if os.path.exists(file_path):
-            log.warning(
-                "Analyzer tried to overwrite an existing file, "
-                "closing connection."
-            )
-            return
-
-        self.fd = open(file_path, "wb")
+        try:
+            self.fd = open_exclusive(file_path)
+        except OSError as e:
+            if e.errno == errno.EEXIST:
+                raise CuckooOperationalError(
+                    "Analyzer tried to overwrite an existing file"
+                )
+            raise
         chunk = self.handler.read_any()
         while chunk:
             self.fd.write(chunk)
@@ -163,11 +162,11 @@ def handle(self):
                 break
 
         with open(self.filelog, "a+b") as f:
-            f.write("%s\n" % json.dumps({
+            print(json.dumps({
                 "path": dump_path,
                 "filepath": filepath,
                 "pids": pids,
-            }))
+            }), file=f)
 
         log.debug("Uploaded file length: %s", self.fd.tell())
         self.fd.close()
@@ -205,9 +204,11 @@ def close(self):
             self.fd.close()
 
     def _open(self):
-        if not os.path.exists(self.logpath):
-            # (Race condition)
-            return open(self.logpath, "wb")
+        try:
+            return open_exclusive(self.logpath)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
 
         log.debug("Log analysis.log already existing, appending data.")
         fd = open(self.logpath, "ab")
@@ -231,6 +232,13 @@ def init(self):
         # acceptable and backwards compatible (for now). Backwards compatible
         # in the sense that newer Cuckoo Monitor binaries work with older
         # versions of Cuckoo, the other way around doesn't apply here.
+        if self.version is None:
+            log.warning("Agent is sending BSON files without PID parameter, "
+                        "you should probably update it")
+            self.f = None
+            return
+
+        Folders.create(self.handler.storagepath, 'logs')
         self.f = open(os.path.join(self.handler.storagepath,
                                    "logs", "%d.bson" % self.version), "wb")
 
@@ -245,24 +253,33 @@ def handle(self):
             except EOFError:
                 break
 
-            self.f.write(lenbuf)
+            if self.f:
+                self.f.write(lenbuf)
 
             length = struct.unpack("<I", lenbuf)[0]
             remain = length
             while remain > 0:
                 size = min(BUFSIZE, remain)
-                buf = self.handler.read(size)
+                try:
+                    buf = self.handler.read(size)
+                except EOFError:
+                    logging.warning("BSON received EOF, but still expecting "
+                                    "%s byte(s)", remain)
+                    break
                 remain -= len(buf)
 
                 # TODO Handle out of disk space.
-                self.f.write(buf)
+                if self.f:
+                    self.f.write(buf)
 
             log.debug("Task %s: uploaded BSON part for PID %s length: %s",
                       self.task_id,
                       self.version, length)
 
     def close(self):
-        self.f.close()
+        if self.f:
+            self.f.close()
+            self.f = None
 
 
 class GeventResultServerWorker(gevent.server.StreamServer):
@@ -299,7 +316,6 @@ def do_run(self):
         self.serve_forever()
 
     def add_task(self, task_id, ipaddr):
-        Folders.create(cwd(analysis=task_id), 'logs')
         self.tasks[ipaddr] = task_id
 
     def del_task(self, task_id, ipaddr):
@@ -311,8 +327,8 @@ def del_task(self, task_id, ipaddr):
         with self.handler_lock:
             socks = self.handlers.pop(task_id, set())
             if socks:
-                log.debug("Cancel %s socket(s) for task %r", len(socks),
-                          task_id)
+                log.warning("Cancel %s socket(s) for task %r",
+                            len(socks), task_id)
             for sock in socks:
                 sock.close()
 
@@ -355,6 +371,7 @@ def negotiate_protocol(self, ctx):
             version = int(version)
         else:
             command, version = header, None
+        log.debug("Incoming command: %r", header)
         if command not in self.commands:
             log.warning(
                 "Unknown netlog protocol requested (%r), "
diff --git a/setup.py b/setup.py
index c442d72a71..abdc87ed15 100755
--- a/setup.py
+++ b/setup.py
@@ -198,6 +198,7 @@ def do_setup(**kwargs):
         "flask==0.12.2",
         "flask-sqlalchemy==2.4.0",
         "httpreplay>=0.2.4, <0.3",
+        "gevent>=1.2, <1.3",
         "jinja2==2.9.6",
         "jsbeautifier==1.6.2",
         "oletools==0.51",
@@ -229,7 +230,6 @@ def do_setup(**kwargs):
             "scapy==2.3.2",
         ],
         "distributed": [
-            "gevent==1.1.1",
             "psycopg2==2.6.2",
         ],
         "postgresql": [
diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py
index 3b44b3fcc7..1231c5c2db 100644
--- a/tests/test_resultserver.py
+++ b/tests/test_resultserver.py
@@ -11,10 +11,10 @@
 # - Double LOG command
 
 import logging
-import mock
 import pytest
 import tempfile
 import shutil
+import json
 
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.core.log import task_log_start, task_log_stop
@@ -23,33 +23,6 @@
 from cuckoo.main import cuckoo_create
 from cuckoo.misc import mkdir, set_cwd, cwd
 
-# TODO: restore this test
-'''
-@mock.patch("cuckoo.core.resultserver.select")
-def test_open_process_log_unicode(p):
-    set_cwd(tempfile.mkdtemp())
-    cuckoo_create()
-    mkdir(cwd(analysis=1))
-    mkdir(cwd("logs", analysis=1))
-
-    request = server = mock.MagicMock()
-
-    class Handler(ResultHandler):
-        storagepath = cwd(analysis=1)
-
-        def handle(self):
-            pass
-
-    init_logging(logging.DEBUG)
-
-    try:
-        task_log_start(1)
-        Handler(request, (None, None), server).open_process_log({
-            "pid": 1, "ppid": 2, "process_name": u"\u202e", "track": True,
-        })
-    finally:
-        task_log_stop(1)
-'''
 
 @pytest.fixture(scope='module')
 def cuckoo_cwd():
@@ -89,21 +62,60 @@ def read(self, size):
 
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestFileUpload(object):
-    def test_success(self):
-        mock_handler_context(FileUpload,
-                             cwd(analysis=1),
-                             ['logs/1.log'],
-                             ['this', 'is', 'a', 'test'])
-
-        with open(cwd("logs", "1.log", analysis=1), "rb") as f:
+    @pytest.mark.order1
+    def test_success_noversion(self):
+        fu = mock_handler_context(FileUpload,
+                                  cwd(analysis=1),
+                                  ['files/1.exe'],
+                                  ['this', 'is', 'a', 'test'])
+
+        with open(cwd("files", "1.exe", analysis=1), "rb") as f:
             assert f.read() == "thisisatest"
 
+        with open(fu.filelog) as f:
+            lines = f.readlines()
+            blob = json.loads(lines[-1])
+            assert blob['filepath'] is None
+            assert blob['path'] == "files/1.exe"
+            assert blob["pids"] == []
+
+    @pytest.mark.order2
+    def test_overwrite(self):
+        with pytest.raises(CuckooOperationalError) as e:
+            mock_handler_context(FileUpload,
+                                 cwd(analysis=1),
+                                 ['files/1.exe'],
+                                 [])
+        e.match("overwrite an existing file")
+
+
+    def test_success_v2(self):
+        fu = mock_handler_context(FileUpload,
+                                  cwd(analysis=1),
+                                  ['files/2.exe', 'C:\\RealFilename.exe',
+                                   '11 12'],
+                                  ['second', 'test'],
+                                  2)
+
+        with open(cwd("files", "2.exe", analysis=1), "rb") as f:
+            assert f.read() == "secondtest"
+
+        with open(fu.filelog) as f:
+            lines = f.readlines()
+            blob = json.loads(lines[-1])
+            assert blob['filepath'] == 'C:\\RealFilename.exe'
+            assert blob['path'] == "files/2.exe"
+            assert blob["pids"] == [11, 12]
+
+
     def invalid_path(self, path):
         with pytest.raises(CuckooOperationalError) as e:
             mock_handler_context(FileUpload, cwd(analysis=1), [path], [])
         e.match("banned path")
 
     def test_invalid_paths(self):
+        self.invalid_path("dummy")
+        self.invalid_path("reports/report.json")
         self.invalid_path("/tmp/foobar")
         self.invalid_path("../hello")
         self.invalid_path("../../foobar")
@@ -111,6 +123,7 @@ def test_invalid_paths(self):
 
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestLogHandler(object):
+    @pytest.mark.order1
     def test_success(self):
         mock_handler_context(LogHandler,
                              cwd(analysis=1),
@@ -120,6 +133,18 @@ def test_success(self):
         with open(cwd("analysis.log", analysis=1), "rb") as f:
             assert f.read() == "first\nsecond\n"
 
+    @pytest.mark.order2
+    def test_reopen(self):
+        mock_handler_context(LogHandler,
+                             cwd(analysis=1),
+                             [],
+                             ['reopen\n'])
+
+        with open(cwd("analysis.log", analysis=1), "rb") as f:
+            data = f.read()
+            assert 'WARNING: This log file was re-opened' in data
+            assert data.endswith('reopen\n')
+
 
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestBsonStore(object):

From 7b00908faf39858faa609e43d049aa87fcf79e9d Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Tue, 6 Mar 2018 12:53:14 +0100
Subject: [PATCH 054/138] Create analysis directories early, use whitelist
 approach

---
 cuckoo/core/resultserver.py | 59 ++++++++++---------------------------
 cuckoo/core/scheduler.py    |  7 +++--
 tests/test_resultserver.py  |  9 +++++-
 3 files changed, 29 insertions(+), 46 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 009f67c7d0..f6346ef555 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -33,7 +33,10 @@
 
 BUFSIZE = 16 * 1024
 
-NETLOG_RECV_TIMEOUT = 60
+# Directories in which analysis-related files will be stored; also acts as
+# whitelist
+RESULT_UPLOADABLE = ("files", "shots", "buffer",  "extracted")
+RESULT_DIRECTORIES = RESULT_UPLOADABLE + ("reports", "logs")
 
 # Prevent malicious clients from using potentially dangerious filenames
 # E.g. C API confusion by using null, or using the colon on NTFS (Alternate
@@ -41,14 +44,17 @@
 BANNED_PATH_CHARS = b'\x00:'
 
 
-def netlog_sanitize_fname(fname):
+def netlog_sanitize_fname(path):
     """Validate agent-provided path for result files"""
-    fname = fname.replace("\\", "/")
-    if (fname.startswith('/') or './' in fname or
-            any(c in BANNED_PATH_CHARS for c in fname)):
-        raise CuckooOperationalError("Netlog client supplied banned path: %s"
-                                     % fname)
-    return fname
+    path = path.replace("\\", "/")
+    dir_part, name = os.path.split(path)
+    if dir_part not in RESULT_UPLOADABLE:
+        raise CuckooOperationalError("Netlog client requested banned path: %s"
+                                     % path)
+    if any(c in BANNED_PATH_CHARS for c in name):
+        raise CuckooOperationalError("Netlog client requested banned path: %s"
+                                     % path)
+    return path
 
 
 class HandlerContext:
@@ -58,9 +64,6 @@ def __init__(self, storagepath, sock):
         self.storagepath = storagepath
         self.sock = sock.makefile(mode='rb')
 
-    def __del__(self):
-        self.sock.close()
-
     def read(self, size):
         try:
             buf = self.sock.read(size)
@@ -76,8 +79,8 @@ def read_newline(self):
         line = self.sock.readline(MAX_NETLOG_LINE)
         if not line:
             raise EOFError
-        elif not line.endswith('\n'):
-            raise CuckooOperationalError('Received overly long line')
+        elif not line.endswith("\n"):
+            raise CuckooOperationalError("Received overly long line")
         return line[:-1]
 
     def read_any(self):
@@ -85,9 +88,6 @@ def read_any(self):
 
 
 class FileUpload(ProtocolHandler):
-    # TODO: this should be a whitelist
-    RESTRICTED_DIRECTORIES = "reports/",
-
     def init(self):
         self.upload_max_size = config("cuckoo:resultserver:upload_max_size")
         self.storagepath = self.handler.storagepath
@@ -108,34 +108,8 @@ def handle(self):
             filepath, pids = None, []
 
         log.debug("File upload request for %s", dump_path)
-        dir_part, filename = os.path.split(dump_path)
-
-        if not dir_part:
-            raise CuckooOperationalError(
-                "FileUpload failure, banned path: %s" % dump_path
-            )
-
-        dir_part += "/"
-
-        for restricted in self.RESTRICTED_DIRECTORIES:
-            if dir_part.startswith(restricted):
-                raise CuckooOperationalError(
-                    "FileUpload failure, banned path: %s" % dump_path
-                )
-
-        try:
-            Folders.create(self.storagepath, dir_part)
-        except CuckooOperationalError:
-            log.error("Unable to create folder %s", dir_part)
-            return
-
         file_path = os.path.join(self.storagepath, dump_path)
 
-        if not file_path.startswith(self.storagepath):
-            raise CuckooOperationalError(
-                "FileUpload failure, path sanitization failed."
-            )
-
         try:
             self.fd = open_exclusive(file_path)
         except OSError as e:
@@ -238,7 +212,6 @@ def init(self):
             self.f = None
             return
 
-        Folders.create(self.handler.storagepath, 'logs')
         self.f = open(os.path.join(self.handler.storagepath,
                                    "logs", "%d.bson" % self.version), "wb")
 
diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py
index dad8e07ec2..d8ddc59ad8 100644
--- a/cuckoo/core/scheduler.py
+++ b/cuckoo/core/scheduler.py
@@ -25,7 +25,7 @@
 from cuckoo.core.plugins import RunAuxiliary, RunProcessing
 from cuckoo.core.plugins import RunSignatures, RunReporting
 from cuckoo.core.log import task_log_start, task_log_stop, logger
-from cuckoo.core.resultserver import ResultServer
+from cuckoo.core.resultserver import ResultServer, RESULT_DIRECTORIES
 from cuckoo.core.rooter import rooter
 from cuckoo.misc import cwd
 
@@ -37,6 +37,7 @@
 
 active_analysis_count = 0
 
+
 class AnalysisManager(threading.Thread):
     """Analysis Manager.
 
@@ -77,8 +78,10 @@ def init(self):
 
         # If we're not able to create the analysis storage folder, we have to
         # abort the analysis.
+        # Also create all directories that the ResultServer can use for file
+        # uploads.
         try:
-            Folders.create(self.storage)
+            Folders.create(self.storage, RESULT_DIRECTORIES)
         except CuckooOperationalError:
             log.error("Unable to create analysis folder %s", self.storage)
             return False
diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py
index 1231c5c2db..2314592b70 100644
--- a/tests/test_resultserver.py
+++ b/tests/test_resultserver.py
@@ -17,7 +17,9 @@
 import json
 
 from cuckoo.common.exceptions import CuckooOperationalError
+from cuckoo.common.files import Folders
 from cuckoo.core.log import task_log_start, task_log_stop
+from cuckoo.core.resultserver import RESULT_DIRECTORIES
 from cuckoo.core.resultserver import FileUpload, LogHandler, BsonStore
 from cuckoo.core.startup import init_logging
 from cuckoo.main import cuckoo_create
@@ -31,7 +33,8 @@ def cuckoo_cwd():
     print('Temporary path:', path)
     set_cwd(path)
     cuckoo_create()
-    mkdir(cwd(analysis=1))
+    anal_path = cwd(analysis=1)
+    Folders.create(anal_path, RESULT_DIRECTORIES)
     yield path
     shutil.rmtree(path)
 
@@ -115,6 +118,10 @@ def invalid_path(self, path):
 
     def test_invalid_paths(self):
         self.invalid_path("dummy")
+        self.invalid_path("files/p\x00ath.exe")
+        self.invalid_path("files/path.exe:$DATA")
+        self.invalid_path("notallowed/path.exe")
+        self.invalid_path("shots/notallowed/path.jpg")
         self.invalid_path("reports/report.json")
         self.invalid_path("/tmp/foobar")
         self.invalid_path("../hello")

From ffc9921d32d96f9abd60e55d36c3534bfe33efa0 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Tue, 6 Mar 2018 16:53:29 +0100
Subject: [PATCH 055/138] Implement buffering instead of makefile to avoid
 losing data

---
 cuckoo/core/resultserver.py | 44 ++++++++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 13 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index f6346ef555..575a2f76a4 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -10,6 +10,7 @@
 import datetime
 import gevent.server
 import gevent.pool
+import gevent.socket
 import json
 import logging
 import os
@@ -60,28 +61,45 @@ def netlog_sanitize_fname(path):
 class HandlerContext:
     """Holds context for protocol handlers"""
     def __init__(self, storagepath, sock):
-        # The part where artifacts will be stored
+        # The path where artifacts will be stored
         self.storagepath = storagepath
-        self.sock = sock.makefile(mode='rb')
+        self.sock = sock
+        self.buf = ''
 
-    def read(self, size):
+    def _read_ahead(self, size):
         try:
-            buf = self.sock.read(size)
+            buf = self.sock.recv(size)
+        except gevent.socket.cancel_wait_ex:
+            # We were cancelled elsewhere via .close()
+            raise EOFError
         except socket.error as e:
             if e.errno == errno.ECONNRESET:
-                raise EOFError
-            raise
-        if not buf:
+                if not self.buf:
+                    raise EOFError
+            else:
+                raise
+        self.buf += buf
+        if not self.buf:
             raise EOFError
+
+    def read(self, size):
+        have = len(self.buf)
+        if have < size:
+            self._read_ahead(size - have)
+        buf, self.buf = self.buf[:size], self.buf[size:]
         return buf
 
     def read_newline(self):
-        line = self.sock.readline(MAX_NETLOG_LINE)
-        if not line:
-            raise EOFError
-        elif not line.endswith("\n"):
-            raise CuckooOperationalError("Received overly long line")
-        return line[:-1]
+        while True:
+            pos = self.buf.find("\n")
+            if pos < 0:
+                if len(self.buf) >= MAX_NETLOG_LINE:
+                    raise CuckooOperationalError("Received overly long line")
+                fill = MAX_NETLOG_LINE - len(self.buf)
+                self._read_ahead(fill)
+                continue
+            line, self.buf = self.buf[:pos], self.buf[pos + 1:]
+            return line
 
     def read_any(self):
         return self.read(BUFSIZE)

From f6af16179ea4b35932c64047ab79e18eec3cf587 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Wed, 7 Mar 2018 15:55:17 +0100
Subject: [PATCH 056/138] Make task management thread safe; cleanup

---
 cuckoo/common/abstracts.py            |  27 +--
 cuckoo/common/netlog.py               |  12 +-
 cuckoo/core/resultserver.py           | 294 +++++++++++++-------------
 cuckoo/processing/platform/windows.py |  11 +-
 tests/test_resultserver.py            |  15 +-
 5 files changed, 183 insertions(+), 176 deletions(-)

diff --git a/cuckoo/common/abstracts.py b/cuckoo/common/abstracts.py
index 64b723acfb..1b163b8885 100644
--- a/cuckoo/common/abstracts.py
+++ b/cuckoo/common/abstracts.py
@@ -1427,26 +1427,29 @@ def run(self):
         behavior[self.key]."""
         raise NotImplementedError
 
+
 class ProtocolHandler(object):
     """Abstract class for protocol handlers coming out of the analysis."""
-    def __init__(self, handler, version=None):
-        self.handler = handler
+    def __init__(self, task_id, ctx, version=None):
+        self.task_id = task_id
+        self.handler = ctx
+        self.fd = None
         self.version = version
-        self.sock = None
-        self.task_id = None
-        self.running = True
 
-    def init(self):
-        pass
+    def __enter__(self):
+        self.init()
 
-    def handle(self):
-        pass
+    def __exit__(self, type, value, traceback):
+        self.close()
 
     def close(self):
-        pass
+        if self.fd:
+            self.fd.close()
+            self.fd = None
+
+    def handle(self):
+        raise NotImplementedError
 
-    def read(self, length):
-        return self.handler.read(self, length)
 
 class Extractor(object):
     """One piece in a series of recursive extractors & unpackers."""
diff --git a/cuckoo/common/netlog.py b/cuckoo/common/netlog.py
index 9fb65ae7c4..be1b687980 100644
--- a/cuckoo/common/netlog.py
+++ b/cuckoo/common/netlog.py
@@ -1,5 +1,5 @@
 # Copyright (C) 2010-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2016 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -18,7 +18,6 @@
 elif hasattr(bson, "loads"):
     bson_decode = lambda d: bson.loads(d)
 
-from cuckoo.common.abstracts import ProtocolHandler
 from cuckoo.common.files import Storage
 from cuckoo.common.exceptions import CuckooResultError
 
@@ -54,8 +53,8 @@ def default_converter_64bit(v):
         return v.decode("latin-1")
     return v
 
-class BsonParser(ProtocolHandler):
-    """Receive and interpret .bson logs from the monitor.
+class BsonParser(object):
+    """Interprets .bson logs from the monitor.
 
     The monitor provides us with "info" messages that explain how the function
     arguments will come through later on. This class remembers these info
@@ -76,9 +75,8 @@ class BsonParser(ProtocolHandler):
         "x": pointer_converter_32bit,
     }
 
-    def init(self):
-        self.fd = self.handler
-
+    def __init__(self, fd):
+        self.fd = fd
         self.infomap = {}
         self.flags_value = {}
         self.flags_bitmask = {}
diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 575a2f76a4..6548c1e151 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -32,6 +32,7 @@
 # Maximum line length to read for netlog messages, to avoid memory exhaustion
 MAX_NETLOG_LINE = 4 * 1024
 
+# Maximum number of bytes to buffer for a single connection
 BUFSIZE = 16 * 1024
 
 # Directories in which analysis-related files will be stored; also acts as
@@ -59,50 +60,96 @@ def netlog_sanitize_fname(path):
 
 
 class HandlerContext:
-    """Holds context for protocol handlers"""
-    def __init__(self, storagepath, sock):
+    """Holds context for protocol handlers.
+
+    To avoid losing data that was already buffered, handlers should check if
+    the buffer is empty when handling EOFError.
+
+    Can safely be cancelled from another thread, though in practice this will
+    not occur often -- usually the connection between VM and the ResultServer
+    will be reset during shutdown."""
+    def __init__(self, task_id, storagepath, sock):
+        self.task_id = task_id
+        self.command = None
+
         # The path where artifacts will be stored
         self.storagepath = storagepath
         self.sock = sock
         self.buf = ''
 
-    def _read_ahead(self, size):
+    def __repr__(self):
+        return '<Context for %s>' % self.command
+
+    def cancel(self):
+        """Cancel this context; gevent might complain about this with an
+        exception later on."""
         try:
-            buf = self.sock.recv(size)
-        except gevent.socket.cancel_wait_ex:
-            # We were cancelled elsewhere via .close()
-            raise EOFError
+            self.sock.shutdown(socket.SHUT_RD)
+        except socket.error:
+            pass
+
+    def read(self):
+        try:
+            return self.sock.recv(16384)
         except socket.error as e:
-            if e.errno == errno.ECONNRESET:
-                if not self.buf:
-                    raise EOFError
-            else:
+            if e.errno != errno.ECONNRESET:
                 raise
+            return ''
+
+    def buffered_read(self, size):
+        """Try to fill the buffer with at most `size` bytes
+        Will keep waiting until we're either cancelled, or until we received
+        something.
+        """
+        while True:
+            try:
+                buf = self.sock.recv(size)
+            except socket.timeout:
+                if self.cancelled:
+                    log.debug("Task #%s: connection was cancelled",
+                              self.task_id)
+                    raise Cancelled
+                continue
+            except socket.error as e:
+                log.debug("Task #%s encountered a socket error: %s",
+                          self.task_id, e)
+                if e.errno != errno.ECONNRESET:
+                    raise
+                raise EOFError
+            break
         self.buf += buf
         if not self.buf:
             raise EOFError
 
-    def read(self, size):
-        have = len(self.buf)
-        if have < size:
-            self._read_ahead(size - have)
-        buf, self.buf = self.buf[:size], self.buf[size:]
+    def drain_buffer(self):
+        """Drain buffer and end buffering"""
+        buf, self.buf = "", None
         return buf
 
     def read_newline(self):
+        """Read until the next newline character, but never more than
+        `MAX_NETLOG_LINE`."""
         while True:
             pos = self.buf.find("\n")
             if pos < 0:
                 if len(self.buf) >= MAX_NETLOG_LINE:
                     raise CuckooOperationalError("Received overly long line")
-                fill = MAX_NETLOG_LINE - len(self.buf)
-                self._read_ahead(fill)
+                buf = self.read()
+                if buf == '':
+                    raise EOFError
+                self.buf += buf
                 continue
             line, self.buf = self.buf[:pos], self.buf[pos + 1:]
             return line
 
-    def read_any(self):
-        return self.read(BUFSIZE)
+    def copy_to_fd(self, fd, max_size=None):
+        fd.write(self.drain_buffer())
+        while True:
+            buf = self.read()
+            if buf == '':
+                break
+            fd.write(buf)
+        fd.flush()
 
 
 class FileUpload(ProtocolHandler):
@@ -115,7 +162,6 @@ def init(self):
     def handle(self):
         # Read until newline for file path, e.g.,
         # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin
-
         dump_path = netlog_sanitize_fname(self.handler.read_newline())
 
         if self.version >= 2:
@@ -125,47 +171,30 @@ def handle(self):
         else:
             filepath, pids = None, []
 
-        log.debug("File upload request for %s", dump_path)
+        log.debug("Task #%s: File upload for %s", self.task_id, dump_path)
         file_path = os.path.join(self.storagepath, dump_path)
 
         try:
             self.fd = open_exclusive(file_path)
         except OSError as e:
             if e.errno == errno.EEXIST:
-                raise CuckooOperationalError(
-                    "Analyzer tried to overwrite an existing file"
-                )
+                raise CuckooOperationalError("Analyzer for task #%s tried to "
+                                             "overwrite an existing file" %
+                                             self.task_id)
             raise
-        chunk = self.handler.read_any()
-        while chunk:
-            self.fd.write(chunk)
-
-            if self.fd.tell() >= self.upload_max_size:
-                log.warning(
-                    "Uploaded file length larger than upload_max_size, "
-                    "stopping upload."
-                )
-                self.fd.write("... (truncated)")
-                break
-
-            try:
-                chunk = self.handler.read_any()
-            except:
-                break
 
+        # !! race condition !!
         with open(self.filelog, "a+b") as f:
             print(json.dumps({
                 "path": dump_path,
                 "filepath": filepath,
                 "pids": pids,
             }), file=f)
-
-        log.debug("Uploaded file length: %s", self.fd.tell())
-        self.fd.close()
-
-    def close(self):
-        if self.fd:
-            self.fd.close()
+        try:
+            return self.handler.copy_to_fd(self.fd)
+        finally:
+            log.debug("Task #%s uploaded file length: %s", self.task_id,
+                      self.fd.tell())
 
 
 class LogHandler(ProtocolHandler):
@@ -173,27 +202,12 @@ class LogHandler(ProtocolHandler):
     def init(self):
         self.logpath = os.path.join(self.handler.storagepath, "analysis.log")
         self.fd = self._open()
-        log.debug("LogHandler for live analysis.log initialized.")
+        log.debug("Task #%s: live log analysis.log initialized.",
+                  self.task_id)
 
     def handle(self):
-        if not self.fd:
-            return
-
-        while True:
-            try:
-                buf = self.handler.read_any()
-            except EOFError:
-                break
-
-            if not buf:
-                break
-
-            self.fd.write(buf)
-            self.fd.flush()  # Expensive...
-
-    def close(self):
         if self.fd:
-            self.fd.close()
+            return self.handler.copy_to_fd(self.fd)
 
     def _open(self):
         try:
@@ -202,7 +216,8 @@ def _open(self):
             if e.errno != errno.EEXIST:
                 raise
 
-        log.debug("Log analysis.log already existing, appending data.")
+        log.debug("Task #%s: Log analysis.log already existing, "
+                  "appending data.", self.task_id)
         fd = open(self.logpath, "ab")
 
         # Add a fake log entry, saying this had to be re-opened. Use the same
@@ -227,50 +242,18 @@ def init(self):
         if self.version is None:
             log.warning("Agent is sending BSON files without PID parameter, "
                         "you should probably update it")
-            self.f = None
+            self.fd = None
             return
 
-        self.f = open(os.path.join(self.handler.storagepath,
-                                   "logs", "%d.bson" % self.version), "wb")
+        self.fd = open(os.path.join(self.handler.storagepath,
+                                    "logs", "%d.bson" % self.version), "wb")
 
     def handle(self):
-        while self.running:
-            # TODO: just loop read_any
-            try:
-                lenbuf = self.handler.read(4)
-                if len(lenbuf) != 4:
-                    log.warning("BsonStore short read")
-                    break
-            except EOFError:
-                break
-
-            if self.f:
-                self.f.write(lenbuf)
-
-            length = struct.unpack("<I", lenbuf)[0]
-            remain = length
-            while remain > 0:
-                size = min(BUFSIZE, remain)
-                try:
-                    buf = self.handler.read(size)
-                except EOFError:
-                    logging.warning("BSON received EOF, but still expecting "
-                                    "%s byte(s)", remain)
-                    break
-                remain -= len(buf)
-
-                # TODO Handle out of disk space.
-                if self.f:
-                    self.f.write(buf)
-
-            log.debug("Task %s: uploaded BSON part for PID %s length: %s",
-                      self.task_id,
-                      self.version, length)
-
-    def close(self):
-        if self.f:
-            self.f.close()
-            self.f = None
+        """Read a BSON stream, attempting at least basic validation, and
+        log failures."""
+        log.debug("Task #%s is sending a BSON stream", self.task_id)
+        if self.fd:
+            return self.handler.copy_to_fd(self.fd)
 
 
 class GeventResultServerWorker(gevent.server.StreamServer):
@@ -292,7 +275,7 @@ class GeventResultServerWorker(gevent.server.StreamServer):
         "FILE": FileUpload,
         "LOG": LogHandler,
     }
-    handler_lock = threading.Lock()
+    task_mgmt_lock = threading.Lock()
 
     def __init__(self, *args, **kwargs):
         super(GeventResultServerWorker, self).__init__(*args, **kwargs)
@@ -307,69 +290,85 @@ def do_run(self):
         self.serve_forever()
 
     def add_task(self, task_id, ipaddr):
-        self.tasks[ipaddr] = task_id
+        with self.task_mgmt_lock:
+            self.tasks[ipaddr] = task_id
 
     def del_task(self, task_id, ipaddr):
-        """Delete ResultServer state and wait for pending RequestHandlers."""
-        if self.tasks.pop(ipaddr, None) is None:
-            log.warning("ResultServer did not have a task with ID %s",
-                        task_id)
-
-        with self.handler_lock:
-            socks = self.handlers.pop(task_id, set())
-            if socks:
-                log.warning("Cancel %s socket(s) for task %r",
-                            len(socks), task_id)
-            for sock in socks:
-                sock.close()
+        """Delete ResultServer state and abort pending RequestHandlers. Since
+        we're about to shutdown the VM, any remaining open connections can
+        be considered a bug from the VM side, since all connections should
+        have been closed after the analyzer signalled completion."""
+        with self.task_mgmt_lock:
+            if self.tasks.pop(ipaddr, None) is None:
+                log.warning("ResultServer did not have a task with ID %s",
+                            task_id)
+            ctxs = self.handlers.pop(task_id, set())
+            for ctx in ctxs:
+                log.warning("Cancel %s for task %r", ctx, task_id)
+                ctx.cancel()
 
     def handle(self, sock, addr):
+        """Handle the incoming connection.
+        Gevent will close the socket when the function returns."""
         ipaddr = addr[0]
-        task_id = self.tasks.get(ipaddr)
-        if not task_id:
-            log.warning("ResultServer did not have a task for IP %s", ipaddr)
-            return
+
+        with self.task_mgmt_lock:
+            task_id = self.tasks.get(ipaddr)
+            if not task_id:
+                log.warning("ResultServer did not have a task for IP %s",
+                            ipaddr)
+                return
 
         storagepath = cwd(analysis=task_id)
-        ctx = HandlerContext(storagepath, sock)
+        ctx = HandlerContext(task_id, storagepath, sock)
         task_log_start(task_id)
         try:
-            protocol = self.negotiate_protocol(ctx)
-
-            # Registering the socket allows us to cancel the handler by
-            # closing the socket when the task is deleted
-            with self.handler_lock:
+            protocol = self.negotiate_protocol(task_id, ctx)
+
+            # Registering the context allows us to abort the handler by
+            # shutting down its socket when the task is deleted; this should
+            # prevent lingering sockets
+            with self.task_mgmt_lock:
+                # NOTE: the task may have been cancelled during the negotation
+                # protocol and a different task for that IP address may have
+                # been registered
+                if self.tasks.get(ipaddr) != task_id:
+                    log.warning("Task #%s for IP %s was cancelled during "
+                                "negotiation", task_id, ipaddr)
+                    return
                 s = self.handlers.setdefault(task_id, set())
-                s.add(sock)
+                s.add(ctx)
 
-            protocol.task_id = task_id  # TODO
-            protocol.init()
             try:
-                protocol.handle()
+                with protocol:
+                    protocol.handle()
             finally:
-                protocol.close()
-                with self.handler_lock:
-                    s.discard(sock)
+                with self.task_mgmt_lock:
+                    s.discard(ctx)
+                ctx.cancel()
+                if ctx.buf:
+                    # This is usually not a good sign
+                    log.warning("Task #%s with protocol %s has unprocessed "
+                                "data before getting disconnected",
+                                task_id, protocol)
 
         finally:
-            sock.close()
             task_log_stop(task_id)
 
-    def negotiate_protocol(self, ctx):
+    def negotiate_protocol(self, task_id, ctx):
         header = ctx.read_newline()
         if " " in header:
             command, version = header.split()
             version = int(version)
         else:
             command, version = header, None
-        log.debug("Incoming command: %r", header)
-        if command not in self.commands:
-            log.warning(
-                "Unknown netlog protocol requested (%r), "
-                "terminating connection.", command
-            )
+        klass = self.commands.get(command)
+        if not klass:
+            log.warning("Task #%s: unknown netlog protocol requested (%r), "
+                        "terminating connection.", self.task_id, command)
             return
-        return self.commands[command](ctx, version)
+        ctx.command = command
+        return klass(task_id, ctx, version)
 
 
 class ResultServer(object):
@@ -405,6 +404,7 @@ def create_bg_server(self):
                                                      spawn=pool)
             self.instance.do_run()
         except OSError as e:
+            # TODO: this currently does not kill the process itself
             if e.errno == errno.EADDRINUSE:
                 raise CuckooCriticalError(
                     "Cannot bind ResultServer on port %d "
diff --git a/cuckoo/processing/platform/windows.py b/cuckoo/processing/platform/windows.py
index 30aee00e6c..9ec7b9a9c8 100644
--- a/cuckoo/processing/platform/windows.py
+++ b/cuckoo/processing/platform/windows.py
@@ -226,8 +226,8 @@ def handles_path(self, path):
 
     def parse(self, path):
         # Invoke parsing of current log file.
-        parser = BsonParser(open(path, "rb"))
-        parser.init()
+        self.fp = open(path, "rb")   # TODO: no proper cleanup
+        parser = BsonParser(self.fp)
 
         for event in parser:
             if event["type"] == "process":
@@ -255,8 +255,8 @@ def parse(self, path):
 
                 # Process the reboot reconstructor.
                 for category, args in reboot.process_apicall(event):
-                    # TODO Improve this where we have to calculate the "real"
-                    # time again even though we already do this in
+                    # TODO Improve this where we have to calculate the
+                    # "real" time again even though we already do this in
                     # MonitorProcessLog.
                     ts = process["first_seen"] + \
                         datetime.timedelta(0, 0, event["time"] * 1000)
@@ -269,7 +269,8 @@ def parse(self, path):
                     }
 
                 # Indicate that the process has API calls. For more
-                # information on this matter, see also the __nonzero__ above.
+                # information on this matter, see also the __nonzero__
+                # above.
                 process["calls"].has_apicalls = True
 
             yield event
diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py
index 2314592b70..39dfebd995 100644
--- a/tests/test_resultserver.py
+++ b/tests/test_resultserver.py
@@ -42,22 +42,27 @@ def cuckoo_cwd():
 def mock_handler_context(klass, path, lines, data, version=None):
     class FakeContext:
         storagepath = path
+        buf = ''
+        task_id = 1
 
         def read_newline(self):
             if not lines:
                 raise EOFError
             return lines.pop(0)
 
-        def read_any(self):
+        def read(self, size=None):
             if not data:
                 raise EOFError
             return data.pop(0)
 
-        def read(self, size):
-            # TODO: we can test expected sizes here
-            return self.read_any()
+        def copy_to_fd(self, fd, max_size=None):
+            while True:
+                try:
+                    fd.write(self.read())
+                except EOFError:
+                    break
 
-    h = klass(FakeContext(), version)
+    h = klass(1, FakeContext(), version)
     h.init()
     h.handle()
     h.close()

From 66f773274b7774075cd16b4f43836052bb77a590 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Wed, 7 Mar 2018 16:13:08 +0100
Subject: [PATCH 057/138] Reimplement upload limit

---
 cuckoo/core/resultserver.py | 58 ++++++++++++++++++-------------------
 tests/test_resultserver.py  |  5 +++-
 2 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 6548c1e151..06bf8f8240 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -62,9 +62,6 @@ def netlog_sanitize_fname(path):
 class HandlerContext:
     """Holds context for protocol handlers.
 
-    To avoid losing data that was already buffered, handlers should check if
-    the buffer is empty when handling EOFError.
-
     Can safely be cancelled from another thread, though in practice this will
     not occur often -- usually the connection between VM and the ResultServer
     will be reset during shutdown."""
@@ -96,31 +93,6 @@ def read(self):
                 raise
             return ''
 
-    def buffered_read(self, size):
-        """Try to fill the buffer with at most `size` bytes
-        Will keep waiting until we're either cancelled, or until we received
-        something.
-        """
-        while True:
-            try:
-                buf = self.sock.recv(size)
-            except socket.timeout:
-                if self.cancelled:
-                    log.debug("Task #%s: connection was cancelled",
-                              self.task_id)
-                    raise Cancelled
-                continue
-            except socket.error as e:
-                log.debug("Task #%s encountered a socket error: %s",
-                          self.task_id, e)
-                if e.errno != errno.ECONNRESET:
-                    raise
-                raise EOFError
-            break
-        self.buf += buf
-        if not self.buf:
-            raise EOFError
-
     def drain_buffer(self):
         """Drain buffer and end buffering"""
         buf, self.buf = "", None
@@ -143,6 +115,8 @@ def read_newline(self):
             return line
 
     def copy_to_fd(self, fd, max_size=None):
+        if max_size:
+            fd = WriteLimiter(fd, max_size)
         fd.write(self.drain_buffer())
         while True:
             buf = self.read()
@@ -152,6 +126,29 @@ def copy_to_fd(self, fd, max_size=None):
         fd.flush()
 
 
+class WriteLimiter(object):
+    def __init__(self, fd, remain):
+        self.fd = fd
+        self.remain = remain
+        self.warned = False
+
+    def write(self, buf):
+        size = len(buf)
+        write = min(size, self.remain)
+        if write:
+            self.fd.write(buf[:write])
+            self.remain -= write
+        if size and size != write:
+            if not self.warned:
+                log.warning("Uploaded file length larger than upload_max_size, "
+                            "stopping upload.")
+                self.fd.write("... (truncated)")
+                self.warned = True
+
+    def flush(self):
+        self.fd.flush()
+
+
 class FileUpload(ProtocolHandler):
     def init(self):
         self.upload_max_size = config("cuckoo:resultserver:upload_max_size")
@@ -162,6 +159,7 @@ def init(self):
     def handle(self):
         # Read until newline for file path, e.g.,
         # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin
+        self.handler.sock.settimeout(30)
         dump_path = netlog_sanitize_fname(self.handler.read_newline())
 
         if self.version >= 2:
@@ -190,8 +188,10 @@ def handle(self):
                 "filepath": filepath,
                 "pids": pids,
             }), file=f)
+
+        self.handler.sock.settimeout(None)
         try:
-            return self.handler.copy_to_fd(self.fd)
+            return self.handler.copy_to_fd(self.fd, self.upload_max_size)
         finally:
             log.debug("Task #%s uploaded file length: %s", self.task_id,
                       self.fd.tell())
diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py
index 39dfebd995..5537deb731 100644
--- a/tests/test_resultserver.py
+++ b/tests/test_resultserver.py
@@ -15,6 +15,7 @@
 import tempfile
 import shutil
 import json
+from mock import Mock
 
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.common.files import Folders
@@ -62,7 +63,9 @@ def copy_to_fd(self, fd, max_size=None):
                 except EOFError:
                     break
 
-    h = klass(1, FakeContext(), version)
+    ctx = FakeContext()
+    ctx.sock = Mock()
+    h = klass(1, ctx, version)
     h.init()
     h.handle()
     h.close()

From 41ba44e71cee38c969f08a4a5e2d04526e32ca65 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Wed, 7 Mar 2018 17:34:33 +0100
Subject: [PATCH 058/138] Improve coverage and fix bug in drain_buffer

---
 cuckoo/core/resultserver.py |   2 +-
 tests/test_resultserver.py  | 109 +++++++++++++++++++++++++++++++++++-
 2 files changed, 107 insertions(+), 4 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 06bf8f8240..7a8832368c 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -95,7 +95,7 @@ def read(self):
 
     def drain_buffer(self):
         """Drain buffer and end buffering"""
-        buf, self.buf = "", None
+        buf, self.buf = self.buf, None
         return buf
 
     def read_newline(self):
diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py
index 5537deb731..2a9003090e 100644
--- a/tests/test_resultserver.py
+++ b/tests/test_resultserver.py
@@ -15,12 +15,16 @@
 import tempfile
 import shutil
 import json
-from mock import Mock
+import socket
+import mock
+import errno
 
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.common.files import Folders
 from cuckoo.core.log import task_log_start, task_log_stop
-from cuckoo.core.resultserver import RESULT_DIRECTORIES
+from cuckoo.core.resultserver import RESULT_DIRECTORIES, MAX_NETLOG_LINE
+from cuckoo.core.resultserver import HandlerContext
+from cuckoo.core.resultserver import GeventResultServerWorker
 from cuckoo.core.resultserver import FileUpload, LogHandler, BsonStore
 from cuckoo.core.startup import init_logging
 from cuckoo.main import cuckoo_create
@@ -64,13 +68,73 @@ def copy_to_fd(self, fd, max_size=None):
                     break
 
     ctx = FakeContext()
-    ctx.sock = Mock()
+    ctx.sock = mock.Mock()
     h = klass(1, ctx, version)
     h.init()
     h.handle()
     h.close()
     return h
 
+
+class TestHandlerContext(object):
+    def test_pointless_busywork(self):
+        sock = mock.Mock()
+        h = HandlerContext(1, 'does-not-exist', sock)
+        assert repr(h) == '<Context for None>'
+
+        h.cancel()
+        sock.shutdown.assert_called_with(socket.SHUT_RD)
+
+        # Should not raise
+        sock.shutdown.side_effect = socket.error()
+        h.cancel()
+
+        err = socket.error()
+        err.errno = errno.ECONNRESET
+        sock.recv.side_effect = err
+        assert h.read() == ''
+
+        err = socket.error()
+        err.errno = errno.EPIPE
+        sock.recv.side_effect = err
+        with pytest.raises(socket.error) as e:
+            h.read()
+
+    def test_long_line(self):
+        sock = mock.Mock()
+        h = HandlerContext(1, 'does-not-exist', sock)
+        sock.recv.return_value = 'A' * (MAX_NETLOG_LINE + 1)
+        with pytest.raises(CuckooOperationalError) as e:
+            h.read_newline()
+        assert h.buf is sock.recv.return_value
+
+    def test_line_eof(self):
+        sock = mock.Mock()
+        h = HandlerContext(1, 'does-not-exist', sock)
+        sock.recv.return_value = ''
+        with pytest.raises(EOFError) as e:
+            h.read_newline()
+
+    def test_buffer(self):
+        sock = mock.Mock()
+        h = HandlerContext(1, 'does-not-exist', sock)
+        sock.recv.return_value = 'first\nsecond\nthird'
+        assert h.read_newline() == 'first'
+        assert h.buf == 'second\nthird'
+        assert h.drain_buffer() == 'second\nthird'
+        assert h.buf is None
+
+    def test_copy_limited(self):
+        sock = mock.Mock()
+        fd = mock.Mock()
+        h = HandlerContext(1, 'does-not-exist', sock)
+        sock.recv.side_effect = ['A' * 64, '']
+        h.copy_to_fd(fd, 32)
+        fd.write.assert_has_calls([mock.call('A' * 32),
+                                   mock.call('... (truncated)')])
+        assert fd.flush.called
+
+
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestFileUpload(object):
     @pytest.mark.order1
@@ -99,6 +163,16 @@ def test_overwrite(self):
                                  [])
         e.match("overwrite an existing file")
 
+    @mock.patch('cuckoo.core.resultserver.open_exclusive')
+    def test_open_error(self, open_exclusive):
+        err = OSError()
+        err.errno = errno.EACCES
+        open_exclusive.side_effect = err
+        with pytest.raises(OSError):
+            mock_handler_context(FileUpload,
+                                 cwd(analysis=1),
+                                 ['files/any.exe'],
+                                 [])
 
     def test_success_v2(self):
         fu = mock_handler_context(FileUpload,
@@ -160,6 +234,14 @@ def test_reopen(self):
             assert 'WARNING: This log file was re-opened' in data
             assert data.endswith('reopen\n')
 
+    @mock.patch('cuckoo.core.resultserver.open_exclusive')
+    def test_open_error(self, open_exclusive):
+        err = OSError()
+        err.errno = errno.EACCES
+        open_exclusive.side_effect = err
+        with pytest.raises(OSError):
+            mock_handler_context(LogHandler, cwd(analysis=1), [], [])
+
 
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestBsonStore(object):
@@ -172,3 +254,24 @@ def test_success(self):
 
         with open(cwd("logs/1.bson", analysis=1), "rb") as f:
             assert f.read() == "\x01\x00\x00\x00a"
+
+    def test_unversioned(self):
+        h = mock_handler_context(BsonStore, cwd(analysis=1), [], [], None)
+        assert h.fd is None
+
+# Work in progress
+class TestWorkerServer(object):
+    def test_unregistered(self):
+        g = GeventResultServerWorker(('127.0.0.1', 1))
+        sock = mock.Mock()
+        sock.recv.side_effect = IOError
+        g.handle(sock, ('127.0.0.1', 41337))
+        # <no effect>
+
+    def test_negotiate(self):
+        g = GeventResultServerWorker(('127.0.0.1', 1))
+        g.add_task(1, '127.0.0.1')
+        assert g.tasks == {'127.0.0.1': 1}
+        sock = mock.Mock()
+        sock.recv.side_effect = ["LOG\n", "Hello\n", ""]
+        g.handle(sock, ('127.0.0.1', 41337))

From 5f3305ed7cc880e1f169f6818f20e886aea653f0 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Thu, 8 Mar 2018 11:07:12 +0100
Subject: [PATCH 059/138] Close Netlog LOG connection at end of analysis

---
 cuckoo/data/analyzer/windows/analyzer.py          |  5 ++++-
 .../data/analyzer/windows/lib/common/results.py   |  1 +
 cuckoo/data/analyzer/windows/lib/core/startup.py  | 15 +++++++++++----
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/cuckoo/data/analyzer/windows/analyzer.py b/cuckoo/data/analyzer/windows/analyzer.py
index 0a3868ae71..a8ae1641d6 100644
--- a/cuckoo/data/analyzer/windows/analyzer.py
+++ b/cuckoo/data/analyzer/windows/analyzer.py
@@ -32,7 +32,7 @@
 from lib.core.packages import choose_package
 from lib.core.pipe import PipeServer, PipeForwarder, PipeDispatcher
 from lib.core.privileges import grant_privilege
-from lib.core.startup import init_logging, set_clock
+from lib.core.startup import init_logging, disconnect_logger, set_clock
 from modules import auxiliary
 
 log = logging.getLogger("analyzer")
@@ -524,6 +524,9 @@ def complete(self):
         # Hell yeah.
         log.info("Analysis completed.")
 
+        # Cleanly close remaining connections
+        disconnect_logger()
+
     def run(self):
         """Run analysis.
         @return: operation status.
diff --git a/cuckoo/data/analyzer/windows/lib/common/results.py b/cuckoo/data/analyzer/windows/lib/common/results.py
index ba5a602c68..0a506a0455 100644
--- a/cuckoo/data/analyzer/windows/lib/common/results.py
+++ b/cuckoo/data/analyzer/windows/lib/common/results.py
@@ -77,6 +77,7 @@ def send(self, data, retry=True):
     def close(self):
         try:
             self.sock.close()
+            self.sock = None
         except Exception:
             pass
 
diff --git a/cuckoo/data/analyzer/windows/lib/core/startup.py b/cuckoo/data/analyzer/windows/lib/core/startup.py
index adc102bdac..c5f514d7f4 100644
--- a/cuckoo/data/analyzer/windows/lib/core/startup.py
+++ b/cuckoo/data/analyzer/windows/lib/core/startup.py
@@ -7,25 +7,32 @@
 import logging
 
 from lib.common.defines import KERNEL32, SYSTEMTIME
-from lib.common.results import NetlogHandler
+from lib.common.results import NetlogHandler, NetlogConnection
 
 log = logging.getLogger()
+netlog_handler = None
 
 def init_logging():
     """Initialize logger."""
     formatter = logging.Formatter(
         "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
     )
+
     sh = logging.StreamHandler()
     sh.setFormatter(formatter)
     log.addHandler(sh)
 
-    nh = NetlogHandler()
-    nh.setFormatter(formatter)
-    log.addHandler(nh)
+    global netlog_handler
+    netlog_handler = NetlogHandler()
+    netlog_handler.setFormatter(formatter)
+    log.addHandler(netlog_handler)
 
     log.setLevel(logging.DEBUG)
 
+def disconnect_logger():
+    """Cleanly close the logger. Note that LogHandler also implements close."""
+    NetlogConnection.close(netlog_handler)
+
 def set_clock(clock):
     st = SYSTEMTIME()
     st.wYear = clock.year

From f3abcd0260d880c2ce4cb934daa7812e08882eec Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Thu, 8 Mar 2018 11:46:57 +0100
Subject: [PATCH 060/138] Open socket in main thread; do not allow LOG reopen

---
 cuckoo/core/resultserver.py | 97 ++++++++++++++-----------------------
 tests/test_resultserver.py  | 37 ++++++--------
 2 files changed, 52 insertions(+), 82 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 7a8832368c..d9b5158b30 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -45,7 +45,6 @@
 # Data Streams); XXX: just replace illegal chars?
 BANNED_PATH_CHARS = b'\x00:'
 
-
 def netlog_sanitize_fname(path):
     """Validate agent-provided path for result files"""
     path = path.replace("\\", "/")
@@ -58,8 +57,7 @@ def netlog_sanitize_fname(path):
                                      % path)
     return path
 
-
-class HandlerContext:
+class HandlerContext(object):
     """Holds context for protocol handlers.
 
     Can safely be cancelled from another thread, though in practice this will
@@ -72,10 +70,10 @@ def __init__(self, task_id, storagepath, sock):
         # The path where artifacts will be stored
         self.storagepath = storagepath
         self.sock = sock
-        self.buf = ''
+        self.buf = ""
 
     def __repr__(self):
-        return '<Context for %s>' % self.command
+        return "<Context for %s>" % self.command
 
     def cancel(self):
         """Cancel this context; gevent might complain about this with an
@@ -91,7 +89,9 @@ def read(self):
         except socket.error as e:
             if e.errno != errno.ECONNRESET:
                 raise
-            return ''
+            log.debug("Task #%s had connection reset for %r", self.task_id,
+                      self)
+            return ""
 
     def drain_buffer(self):
         """Drain buffer and end buffering"""
@@ -107,7 +107,7 @@ def read_newline(self):
                 if len(self.buf) >= MAX_NETLOG_LINE:
                     raise CuckooOperationalError("Received overly long line")
                 buf = self.read()
-                if buf == '':
+                if buf == "":
                     raise EOFError
                 self.buf += buf
                 continue
@@ -120,12 +120,11 @@ def copy_to_fd(self, fd, max_size=None):
         fd.write(self.drain_buffer())
         while True:
             buf = self.read()
-            if buf == '':
+            if buf == "":
                 break
             fd.write(buf)
         fd.flush()
 
-
 class WriteLimiter(object):
     def __init__(self, fd, remain):
         self.fd = fd
@@ -148,7 +147,6 @@ def write(self, buf):
     def flush(self):
         self.fd.flush()
 
-
 class FileUpload(ProtocolHandler):
     def init(self):
         self.upload_max_size = config("cuckoo:resultserver:upload_max_size")
@@ -181,7 +179,7 @@ def handle(self):
                                              self.task_id)
             raise
 
-        # !! race condition !!
+        # Race condition! This needs a lock per task.
         with open(self.filelog, "a+b") as f:
             print(json.dumps({
                 "path": dump_path,
@@ -196,12 +194,12 @@ def handle(self):
             log.debug("Task #%s uploaded file length: %s", self.task_id,
                       self.fd.tell())
 
-
 class LogHandler(ProtocolHandler):
-    # TODO: not protected against opening multiple times
+    """The live analysis log. Can only be opened once in a single session."""
+
     def init(self):
         self.logpath = os.path.join(self.handler.storagepath, "analysis.log")
-        self.fd = self._open()
+        self.fd = open_exclusive(self.logpath)
         log.debug("Task #%s: live log analysis.log initialized.",
                   self.task_id)
 
@@ -209,30 +207,6 @@ def handle(self):
         if self.fd:
             return self.handler.copy_to_fd(self.fd)
 
-    def _open(self):
-        try:
-            return open_exclusive(self.logpath)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
-
-        log.debug("Task #%s: Log analysis.log already existing, "
-                  "appending data.", self.task_id)
-        fd = open(self.logpath, "ab")
-
-        # Add a fake log entry, saying this had to be re-opened. Use the same
-        # format as the default logger, in case anyone wants to parse this:
-        #  2015-02-23 12:05:05,092 [lib.api.process] DEBUG: Using QueueUserAPC
-        #   injection.
-        now = datetime.datetime.now()
-        print("\n", now.strftime("%Y-%m-%d %H:%M:%S",),
-              now.microsecond / 1000.0,
-              " [lib.core.resultserver] WARNING: This log file was re-opened,"
-              " log entries will be appended.",
-              sep='', file=fd)
-        return fd
-
-
 class BsonStore(ProtocolHandler):
     def init(self):
         # We cheat a little bit through the "version" variable, but that's
@@ -255,7 +229,6 @@ def handle(self):
         if self.fd:
             return self.handler.copy_to_fd(self.fd)
 
-
 class GeventResultServerWorker(gevent.server.StreamServer):
     """The new ResultServer, providing a huge performance boost as well as
     implementing a new dropped file storage format avoiding small fd limits.
@@ -370,25 +343,12 @@ def negotiate_protocol(self, task_id, ctx):
         ctx.command = command
         return klass(task_id, ctx, version)
 
-
 class ResultServer(object):
     """Manager for the ResultServer worker and task state."""
     __metaclass__ = Singleton
 
     def __init__(self):
-        self.thread = threading.Thread(target=self.create_bg_server)
-        self.thread.daemon = True
-        self.thread.start()
-
-    def add_task(self, task, machine):
-        """Register a task/machine with the ResultServer."""
-        self.instance.add_task(task.id, machine.ip)
-
-    def del_task(self, task, machine):
-        """Delete running task and cancel existing handlers."""
-        self.instance.del_task(task.id, machine.ip)
-
-    def create_bg_server(self):
+        # TODO: support binding to port 0 for random port
         ip = config("cuckoo:resultserver:ip")
         port = self.port = config("cuckoo:resultserver:port")
         pool_size = config('cuckoo:resultserver:poolsize')
@@ -397,14 +357,12 @@ def create_bg_server(self):
         else:
             pool_size = 128
 
-        pool = gevent.pool.Pool(pool_size)
+        sock = gevent.socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
         try:
-            # TODO: support binding to port 0 for random port
-            self.instance = GeventResultServerWorker((ip, port),
-                                                     spawn=pool)
-            self.instance.do_run()
-        except OSError as e:
-            # TODO: this currently does not kill the process itself
+            sock.bind((ip, port))
+        except (OSError, socket.error) as e:
             if e.errno == errno.EADDRINUSE:
                 raise CuckooCriticalError(
                     "Cannot bind ResultServer on port %d "
@@ -424,3 +382,22 @@ def create_bg_server(self):
                     "Unable to bind ResultServer on %s:%s: %s" %
                     (ip, port, e)
                 )
+        sock.listen(pool_size)
+
+        self.thread = threading.Thread(target=self.create_server,
+                                       args=(sock, pool_size))
+        self.thread.daemon = True
+        self.thread.start()
+
+    def add_task(self, task, machine):
+        """Register a task/machine with the ResultServer."""
+        self.instance.add_task(task.id, machine.ip)
+
+    def del_task(self, task, machine):
+        """Delete running task and cancel existing handlers."""
+        self.instance.del_task(task.id, machine.ip)
+
+    def create_server(self, sock, pool_size):
+        pool = gevent.pool.Pool(pool_size)
+        self.instance = GeventResultServerWorker(sock, spawn=pool)
+        self.instance.do_run()
diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py
index 2a9003090e..92251c1b77 100644
--- a/tests/test_resultserver.py
+++ b/tests/test_resultserver.py
@@ -10,27 +10,26 @@
 # - Invalid path tests
 # - Double LOG command
 
+import errno
+import json
 import logging
+import mock
 import pytest
-import tempfile
 import shutil
-import json
 import socket
-import mock
-import errno
+import tempfile
 
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.common.files import Folders
 from cuckoo.core.log import task_log_start, task_log_stop
-from cuckoo.core.resultserver import RESULT_DIRECTORIES, MAX_NETLOG_LINE
-from cuckoo.core.resultserver import HandlerContext
-from cuckoo.core.resultserver import GeventResultServerWorker
 from cuckoo.core.resultserver import FileUpload, LogHandler, BsonStore
+from cuckoo.core.resultserver import GeventResultServerWorker
+from cuckoo.core.resultserver import HandlerContext
+from cuckoo.core.resultserver import RESULT_DIRECTORIES, MAX_NETLOG_LINE
 from cuckoo.core.startup import init_logging
 from cuckoo.main import cuckoo_create
 from cuckoo.misc import mkdir, set_cwd, cwd
 
-
 @pytest.fixture(scope='module')
 def cuckoo_cwd():
     """Create a temporary Cuckoo working directory"""
@@ -43,7 +42,6 @@ def cuckoo_cwd():
     yield path
     shutil.rmtree(path)
 
-
 def mock_handler_context(klass, path, lines, data, version=None):
     class FakeContext:
         storagepath = path
@@ -75,7 +73,6 @@ def copy_to_fd(self, fd, max_size=None):
     h.close()
     return h
 
-
 class TestHandlerContext(object):
     def test_pointless_busywork(self):
         sock = mock.Mock()
@@ -134,7 +131,6 @@ def test_copy_limited(self):
                                    mock.call('... (truncated)')])
         assert fd.flush.called
 
-
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestFileUpload(object):
     @pytest.mark.order1
@@ -209,7 +205,6 @@ def test_invalid_paths(self):
         self.invalid_path("../hello")
         self.invalid_path("../../foobar")
 
-
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestLogHandler(object):
     @pytest.mark.order1
@@ -224,15 +219,13 @@ def test_success(self):
 
     @pytest.mark.order2
     def test_reopen(self):
-        mock_handler_context(LogHandler,
-                             cwd(analysis=1),
-                             [],
-                             ['reopen\n'])
+        with pytest.raises(OSError) as e:
+            mock_handler_context(LogHandler,
+                                 cwd(analysis=1),
+                                 [],
+                                 ['reopen\n'])
 
-        with open(cwd("analysis.log", analysis=1), "rb") as f:
-            data = f.read()
-            assert 'WARNING: This log file was re-opened' in data
-            assert data.endswith('reopen\n')
+        assert e.value.errno == errno.EEXIST
 
     @mock.patch('cuckoo.core.resultserver.open_exclusive')
     def test_open_error(self, open_exclusive):
@@ -242,7 +235,6 @@ def test_open_error(self, open_exclusive):
         with pytest.raises(OSError):
             mock_handler_context(LogHandler, cwd(analysis=1), [], [])
 
-
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestBsonStore(object):
     def test_success(self):
@@ -260,6 +252,7 @@ def test_unversioned(self):
         assert h.fd is None
 
 # Work in progress
+@pytest.mark.usefixtures('cuckoo_cwd')
 class TestWorkerServer(object):
     def test_unregistered(self):
         g = GeventResultServerWorker(('127.0.0.1', 1))
@@ -273,5 +266,5 @@ def test_negotiate(self):
         g.add_task(1, '127.0.0.1')
         assert g.tasks == {'127.0.0.1': 1}
         sock = mock.Mock()
-        sock.recv.side_effect = ["LOG\n", "Hello\n", ""]
+        sock.recv.side_effect = ["FILE\n", "files/example.txt\n", "hello", ""]
         g.handle(sock, ('127.0.0.1', 41337))

From 7ce992b87f5e447acd9057c43fc8099c127d7c55 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Thu, 8 Mar 2018 12:35:01 +0100
Subject: [PATCH 061/138] Use line-buffering for live analysis log

---
 cuckoo/common/files.py      | 6 ++----
 cuckoo/core/resultserver.py | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/cuckoo/common/files.py b/cuckoo/common/files.py
index ddaf8d57e7..469f66d8b1 100644
--- a/cuckoo/common/files.py
+++ b/cuckoo/common/files.py
@@ -25,18 +25,16 @@ def temppath():
 
     return tmppath
 
-
-def open_exclusive(path, mode='wb'):
+def open_exclusive(path, mode='wb', bufsize=-1):
     """Open a file with O_EXCL, failing if it already exists
     [In Python 3, use open with x]"""
     fd = os.open(path, os.O_CREAT|os.O_EXCL|os.O_WRONLY)
     try:
-        return os.fdopen(fd, mode)
+        return os.fdopen(fd, mode, bufsize)
     except:
         os.close(fd)
         raise
 
-
 class Storage(object):
     @staticmethod
     def get_filename_from_path(path):
diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index d9b5158b30..98a7226080 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -199,7 +199,7 @@ class LogHandler(ProtocolHandler):
 
     def init(self):
         self.logpath = os.path.join(self.handler.storagepath, "analysis.log")
-        self.fd = open_exclusive(self.logpath)
+        self.fd = open_exclusive(self.logpath, bufsize=1)
         log.debug("Task #%s: live log analysis.log initialized.",
                   self.task_id)
 

From f10549cbf70b58f47b8e1e76d2bb486c2fff8272 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Thu, 8 Mar 2018 13:01:12 +0100
Subject: [PATCH 062/138] Do not make files created by open_exclusive
 executable

---
 cuckoo/common/files.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cuckoo/common/files.py b/cuckoo/common/files.py
index 469f66d8b1..f67de03f15 100644
--- a/cuckoo/common/files.py
+++ b/cuckoo/common/files.py
@@ -28,7 +28,7 @@ def temppath():
 def open_exclusive(path, mode='wb', bufsize=-1):
     """Open a file with O_EXCL, failing if it already exists
     [In Python 3, use open with x]"""
-    fd = os.open(path, os.O_CREAT|os.O_EXCL|os.O_WRONLY)
+    fd = os.open(path, os.O_CREAT|os.O_EXCL|os.O_WRONLY, 0644)
     try:
         return os.fdopen(fd, mode, bufsize)
     except:

From 28fe5307d5237fbc661919e23d87b831927dca83 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Thu, 8 Mar 2018 14:04:04 +0100
Subject: [PATCH 063/138] Close all PipeServer sockets when stopping Windows
 analyzer

---
 cuckoo/core/resultserver.py                   |  8 ++++----
 cuckoo/data/analyzer/windows/analyzer.py      |  2 ++
 cuckoo/data/analyzer/windows/lib/core/pipe.py | 19 ++++++++++++++-----
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 98a7226080..a4b650ed0d 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -5,22 +5,22 @@
 
 from __future__ import print_function
 
-import socket
-import errno
 import datetime
-import gevent.server
+import errno
 import gevent.pool
+import gevent.server
 import gevent.socket
 import json
 import logging
 import os
+import socket
 import struct
 import threading
 
 from cuckoo.common.abstracts import ProtocolHandler
 from cuckoo.common.config import config
-from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.common.exceptions import CuckooCriticalError
+from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.common.exceptions import CuckooResultError
 from cuckoo.common.files import Folders, open_exclusive
 from cuckoo.common.utils import Singleton
diff --git a/cuckoo/data/analyzer/windows/analyzer.py b/cuckoo/data/analyzer/windows/analyzer.py
index a8ae1641d6..1ba3066821 100644
--- a/cuckoo/data/analyzer/windows/analyzer.py
+++ b/cuckoo/data/analyzer/windows/analyzer.py
@@ -31,6 +31,7 @@
 from lib.core.ioctl import zer0m0n
 from lib.core.packages import choose_package
 from lib.core.pipe import PipeServer, PipeForwarder, PipeDispatcher
+from lib.core.pipe import disconnect_pipes
 from lib.core.privileges import grant_privilege
 from lib.core.startup import init_logging, disconnect_logger, set_clock
 from modules import auxiliary
@@ -525,6 +526,7 @@ def complete(self):
         log.info("Analysis completed.")
 
         # Cleanly close remaining connections
+        disconnect_pipes()
         disconnect_logger()
 
     def run(self):
diff --git a/cuckoo/data/analyzer/windows/lib/core/pipe.py b/cuckoo/data/analyzer/windows/lib/core/pipe.py
index 26594266ee..32a24fabee 100644
--- a/cuckoo/data/analyzer/windows/lib/core/pipe.py
+++ b/cuckoo/data/analyzer/windows/lib/core/pipe.py
@@ -19,6 +19,7 @@
 log = logging.getLogger(__name__)
 
 BUFSIZE = 0x10000
+open_handles = set()
 
 class PipeForwarder(threading.Thread):
     """Forward all data received from a local pipe to the Cuckoo
@@ -63,16 +64,17 @@ def run(self):
             return
 
         if pid.value:
-            if pid.value not in self.sockets:
-                self.sockets[pid.value] = (
-                    socket.create_connection(self.destination)
-                )
+            sock = self.sockets.get(pid.value)
+            if not sock:
+                sock = socket.create_connection(self.destination)
+                self.sockets[pid.value] = sock
 
-            sock = self.sockets[pid.value]
             self.active[pid.value] = True
         else:
             sock = socket.create_connection(self.destination)
 
+        open_handles.add(sock)
+
         while True:
             success = KERNEL32.ReadFile(
                 self.pipe_handle, byref(buf), sizeof(buf),
@@ -187,3 +189,10 @@ def run(self):
 
     def stop(self):
         self.do_run = False
+
+def disconnect_pipes():
+    for sock in open_handles:
+        try:
+            sock.close()
+        except:
+            log.exception("Could not close socket")

From dd689daf668d73e4a61604b739712340d3f4b174 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Thu, 8 Mar 2018 14:12:55 +0100
Subject: [PATCH 064/138] Use global lock for files.json for safety

---
 cuckoo/core/resultserver.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index a4b650ed0d..9dcd054b09 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -45,6 +45,10 @@
 # Data Streams); XXX: just replace illegal chars?
 BANNED_PATH_CHARS = b'\x00:'
 
+# Safeguard againt multiple writes to `files.json` -- note that this really
+# should have per-task granularity
+filelist_lock = threading.Lock()
+
 def netlog_sanitize_fname(path):
     """Validate agent-provided path for result files"""
     path = path.replace("\\", "/")
@@ -179,13 +183,13 @@ def handle(self):
                                              self.task_id)
             raise
 
-        # Race condition! This needs a lock per task.
-        with open(self.filelog, "a+b") as f:
-            print(json.dumps({
-                "path": dump_path,
-                "filepath": filepath,
-                "pids": pids,
-            }), file=f)
+        with filelist_lock:
+            with open(self.filelog, "a+b") as f:
+                print(json.dumps({
+                    "path": dump_path,
+                    "filepath": filepath,
+                    "pids": pids,
+                }), file=f)
 
         self.handler.sock.settimeout(None)
         try:

From 65dba7a4922997c1d68f5d5d413279d8c199cd4f Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Thu, 8 Mar 2018 16:10:56 +0100
Subject: [PATCH 065/138] Remove unused `force_port` for ResultServer and allow
 `port = 0`

---
 cuckoo/common/config.py                 | 2 +-
 cuckoo/compat/config.py                 | 1 -
 cuckoo/core/resultserver.py             | 7 +++++--
 cuckoo/private/cwd/conf/cuckoo.conf     | 8 ++------
 docs/book/_files/conf/cuckoo.conf       | 8 ++------
 tests/files/conf/20c2_plain/cuckoo.conf | 5 -----
 tests/test_config.py                    | 6 +++---
 7 files changed, 13 insertions(+), 24 deletions(-)

diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py
index 31e711c475..99d83e3844 100644
--- a/cuckoo/common/config.py
+++ b/cuckoo/common/config.py
@@ -243,7 +243,7 @@ class Config(object):
             "resultserver": {
                 "ip": String("192.168.56.1"),
                 "port": Int(2042),
-                "force_port": Boolean(False),
+                "force_port": Boolean(False, False),  # Unused
                 "upload_max_size": Int(128 * 1024 * 1024),
             },
             "processing": {
diff --git a/cuckoo/compat/config.py b/cuckoo/compat/config.py
index b690a22bf2..cb90d9b881 100644
--- a/cuckoo/compat/config.py
+++ b/cuckoo/compat/config.py
@@ -481,7 +481,6 @@ def _20c1_20c2(c):
     }
     c["cuckoo"]["routing"]["rt_table"] = "main"
     c["cuckoo"]["routing"]["auto_rt"] = True
-    c["cuckoo"]["resultserver"]["force_port"] = False
     if c["cuckoo"]["timeouts"]["critical"] == 600:
         c["cuckoo"]["timeouts"]["critical"] = 60
     c["processing"]["misp"] = {
diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 9dcd054b09..ec036932b4 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -352,9 +352,8 @@ class ResultServer(object):
     __metaclass__ = Singleton
 
     def __init__(self):
-        # TODO: support binding to port 0 for random port
         ip = config("cuckoo:resultserver:ip")
-        port = self.port = config("cuckoo:resultserver:port")
+        port = config("cuckoo:resultserver:port")
         pool_size = config('cuckoo:resultserver:poolsize')
         if pool_size:
             pool_size = int(pool_size)
@@ -386,6 +385,10 @@ def __init__(self):
                     "Unable to bind ResultServer on %s:%s: %s" %
                     (ip, port, e)
                 )
+
+        # We allow user to specify port 0 to get a random port, report it back
+        # here
+        _, self.port = sock.getsockname()
         sock.listen(pool_size)
 
         self.thread = threading.Thread(target=self.create_server,
diff --git a/cuckoo/private/cwd/conf/cuckoo.conf b/cuckoo/private/cwd/conf/cuckoo.conf
index 2d77890035..2b8095c246 100644
--- a/cuckoo/private/cwd/conf/cuckoo.conf
+++ b/cuckoo/private/cwd/conf/cuckoo.conf
@@ -104,14 +104,10 @@ email = {{ cuckoo.feedback.email }}
 # `resultserver_ip` for all your virtual machines in machinery configuration.
 ip = {{ cuckoo.resultserver.ip }}
 
-# Specify a port number to bind the result server on.
+# Specify a port number to bind the result server on. Set to 0 to use a random
+# port.
 port = {{ cuckoo.resultserver.port }}
 
-# Force the port chosen above, don't try another one (we can select another
-# port dynamically if we can not bind this one, but that is not an option
-# in some setups)
-force_port = {{ cuckoo.resultserver.force_port }}
-
 # Maximum size of uploaded files from VM (screenshots, dropped files, log).
 # The value is expressed in bytes, by default 128 MB.
 upload_max_size = {{ cuckoo.resultserver.upload_max_size }}
diff --git a/docs/book/_files/conf/cuckoo.conf b/docs/book/_files/conf/cuckoo.conf
index d7da543187..201f3b4ca3 100644
--- a/docs/book/_files/conf/cuckoo.conf
+++ b/docs/book/_files/conf/cuckoo.conf
@@ -92,14 +92,10 @@ email =
 # `resultserver_ip` for all your virtual machines in machinery configuration.
 ip = 192.168.56.1
 
-# Specify a port number to bind the result server on.
+# Specify a port number to bind the result server on. Set to 0 to use a random
+# port.
 port = 2042
 
-# Force the port chosen above, don't try another one (we can select another
-# port dynamically if we can not bind this one, but that is not an option
-# in some setups)
-force_port = no
-
 # Maximum size of uploaded files from VM (screenshots, dropped files, log).
 # The value is expressed in bytes, by default 128 MB.
 upload_max_size = 134217728
diff --git a/tests/files/conf/20c2_plain/cuckoo.conf b/tests/files/conf/20c2_plain/cuckoo.conf
index ca14060473..09d98b45f3 100644
--- a/tests/files/conf/20c2_plain/cuckoo.conf
+++ b/tests/files/conf/20c2_plain/cuckoo.conf
@@ -113,11 +113,6 @@ ip = 192.168.56.1
 # Specify a port number to bind the result server on.
 port = 2042
 
-# Force the port chosen above, don't try another one (we can select another
-# port dynamically if we can not bind this one, but that is not an option
-# in some setups)
-force_port = no
-
 # Maximum size of uploaded files from VM (screenshots, dropped files, log)
 # The value is expressed in bytes, by default 10Mb.
 upload_max_size = 10485760
diff --git a/tests/test_config.py b/tests/test_config.py
index cfd0b99be3..b52f4638e8 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -121,7 +121,7 @@ def test_env():
 rooter = /tmp/cuckoo-rooter
 tmppath =
 [resultserver]
-force_port = no
+port = 1234
 [database]
 connection =
 timeout =
@@ -169,7 +169,7 @@ def test_boolean_parse(self):
         """Testing the boolean parsing in the configuration file parsing."""
         assert self.cuckoo.get("cuckoo")["version_check"] is True
         assert self.cuckoo.get("cuckoo")["max_analysis_count"] is not False
-        assert self.cuckoo.get("resultserver")["force_port"] is False
+        assert self.cuckoo.get("resultserver")["port"] == 1234
 
     def test_path_parse(self):
         """Testing the Path parsing in the configuration file parsing."""
@@ -876,7 +876,7 @@ def test_migration_20c1_20c2():
     assert cfg["auxiliary"]["reboot"]["enabled"] is True
     assert cfg["cuckoo"]["routing"]["rt_table"] == "main"
     assert cfg["cuckoo"]["routing"]["auto_rt"] is True
-    assert cfg["cuckoo"]["resultserver"]["force_port"] is False
+    assert cfg["cuckoo"]["resultserver"]["port"] == 2042
     assert cfg["cuckoo"]["timeouts"]["critical"] == 60
     assert cfg["processing"]["misp"]["enabled"] is False
     assert cfg["processing"]["misp"]["url"] is None

From 9b8cf6a3acf8543232819d255e6bf6275c88559d Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Fri, 9 Mar 2018 12:15:03 +0100
Subject: [PATCH 066/138] Increase & warn about resource limits; add hidden
 pool_size option for gevent

---
 cuckoo/common/config.py     |  1 +
 cuckoo/core/resultserver.py | 13 ++++++-------
 cuckoo/main.py              | 31 +++++++++++++++++++++++++++++++
 3 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py
index 99d83e3844..48fb7e3dd9 100644
--- a/cuckoo/common/config.py
+++ b/cuckoo/common/config.py
@@ -244,6 +244,7 @@ class Config(object):
                 "ip": String("192.168.56.1"),
                 "port": Int(2042),
                 "force_port": Boolean(False, False),  # Unused
+                "pool_size": Int(0, False),
                 "upload_max_size": Int(128 * 1024 * 1024),
             },
             "processing": {
diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index ec036932b4..90dacdc4ee 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -354,11 +354,7 @@ class ResultServer(object):
     def __init__(self):
         ip = config("cuckoo:resultserver:ip")
         port = config("cuckoo:resultserver:port")
-        pool_size = config('cuckoo:resultserver:poolsize')
-        if pool_size:
-            pool_size = int(pool_size)
-        else:
-            pool_size = 128
+        pool_size = config('cuckoo:resultserver:pool_size')
 
         sock = gevent.socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -389,7 +385,7 @@ def __init__(self):
         # We allow user to specify port 0 to get a random port, report it back
         # here
         _, self.port = sock.getsockname()
-        sock.listen(pool_size)
+        sock.listen(128)
 
         self.thread = threading.Thread(target=self.create_server,
                                        args=(sock, pool_size))
@@ -405,6 +401,9 @@ def del_task(self, task, machine):
         self.instance.del_task(task.id, machine.ip)
 
     def create_server(self, sock, pool_size):
-        pool = gevent.pool.Pool(pool_size)
+        if pool_size:
+            pool = gevent.pool.Pool(pool_size)
+        else:
+            pool = 'default'
         self.instance = GeventResultServerWorker(sock, spawn=pool)
         self.instance.do_run()
diff --git a/cuckoo/main.py b/cuckoo/main.py
index 03958f3e4d..040466d70f 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -89,6 +89,34 @@ def _ignore_first_makedirs(dst):
             open(cwd("cwd", "init-post.jinja2", private=True), "rb").read()
         ).render()
 
+def cuckoo_resources():
+    try:
+        import resource
+    except ImportError:
+        # Only Unix platforms have this option; should not be an issue on
+        # Windows
+        return
+
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if soft != hard:
+        log.debug("Increasing resource limit for number of open files to %s",
+                  hard if hard != resource.RLIM_INFINITY else '[unlimited]')
+        resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+
+    if hard != resource.RLIM_INFINITY:
+        # This can really affect the stability of Cuckoo, so the user should
+        # really fix it.  TODO: find a good minimum.
+        if hard <= 4096:
+            log.error("The maximum number of open files is low (%s). If you "
+                      "do not increase it, you may run into errors later "
+                      "on.", hard)
+            log.error("See also: https://cuckoo.sh/docs/faq/index.html#"
+                      "ioerror-errno-24-too-many-open-files")
+
+    # IDEAS:
+    # Check if limit is realistic versus the number of VMs
+    # Same for pool_size
+
 def cuckoo_init(level, ctx, cfg=None):
     """Initialize Cuckoo configuration.
     @param quiet: enable quiet mode.
@@ -113,6 +141,9 @@ def cuckoo_init(level, ctx, cfg=None):
 
     init_console_logging(level)
 
+    # Make sure user is aware of potential resource limits
+    cuckoo_resources()
+
     # Only one Cuckoo process should exist per CWD. Run this check before any
     # files are possibly modified. Note that we mkdir $CWD/pidfiles/ here as
     # its CWD migration rules only kick in after the pidfile check.

From 057e7b7cc143df6dfcdbef0f8bdaccb436faeab0 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Mon, 12 Mar 2018 11:57:07 +0100
Subject: [PATCH 067/138] Fix current task-to-thread mapping for gevent

---
 cuckoo/core/log.py          | 24 +++++++++++++++---------
 cuckoo/core/resultserver.py | 18 +++++++-----------
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/cuckoo/core/log.py b/cuckoo/core/log.py
index 7c3287c4db..54a13b268c 100644
--- a/cuckoo/core/log.py
+++ b/cuckoo/core/log.py
@@ -3,6 +3,7 @@
 # See the file 'docs/LICENSE' for copying permission.
 
 import copy
+import gevent.thread
 import json
 import logging
 import logging.handlers
@@ -22,6 +23,10 @@
 else:
     tz = time.timezone / -3600.
 
+# The greenlet library (used by Gevent) also creates some state per thread,
+# so we can (ab)use this for both multi-threading and Gevent code
+task_key = gevent.thread.get_ident
+
 class DatabaseHandler(logging.Handler):
     """Logging to database handler.
     Used to log errors related to tasks in database.
@@ -41,12 +46,11 @@ class TaskHandler(logging.Handler):
     """
 
     def emit(self, record):
-        task_id = _tasks.get(thread.get_ident())
-        if not task_id:
+        task = _tasks.get(task_key())
+        if not task:
             return
 
-        with open(cwd("cuckoo.log", analysis=task_id), "a+b") as f:
-            f.write("%s\n" % self.format(record))
+        task[1].write("%s\n" % self.format(record))
 
 class ConsoleHandler(logging.StreamHandler):
     """Logging to console handler."""
@@ -72,9 +76,8 @@ class JsonFormatter(logging.Formatter):
     def format(self, record):
         action = record.__dict__.get("action")
         status = record.__dict__.get("status")
-        task_id = _tasks.get(
-            thread.get_ident(), record.__dict__.get("task_id")
-        )
+        task = _tasks.get(task_key())
+        task_id = task[0] if task else record.__dict__.get("task_id")
         d = {
             "action": action,
             "task_id": task_id,
@@ -96,11 +99,14 @@ def filter(self, record):
 
 def task_log_start(task_id):
     """Associate a thread with a task."""
-    _tasks[thread.get_ident()] = task_id
+    fp = open(cwd("cuckoo.log", analysis=task_id), "a+b")
+    _tasks[task_key()] = (task_id, fp)
 
 def task_log_stop(task_id):
     """Disassociate a thread from a task."""
-    _tasks.pop(thread.get_ident(), None)
+    task = _tasks.pop(task_key(), None)
+    if task:
+        task[1].close()
 
 def init_logger(name, level=None):
     formatter = logging.Formatter(
diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 90dacdc4ee..b17b324145 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -45,10 +45,6 @@
 # Data Streams); XXX: just replace illegal chars?
 BANNED_PATH_CHARS = b'\x00:'
 
-# Safeguard againt multiple writes to `files.json` -- note that this really
-# should have per-task granularity
-filelist_lock = threading.Lock()
-
 def netlog_sanitize_fname(path):
     """Validate agent-provided path for result files"""
     path = path.replace("\\", "/")
@@ -183,13 +179,13 @@ def handle(self):
                                              self.task_id)
             raise
 
-        with filelist_lock:
-            with open(self.filelog, "a+b") as f:
-                print(json.dumps({
-                    "path": dump_path,
-                    "filepath": filepath,
-                    "pids": pids,
-                }), file=f)
+        # Append-writes are atomic
+        with open(self.filelog, "a+b") as f:
+            print(json.dumps({
+                "path": dump_path,
+                "filepath": filepath,
+                "pids": pids,
+            }), file=f)
 
         self.handler.sock.settimeout(None)
         try:

From 7bd9dd46d18731e9860377f602913b59f890560c Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Mon, 12 Mar 2018 11:57:32 +0100
Subject: [PATCH 068/138] Prevent an exception when dealing with old FILE
 commands

---
 cuckoo/core/resultserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index b17b324145..12ba1b6957 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -160,7 +160,7 @@ def handle(self):
         self.handler.sock.settimeout(30)
         dump_path = netlog_sanitize_fname(self.handler.read_newline())
 
-        if self.version >= 2:
+        if self.version and self.version >= 2:
             # NB: filepath is only used as metadata
             filepath = self.handler.read_newline()
             pids = map(int, self.handler.read_newline().split())

From aaaeaa2fca6aa006a7ac64ee28e6474aaaf3bf13 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 17 May 2019 17:42:51 +0200
Subject: [PATCH 069/138] Handle analysis log re-open attempt

---
 cuckoo/core/resultserver.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 12ba1b6957..7e806c4515 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -199,7 +199,12 @@ class LogHandler(ProtocolHandler):
 
     def init(self):
         self.logpath = os.path.join(self.handler.storagepath, "analysis.log")
-        self.fd = open_exclusive(self.logpath, bufsize=1)
+        try:
+            self.fd = open_exclusive(self.logpath, bufsize=1)
+        except OSError:
+            log.error("Task #%s: attempted to reopen live log analysis.log.",
+                      self.task_id)
+            return
         log.debug("Task #%s: live log analysis.log initialized.",
                   self.task_id)
 

From cf8b70b4b5eb99bb352c9ccfc1691793ff23e9f0 Mon Sep 17 00:00:00 2001
From: Ben de Graaff <ben@cuckoo.sh>
Date: Mon, 13 Aug 2018 11:47:51 +0200
Subject: [PATCH 070/138] Fix on_call detection for signatures that always mark

---
 cuckoo/core/plugins.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/cuckoo/core/plugins.py b/cuckoo/core/plugins.py
index 36c16fd89c..ae58ec9fd0 100644
--- a/cuckoo/core/plugins.py
+++ b/cuckoo/core/plugins.py
@@ -9,6 +9,7 @@
 import logging
 import os
 import pkgutil
+import sys
 
 import cuckoo
 
@@ -363,15 +364,14 @@ def __init__(self, results):
     def _on_call_defined(self, sig):
         """Test if on_call is defined.  This is not pretty, but it allows
         on_call to be defined in `abstracts` for documentation purposes.
+        """
 
-        NB: In Python 3, we can just use `sig.on_call is Signature.on_call`."""
-        try:
-            sig.on_call(None, None)
-        except NotImplementedError:
-            return False
-        except:
-            pass
-        return True
+        # In Python 3, we can just use a simple check
+        if sys.version_info[0] >= 3:
+            return sig.on_call is not Signature.on_call
+
+        # Check where the method was defined
+        return sig.on_call.__func__.__module__ != Signature.on_call.__func__.__module__
 
     @classmethod
     def init_once(cls):

From e39a17fbc6db1f93fa6409321f45795494fe08ce Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sat, 18 May 2019 23:21:48 +0200
Subject: [PATCH 071/138] Use single fd for task logs

---
 cuckoo/core/log.py | 34 +++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/cuckoo/core/log.py b/cuckoo/core/log.py
index 54a13b268c..a029cd9a8e 100644
--- a/cuckoo/core/log.py
+++ b/cuckoo/core/log.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2017 Cuckoo Foundation.
+# Copyright (C) 2016-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -7,16 +7,20 @@
 import json
 import logging
 import logging.handlers
-import thread
 import time
 
+from threading import Lock
+
 from cuckoo.common.colors import red, yellow, cyan
 from cuckoo.core.database import Database
 from cuckoo.misc import cwd
 
+_task_threads = {}
 _tasks = {}
 _loggers = {}
 
+_tasks_lock = Lock()
+
 # Current GMT+x.
 if time.localtime().tm_isdst:
     tz = time.altzone / -3600.
@@ -99,14 +103,30 @@ def filter(self, record):
 
 def task_log_start(task_id):
     """Associate a thread with a task."""
-    fp = open(cwd("cuckoo.log", analysis=task_id), "a+b")
-    _tasks[task_key()] = (task_id, fp)
+    _tasks_lock.acquire()
+    try:
+        if task_id not in _task_threads:
+            _task_threads[task_id] = []
+            fp = open(cwd("cuckoo.log", analysis=task_id), "a+b")
+            _tasks[task_key()] = (task_id, fp)
+        else:
+            existing_key = _task_threads[task_id][0]
+            _tasks[task_key()] = _tasks[existing_key]
+
+        _task_threads[task_id].append(task_key())
+    finally:
+        _tasks_lock.release()
 
 def task_log_stop(task_id):
     """Disassociate a thread from a task."""
-    task = _tasks.pop(task_key(), None)
-    if task:
-        task[1].close()
+    _tasks_lock.acquire()
+    try:
+        _, fp =_tasks.pop(task_key())
+        _task_threads[task_id].remove(task_key())
+        if not _task_threads[task_id]:
+            fp.close()
+    finally:
+        _tasks_lock.release()
 
 def init_logger(name, level=None):
     formatter = logging.Formatter(

From db5ad1519c83e1a8120b1ed3bb02535c7e036d6d Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sat, 18 May 2019 23:22:17 +0200
Subject: [PATCH 072/138] Add memory path to allowed paths

---
 cuckoo/core/resultserver.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 7e806c4515..8fd6b2beb4 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -5,7 +5,6 @@
 
 from __future__ import print_function
 
-import datetime
 import errno
 import gevent.pool
 import gevent.server
@@ -14,15 +13,13 @@
 import logging
 import os
 import socket
-import struct
 import threading
 
 from cuckoo.common.abstracts import ProtocolHandler
 from cuckoo.common.config import config
 from cuckoo.common.exceptions import CuckooCriticalError
 from cuckoo.common.exceptions import CuckooOperationalError
-from cuckoo.common.exceptions import CuckooResultError
-from cuckoo.common.files import Folders, open_exclusive
+from cuckoo.common.files import open_exclusive
 from cuckoo.common.utils import Singleton
 from cuckoo.core.log import task_log_start, task_log_stop
 from cuckoo.misc import cwd
@@ -37,7 +34,7 @@
 
 # Directories in which analysis-related files will be stored; also acts as
 # whitelist
-RESULT_UPLOADABLE = ("files", "shots", "buffer",  "extracted")
+RESULT_UPLOADABLE = ("files", "shots", "buffer",  "extracted", "memory")
 RESULT_DIRECTORIES = RESULT_UPLOADABLE + ("reports", "logs")
 
 # Prevent malicious clients from using potentially dangerious filenames
@@ -205,6 +202,7 @@ def init(self):
             log.error("Task #%s: attempted to reopen live log analysis.log.",
                       self.task_id)
             return
+
         log.debug("Task #%s: live log analysis.log initialized.",
                   self.task_id)
 

From 25eb3d25f7613e547063eb34baa5663dc2f3801f Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sat, 18 May 2019 23:22:32 +0200
Subject: [PATCH 073/138] Add tests

---
 tests/test_resultserver.py | 18 ------------------
 tests/test_utils.py        | 12 +++++++++++-
 2 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py
index 92251c1b77..9ed3581349 100644
--- a/tests/test_resultserver.py
+++ b/tests/test_resultserver.py
@@ -217,24 +217,6 @@ def test_success(self):
         with open(cwd("analysis.log", analysis=1), "rb") as f:
             assert f.read() == "first\nsecond\n"
 
-    @pytest.mark.order2
-    def test_reopen(self):
-        with pytest.raises(OSError) as e:
-            mock_handler_context(LogHandler,
-                                 cwd(analysis=1),
-                                 [],
-                                 ['reopen\n'])
-
-        assert e.value.errno == errno.EEXIST
-
-    @mock.patch('cuckoo.core.resultserver.open_exclusive')
-    def test_open_error(self, open_exclusive):
-        err = OSError()
-        err.errno = errno.EACCES
-        open_exclusive.side_effect = err
-        with pytest.raises(OSError):
-            mock_handler_context(LogHandler, cwd(analysis=1), [], [])
-
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestBsonStore(object):
     def test_success(self):
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 4bb2263451..8d58f84461 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -15,7 +15,9 @@
 import cuckoo
 
 from cuckoo.common.exceptions import CuckooOperationalError
-from cuckoo.common.files import Folders, Files, Storage, temppath
+from cuckoo.common.files import (
+    Folders, Files, Storage, temppath, open_exclusive
+)
 from cuckoo.common.whitelist import is_whitelisted_domain
 from cuckoo.common import utils
 from cuckoo.main import cuckoo_create
@@ -437,3 +439,11 @@ def test_is_whitelisted_domain():
     assert is_whitelisted_domain("java.com") is True
     assert is_whitelisted_domain("java2.com") is False
     assert is_whitelisted_domain("crl.microsoft.com") is True
+
+def test_open_exclusive():
+    fpath = os.path.join(tempfile.mkdtemp(), "yeet.exclusive")
+    with open(fpath, "wb") as fp:
+        fp.write("42421337Test")
+
+    with pytest.raises(OSError):
+        open_exclusive(fpath, bufsize=1)

From ad5bf8939fb4b86d03c4d96014b174b8b56885e3 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sat, 18 May 2019 23:52:47 +0200
Subject: [PATCH 074/138] Fix buffer extr with new resultserver

---
 cuckoo/common/netlog.py               | 20 ++++++++------------
 cuckoo/processing/behavior.py         |  2 +-
 cuckoo/processing/platform/windows.py |  3 ++-
 3 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/cuckoo/common/netlog.py b/cuckoo/common/netlog.py
index be1b687980..69f70cbf38 100644
--- a/cuckoo/common/netlog.py
+++ b/cuckoo/common/netlog.py
@@ -20,6 +20,7 @@
 
 from cuckoo.common.files import Storage
 from cuckoo.common.exceptions import CuckooResultError
+from cuckoo.misc import cwd
 
 log = logging.getLogger(__name__)
 
@@ -75,7 +76,7 @@ class BsonParser(object):
         "x": pointer_converter_32bit,
     }
 
-    def __init__(self, fd):
+    def __init__(self, fd, task_id=None):
         self.fd = fd
         self.infomap = {}
         self.flags_value = {}
@@ -83,6 +84,7 @@ def __init__(self, fd):
         self.pid = None
         self.is_64bit = False
         self.buffer_sha1 = None
+        self.task_id = task_id
 
     def resolve_flags(self, apiname, argdict, flags):
         # Resolve 1:1 values.
@@ -203,17 +205,11 @@ def __iter__(self):
                 if sha1 != self.buffer_sha1:
                     log.warning("Incorrect sha1 passed along for a buffer.")
 
-                # If the parent is netlogs ResultHandler then we actually dump
-                # it - this should only be the case during the analysis, any
-                # after processing will then be ignored.
-                from cuckoo.core.resultserver import ResultHandler
-
-                if isinstance(self.fd, ResultHandler):
-                    filepath = os.path.join(
-                        self.fd.storagepath, "buffer", self.buffer_sha1
-                    )
-                    with open(filepath, "wb") as f:
-                        f.write(buf)
+                filepath = cwd(
+                    "buffer", self.buffer_sha1, analysis=self.task_id
+                )
+                with open(filepath, "wb") as f:
+                    f.write(buf)
 
                 continue
 
diff --git a/cuckoo/processing/behavior.py b/cuckoo/processing/behavior.py
index ab24898060..5d07a232b8 100644
--- a/cuckoo/processing/behavior.py
+++ b/cuckoo/processing/behavior.py
@@ -294,7 +294,7 @@ def run(self):
             ApiStats(self),
 
             # platform specific stuff
-            WindowsMonitor(self),
+            WindowsMonitor(self, task_id=self.task["id"]),
             LinuxSystemTap(self),
 
             # Reboot information.
diff --git a/cuckoo/processing/platform/windows.py b/cuckoo/processing/platform/windows.py
index 9ec7b9a9c8..85bcfa9d91 100644
--- a/cuckoo/processing/platform/windows.py
+++ b/cuckoo/processing/platform/windows.py
@@ -213,6 +213,7 @@ class WindowsMonitor(BehaviorHandler):
     key = "processes"
 
     def __init__(self, *args, **kwargs):
+        self.task_id = kwargs.pop("task_id")
         super(WindowsMonitor, self).__init__(*args, **kwargs)
         self.processes = []
         self.behavior = {}
@@ -227,7 +228,7 @@ def handles_path(self, path):
     def parse(self, path):
         # Invoke parsing of current log file.
         self.fp = open(path, "rb")   # TODO: no proper cleanup
-        parser = BsonParser(self.fp)
+        parser = BsonParser(self.fp, self.task_id)
 
         for event in parser:
             if event["type"] == "process":

From 58345f0eeaaf2aaa1cbe86a4bb0f42b4a6af6c47 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sat, 18 May 2019 23:57:14 +0200
Subject: [PATCH 075/138] Cleanup task ref when no more log threads exist

---
 cuckoo/core/log.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/cuckoo/core/log.py b/cuckoo/core/log.py
index a029cd9a8e..8e1577a9e1 100644
--- a/cuckoo/core/log.py
+++ b/cuckoo/core/log.py
@@ -121,10 +121,15 @@ def task_log_stop(task_id):
     """Disassociate a thread from a task."""
     _tasks_lock.acquire()
     try:
-        _, fp =_tasks.pop(task_key())
-        _task_threads[task_id].remove(task_key())
+        thread_key = task_key()
+        if thread_key not in _tasks:
+            return
+
+        _, fp =_tasks.pop(thread_key)
+        _task_threads[task_id].remove(thread_key)
         if not _task_threads[task_id]:
             fp.close()
+            _task_threads.pop(task_id)
     finally:
         _tasks_lock.release()
 

From 7266d9422ae03d5160252cd030457d873362ec0f Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 19 May 2019 00:44:55 +0200
Subject: [PATCH 076/138] Don't start logger for thread if path does not exist

---
 cuckoo/core/log.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cuckoo/core/log.py b/cuckoo/core/log.py
index 8e1577a9e1..6a081dc0ce 100644
--- a/cuckoo/core/log.py
+++ b/cuckoo/core/log.py
@@ -8,6 +8,7 @@
 import logging
 import logging.handlers
 import time
+import os
 
 from threading import Lock
 
@@ -107,7 +108,11 @@ def task_log_start(task_id):
     try:
         if task_id not in _task_threads:
             _task_threads[task_id] = []
-            fp = open(cwd("cuckoo.log", analysis=task_id), "a+b")
+            task_path = cwd(analysis=task_id)
+            if not os.path.exists(task_path):
+                return
+
+            fp = open(os.path.join(task_path, "cuckoo.log"), "a+b")
             _tasks[task_key()] = (task_id, fp)
         else:
             existing_key = _task_threads[task_id][0]

From dc6ce698e60e7d30462560243760f49074ee26cc Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 19 May 2019 00:45:19 +0200
Subject: [PATCH 077/138] Update tests for new rs logic

---
 tests/test_apps.py | 2 ++
 tests/test_init.py | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/tests/test_apps.py b/tests/test_apps.py
index f0f90e1cc2..aaf9389760 100644
--- a/tests/test_apps.py
+++ b/tests/test_apps.py
@@ -414,6 +414,7 @@ def test_process_many(self, p, q):
     @mock.patch("cuckoo.apps.apps.process")
     @mock.patch("cuckoo.apps.apps.logger")
     def test_logger(self, p, q, r):
+        mkdir(cwd(analysis=123))
         process_task({
             "id": 123,
             "target": "foo",
@@ -499,6 +500,7 @@ def test_process_dodelete(r, s, p):
 def test_process_log_taskid(p, q):
     set_cwd(tempfile.mkdtemp())
     cuckoo_create()
+    mkdir(cwd(analysis=12345))
 
     init_console_logging(logging.DEBUG)
     init_logfile("process-p0.json")
diff --git a/tests/test_init.py b/tests/test_init.py
index ca7dd8988f..d8272bdfaf 100644
--- a/tests/test_init.py
+++ b/tests/test_init.py
@@ -371,6 +371,12 @@ def lookup_config(s):
         p.Template = Template
         write_cuckoo_conf(cfg)
 
+    print cfg["cuckoo"]
+    # Force port was removed/now unused for backwards compatibility
+    cfg["cuckoo"]["resultserver"].pop("force_port", None)
+
+    # Pool size is a hidden option for now
+    cfg["cuckoo"]["resultserver"].pop("pool_size", None)
     for key, value in cfg.items():
         for key2, value2 in value.items():
             for key3, value3 in value2.items():

From b01ff6bd36ac8607df88274b57876dfebef519fa Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 19 May 2019 01:19:28 +0200
Subject: [PATCH 078/138] Only create thread id tracker if path exists

---
 cuckoo/core/log.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cuckoo/core/log.py b/cuckoo/core/log.py
index 6a081dc0ce..454d5e3571 100644
--- a/cuckoo/core/log.py
+++ b/cuckoo/core/log.py
@@ -107,11 +107,11 @@ def task_log_start(task_id):
     _tasks_lock.acquire()
     try:
         if task_id not in _task_threads:
-            _task_threads[task_id] = []
             task_path = cwd(analysis=task_id)
             if not os.path.exists(task_path):
                 return
 
+            _task_threads[task_id] = []
             fp = open(os.path.join(task_path, "cuckoo.log"), "a+b")
             _tasks[task_key()] = (task_id, fp)
         else:
@@ -130,7 +130,7 @@ def task_log_stop(task_id):
         if thread_key not in _tasks:
             return
 
-        _, fp =_tasks.pop(thread_key)
+        _, fp = _tasks.pop(thread_key)
         _task_threads[task_id].remove(thread_key)
         if not _task_threads[task_id]:
             fp.close()

From 6b0bd44e21733723c8af4267a4f83a2a6de59ac0 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 19 May 2019 01:19:43 +0200
Subject: [PATCH 079/138] Decode file paths as utf8

---
 cuckoo/core/resultserver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 8fd6b2beb4..b64928afb2 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -164,8 +164,8 @@ def handle(self):
         else:
             filepath, pids = None, []
 
-        log.debug("Task #%s: File upload for %s", self.task_id, dump_path)
-        file_path = os.path.join(self.storagepath, dump_path)
+        log.debug("Task #%s: File upload for %r", self.task_id, dump_path)
+        file_path = os.path.join(self.storagepath, dump_path.decode("utf-8"))
 
         try:
             self.fd = open_exclusive(file_path)

From ea493eedac83cf6d92eb796838981f91a026968e Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 19 May 2019 02:00:16 +0200
Subject: [PATCH 080/138] Windows specific test

---
 tests/test_resultserver.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py
index 9ed3581349..276a1b46f7 100644
--- a/tests/test_resultserver.py
+++ b/tests/test_resultserver.py
@@ -12,8 +12,8 @@
 
 import errno
 import json
-import logging
 import mock
+import platform
 import pytest
 import shutil
 import socket
@@ -21,12 +21,10 @@
 
 from cuckoo.common.exceptions import CuckooOperationalError
 from cuckoo.common.files import Folders
-from cuckoo.core.log import task_log_start, task_log_stop
 from cuckoo.core.resultserver import FileUpload, LogHandler, BsonStore
 from cuckoo.core.resultserver import GeventResultServerWorker
 from cuckoo.core.resultserver import HandlerContext
 from cuckoo.core.resultserver import RESULT_DIRECTORIES, MAX_NETLOG_LINE
-from cuckoo.core.startup import init_logging
 from cuckoo.main import cuckoo_create
 from cuckoo.misc import mkdir, set_cwd, cwd
 
@@ -215,7 +213,10 @@ def test_success(self):
                              ['first\n', 'second\n'])
 
         with open(cwd("analysis.log", analysis=1), "rb") as f:
-            assert f.read() == "first\nsecond\n"
+            if platform.system() == "Windows":
+                assert f.read() == "first\r\nsecond\r\n"
+            else:
+                assert f.read() == "first\nsecond\n"
 
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestBsonStore(object):

From df00d389d0ec752d9da792d2cc824988293f90ab Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 19 May 2019 22:17:11 +0200
Subject: [PATCH 081/138] Catch all exceptions when trying to post status to
 agent

---
 cuckoo/data/analyzer/windows/analyzer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cuckoo/data/analyzer/windows/analyzer.py b/cuckoo/data/analyzer/windows/analyzer.py
index 1ba3066821..a04cab8a4e 100644
--- a/cuckoo/data/analyzer/windows/analyzer.py
+++ b/cuckoo/data/analyzer/windows/analyzer.py
@@ -842,6 +842,6 @@ def run(self):
         try:
             server = xmlrpclib.Server("http://127.0.0.1:8000")
             server.complete(success, error, "unused_path")
-        except xmlrpclib.ProtocolError:
+        except Exception as e:
             urllib2.urlopen("http://127.0.0.1:8000/status",
                             urllib.urlencode(data)).read()

From c67ef775444244eacf0fdfc1c81f646cac97b0ae Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 19 May 2019 23:28:37 +0200
Subject: [PATCH 082/138] Catch exception for bad fds after sock was closed

---
 cuckoo/core/resultserver.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index b64928afb2..4d7cd7e9dd 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -47,10 +47,10 @@ def netlog_sanitize_fname(path):
     path = path.replace("\\", "/")
     dir_part, name = os.path.split(path)
     if dir_part not in RESULT_UPLOADABLE:
-        raise CuckooOperationalError("Netlog client requested banned path: %s"
+        raise CuckooOperationalError("Netlog client requested banned path: %r"
                                      % path)
     if any(c in BANNED_PATH_CHARS for c in name):
-        raise CuckooOperationalError("Netlog client requested banned path: %s"
+        raise CuckooOperationalError("Netlog client requested banned path: %r"
                                      % path)
     return path
 
@@ -84,6 +84,9 @@ def read(self):
         try:
             return self.sock.recv(16384)
         except socket.error as e:
+            if e.errno == errno.EBADF:
+                return ""
+
             if e.errno != errno.ECONNRESET:
                 raise
             log.debug("Task #%s had connection reset for %r", self.task_id,
@@ -299,7 +302,10 @@ def handle(self, sock, addr):
         ctx = HandlerContext(task_id, storagepath, sock)
         task_log_start(task_id)
         try:
-            protocol = self.negotiate_protocol(task_id, ctx)
+            try:
+                protocol = self.negotiate_protocol(task_id, ctx)
+            except EOFError:
+                return
 
             # Registering the context allows us to abort the handler by
             # shutting down its socket when the task is deleted; this should
@@ -341,7 +347,7 @@ def negotiate_protocol(self, task_id, ctx):
         klass = self.commands.get(command)
         if not klass:
             log.warning("Task #%s: unknown netlog protocol requested (%r), "
-                        "terminating connection.", self.task_id, command)
+                        "terminating connection.", task_id, command)
             return
         ctx.command = command
         return klass(task_id, ctx, version)

From 0315e17accffacb0276d3f716ea12b699887d5a9 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 19 May 2019 23:38:30 +0200
Subject: [PATCH 083/138] Always run complete/cleanup in analyzer

---
 cuckoo/data/analyzer/windows/analyzer.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/cuckoo/data/analyzer/windows/analyzer.py b/cuckoo/data/analyzer/windows/analyzer.py
index a04cab8a4e..4bc99ef3e8 100644
--- a/cuckoo/data/analyzer/windows/analyzer.py
+++ b/cuckoo/data/analyzer/windows/analyzer.py
@@ -519,12 +519,6 @@ def complete(self):
         self.command_pipe.stop()
         self.log_pipe_server.stop()
 
-        # Dump all the notified files.
-        self.files.dump_files()
-
-        # Hell yeah.
-        log.info("Analysis completed.")
-
         # Cleanly close remaining connections
         disconnect_pipes()
         disconnect_logger()
@@ -795,8 +789,11 @@ def run(self):
                 log.warning("Exception running finish callback of auxiliary "
                             "module %s: %s", aux.__class__.__name__, e)
 
-        # Let's invoke the completion procedure.
-        self.complete()
+        # Dump all the notified files.
+        self.files.dump_files()
+
+        # Hell yeah.
+        log.info("Analysis completed.")
         return True
 
 if __name__ == "__main__":
@@ -837,6 +834,17 @@ def run(self):
             "description": error_exc,
         }
     finally:
+        try:
+            # Let's invoke the completion procedure.
+            analyzer.complete()
+        except Exception as e:
+            complete_excp = traceback.format_exc()
+            data["status"] = "exception"
+            if "description" in data:
+                data["description"] += "\n%s" % complete_excp
+            else:
+                data["description"] = complete_excp
+
         # Report that we're finished. First try with the XML RPC thing and
         # if that fails, attempt the new Agent.
         try:

From d3ac3e70a04f2061239fe1a0d6b7590182e9d5d6 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 20 May 2019 00:04:59 +0200
Subject: [PATCH 084/138] Replace banned chars with x

---
 cuckoo/core/resultserver.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 4d7cd7e9dd..e4565bfc8c 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -50,8 +50,9 @@ def netlog_sanitize_fname(path):
         raise CuckooOperationalError("Netlog client requested banned path: %r"
                                      % path)
     if any(c in BANNED_PATH_CHARS for c in name):
-        raise CuckooOperationalError("Netlog client requested banned path: %r"
-                                     % path)
+        for c in BANNED_PATH_CHARS:
+            path = path.replace(c, "X")
+
     return path
 
 class HandlerContext(object):

From cce54e62c94455f4642b3e11f6c0d4bd193d3c20 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 20 May 2019 00:07:47 +0200
Subject: [PATCH 085/138] Replace banned chars with x

---
 cuckoo/core/resultserver.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index e4565bfc8c..477c379521 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -51,7 +51,9 @@ def netlog_sanitize_fname(path):
                                      % path)
     if any(c in BANNED_PATH_CHARS for c in name):
         for c in BANNED_PATH_CHARS:
-            path = path.replace(c, "X")
+            name = name.replace(c, "X")
+
+        path = os.path.join(dir_part, name)
 
     return path
 

From b99ff172bc418fe5fa60bad843894b497fe05db1 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 20 May 2019 11:12:31 +0200
Subject: [PATCH 086/138] Test banned name char replacement

---
 tests/test_resultserver.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py
index 276a1b46f7..fa9900f4b5 100644
--- a/tests/test_resultserver.py
+++ b/tests/test_resultserver.py
@@ -23,10 +23,10 @@
 from cuckoo.common.files import Folders
 from cuckoo.core.resultserver import FileUpload, LogHandler, BsonStore
 from cuckoo.core.resultserver import GeventResultServerWorker
-from cuckoo.core.resultserver import HandlerContext
+from cuckoo.core.resultserver import HandlerContext, netlog_sanitize_fname
 from cuckoo.core.resultserver import RESULT_DIRECTORIES, MAX_NETLOG_LINE
 from cuckoo.main import cuckoo_create
-from cuckoo.misc import mkdir, set_cwd, cwd
+from cuckoo.misc import set_cwd, cwd
 
 @pytest.fixture(scope='module')
 def cuckoo_cwd():
@@ -186,7 +186,6 @@ def test_success_v2(self):
             assert blob['path'] == "files/2.exe"
             assert blob["pids"] == [11, 12]
 
-
     def invalid_path(self, path):
         with pytest.raises(CuckooOperationalError) as e:
             mock_handler_context(FileUpload, cwd(analysis=1), [path], [])
@@ -194,8 +193,6 @@ def invalid_path(self, path):
 
     def test_invalid_paths(self):
         self.invalid_path("dummy")
-        self.invalid_path("files/p\x00ath.exe")
-        self.invalid_path("files/path.exe:$DATA")
         self.invalid_path("notallowed/path.exe")
         self.invalid_path("shots/notallowed/path.jpg")
         self.invalid_path("reports/report.json")
@@ -203,6 +200,12 @@ def test_invalid_paths(self):
         self.invalid_path("../hello")
         self.invalid_path("../../foobar")
 
+    def test_banned_names(self):
+        assert netlog_sanitize_fname("files/file1.exe") == "files/file1.exe"
+        assert netlog_sanitize_fname("files/p\x00ath.exe") == "files/pXath.exe"
+        assert netlog_sanitize_fname("files/path.exe:$DATA") == "files/path.exeX$DATA"
+        assert netlog_sanitize_fname("files/file.\x00.exe:$DATA") == "files/file.X.exeX$DATA"
+
 @pytest.mark.usefixtures('cuckoo_cwd')
 class TestLogHandler(object):
     @pytest.mark.order1

From f49f97b7dffc8ef1ef7b0b7bcfffd50ba0c6ade5 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 20 May 2019 11:46:46 +0200
Subject: [PATCH 087/138] Log database errors as exception

---
 cuckoo/core/database.py | 90 ++++++++++++++++++++++-------------------
 1 file changed, 49 insertions(+), 41 deletions(-)

diff --git a/cuckoo/core/database.py b/cuckoo/core/database.py
index 364e8973d3..d4994fb9f5 100644
--- a/cuckoo/core/database.py
+++ b/cuckoo/core/database.py
@@ -576,7 +576,7 @@ def clean_machines(self):
             session.query(Machine).delete()
             session.commit()
         except SQLAlchemyError as e:
-            log.debug("Database error cleaning machines: {0}".format(e))
+            log.exception("Database error cleaning machines: {0}".format(e))
             session.rollback()
         finally:
             session.close()
@@ -622,7 +622,7 @@ def add_machine(self, name, label, ip, platform, options, tags, interface,
         try:
             session.commit()
         except SQLAlchemyError as e:
-            log.debug("Database error adding machine: {0}".format(e))
+            log.exception("Database error adding machine: {0}".format(e))
             session.rollback()
         finally:
             session.close()
@@ -649,7 +649,7 @@ def set_status(self, task_id, status):
 
             session.commit()
         except SQLAlchemyError as e:
-            log.debug("Database error setting status: {0}".format(e))
+            log.exception("Database error setting status: {0}".format(e))
             session.rollback()
         finally:
             session.close()
@@ -670,7 +670,7 @@ def set_route(self, task_id, route):
             row.route = route
             session.commit()
         except SQLAlchemyError as e:
-            log.debug("Database error setting route: {0}".format(e))
+            log.exception("Database error setting route: {0}".format(e))
             session.rollback()
         finally:
             session.close()
@@ -697,7 +697,7 @@ def fetch(self, machine=None, service=True):
 
             return row
         except SQLAlchemyError as e:
-            log.debug("Database error fetching task: {0}".format(e))
+            log.exception("Database error fetching task: {0}".format(e))
             session.rollback()
         finally:
             session.close()
@@ -720,7 +720,7 @@ def guest_start(self, task_id, name, label, manager):
             session.refresh(guest)
             return guest.id
         except SQLAlchemyError as e:
-            log.debug("Database error logging guest start: {0}".format(e))
+            log.exception("Database error logging guest start: {0}".format(e))
             session.rollback()
             return None
         finally:
@@ -737,7 +737,7 @@ def guest_get_status(self, task_id):
             guest = session.query(Guest).filter_by(task_id=task_id).first()
             return guest.status if guest else None
         except SQLAlchemyError as e:
-            log.debug("Database error logging guest start: {0}".format(e))
+            log.exception("Database error logging guest start: {0}".format(e))
             session.rollback()
             return
         finally:
@@ -756,7 +756,7 @@ def guest_set_status(self, task_id, status):
             session.commit()
             session.refresh(guest)
         except SQLAlchemyError as e:
-            log.debug("Database error logging guest start: {0}".format(e))
+            log.exception("Database error logging guest start: {0}".format(e))
             session.rollback()
             return None
         finally:
@@ -771,7 +771,7 @@ def guest_remove(self, guest_id):
             session.delete(guest)
             session.commit()
         except SQLAlchemyError as e:
-            log.debug("Database error logging guest remove: {0}".format(e))
+            log.exception("Database error logging guest remove: {0}".format(e))
             session.rollback()
             return None
         finally:
@@ -789,7 +789,7 @@ def guest_stop(self, guest_id):
             guest.shutdown_on = datetime.datetime.now()
             session.commit()
         except SQLAlchemyError as e:
-            log.debug("Database error logging guest stop: {0}".format(e))
+            log.exception("Database error logging guest stop: {0}".format(e))
             session.rollback()
         except TypeError:
             log.warning("Data inconsistency in guests table detected, it might be a crash leftover. Continue")
@@ -810,7 +810,7 @@ def list_machines(self, locked=False):
                 machines = session.query(Machine).options(joinedload("tags")).all()
             return machines
         except SQLAlchemyError as e:
-            log.debug("Database error listing machines: {0}".format(e))
+            log.exception("Database error listing machines: {0}".format(e))
             return []
         finally:
             session.close()
@@ -853,7 +853,7 @@ def lock_machine(self, label=None, platform=None, tags=None):
             # Get the first free machine.
             machine = machines.filter_by(locked=False).first()
         except SQLAlchemyError as e:
-            log.debug("Database error locking machine: {0}".format(e))
+            log.exception("Database error locking machine: {0}".format(e))
             session.close()
             return None
 
@@ -864,7 +864,7 @@ def lock_machine(self, label=None, platform=None, tags=None):
                 session.commit()
                 session.refresh(machine)
             except SQLAlchemyError as e:
-                log.debug("Database error locking machine: {0}".format(e))
+                log.exception("Database error locking machine: {0}".format(e))
                 session.rollback()
                 return None
             finally:
@@ -884,7 +884,7 @@ def unlock_machine(self, label):
         try:
             machine = session.query(Machine).filter_by(label=label).first()
         except SQLAlchemyError as e:
-            log.debug("Database error unlocking machine: {0}".format(e))
+            log.exception("Database error unlocking machine: {0}".format(e))
             session.close()
             return None
 
@@ -895,7 +895,7 @@ def unlock_machine(self, label):
                 session.commit()
                 session.refresh(machine)
             except SQLAlchemyError as e:
-                log.debug("Database error locking machine: {0}".format(e))
+                log.exception("Database error locking machine: {0}".format(e))
                 session.rollback()
                 return None
             finally:
@@ -913,7 +913,7 @@ def count_machines_available(self):
             machines_count = session.query(Machine).filter_by(locked=False).count()
             return machines_count
         except SQLAlchemyError as e:
-            log.debug("Database error counting machines: {0}".format(e))
+            log.exception("Database error counting machines: {0}".format(e))
             return 0
         finally:
             session.close()
@@ -928,7 +928,9 @@ def get_available_machines(self):
             machines = session.query(Machine).options(joinedload("tags")).filter_by(locked=False).all()
             return machines
         except SQLAlchemyError as e:
-            log.debug("Database error getting available machines: {0}".format(e))
+            log.exception(
+                "Database error getting available machines: {0}".format(e)
+            )
             return []
         finally:
             session.close()
@@ -943,7 +945,9 @@ def set_machine_status(self, label, status):
         try:
             machine = session.query(Machine).filter_by(label=label).first()
         except SQLAlchemyError as e:
-            log.debug("Database error setting machine status: {0}".format(e))
+            log.exception(
+                "Database error setting machine status: {0}".format(e)
+            )
             session.close()
             return
 
@@ -954,7 +958,7 @@ def set_machine_status(self, label, status):
                 session.commit()
                 session.refresh(machine)
             except SQLAlchemyError as e:
-                log.debug("Database error setting machine status: %s", e)
+                log.exception("Database error setting machine status: %s", e)
                 session.rollback()
             finally:
                 session.close()
@@ -971,7 +975,7 @@ def set_machine_rcparams(self, label, rcparams):
         try:
             machine = session.query(Machine).filter_by(label=label).first()
         except SQLAlchemyError as e:
-            log.debug("Database error setting machine rcparams: %s", e)
+            log.exception("Database error setting machine rcparams: %s", e)
             session.close()
             return
 
@@ -981,7 +985,7 @@ def set_machine_rcparams(self, label, rcparams):
                 session.commit()
                 session.refresh(machine)
             except SQLAlchemyError as e:
-                log.debug("Database error setting machine rcparams: %s", e)
+                log.exception("Database error setting machine rcparams: %s", e)
                 session.rollback()
             finally:
                 session.close()
@@ -1000,7 +1004,7 @@ def add_error(self, message, task_id, action=None):
         try:
             session.commit()
         except SQLAlchemyError as e:
-            log.debug("Database error adding error log: {0}".format(e))
+            log.exception("Database error adding error log: {0}".format(e))
             session.rollback()
         finally:
             session.close()
@@ -1064,11 +1068,13 @@ def add(self, obj, timeout=0, package="", options="", priority=1,
                 try:
                     sample = session.query(Sample).filter_by(md5=obj.get_md5()).first()
                 except SQLAlchemyError as e:
-                    log.debug("Error querying sample for hash: {0}".format(e))
+                    log.exception(
+                        "Error querying sample for hash: {0}".format(e)
+                    )
                     session.close()
                     return None
             except SQLAlchemyError as e:
-                log.debug("Database error adding task: {0}".format(e))
+                log.exception("Database error adding task: {0}".format(e))
                 session.close()
                 return None
 
@@ -1123,7 +1129,7 @@ def add(self, obj, timeout=0, package="", options="", priority=1,
             session.commit()
             task_id = task.id
         except SQLAlchemyError as e:
-            log.debug("Database error adding task: {0}".format(e))
+            log.exception("Database error adding task: {0}".format(e))
             session.rollback()
             return None
         finally:
@@ -1291,7 +1297,7 @@ def add_submit(self, tmp_path, submit_type, data):
             session.refresh(submit)
             submit_id = submit.id
         except SQLAlchemyError as e:
-            log.debug("Database error adding submit entry: %s", e)
+            log.exception("Database error adding submit entry: %s", e)
             session.rollback()
         finally:
             session.close()
@@ -1306,7 +1312,7 @@ def view_submit(self, submit_id, tasks=False):
                 q = q.options(joinedload("tasks"))
             submit = q.get(submit_id)
         except SQLAlchemyError as e:
-            log.debug("Database error viewing submit: %s", e)
+            log.exception("Database error viewing submit: %s", e)
             return
         finally:
             session.close()
@@ -1335,7 +1341,7 @@ def reschedule(self, task_id, priority=None):
         try:
             session.commit()
         except SQLAlchemyError as e:
-            log.debug("Database error rescheduling task: {0}".format(e))
+            log.exception("Database error rescheduling task: {0}".format(e))
             session.rollback()
             return False
         finally:
@@ -1400,7 +1406,7 @@ def list_tasks(self, limit=None, details=True, category=None, owner=None,
             tasks = search.limit(limit).offset(offset).all()
             return tasks
         except SQLAlchemyError as e:
-            log.debug("Database error listing tasks: {0}".format(e))
+            log.exception("Database error listing tasks: {0}".format(e))
             return []
         finally:
             session.close()
@@ -1419,7 +1425,7 @@ def minmax_tasks(self):
 
             return int(_min[0].strftime("%s")), int(_max[0].strftime("%s"))
         except SQLAlchemyError as e:
-            log.debug("Database error counting tasks: {0}".format(e))
+            log.exception("Database error counting tasks: {0}".format(e))
             return
         finally:
             session.close()
@@ -1438,7 +1444,7 @@ def count_tasks(self, status=None):
                 tasks_count = session.query(Task).count()
             return tasks_count
         except SQLAlchemyError as e:
-            log.debug("Database error counting tasks: {0}".format(e))
+            log.exception("Database error counting tasks: {0}".format(e))
             return 0
         finally:
             session.close()
@@ -1460,7 +1466,7 @@ def view_task(self, task_id, details=True):
             else:
                 task = session.query(Task).get(task_id)
         except SQLAlchemyError as e:
-            log.debug("Database error viewing task: {0}".format(e))
+            log.exception("Database error viewing task: {0}".format(e))
             return None
         else:
             if task:
@@ -1483,7 +1489,7 @@ def view_tasks(self, task_ids):
                 joinedload("tags")
             ).filter(Task.id.in_(task_ids)).order_by(Task.id).all()
         except SQLAlchemyError as e:
-            log.debug("Database error viewing tasks: {0}".format(e))
+            log.exception("Database error viewing tasks: {0}".format(e))
             return []
         else:
             for task in tasks:
@@ -1504,7 +1510,7 @@ def delete_task(self, task_id):
             session.delete(task)
             session.commit()
         except SQLAlchemyError as e:
-            log.debug("Database error deleting task: {0}".format(e))
+            log.exception("Database error deleting task: {0}".format(e))
             session.rollback()
             return False
         finally:
@@ -1523,7 +1529,7 @@ def view_sample(self, sample_id):
         except AttributeError:
             return None
         except SQLAlchemyError as e:
-            log.debug("Database error viewing task: {0}".format(e))
+            log.exception("Database error viewing task: {0}".format(e))
             return None
         else:
             if sample:
@@ -1546,7 +1552,7 @@ def find_sample(self, md5=None, sha256=None):
             elif sha256:
                 sample = session.query(Sample).filter_by(sha256=sha256).first()
         except SQLAlchemyError as e:
-            log.debug("Database error searching sample: {0}".format(e))
+            log.exception("Database error searching sample: {0}".format(e))
             return None
         else:
             if sample:
@@ -1562,7 +1568,7 @@ def count_samples(self):
         try:
             sample_count = session.query(Sample).count()
         except SQLAlchemyError as e:
-            log.debug("Database error counting samples: {0}".format(e))
+            log.exception("Database error counting samples: {0}".format(e))
             return 0
         finally:
             session.close()
@@ -1578,7 +1584,7 @@ def view_machine(self, name):
         try:
             machine = session.query(Machine).options(joinedload("tags")).filter_by(name=name).first()
         except SQLAlchemyError as e:
-            log.debug("Database error viewing machine: {0}".format(e))
+            log.exception("Database error viewing machine: {0}".format(e))
             return None
         else:
             if machine:
@@ -1597,7 +1603,9 @@ def view_machine_by_label(self, label):
         try:
             machine = session.query(Machine).options(joinedload("tags")).filter_by(label=label).first()
         except SQLAlchemyError as e:
-            log.debug("Database error viewing machine by label: {0}".format(e))
+            log.exception(
+                "Database error viewing machine by label: {0}".format(e)
+            )
             return None
         else:
             if machine:
@@ -1617,7 +1625,7 @@ def view_errors(self, task_id):
             q = session.query(Error).filter_by(task_id=task_id)
             errors = q.order_by(Error.id).all()
         except SQLAlchemyError as e:
-            log.debug("Database error viewing errors: {0}".format(e))
+            log.exception("Database error viewing errors: {0}".format(e))
             return []
         finally:
             session.close()
@@ -1656,6 +1664,6 @@ def processing_get_task(self, instance):
             if task.processing == instance:
                 return task.id
         except SQLAlchemyError as e:
-            log.debug("Database error getting new processing tasks: %s", e)
+            log.exception("Database error getting new processing tasks: %s", e)
         finally:
             session.close()

From c32835333ee873a363e4d178dfbce3fe462fc203 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Tue, 21 May 2019 18:16:14 +0200
Subject: [PATCH 088/138] Shutdown socket before closing

---
 cuckoo/core/log.py                                |  8 ++++----
 .../data/analyzer/windows/lib/common/results.py   |  1 +
 .../windows/modules/auxiliary/screenshots.py      | 15 ++++++++-------
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/cuckoo/core/log.py b/cuckoo/core/log.py
index 454d5e3571..64437844d0 100644
--- a/cuckoo/core/log.py
+++ b/cuckoo/core/log.py
@@ -3,14 +3,14 @@
 # See the file 'docs/LICENSE' for copying permission.
 
 import copy
-import gevent.thread
 import json
 import logging
 import logging.handlers
-import time
 import os
+import threading
+import time
 
-from threading import Lock
+import gevent.thread
 
 from cuckoo.common.colors import red, yellow, cyan
 from cuckoo.core.database import Database
@@ -20,7 +20,7 @@
 _tasks = {}
 _loggers = {}
 
-_tasks_lock = Lock()
+_tasks_lock = threading.Lock()
 
 # Current GMT+x.
 if time.localtime().tm_isdst:
diff --git a/cuckoo/data/analyzer/windows/lib/common/results.py b/cuckoo/data/analyzer/windows/lib/common/results.py
index 0a506a0455..e0455458e6 100644
--- a/cuckoo/data/analyzer/windows/lib/common/results.py
+++ b/cuckoo/data/analyzer/windows/lib/common/results.py
@@ -76,6 +76,7 @@ def send(self, data, retry=True):
 
     def close(self):
         try:
+            self.sock.shutdown(socket.SHUT_RDWR)
             self.sock.close()
             self.sock = None
         except Exception:
diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/screenshots.py b/cuckoo/data/analyzer/windows/modules/auxiliary/screenshots.py
index 2aae0f8727..ad7f4e4f1d 100644
--- a/cuckoo/data/analyzer/windows/modules/auxiliary/screenshots.py
+++ b/cuckoo/data/analyzer/windows/modules/auxiliary/screenshots.py
@@ -1,5 +1,5 @@
 # Copyright (C) 2012-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2017 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -74,13 +74,14 @@ def run(self):
             tmpio.seek(0)
 
             # now upload to host from the StringIO
-            nf = NetlogFile()
-            nf.init("shots/%04d.jpg" % img_counter)
-
-            for chunk in tmpio:
-                nf.sock.sendall(chunk)
+            try:
+                nf = NetlogFile()
+                nf.init("shots/%04d.jpg" % img_counter)
 
-            nf.close()
+                for chunk in tmpio:
+                    nf.sock.sendall(chunk)
+            finally:
+                nf.close()
 
             img_last = img_current
 

From bf001ef0d4ed443e45c87f558a1c9f1579eb8327 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Tue, 21 May 2019 23:33:48 +0200
Subject: [PATCH 089/138] Replace chars in given path

---
 cuckoo/core/resultserver.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 477c379521..d26d19a618 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -51,9 +51,7 @@ def netlog_sanitize_fname(path):
                                      % path)
     if any(c in BANNED_PATH_CHARS for c in name):
         for c in BANNED_PATH_CHARS:
-            name = name.replace(c, "X")
-
-        path = os.path.join(dir_part, name)
+            path = path.replace(c, "X")
 
     return path
 
@@ -286,7 +284,7 @@ def del_task(self, task_id, ipaddr):
                             task_id)
             ctxs = self.handlers.pop(task_id, set())
             for ctx in ctxs:
-                log.warning("Cancel %s for task %r", ctx, task_id)
+                log.debug("Cancel %s for task %r", ctx, task_id)
                 ctx.cancel()
 
     def handle(self, sock, addr):
@@ -327,6 +325,8 @@ def handle(self, sock, addr):
             try:
                 with protocol:
                     protocol.handle()
+            except CuckooOperationalError as e:
+                log.error(e)
             finally:
                 with self.task_mgmt_lock:
                     s.discard(ctx)

From 8995ae91e227b4f2c464122f3c248565ec38cfb1 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Tue, 21 May 2019 23:34:10 +0200
Subject: [PATCH 090/138] Ensure cleanup on Cuckoo stop

---
 cuckoo/main.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/cuckoo/main.py b/cuckoo/main.py
index 040466d70f..c599420d05 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -208,13 +208,15 @@ def cuckoo_main(max_analysis_count=0):
     @param max_analysis_count: kill cuckoo after this number of analyses
     """
     try:
-        ResultServer()
+        rs = ResultServer()
         sched = Scheduler(max_analysis_count)
         sched.start()
     except KeyboardInterrupt:
+        log.info("CTRL+C detected! Stopping..")
+    finally:
         sched.stop()
-
-    Pidfile("cuckoo").remove()
+        Pidfile("cuckoo").remove()
+        rs.instance.stop()
 
 @click.group(invoke_without_command=True)
 @click.option("-d", "--debug", is_flag=True, help="Enable verbose logging")

From 7451a583c73c9dc78d6b27bb13c715f87708fad2 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Tue, 21 May 2019 23:35:42 +0200
Subject: [PATCH 091/138] Ensure pipe handlers stop sending data to
 resultserver

---
 cuckoo/data/analyzer/windows/analyzer.py      |  4 ++-
 cuckoo/data/analyzer/windows/lib/core/pipe.py | 27 +++++++++++++++++--
 .../data/analyzer/windows/lib/core/startup.py |  4 +--
 3 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/cuckoo/data/analyzer/windows/analyzer.py b/cuckoo/data/analyzer/windows/analyzer.py
index 4bc99ef3e8..de1be36162 100644
--- a/cuckoo/data/analyzer/windows/analyzer.py
+++ b/cuckoo/data/analyzer/windows/analyzer.py
@@ -841,7 +841,9 @@ def run(self):
             complete_excp = traceback.format_exc()
             data["status"] = "exception"
             if "description" in data:
-                data["description"] += "\n%s" % complete_excp
+                data["description"] += "%s\n%s" % (
+                    data["description"], complete_excp
+                )
             else:
                 data["description"] = complete_excp
 
diff --git a/cuckoo/data/analyzer/windows/lib/core/pipe.py b/cuckoo/data/analyzer/windows/lib/core/pipe.py
index 32a24fabee..20645530b2 100644
--- a/cuckoo/data/analyzer/windows/lib/core/pipe.py
+++ b/cuckoo/data/analyzer/windows/lib/core/pipe.py
@@ -6,6 +6,7 @@
 import logging
 import socket
 import threading
+import errno
 
 from ctypes import create_string_buffer, c_uint, byref, sizeof
 
@@ -31,6 +32,7 @@ def __init__(self, pipe_handle, destination):
         threading.Thread.__init__(self)
         self.pipe_handle = pipe_handle
         self.destination = destination
+        self.do_run = True
 
     def run(self):
         buf = create_string_buffer(BUFSIZE)
@@ -75,14 +77,20 @@ def run(self):
 
         open_handles.add(sock)
 
-        while True:
+        while self.do_run:
             success = KERNEL32.ReadFile(
                 self.pipe_handle, byref(buf), sizeof(buf),
                 byref(bytes_read), None
             )
 
             if success or KERNEL32.GetLastError() == ERROR_MORE_DATA:
-                sock.sendall(buf.raw[:bytes_read.value])
+                try:
+                    sock.sendall(buf.raw[:bytes_read.value])
+                except socket.error as e:
+                    if e.errno != errno.EBADF:
+                        log.warning("Failed socket operation: %s", e)
+                    break
+
             # If we get the broken pipe error then this pipe connection has
             # been terminated for one reason or another. So break from the
             # loop and make the socket "inactive", that is, another pipe
@@ -100,6 +108,9 @@ def run(self):
         if pid.value:
             self.active[pid.value] = False
 
+    def stop(self):
+        self.do_run = False
+
 class PipeDispatcher(threading.Thread):
     """Receive commands through a local pipe, forward them to the
     dispatcher, and return the response."""
@@ -147,6 +158,9 @@ def run(self):
 
         KERNEL32.CloseHandle(self.pipe_handle)
 
+    def stop(self):
+        self.do_run = False
+
 class PipeServer(threading.Thread):
     """Accept incoming pipe handlers and initialize them in
     a new thread."""
@@ -158,6 +172,7 @@ def __init__(self, pipe_handler, pipe_name, message=False, **kwargs):
         self.message = message
         self.kwargs = kwargs
         self.do_run = True
+        self.handlers = set()
 
     def run(self):
         while self.do_run:
@@ -184,15 +199,23 @@ def run(self):
                 handler = self.pipe_handler(pipe_handle, **self.kwargs)
                 handler.daemon = True
                 handler.start()
+                self.handlers.add(handler)
             else:
                 KERNEL32.CloseHandle(pipe_handle)
 
     def stop(self):
         self.do_run = False
+        for h in self.handlers:
+            try:
+                if h.isAlive():
+                    h.stop()
+            except:
+                pass
 
 def disconnect_pipes():
     for sock in open_handles:
         try:
+            sock.shutdown(socket.SHUT_RDWR)
             sock.close()
         except:
             log.exception("Could not close socket")
diff --git a/cuckoo/data/analyzer/windows/lib/core/startup.py b/cuckoo/data/analyzer/windows/lib/core/startup.py
index c5f514d7f4..d644fd6c3d 100644
--- a/cuckoo/data/analyzer/windows/lib/core/startup.py
+++ b/cuckoo/data/analyzer/windows/lib/core/startup.py
@@ -7,7 +7,7 @@
 import logging
 
 from lib.common.defines import KERNEL32, SYSTEMTIME
-from lib.common.results import NetlogHandler, NetlogConnection
+from lib.common.results import NetlogHandler
 
 log = logging.getLogger()
 netlog_handler = None
@@ -31,7 +31,7 @@ def init_logging():
 
 def disconnect_logger():
     """Cleanly close the logger. Note that LogHandler also implements close."""
-    NetlogConnection.close(netlog_handler)
+    netlog_handler.close()
 
 def set_clock(clock):
     st = SYSTEMTIME()

From d82e63aa599e409a01f882585e973559d6b65ef4 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Tue, 21 May 2019 23:36:01 +0200
Subject: [PATCH 092/138] Ensure cuckoo tmp dir is created for tests

---
 conftest.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index b7a3ca691f..85057555dd 100644
--- a/conftest.py
+++ b/conftest.py
@@ -5,8 +5,9 @@
 import os
 import shutil
 import sys
+import tempfile
 
-from cuckoo.misc import is_windows, is_linux, is_macosx
+from cuckoo.misc import is_windows, is_linux, is_macosx, getuser, mkdir
 
 # Note that collect_ignore is a parameter for pytest so that it knows which
 # unit tests to skip etc. In other words, perform platform-specific unit tests
@@ -35,3 +36,6 @@
     sys.path.insert(0, "cuckoo/data/analyzer/darwin")
     collect_ignore.append("tests/windows")
     collect_ignore.append("tests/linux")
+
+# Ensure the Cuckoo TMP dir exists, as some tests rely on it.
+mkdir(os.path.join(tempfile.gettempdir(), "cuckoo-tmp-%s" % getuser()))

From 1a91187e425a9c099313e99d85abe371e33b713b Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 22 May 2019 00:27:08 +0200
Subject: [PATCH 093/138] Handle failure of limit increasing

---
 cuckoo/main.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/cuckoo/main.py b/cuckoo/main.py
index c599420d05..73d14660af 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -98,15 +98,22 @@ def cuckoo_resources():
         return
 
     soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    limit = soft
     if soft != hard:
         log.debug("Increasing resource limit for number of open files to %s",
                   hard if hard != resource.RLIM_INFINITY else '[unlimited]')
-        resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+        try:
+            resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+            limit = hard
+        except ValueError:
+            log.debug(
+                "Failed to increase open file limit from %s to %s", soft, hard
+            )
 
-    if hard != resource.RLIM_INFINITY:
+    if limit != resource.RLIM_INFINITY:
         # This can really affect the stability of Cuckoo, so the user should
         # really fix it.  TODO: find a good minimum.
-        if hard <= 4096:
+        if limit <= 4096:
             log.error("The maximum number of open files is low (%s). If you "
                       "do not increase it, you may run into errors later "
                       "on.", hard)
@@ -207,6 +214,7 @@ def cuckoo_main(max_analysis_count=0):
     """Cuckoo main loop.
     @param max_analysis_count: kill cuckoo after this number of analyses
     """
+    rs, sched = None, None
     try:
         rs = ResultServer()
         sched = Scheduler(max_analysis_count)
@@ -214,9 +222,12 @@ def cuckoo_main(max_analysis_count=0):
     except KeyboardInterrupt:
         log.info("CTRL+C detected! Stopping..")
     finally:
-        sched.stop()
+        if sched:
+            sched.stop()
+
         Pidfile("cuckoo").remove()
-        rs.instance.stop()
+        if rs:
+            rs.instance.stop()
 
 @click.group(invoke_without_command=True)
 @click.option("-d", "--debug", is_flag=True, help="Enable verbose logging")

From b439b69945a408610346284191581e0b00158076 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 26 May 2019 00:00:48 +0200
Subject: [PATCH 094/138] Add lock to vbox start/stop operations The scheduler
 thread/lock creating thread can ignore this lock. This is so that it does not
 have to wait when Cuckoo is stopped and all machines should be shut down.

---
 cuckoo/machinery/virtualbox.py | 64 +++++++++++++++++++++++++++++-----
 1 file changed, 55 insertions(+), 9 deletions(-)

diff --git a/cuckoo/machinery/virtualbox.py b/cuckoo/machinery/virtualbox.py
index a4255abf6f..dbd5b4649b 100644
--- a/cuckoo/machinery/virtualbox.py
+++ b/cuckoo/machinery/virtualbox.py
@@ -7,6 +7,8 @@
 import os
 import re
 import subprocess
+import thread
+import threading
 import time
 
 from cuckoo.common.abstracts import Machinery
@@ -19,6 +21,43 @@
 
 log = logging.getLogger(__name__)
 
+class IgnoreLock(object):
+    """Behaves like a Lock object. Always allows the creating thread to
+    ignore the lock. The lock is used to prevent Virtualbox start/stop race
+    conditions. In the event of Cuckoo stopping, the scheduler should always
+    be able to perform the stopping operation."""
+
+    def __init__(self):
+        self.parent = thread.get_ident()
+        self._takers = []
+        self._ev = threading.Event()
+
+    def acquire(self):
+        th_ident = thread.get_ident()
+        self._takers.append(th_ident)
+
+        if th_ident == self.parent:
+            return True
+
+        while self._takers[0] != th_ident:
+            self._ev.wait(timeout=0.1)
+
+        self._ev.clear()
+
+        return True
+
+    def release(self):
+        self._takers.remove(thread.get_ident())
+        self._ev.set()
+
+    def __enter__(self):
+        self.acquire()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.release()
+
+_power_lock = IgnoreLock()
+
 class VirtualBox(Machinery):
     """Virtualization layer for VirtualBox."""
 
@@ -128,10 +167,12 @@ def start(self, label, task):
                 self.options.virtualbox.path, "startvm", label,
                 "--type", self.options.virtualbox.mode
             ]
-            _, err = Popen(
-                args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                close_fds=True
-            ).communicate()
+
+            with _power_lock:
+                _, err = Popen(
+                    args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                    close_fds=True
+                ).communicate()
             if err:
                 raise OSError(err)
         except OSError as e:
@@ -207,10 +248,12 @@ def stop(self, label):
             args = [
                 self.options.virtualbox.path, "controlvm", label, "poweroff"
             ]
-            proc = Popen(
-                args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                close_fds=True
-            )
+
+            with _power_lock:
+                proc = Popen(
+                    args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                    close_fds=True
+                )
 
             # Sometimes VBoxManage stucks when stopping vm so we needed
             # to add a timeout and kill it after that.
@@ -224,9 +267,12 @@ def stop(self, label):
                     proc.terminate()
 
             if proc.returncode != 0 and stop_me < vm_state_timeout:
+                _, err = proc.communicate()
                 log.debug(
-                    "VBoxManage exited with error powering off the machine"
+                    "VBoxManage exited with error powering off the "
+                    "machine: %s", err
                 )
+                raise OSError(err)
         except OSError as e:
             raise CuckooMachineError(
                 "VBoxManage failed powering off the machine: %s" % e

From 657d90d37a97ff419cf8051082ba379d114a2e5d Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 26 May 2019 00:03:59 +0200
Subject: [PATCH 095/138] Improve scheduler stopping and cleanup routine

---
 cuckoo/core/guest.py        | 19 ++++++++---
 cuckoo/core/resultserver.py | 12 +++++--
 cuckoo/core/scheduler.py    | 64 +++++++++++++++++++++++++++++++++----
 cuckoo/main.py              | 11 ++++---
 4 files changed, 88 insertions(+), 18 deletions(-)

diff --git a/cuckoo/core/guest.py b/cuckoo/core/guest.py
index 5f148d58f1..945a62365d 100644
--- a/cuckoo/core/guest.py
+++ b/cuckoo/core/guest.py
@@ -106,6 +106,7 @@ def __init__(self, vm_id, ip, platform, task_id):
         # TODO, pull options parameter into __init__ so we can do this here
         self.timeout = None
         self.server = None
+        self.do_run = True
 
     def wait(self, status):
         """Waiting for status.
@@ -117,7 +118,7 @@ def wait(self, status):
         end = time.time() + self.timeout
         self.server._set_timeout(self.timeout)
 
-        while db.guest_get_status(self.task_id) == "starting":
+        while db.guest_get_status(self.task_id) == "starting" and self.do_run:
             # Check if we've passed the timeout.
             if time.time() > end:
                 raise CuckooGuestCriticalTimeout(
@@ -179,6 +180,8 @@ def start_analysis(self, options, monitor):
             # availability of the agent and verify that it's ready to receive
             # data.
             self.wait(CUCKOO_GUEST_INIT)
+            if not self.do_run:
+                return
 
             # Invoke the upload of the analyzer to the guest.
             self.upload_analyzer(monitor)
@@ -233,7 +236,7 @@ def wait_for_completion(self):
         end = time.time() + self.timeout
         self.server._set_timeout(self.timeout)
 
-        while db.guest_get_status(self.task_id) == "running":
+        while db.guest_get_status(self.task_id) == "running" and self.do_run:
             time.sleep(1)
 
             # If the analysis hits the critical timeout, just return straight
@@ -286,11 +289,17 @@ def __init__(self, vmid, ipaddr, platform, task_id, analysis_manager):
         self.environ = {}
 
         self.options = {}
+        self.do_run = True
 
     @property
     def aux(self):
         return self.analysis_manager.aux
 
+    def stop(self):
+        self.do_run = False
+        if self.is_old:
+            self.old.do_run = False
+
     def get(self, method, *args, **kwargs):
         """Simple wrapper around requests.get()."""
         do_raise = kwargs.pop("do_raise", True)
@@ -334,7 +343,7 @@ def wait_available(self):
         """Wait until the Virtual Machine is available for usage."""
         end = time.time() + self.timeout
 
-        while db.guest_get_status(self.task_id) == "starting":
+        while db.guest_get_status(self.task_id) == "starting" and self.do_run:
             try:
                 socket.create_connection((self.ipaddr, self.port), 1).close()
                 break
@@ -424,6 +433,8 @@ def start_analysis(self, options, monitor):
 
         # Wait for the agent to come alive.
         self.wait_available()
+        if not self.do_run:
+            return
 
         # Could be beautified a bit, but basically we have to perform the
         # same check here as we did in wait_available().
@@ -523,7 +534,7 @@ def wait_for_completion(self):
 
         end = time.time() + self.timeout
 
-        while db.guest_get_status(self.task_id) == "running":
+        while db.guest_get_status(self.task_id) == "running" and self.do_run:
             log.debug("%s: analysis still processing", self.vmid)
 
             time.sleep(1)
diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index d26d19a618..c473e2d38c 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -272,6 +272,7 @@ def do_run(self):
     def add_task(self, task_id, ipaddr):
         with self.task_mgmt_lock:
             self.tasks[ipaddr] = task_id
+            log.debug("Now tracking machine %s for task #%s", ipaddr, task_id)
 
     def del_task(self, task_id, ipaddr):
         """Delete ResultServer state and abort pending RequestHandlers. Since
@@ -280,8 +281,15 @@ def del_task(self, task_id, ipaddr):
         have been closed after the analyzer signalled completion."""
         with self.task_mgmt_lock:
             if self.tasks.pop(ipaddr, None) is None:
-                log.warning("ResultServer did not have a task with ID %s",
-                            task_id)
+                log.warning(
+                    "ResultServer did not have a task with ID %s and IP %s",
+                    task_id, ipaddr
+                )
+            else:
+                log.debug(
+                    "Stopped tracking machine %s for task #%s",
+                    ipaddr, task_id
+                )
             ctxs = self.handlers.pop(task_id, set())
             for ctx in ctxs:
                 log.debug("Cancel %s for task %r", ctx, task_id)
diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py
index d8ddc59ad8..40d99b580e 100644
--- a/cuckoo/core/scheduler.py
+++ b/cuckoo/core/scheduler.py
@@ -64,6 +64,7 @@ def __init__(self, task_id, error_queue):
         self.interface = None
         self.rt_table = None
         self.unrouted_network = False
+        self.stopped_aux = False
 
     def init(self):
         """Initialize the analysis."""
@@ -587,7 +588,9 @@ def launch_analysis(self):
             })
         finally:
             # Stop Auxiliary modules.
-            self.aux.stop()
+            if not self.stopped_aux:
+                self.aux.stop()
+                self.stopped_aux = True
 
             # Take a memory dump of the machine before shutting it off.
             if self.cfg.cuckoo.memory_dump or self.task.memory:
@@ -666,7 +669,8 @@ def launch_analysis(self):
             ResultServer().del_task(self.task, self.machine)
 
             # Drop the network routing rules if any.
-            self.unroute_network()
+            if not self.unrouted_network:
+                self.unroute_network()
 
             try:
                 # Release the analysis machine. But only if the machine has
@@ -792,11 +796,6 @@ def run(self):
                 "status": "error",
             })
         finally:
-            # In case the analysis manager crashes, the network cleanup
-            # should still be performed.
-            if not self.unrouted_network:
-                self.unroute_network()
-
             if self.cfg.cuckoo.process_results:
                 self.db.set_status(self.task.id, TASK_REPORTED)
             else:
@@ -804,6 +803,22 @@ def run(self):
             task_log_stop(self.task.id)
             active_analysis_count -= 1
 
+    def cleanup(self):
+        # In case the analysis manager crashes, the network cleanup
+        # should still be performed.
+        if not self.unrouted_network:
+            self.unroute_network()
+
+        if not self.stopped_aux:
+            self.aux.stop()
+
+    def force_stop(self):
+        # Make the guest manager stop the status checking loop and return
+        # to the main analysis manager routine.
+        self.db.guest_set_status(self.task.id, "stopping")
+        self.guest_manager.stop()
+        log.debug("Force stopping task #%s", self.task.id)
+
 class Scheduler(object):
     """Tasks Scheduler.
 
@@ -820,6 +835,7 @@ def __init__(self, maxcount=None):
         self.db = Database()
         self.maxcount = maxcount
         self.total_analysis_count = 0
+        self.analysis_managers = set()
 
     def initialize(self):
         """Initialize the machine manager."""
@@ -908,9 +924,38 @@ def initialize(self):
     def stop(self):
         """Stop scheduler."""
         self.running = False
+
+        # Force stop all analysis managers.
+        for am in self.analysis_managers:
+            try:
+                am.force_stop()
+            except Exception as e:
+                log.exception("Error force stopping analysis manager: %s", e)
+
         # Shutdown machine manager (used to kill machines that still alive).
         machinery.shutdown()
 
+        # Remove network rules if any are present and stop auxiliary modules
+        for am in self.analysis_managers:
+            try:
+                am.cleanup()
+            except Exception as e:
+                log.exception(
+                    "Error while cleaning up analysis manager: %s", e
+                )
+
+    def _cleanup_managers(self):
+        cleaned = set()
+        for am in self.analysis_managers:
+            if not am.isAlive():
+                try:
+                    am.cleanup()
+                except Exception as e:
+                    log.exception("Error in analysis manager cleanup: %s", e)
+
+                cleaned.add(am)
+        return cleaned
+
     def start(self):
         """Start scheduler."""
         self.initialize()
@@ -928,6 +973,10 @@ def start(self):
         while self.running:
             time.sleep(1)
 
+            # Run cleanup on finished analysis managers and untrack them
+            for am in self._cleanup_managers():
+                self.analysis_managers.discard(am)
+
             # Wait until the machine lock is not locked. This is only the case
             # when all machines are fully running, rather that about to start
             # or still busy starting. This way we won't have race conditions
@@ -1035,6 +1084,7 @@ def start(self):
                 analysis = AnalysisManager(task.id, errors)
                 analysis.daemon = True
                 analysis.start()
+                self.analysis_managers.add(analysis)
 
             # Deal with errors.
             try:
diff --git a/cuckoo/main.py b/cuckoo/main.py
index 73d14660af..d13e4a3f92 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -220,15 +220,16 @@ def cuckoo_main(max_analysis_count=0):
         sched = Scheduler(max_analysis_count)
         sched.start()
     except KeyboardInterrupt:
-        log.info("CTRL+C detected! Stopping..")
+        log.info("CTRL+C detected! Stopping.. This can take a few seconds")
     finally:
-        if sched:
-            sched.stop()
-
-        Pidfile("cuckoo").remove()
+        sched.running = False
         if rs:
             rs.instance.stop()
 
+        Pidfile("cuckoo").remove()
+        if sched:
+            sched.stop()
+
 @click.group(invoke_without_command=True)
 @click.option("-d", "--debug", is_flag=True, help="Enable verbose logging")
 @click.option("-q", "--quiet", is_flag=True, help="Only log warnings and critical messages")

From dd042399365df9e6f63bf7fa33e5ade152c88bad Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 26 May 2019 00:27:36 +0200
Subject: [PATCH 096/138] Update vbox test

---
 tests/test_machinery.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tests/test_machinery.py b/tests/test_machinery.py
index 3f787d839e..066efd4cf3 100644
--- a/tests/test_machinery.py
+++ b/tests/test_machinery.py
@@ -13,7 +13,7 @@
     CuckooMachineError, CuckooCriticalError, CuckooMachineSnapshotError,
     CuckooDependencyError, CuckooMissingMachineError
 )
-from cuckoo.common.files import Folders, Files
+from cuckoo.common.files import Folders
 from cuckoo.common.objects import Dictionary
 from cuckoo.core.database import Database
 from cuckoo.core.init import write_cuckoo_conf
@@ -446,8 +446,10 @@ def test_stop_success(self):
         self.m._wait_status = mock.MagicMock(return_value=None)
 
         with mock.patch("cuckoo.machinery.virtualbox.Popen") as p:
-            p.poll.return_value = True
-            p.returncode = 0
+            proc = mock.MagicMock()
+            proc.returncode = 0
+            proc.poll.return_value = True
+            p.return_value = proc
             self.m.stop("label")
 
         p.assert_called_once_with(
@@ -459,6 +461,10 @@ def test_stop_success(self):
         )
 
         with mock.patch("cuckoo.machinery.virtualbox.Popen") as p:
+            proc = mock.MagicMock()
+            proc.returncode = 0
+            proc.poll.return_value = True
+            p.return_value = proc
             self.m._status.return_value = self.m.SAVED
             self.m.stop("label")
             p.assert_not_called()

From 709d65a721ec00d05e2407b66e947ecafe93904f Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 26 May 2019 01:24:03 +0200
Subject: [PATCH 097/138] Update agent versions in logging

---
 cuckoo/core/guest.py | 30 +++++++++++++++++++-----------
 cuckoo/main.py       |  3 ++-
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/cuckoo/core/guest.py b/cuckoo/core/guest.py
index 945a62365d..ae9f95ba90 100644
--- a/cuckoo/core/guest.py
+++ b/cuckoo/core/guest.py
@@ -313,7 +313,7 @@ def get(self, method, *args, **kwargs):
         except requests.ConnectionError:
             raise CuckooGuestError(
                 "Cuckoo Agent failed without error status, please try "
-                "upgrading to the latest version of agent.py (>= 0.8) and "
+                "upgrading to the latest version of agent.py (>= 0.10) and "
                 "notify us if the issue persists."
             )
 
@@ -332,7 +332,7 @@ def post(self, method, *args, **kwargs):
         except requests.ConnectionError:
             raise CuckooGuestError(
                 "Cuckoo Agent failed without error status, please try "
-                "upgrading to the latest version of agent.py (>= 0.8) and "
+                "upgrading to the latest version of agent.py (>= 0.10) and "
                 "notify us if the issue persists."
             )
 
@@ -425,8 +425,8 @@ def start_analysis(self, options, monitor):
         @param options: the task options
         @param monitor: identifier of the monitor to be used.
         """
-        log.info("Starting analysis on guest (id=%s, ip=%s)",
-                 self.vmid, self.ipaddr)
+        log.info("Starting analysis #%s on guest (id=%s, ip=%s)",
+                 self.task_id, self.vmid, self.ipaddr)
 
         self.options = options
         self.timeout = options["timeout"] + config("cuckoo:timeouts:critical")
@@ -535,7 +535,9 @@ def wait_for_completion(self):
         end = time.time() + self.timeout
 
         while db.guest_get_status(self.task_id) == "running" and self.do_run:
-            log.debug("%s: analysis still processing", self.vmid)
+            log.debug(
+                "%s: analysis #%s still processing", self.vmid, self.task_id
+            )
 
             time.sleep(1)
 
@@ -547,11 +549,17 @@ def wait_for_completion(self):
 
             try:
                 status = self.get("/status", timeout=5).json()
+            except CuckooGuestError:
+                # this might fail due to timeouts or just temporary network
+                # issues thus we don't want to abort the analysis just yet and
+                # wait for things to recover
+                log.info(
+                    "Virtual Machine /status failed. This can indicate the "
+                    "guest losing network connectivity"
+                )
+                continue
             except Exception as e:
-                log.info("Virtual Machine /status failed (%r)", e)
-                # this might fail due to timeouts or just temporary network issues
-                # thus we don't want to abort the analysis just yet and wait for things to
-                # recover
+                log.error("Virtual machine /status failed. %s", e)
                 continue
 
             if status["status"] == "complete":
@@ -559,8 +567,8 @@ def wait_for_completion(self):
                 return
             elif status["status"] == "exception":
                 log.warning(
-                    "%s: analysis caught an exception\n%s",
-                    self.vmid, status["description"]
+                    "%s: analysis #%s caught an exception\n%s",
+                    self.vmid, self.task_id, status["description"]
                 )
                 return
 
diff --git a/cuckoo/main.py b/cuckoo/main.py
index d13e4a3f92..6aeaa27555 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -222,7 +222,8 @@ def cuckoo_main(max_analysis_count=0):
     except KeyboardInterrupt:
         log.info("CTRL+C detected! Stopping.. This can take a few seconds")
     finally:
-        sched.running = False
+        if sched:
+            sched.running = False
         if rs:
             rs.instance.stop()
 

From f87a824c902dbac638e6b8034be1ff2291b5ba83 Mon Sep 17 00:00:00 2001
From: Ants Madisson <ants@cert.ee>
Date: Thu, 28 Feb 2019 13:25:54 +0200
Subject: [PATCH 098/138] block public file download

---
 cuckoo/data/web/local_settings.py |  9 ++++++++-
 cuckoo/web/utils.py               | 11 +++++++++++
 cuckoo/web/web/middle.py          | 22 ++++++++++++++++++++++
 cuckoo/web/web/settings.py        |  1 +
 4 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/cuckoo/data/web/local_settings.py b/cuckoo/data/web/local_settings.py
index 7e44d34810..10a9677bb7 100644
--- a/cuckoo/data/web/local_settings.py
+++ b/cuckoo/data/web/local_settings.py
@@ -6,7 +6,7 @@
 import web.errors
 
 # Maximum upload size (10GB, so there's basically no limit).
-MAX_UPLOAD_SIZE = 10*1024*1024*1024
+MAX_UPLOAD_SIZE = 10 * 1024 * 1024 * 1024
 
 # Override default secret key stored in $CWD/web/.secret_key
 # Make this unique, and don't share it with anybody.
@@ -37,3 +37,10 @@
 
 handler404 = web.errors.handler404
 handler500 = web.errors.handler500
+
+#A list of strings representing the subnets or ipaddresses that can download 
+#samples and dropped files 
+#Values in this list can be ipv4 or ipv6 separated by "," 
+#(e.g. '127.0.0.0/8,10.0.0.0/8,fd00::/8').
+ALLOWED_FILEDOWNLOAD_SUBNETS = '127.0.0.0/8,10.0.0.0/8,fd00::/8'
+
diff --git a/cuckoo/web/utils.py b/cuckoo/web/utils.py
index 46ba0adc51..c48a581806 100644
--- a/cuckoo/web/utils.py
+++ b/cuckoo/web/utils.py
@@ -142,3 +142,14 @@ def normalize_task(task):
             os.path.basename(task["target"])
         )
     return task
+
+def get_client_ip(request):
+    try:
+        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
+        if x_forwarded_for:
+            ip = x_forwarded_for.split(',')[0]
+        else:
+            ip = request.META.get('REMOTE_ADDR')
+        return ip
+    except Exception as e:
+        return None
diff --git a/cuckoo/web/web/middle.py b/cuckoo/web/web/middle.py
index 68380910cc..d8167f70e8 100644
--- a/cuckoo/web/web/middle.py
+++ b/cuckoo/web/web/middle.py
@@ -4,9 +4,11 @@
 # See the file 'docs/LICENSE' for copying permission.
 
 from django.shortcuts import redirect
+from ipaddress import ip_network,ip_address
 
 from cuckoo.common.config import config
 from cuckoo.misc import version
+from cuckoo.web.utils import get_client_ip
 
 class CuckooAuthentication(object):
     def process_request(self, request):
@@ -31,3 +33,23 @@ def process_response(self, request, response):
         response["Cache-Control"] = "no-cache"
         response["Expires"] = "0"
         return response
+
+class CuckooFileDownloadAuthentication(object):
+    def process_request(self, request):
+        if not request.path.startswith(("/file/sample/","/file/dropped/")): 
+            return
+        #if no ALLOWED_FILEDOWNLOAD_SUBNETS in web local_settings, ignore this
+        try:
+            from settings import ALLOWED_FILEDOWNLOAD_SUBNETS
+        except ImportError as e:
+            return        
+        ip = get_client_ip(request)
+        isallowed = False
+        if ip:
+            for network in ALLOWED_FILEDOWNLOAD_SUBNETS.split(','):
+                network = ip_network(unicode(network),strict=False)
+                if ip_address(unicode(ip)) in network:
+                    isallowed = True
+                    return
+        if not isallowed and not request.session.get("auth"):
+            return redirect("/secret/")    
diff --git a/cuckoo/web/web/settings.py b/cuckoo/web/web/settings.py
index 0a2aeb316e..c48ac96f7c 100644
--- a/cuckoo/web/web/settings.py
+++ b/cuckoo/web/web/settings.py
@@ -110,6 +110,7 @@
     "django.middleware.csrf.CsrfViewMiddleware",
     # Cuckoo Authentication & headers.
     "web.middle.CuckooAuthentication",
+    "web.middle.CuckooFileDownloadAuthentication",
     "web.middle.CuckooHeaders",
     # Our custom exception handler.
     "web.errors.ExceptionMiddleware"

From 3ea3fa78b73cba62e1a7c46c67ac2fe5627aa950 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Sun, 26 May 2019 01:56:58 +0200
Subject: [PATCH 099/138] Code cleanup and update dependencies

---
 cuckoo/web/utils.py      | 11 -----------
 cuckoo/web/web/middle.py | 20 +++++++++++---------
 setup.py                 |  1 +
 3 files changed, 12 insertions(+), 20 deletions(-)

diff --git a/cuckoo/web/utils.py b/cuckoo/web/utils.py
index c48a581806..46ba0adc51 100644
--- a/cuckoo/web/utils.py
+++ b/cuckoo/web/utils.py
@@ -142,14 +142,3 @@ def normalize_task(task):
             os.path.basename(task["target"])
         )
     return task
-
-def get_client_ip(request):
-    try:
-        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
-        if x_forwarded_for:
-            ip = x_forwarded_for.split(',')[0]
-        else:
-            ip = request.META.get('REMOTE_ADDR')
-        return ip
-    except Exception as e:
-        return None
diff --git a/cuckoo/web/web/middle.py b/cuckoo/web/web/middle.py
index d8167f70e8..ce0c8cffc3 100644
--- a/cuckoo/web/web/middle.py
+++ b/cuckoo/web/web/middle.py
@@ -4,11 +4,10 @@
 # See the file 'docs/LICENSE' for copying permission.
 
 from django.shortcuts import redirect
-from ipaddress import ip_network,ip_address
+from ipaddress import ip_network, ip_address
 
 from cuckoo.common.config import config
 from cuckoo.misc import version
-from cuckoo.web.utils import get_client_ip
 
 class CuckooAuthentication(object):
     def process_request(self, request):
@@ -36,20 +35,23 @@ def process_response(self, request, response):
 
 class CuckooFileDownloadAuthentication(object):
     def process_request(self, request):
-        if not request.path.startswith(("/file/sample/","/file/dropped/")): 
+        if not request.path.startswith(("/file/sample/", "/file/dropped/")):
             return
+
         #if no ALLOWED_FILEDOWNLOAD_SUBNETS in web local_settings, ignore this
         try:
             from settings import ALLOWED_FILEDOWNLOAD_SUBNETS
-        except ImportError as e:
-            return        
-        ip = get_client_ip(request)
+        except ImportError:
+            return
+
+        ip = request.META.get("REMOTE_ADDR")
         isallowed = False
         if ip:
-            for network in ALLOWED_FILEDOWNLOAD_SUBNETS.split(','):
-                network = ip_network(unicode(network),strict=False)
+            for network in ALLOWED_FILEDOWNLOAD_SUBNETS.split(","):
+                network = ip_network(unicode(network), strict=False)
                 if ip_address(unicode(ip)) in network:
                     isallowed = True
                     return
+
         if not isallowed and not request.session.get("auth"):
-            return redirect("/secret/")    
+            return redirect("/secret/")
diff --git a/setup.py b/setup.py
index abdc87ed15..1440258bd9 100755
--- a/setup.py
+++ b/setup.py
@@ -198,6 +198,7 @@ def do_setup(**kwargs):
         "flask==0.12.2",
         "flask-sqlalchemy==2.4.0",
         "httpreplay>=0.2.4, <0.3",
+        "ipaddress>=1.0.22",
         "gevent>=1.2, <1.3",
         "jinja2==2.9.6",
         "jsbeautifier==1.6.2",

From ba3fa75a8f5333d7b4cb1ee31d4068db3578cb48 Mon Sep 17 00:00:00 2001
From: Ants Madisson <ants@cert.ee>
Date: Fri, 24 May 2019 12:47:12 +0300
Subject: [PATCH 100/138] misp config, whitelist

---
 cuckoo/common/config.py                 |   6 ++
 cuckoo/common/whitelist.py              |  32 ++++++++
 cuckoo/private/whitelist/mispdomain.txt |  11 +++
 cuckoo/private/whitelist/mispip.txt     |   4 +
 cuckoo/reporting/misp.py                | 103 +++++++++++++-----------
 setup.py                                |   2 +-
 6 files changed, 112 insertions(+), 46 deletions(-)
 create mode 100644 cuckoo/private/whitelist/mispdomain.txt
 create mode 100644 cuckoo/private/whitelist/mispip.txt

diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py
index 48fb7e3dd9..ab9714c56f 100644
--- a/cuckoo/common/config.py
+++ b/cuckoo/common/config.py
@@ -752,6 +752,12 @@ class Config(object):
                 "url": String(),
                 "apikey": String(sanitize=True),
                 "mode": String("maldoc ipaddr hashes url"),
+                "distribution": Int(),
+                "analysis": Int(),
+                "threat_level": Int(),
+                "min_malscore": Int(),
+                "tag": String(),
+                "upload_sample": Boolean(False),
             },
             "mongodb": {
                 "enabled": Boolean(False),
diff --git a/cuckoo/common/whitelist.py b/cuckoo/common/whitelist.py
index 3ecba46f4b..21043399c5 100644
--- a/cuckoo/common/whitelist.py
+++ b/cuckoo/common/whitelist.py
@@ -8,6 +8,10 @@
 
 domains = set()
 ips = set()
+mispdomains = set()
+mispips = set()
+mispurls = set()
+misphashes = set()
 
 def _load_whitelist(wlset, wl_file):
     for b in (True, False):
@@ -37,3 +41,31 @@ def is_whitelisted_ip(ip):
         _load_whitelist(ips, "ip.txt")
 
     return ip in ips
+
+def is_whitelisted_mispdomain(domain):
+    if not mispdomains:
+        # Initialize the domain whitelist.
+        _load_whitelist(mispdomains, "mispdomain.txt")
+
+    return domain in mispdomains
+
+def is_whitelisted_mispip(ip):
+    if not mispips:
+        # Initialize the ip whitelist.
+        _load_whitelist(mispips, "mispip.txt")
+
+    return ip in mispips
+
+def is_whitelisted_mispurl(url):
+    if not mispurls:
+        # Initialize the ip whitelist.
+        _load_whitelist(mispurls, "mispurl.txt")
+
+    return ip in mispurls
+
+def is_whitelisted_misphash(hash):
+    if not misphashes:
+        # Initialize the ip whitelist.
+        _load_whitelist(misphashes, "misphash.txt")
+
+    return hash in misphashes
diff --git a/cuckoo/private/whitelist/mispdomain.txt b/cuckoo/private/whitelist/mispdomain.txt
new file mode 100644
index 0000000000..9f4d4a34df
--- /dev/null
+++ b/cuckoo/private/whitelist/mispdomain.txt
@@ -0,0 +1,11 @@
+www.msftncsi.com
+dns.msftncsi.com
+teredo.ipv6.microsoft.com
+time.windows.com
+www.msftconnecttest.com
+v10.vortex-win.data.microsoft.com
+settings-win.data.microsoft.com
+win10.ipv6.microsoft.com
+sls.update.microsoft.com
+fs.microsoft.com
+ctldl.windowsupdate.com
diff --git a/cuckoo/private/whitelist/mispip.txt b/cuckoo/private/whitelist/mispip.txt
new file mode 100644
index 0000000000..5f6d8ff757
--- /dev/null
+++ b/cuckoo/private/whitelist/mispip.txt
@@ -0,0 +1,4 @@
+13.74.179.117
+40.81.120.221
+40.77.226.249
+8.8.8.8
diff --git a/cuckoo/reporting/misp.py b/cuckoo/reporting/misp.py
index 14508d2f71..a1cff8e859 100644
--- a/cuckoo/reporting/misp.py
+++ b/cuckoo/reporting/misp.py
@@ -9,6 +9,7 @@
 
 from cuckoo.common.abstracts import Report
 from cuckoo.common.exceptions import CuckooProcessingError
+from cuckoo.common.whitelist import is_whitelisted_mispdomain,is_whitelisted_mispip,is_whitelisted_mispurl,is_whitelisted_misphash
 
 log = logging.getLogger(__name__)
 
@@ -33,31 +34,25 @@ def all_urls(self, results, event):
         urls = set()
         for protocol in ("http_ex", "https_ex"):
             for entry in results.get("network", {}).get(protocol, []):
-                urls.add("%s://%s%s" % (
-                    entry["protocol"], entry["host"], entry["uri"]
-                ))
+                if not is_whitelisted_mispdomain(entry["host"]) and not is_whitelisted_mispdomain(entry["host"]):
+                    url = "%s://%s%s" % (
+                          entry["protocol"], entry["host"], entry["uri"])
+                    if not is_whitelisted_mispurl(url):
+                        urls.add(url)
 
         self.misp.add_url(event, sorted(list(urls)))
 
     def domain_ipaddr(self, results, event):
-        whitelist = [
-            "www.msftncsi.com", "dns.msftncsi.com", "8.8.8.8", "40.77.226.249",
-            "teredo.ipv6.microsoft.com", "time.windows.com",
-            "www.msftconnecttest.com", "v10.vortex-win.data.microsoft.com",
-            "settings-win.data.microsoft.com", "win10.ipv6.microsoft.com",
-            "sls.update.microsoft.com", "13.74.179.117", "40.81.120.221",
-            "fs.microsoft.com", "ctldl.windowsupdate.com"
-        ]
 
         domains, ips = {}, set()
         for domain in results.get("network", {}).get("domains", []):
-            if domain["domain"] not in whitelist:
+            if not is_whitelisted_mispip(domain["ip"]) and not is_whitelisted_mispdomain(domain["domain"]):
                 domains[domain["domain"]] = domain["ip"]
                 ips.add(domain["ip"])
 
         ipaddrs = set()
         for ipaddr in results.get("network", {}).get("hosts", []):
-            if ipaddr not in whitelist and ipaddr not in ips:
+            if ipaddr not in ips and not is_whitelisted_mispip(ipaddr):
                 ipaddrs.add(ipaddr)
 
         self.misp.add_domains_ips(event, domains)
@@ -66,7 +61,7 @@ def domain_ipaddr(self, results, event):
     def family(self, results, event):
         for config in results.get("metadata", {}).get("cfgextr", []):
             self.misp.add_detection_name(
-                event, config["family"], "Sandbox detection"
+                event, config["family"], "External analysis"
             )
             for cnc in config.get("cnc", []):
                 self.misp.add_url(event, cnc)
@@ -97,42 +92,60 @@ def run(self, results):
         url = self.options.get("url")
         apikey = self.options.get("apikey")
         mode = shlex.split(self.options.get("mode") or "")
+        score = results.get("info", None).get("score")
+        upload_sample = self.options.get("upload_sample")
+        f = results["target"]["file"]
+        hash_whitelisted = is_whitelisted_misphash(f["md5"]) or is_whitelisted_misphash(f["sha1"]) or is_whitelisted_misphash(f["sha256"])
+
+        if score >= self.options.get("min_malscore", 0) and not hash_whitelisted:
+            if not url or not apikey:
+                raise CuckooProcessingError(
+                    "Please configure the URL and API key for your MISP instance."
+                )
 
-        if not url or not apikey:
-            raise CuckooProcessingError(
-                "Please configure the URL and API key for your MISP instance."
-            )
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                import pymisp
+
+            self.misp = pymisp.PyMISP(url, apikey, False, "json")
 
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            import pymisp
-
-        self.misp = pymisp.PyMISP(url, apikey, False, "json")
-
-        event = self.misp.new_event(
-            distribution=pymisp.Distribution.all_communities.value,
-            threat_level_id=pymisp.ThreatLevel.undefined.value,
-            analysis=pymisp.Analysis.completed.value,
-            info="Cuckoo Sandbox analysis #%d" % self.task["id"],
-        )
-
-        if results.get("target", {}).get("category") == "file":
-            self.misp.upload_sample(
-                filename=os.path.basename(self.task["target"]),
-                filepath_or_bytes=self.task["target"],
-                event_id=event["Event"]["id"],
-                category="External analysis",
+            #Get default settings for a new event
+            distribution = self.options.get("distribution") or 0
+            threat_level = self.options.get("threat_level") or 4
+            analysis = self.options.get("analysis") or 0
+            tag = self.options.get("tag") or "Cuckoo"
+
+            event = self.misp.new_event(
+                distribution=distribution,
+                threat_level_id=threat_level,
+                analysis=analysis,
+                info="Cuckoo Sandbox analysis #%d" % self.task["id"]
             )
 
-        self.signature(results, event)
+            # Add a specific tag to flag Cuckoo's event
+            if tag:
+                mispresult = self.misp.tag(event['Event']['uuid'], tag)
+                if mispresult.has_key('message'):
+                    log.debug("tag event: %s" % mispresult['message'])
+
+            if upload_sample and results.get("target", {}).get("category") == "file":
+                if results.get("target", {}).get("file", {}):
+                    self.misp.upload_sample(
+                        filename=os.path.basename(self.task["target"]),
+                        filepath_or_bytes=self.task["target"],
+                        event_id=event["Event"]["id"],
+                        category="External analysis",
+                    )
+
+            self.signature(results, event)
 
-        if "hashes" in mode:
-            self.sample_hashes(results, event)
+            if "hashes" in mode:
+                self.sample_hashes(results, event)
 
-        if "url" in mode:
-            self.all_urls(results, event)
+            if "url" in mode:
+                self.all_urls(results, event)
 
-        if "ipaddr" in mode:
-            self.domain_ipaddr(results, event)
+            if "ipaddr" in mode:
+                self.domain_ipaddr(results, event)
 
-        self.family(results, event)
+            self.family(results, event)
diff --git a/setup.py b/setup.py
index 1440258bd9..29b9744f82 100755
--- a/setup.py
+++ b/setup.py
@@ -208,7 +208,7 @@ def do_setup(**kwargs):
         "pillow==3.2",
         "pyelftools==0.24",
         "pyguacamole==0.6",
-        "pymisp==2.4.103",
+        "pymisp==2.4.106",
         "pymongo==3.0.3",
         "python-dateutil==2.4.2",
         "python-magic==0.4.12",

From eb6d6d75eac6fb6fbaf24c20351d8895200951ea Mon Sep 17 00:00:00 2001
From: Ants Madisson <ants@cert.ee>
Date: Fri, 24 May 2019 12:47:53 +0300
Subject: [PATCH 101/138] signatures marks as misp comment

---
 cuckoo/reporting/misp.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/cuckoo/reporting/misp.py b/cuckoo/reporting/misp.py
index a1cff8e859..5feccf9917 100644
--- a/cuckoo/reporting/misp.py
+++ b/cuckoo/reporting/misp.py
@@ -73,8 +73,29 @@ def family(self, results, event):
                 self.misp.add_useragent(event, user_agent)
 
     def signature(self, results, event):
-        for sig in results["signatures"]:
-            data = "%s - (%s)" % (sig["description"], ",".join(sig["ttp"]))
+        for sig in results.get("signatures", []):
+            try:
+                temp = []
+                for mark in sig.get("marks", []):
+                    if mark["type"] == "generic":
+                        temp.append("%s %s" % (mark.get("parent_process", ""), mark.get("martian_process", "")))
+                        temp.append("%s %s" % (mark.get("reg_key", ""), mark.get("reg_value", "")))
+                        temp.append("%s %s" % (mark.get("option", ""), mark.get("value", "")))
+                        temp.append("%s" % mark.get("domain", ""))
+                        temp.append("%s" % mark.get("description", ""))
+                        temp.append("%s" % mark.get("host", ""))
+                    elif mark["type"] == "call":
+                        if not mark["call"]["api"] in temp:
+                            temp.append(mark["call"]["api"])
+                    elif mark["type"] == "config":
+                        temp.append(mark["config"]["url"])
+                    else:
+                        temp.append(mark[mark["type"]])
+                datainfo = ",".join([x for x in temp if x not in ["", " "]])
+            except:
+                datainfo = ",".join(sig["ttp"])
+                pass
+            data = "%s - (%s)" % (sig["description"], datainfo)
             self.misp.add_internal_comment(event, data)
             for att, description in sig["ttp"].items():
                 if description is None:

From b5fec9fa1d625a7f44b16609661cad9b2249222a Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Tue, 28 May 2019 20:06:00 +0200
Subject: [PATCH 102/138] Code cleanup and only load whitelist from cwd

---
 cuckoo/apps/apps.py                           |   6 +
 cuckoo/common/config.py                       |  12 +-
 cuckoo/common/whitelist.py                    |  31 ++-
 cuckoo/data/whitelist/domain.txt              |  34 +++
 .../whitelist/mispdomain.txt                  |   1 +
 cuckoo/data/whitelist/misphash.txt            |   1 +
 cuckoo/data/whitelist/mispip.txt              |   5 +
 cuckoo/data/whitelist/mispurl.txt             |   1 +
 cuckoo/private/cwd/conf/reporting.conf        |  10 +
 cuckoo/private/whitelist/domain.txt           |  27 ---
 cuckoo/private/whitelist/mispip.txt           |   4 -
 cuckoo/reporting/misp.py                      | 198 +++++++++++-------
 12 files changed, 197 insertions(+), 133 deletions(-)
 rename cuckoo/{private => data}/whitelist/mispdomain.txt (79%)
 create mode 100644 cuckoo/data/whitelist/misphash.txt
 create mode 100644 cuckoo/data/whitelist/mispip.txt
 create mode 100644 cuckoo/data/whitelist/mispurl.txt
 delete mode 100644 cuckoo/private/whitelist/domain.txt
 delete mode 100644 cuckoo/private/whitelist/mispip.txt

diff --git a/cuckoo/apps/apps.py b/cuckoo/apps/apps.py
index 467d0df624..ae5e6425fb 100644
--- a/cuckoo/apps/apps.py
+++ b/cuckoo/apps/apps.py
@@ -509,6 +509,12 @@ def migrate_cwd():
         shutil.copytree(
             cwd("..", "data", "whitelist", private=True), cwd("whitelist")
         )
+    else:
+        data_wl = cwd("..", "data", "whitelist", private=True)
+        for wl_file in os.listdir(data_wl):
+            cwd_wl = cwd("whitelist", wl_file)
+            if not os.path.isfile(cwd_wl):
+                shutil.copy(os.path.join(data_wl, wl_file), cwd_wl)
 
     # Create the new $CWD/yara/dumpmem/ directory.
     if not os.path.exists(cwd("yara", "dumpmem")):
diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py
index ab9714c56f..6ff03c39b9 100644
--- a/cuckoo/common/config.py
+++ b/cuckoo/common/config.py
@@ -752,12 +752,12 @@ class Config(object):
                 "url": String(),
                 "apikey": String(sanitize=True),
                 "mode": String("maldoc ipaddr hashes url"),
-                "distribution": Int(),
-                "analysis": Int(),
-                "threat_level": Int(),
-                "min_malscore": Int(),
-                "tag": String(),
-                "upload_sample": Boolean(False),
+                "distribution": Int(0, required=False),
+                "analysis": Int(0, required=False),
+                "threat_level": Int(4, required=False),
+                "min_malscore": Int(0, required=False),
+                "tag": String("Cuckoo", required=False),
+                "upload_sample": Boolean(False, required=False),
             },
             "mongodb": {
                 "enabled": Boolean(False),
diff --git a/cuckoo/common/whitelist.py b/cuckoo/common/whitelist.py
index 21043399c5..d8d802550b 100644
--- a/cuckoo/common/whitelist.py
+++ b/cuckoo/common/whitelist.py
@@ -14,19 +14,18 @@
 misphashes = set()
 
 def _load_whitelist(wlset, wl_file):
-    for b in (True, False):
-        private = {"private": True} if b else {}
-        wl_path = cwd("whitelist", wl_file, **private)
-        if not os.path.isfile(wl_path):
-            continue
+    wl_path = cwd("whitelist", wl_file)
 
-        with open(wl_path, "rb") as fp:
-            whitelist = fp.read()
+    if not os.path.isfile(wl_path):
+        wl_path = cwd("..", "data", "whitelist", wl_file, private=True)
 
-        for entry in whitelist.split("\n"):
-            entry = entry.strip()
-            if entry and not entry.startswith("#"):
-                wlset.add(entry)
+    with open(wl_path, "rb") as fp:
+        whitelist = fp.read()
+
+    for entry in whitelist.split("\n"):
+        entry = entry.strip()
+        if entry and not entry.startswith("#"):
+            wlset.add(entry)
 
 def is_whitelisted_domain(domain):
     if not domains:
@@ -44,28 +43,28 @@ def is_whitelisted_ip(ip):
 
 def is_whitelisted_mispdomain(domain):
     if not mispdomains:
-        # Initialize the domain whitelist.
+        # Initialize the misp domain whitelist.
         _load_whitelist(mispdomains, "mispdomain.txt")
 
     return domain in mispdomains
 
 def is_whitelisted_mispip(ip):
     if not mispips:
-        # Initialize the ip whitelist.
+        # Initialize the misp ip whitelist.
         _load_whitelist(mispips, "mispip.txt")
 
     return ip in mispips
 
 def is_whitelisted_mispurl(url):
     if not mispurls:
-        # Initialize the ip whitelist.
+        # Initialize the misp url whitelist.
         _load_whitelist(mispurls, "mispurl.txt")
 
-    return ip in mispurls
+    return url in mispurls
 
 def is_whitelisted_misphash(hash):
     if not misphashes:
-        # Initialize the ip whitelist.
+        # Initialize the misp hash whitelist.
         _load_whitelist(misphashes, "misphash.txt")
 
     return hash in misphashes
diff --git a/cuckoo/data/whitelist/domain.txt b/cuckoo/data/whitelist/domain.txt
index 32a0ad0846..b97ef46b7e 100644
--- a/cuckoo/data/whitelist/domain.txt
+++ b/cuckoo/data/whitelist/domain.txt
@@ -1 +1,35 @@
 # You can add whitelisted domains here.
+java.com
+www.msn.com
+www.bing.com
+windows.microsoft.com
+go.microsoft.com
+static-hp-eas.s-msn.com
+img-s-msn-com.akamaized.net
+sdlc-esd.oracle.com
+javadl.sun.com
+res2.windows.microsoft.com
+res1.windows.microsoft.com
+img.s-msn.com
+js.microsoft.com
+fbstatic-a.akamaihd.net
+ajax.microsoft.com
+ajax.aspnetcdn.com
+ieonline.microsoft.com
+api.bing.com
+schemas.microsoft.com
+www.w3.org
+dns.msftncsi.com
+teredo.ipv6.microsoft.com
+time.windows.com
+www.msftncsi.com
+ocsp.msocsp.com
+ocsp.omniroot.com
+crl.microsoft.com
+www.msftconnecttest.com
+v10.vortex-win.data.microsoft.com
+settings-win.data.microsoft.com
+win10.ipv6.microsoft.com
+sls.update.microsoft.com
+fs.microsoft.com
+ctldl.windowsupdate.com
\ No newline at end of file
diff --git a/cuckoo/private/whitelist/mispdomain.txt b/cuckoo/data/whitelist/mispdomain.txt
similarity index 79%
rename from cuckoo/private/whitelist/mispdomain.txt
rename to cuckoo/data/whitelist/mispdomain.txt
index 9f4d4a34df..79c56a4f93 100644
--- a/cuckoo/private/whitelist/mispdomain.txt
+++ b/cuckoo/data/whitelist/mispdomain.txt
@@ -1,3 +1,4 @@
+# Domains that should not be reported to MISP should be added here
 www.msftncsi.com
 dns.msftncsi.com
 teredo.ipv6.microsoft.com
diff --git a/cuckoo/data/whitelist/misphash.txt b/cuckoo/data/whitelist/misphash.txt
new file mode 100644
index 0000000000..c4a3cde4e0
--- /dev/null
+++ b/cuckoo/data/whitelist/misphash.txt
@@ -0,0 +1 @@
+# Hashes of file that should not be reported to MISP should be added here
\ No newline at end of file
diff --git a/cuckoo/data/whitelist/mispip.txt b/cuckoo/data/whitelist/mispip.txt
new file mode 100644
index 0000000000..8837e4c867
--- /dev/null
+++ b/cuckoo/data/whitelist/mispip.txt
@@ -0,0 +1,5 @@
+# IPs that should not be reported to MISP should be added here
+13.74.179.117
+40.81.120.221
+40.77.226.249
+8.8.8.8
diff --git a/cuckoo/data/whitelist/mispurl.txt b/cuckoo/data/whitelist/mispurl.txt
new file mode 100644
index 0000000000..2eb812c76d
--- /dev/null
+++ b/cuckoo/data/whitelist/mispurl.txt
@@ -0,0 +1 @@
+# URLs that should not be reported to MISP should be added here
\ No newline at end of file
diff --git a/cuckoo/private/cwd/conf/reporting.conf b/cuckoo/private/cwd/conf/reporting.conf
index 8742e19c62..752d6ae24f 100644
--- a/cuckoo/private/cwd/conf/reporting.conf
+++ b/cuckoo/private/cwd/conf/reporting.conf
@@ -31,6 +31,16 @@ apikey = {{ reporting.misp.apikey }}
 # separated by whitespace. Available modes: maldoc ipaddr hashes url.
 mode = {{ reporting.misp.mode }}
 
+distribution = {{ reporting.misp.distribution }}
+analysis = {{ reporting.misp.analysis }}
+threat_level = {{ reporting.misp.threat_level }}
+
+# The minimum Cuckoo score for a MISP event to be created
+min_malscore = {{ reporting.misp.min_malscore }}
+
+tag = {{ reporting.misp.tag }}
+upload_sample = {{ reporting.misp.upload_sample }}
+
 [mongodb]
 enabled = {{ reporting.mongodb.enabled }}
 host = {{ reporting.mongodb.host }}
diff --git a/cuckoo/private/whitelist/domain.txt b/cuckoo/private/whitelist/domain.txt
deleted file mode 100644
index d5c30dd02a..0000000000
--- a/cuckoo/private/whitelist/domain.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-java.com
-www.msn.com
-www.bing.com
-windows.microsoft.com
-go.microsoft.com
-static-hp-eas.s-msn.com
-img-s-msn-com.akamaized.net
-sdlc-esd.oracle.com
-javadl.sun.com
-res2.windows.microsoft.com
-res1.windows.microsoft.com
-img.s-msn.com
-js.microsoft.com
-fbstatic-a.akamaihd.net
-ajax.microsoft.com
-ajax.aspnetcdn.com
-ieonline.microsoft.com
-api.bing.com
-schemas.microsoft.com
-www.w3.org
-dns.msftncsi.com
-teredo.ipv6.microsoft.com
-time.windows.com
-www.msftncsi.com
-ocsp.msocsp.com
-ocsp.omniroot.com
-crl.microsoft.com
diff --git a/cuckoo/private/whitelist/mispip.txt b/cuckoo/private/whitelist/mispip.txt
deleted file mode 100644
index 5f6d8ff757..0000000000
--- a/cuckoo/private/whitelist/mispip.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-13.74.179.117
-40.81.120.221
-40.77.226.249
-8.8.8.8
diff --git a/cuckoo/reporting/misp.py b/cuckoo/reporting/misp.py
index 5feccf9917..fb5413b060 100644
--- a/cuckoo/reporting/misp.py
+++ b/cuckoo/reporting/misp.py
@@ -2,14 +2,17 @@
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
+import logging
 import os.path
 import shlex
 import warnings
-import logging
 
 from cuckoo.common.abstracts import Report
 from cuckoo.common.exceptions import CuckooProcessingError
-from cuckoo.common.whitelist import is_whitelisted_mispdomain,is_whitelisted_mispip,is_whitelisted_mispurl,is_whitelisted_misphash
+from cuckoo.common.whitelist import (
+    is_whitelisted_mispdomain, is_whitelisted_mispip, is_whitelisted_mispurl,
+    is_whitelisted_misphash
+)
 
 log = logging.getLogger(__name__)
 
@@ -34,21 +37,30 @@ def all_urls(self, results, event):
         urls = set()
         for protocol in ("http_ex", "https_ex"):
             for entry in results.get("network", {}).get(protocol, []):
-                if not is_whitelisted_mispdomain(entry["host"]) and not is_whitelisted_mispdomain(entry["host"]):
-                    url = "%s://%s%s" % (
-                          entry["protocol"], entry["host"], entry["uri"])
-                    if not is_whitelisted_mispurl(url):
-                        urls.add(url)
+                if is_whitelisted_mispdomain(entry["host"]):
+                    continue
+                if is_whitelisted_mispdomain(entry["host"]):
+                    continue
+
+                url = "%s://%s%s" % (
+                      entry["protocol"], entry["host"], entry["uri"])
+
+                if not is_whitelisted_mispurl(url):
+                    urls.add(url)
 
         self.misp.add_url(event, sorted(list(urls)))
 
     def domain_ipaddr(self, results, event):
-
         domains, ips = {}, set()
         for domain in results.get("network", {}).get("domains", []):
-            if not is_whitelisted_mispip(domain["ip"]) and not is_whitelisted_mispdomain(domain["domain"]):
-                domains[domain["domain"]] = domain["ip"]
-                ips.add(domain["ip"])
+            if is_whitelisted_mispip(domain["ip"]):
+                continue
+
+            if is_whitelisted_mispdomain(domain["domain"]):
+                continue
+
+            domains[domain["domain"]] = domain["ip"]
+            ips.add(domain["ip"])
 
         ipaddrs = set()
         for ipaddr in results.get("network", {}).get("hosts", []):
@@ -74,32 +86,47 @@ def family(self, results, event):
 
     def signature(self, results, event):
         for sig in results.get("signatures", []):
-            try:
-                temp = []
-                for mark in sig.get("marks", []):
-                    if mark["type"] == "generic":
-                        temp.append("%s %s" % (mark.get("parent_process", ""), mark.get("martian_process", "")))
-                        temp.append("%s %s" % (mark.get("reg_key", ""), mark.get("reg_value", "")))
-                        temp.append("%s %s" % (mark.get("option", ""), mark.get("value", "")))
-                        temp.append("%s" % mark.get("domain", ""))
-                        temp.append("%s" % mark.get("description", ""))
-                        temp.append("%s" % mark.get("host", ""))
-                    elif mark["type"] == "call":
-                        if not mark["call"]["api"] in temp:
-                            temp.append(mark["call"]["api"])
-                    elif mark["type"] == "config":
-                        temp.append(mark["config"]["url"])
-                    else:
-                        temp.append(mark[mark["type"]])
-                datainfo = ",".join([x for x in temp if x not in ["", " "]])
-            except:
-                datainfo = ",".join(sig["ttp"])
-                pass
-            data = "%s - (%s)" % (sig["description"], datainfo)
+
+            marks = []
+
+            if sig["ttp"]:
+                marks.append("%s" % ", ".join(sig["ttp"]))
+
+            for mark in sig.get("marks", []):
+                if mark["type"] == "generic":
+                    marks.append(
+                        "%s %s" % (mark.get("parent_process", ""),
+                                   mark.get("martian_process", ""))
+                    )
+                    marks.append(
+                        "%s %s" % (mark.get("reg_key", ""),
+                                   mark.get("reg_value", ""))
+                    )
+                    marks.append(
+                        "%s %s" % (mark.get("option", ""),
+                                   mark.get("value", ""))
+                    )
+                    marks.append("%s" % mark.get("domain", ""))
+                    marks.append("%s" % mark.get("description", ""))
+                    marks.append("%s" % mark.get("host", ""))
+
+                elif mark["type"] == "call":
+                    if not mark["call"]["api"] in marks:
+                        marks.append(mark["call"]["api"])
+
+                elif mark["type"] == "config":
+                    marks.append(mark["config"].get("url", ""))
+
+                else:
+                    marks.append(mark[mark["type"]])
+
+            markslist = ", ".join([x for x in marks if x and x != " "])
+
+            data = "%s - (%s)" % (sig["description"], markslist)
             self.misp.add_internal_comment(event, data)
             for att, description in sig["ttp"].items():
-                if description is None:
-                    log.warning("Description for %s is not found" % (att))
+                if not description:
+                    log.warning("Description for %s is not found", att)
                     continue
 
                 self.misp.add_internal_comment(
@@ -113,60 +140,71 @@ def run(self, results):
         url = self.options.get("url")
         apikey = self.options.get("apikey")
         mode = shlex.split(self.options.get("mode") or "")
-        score = results.get("info", None).get("score")
+        score = results.get("info", {}).get("score", 0)
         upload_sample = self.options.get("upload_sample")
-        f = results["target"]["file"]
-        hash_whitelisted = is_whitelisted_misphash(f["md5"]) or is_whitelisted_misphash(f["sha1"]) or is_whitelisted_misphash(f["sha256"])
-
-        if score >= self.options.get("min_malscore", 0) and not hash_whitelisted:
-            if not url or not apikey:
-                raise CuckooProcessingError(
-                    "Please configure the URL and API key for your MISP instance."
-                )
 
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore")
-                import pymisp
+        if results.get("target", {}).get("category") == "file":
+            f = results.get("target", {}).get("file", {})
+            hash_whitelisted = is_whitelisted_misphash(f["md5"]) or \
+                               is_whitelisted_misphash(f["sha1"]) or \
+                               is_whitelisted_misphash(f["sha256"])
 
-            self.misp = pymisp.PyMISP(url, apikey, False, "json")
+            if hash_whitelisted:
+                return
 
-            #Get default settings for a new event
-            distribution = self.options.get("distribution") or 0
-            threat_level = self.options.get("threat_level") or 4
-            analysis = self.options.get("analysis") or 0
-            tag = self.options.get("tag") or "Cuckoo"
+        if score < self.options.get("min_malscore", 0):
+            return
 
-            event = self.misp.new_event(
-                distribution=distribution,
-                threat_level_id=threat_level,
-                analysis=analysis,
-                info="Cuckoo Sandbox analysis #%d" % self.task["id"]
+        if not url or not apikey:
+            raise CuckooProcessingError(
+                "Please configure the URL and API key for your MISP "
+                "instance."
             )
 
-            # Add a specific tag to flag Cuckoo's event
-            if tag:
-                mispresult = self.misp.tag(event['Event']['uuid'], tag)
-                if mispresult.has_key('message'):
-                    log.debug("tag event: %s" % mispresult['message'])
-
-            if upload_sample and results.get("target", {}).get("category") == "file":
-                if results.get("target", {}).get("file", {}):
-                    self.misp.upload_sample(
-                        filename=os.path.basename(self.task["target"]),
-                        filepath_or_bytes=self.task["target"],
-                        event_id=event["Event"]["id"],
-                        category="External analysis",
-                    )
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            import pymisp
+
+        self.misp = pymisp.PyMISP(url, apikey, False, "json")
+
+        # Get default settings for a new event
+        distribution = self.options.get("distribution") or 0
+        threat_level = self.options.get("threat_level") or 4
+        analysis = self.options.get("analysis") or 0
+        tag = self.options.get("tag") or "Cuckoo"
+
+        event = self.misp.new_event(
+            distribution=distribution,
+            threat_level_id=threat_level,
+            analysis=analysis,
+            info="Cuckoo Sandbox analysis #%d" % self.task["id"]
+        )
+
+        # Add a specific tag to flag Cuckoo's event
+        if tag:
+            mispresult = self.misp.tag(event["Event"]["uuid"], tag)
+            if mispresult.has_key("message"):
+                log.debug("tag event: %s" % mispresult["message"])
+
+        if upload_sample:
+            target = results.get("target", {})
+            if target.get("category") == "file" and target.get("file"):
+                self.misp.upload_sample(
+                    filename=os.path.basename(self.task["target"]),
+                    filepath_or_bytes=self.task["target"],
+                    event_id=event["Event"]["id"],
+                    category="External analysis",
+                )
 
-            self.signature(results, event)
+        self.signature(results, event)
 
-            if "hashes" in mode:
-                self.sample_hashes(results, event)
+        if "hashes" in mode:
+            self.sample_hashes(results, event)
 
-            if "url" in mode:
-                self.all_urls(results, event)
+        if "url" in mode:
+            self.all_urls(results, event)
 
-            if "ipaddr" in mode:
-                self.domain_ipaddr(results, event)
+        if "ipaddr" in mode:
+            self.domain_ipaddr(results, event)
 
-            self.family(results, event)
+        self.family(results, event)

From b67066a17abcb485d1b4da234be088489bb509b8 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Tue, 28 May 2019 20:06:33 +0200
Subject: [PATCH 103/138] Test new misp changes

---
 tests/files/reportsignatures.json | 4131 +++++++++++++++++++++++++++++
 tests/test_reporting.py           |  133 +-
 2 files changed, 4241 insertions(+), 23 deletions(-)
 create mode 100644 tests/files/reportsignatures.json

diff --git a/tests/files/reportsignatures.json b/tests/files/reportsignatures.json
new file mode 100644
index 0000000000..001386e48b
--- /dev/null
+++ b/tests/files/reportsignatures.json
@@ -0,0 +1,4131 @@
+[
+  {
+    "families": [], 
+    "description": "Command line console output was observed", 
+    "ttp": {}, 
+    "name": "console_output", 
+    "markcount": 76, 
+    "references": [], 
+    "marks": [
+      {
+        "pid": 2896, 
+        "cid": 30, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The operation completed successfully.\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.094, 
+          "tid": 2900, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2932, 
+        "cid": 30, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The operation completed successfully.\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.10925, 
+          "tid": 2936, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2992, 
+        "cid": 30, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The operation completed successfully.\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.188125, 
+          "tid": 2996, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 182, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.219, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 184, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "echo", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.219, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 186, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": " WScript.Sleep(50) ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.219, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 190, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp/file.vbs ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.219, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 220, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.219, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 222, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "cscript", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.219, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 224, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": " C:\\Users\\ADMINI~1\\AppData\\Local\\Temp/file.vbs ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.219, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 273, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.609, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 275, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "del", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.609, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 277, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": " /F /Q file.js ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.609, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 293, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "Could Not Find C:\\Users\\Administrator\\AppData\\Local\\Temp\\file.js\r\n", 
+            "console_handle": "0x0000000b"
+          }, 
+          "time": 1558483088.625, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 310, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.641, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 312, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "del", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.641, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 314, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": " /F /Q \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\" ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.641, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 330, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.703, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 336, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The process cannot access the file because it is being used by another process.\r\n", 
+            "console_handle": "0x0000000b"
+          }, 
+          "time": 1558483088.703, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 355, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.703, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 357, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "del", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.719, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 359, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": " /F /Q \"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\YygokMcY.bat\" ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.719, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2112, 
+        "cid": 378, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The batch file cannot be found.\r\n", 
+            "console_handle": "0x0000000b"
+          }, 
+          "time": 1558483088.766, 
+          "tid": 2088, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2216, 
+        "cid": 41, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "Microsoft (R) Windows Script Host Version 5.8\r\nCopyright (C) Microsoft Corporation. All rights reserved.\r\n\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.438125, 
+          "tid": 2228, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2556, 
+        "cid": 30, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The operation completed successfully.\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.79725, 
+          "tid": 2552, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2588, 
+        "cid": 30, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The operation completed successfully.\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.90575, 
+          "tid": 2592, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2624, 
+        "cid": 30, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The operation completed successfully.\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483088.93775, 
+          "tid": 2620, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 181, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.20325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 183, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "echo", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.20325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 185, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": " WScript.Sleep(50) ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.20325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 189, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp/file.vbs ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.20325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 221, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.20325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 223, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "cscript", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.20325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 225, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": " C:\\Users\\ADMINI~1\\AppData\\Local\\Temp/file.vbs ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.20325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 279, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.56325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 281, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "del", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.56325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 283, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": " /F /Q file.js ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.56325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 299, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "Could Not Find C:\\Users\\Administrator\\AppData\\Local\\Temp\\file.js\r\n", 
+            "console_handle": "0x0000000b"
+          }, 
+          "time": 1558483089.56325, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 316, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.57825, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 318, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "del", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.57825, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 320, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": " /F /Q \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\" ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.57825, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 336, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.60925, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 342, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "Access is denied.\r\n", 
+            "console_handle": "0x0000000b"
+          }, 
+          "time": 1558483089.60925, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 360, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.60925, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 362, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "del", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.60925, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 364, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": " /F /Q \"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\PeUwoMwM.bat\" ", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.60925, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2956, 
+        "cid": 383, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The batch file cannot be found.\r\n", 
+            "console_handle": "0x0000000b"
+          }, 
+          "time": 1558483089.64125, 
+          "tid": 2964, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2148, 
+        "cid": 50, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "Microsoft (R) Windows Script Host Version 5.8\r\nCopyright (C) Microsoft Corporation. All rights reserved.\r\n\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.4215, 
+          "tid": 2204, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 3056, 
+        "cid": 30, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The operation completed successfully.\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.6405, 
+          "tid": 2648, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2484, 
+        "cid": 30, 
+        "call": {
+          "category": "misc", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "WriteConsoleW", 
+          "return_value": 1, 
+          "arguments": {
+            "buffer": "The operation completed successfully.\r\n", 
+            "console_handle": "0x00000007"
+          }, 
+          "time": 1558483089.703375, 
+          "tid": 2468, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }
+    ], 
+    "severity": 1
+  }, 
+  {
+    "families": [], 
+    "description": "HTTP traffic contains suspicious features which may be indicative of malware related traffic", 
+    "ttp": {}, 
+    "name": "network_cnc_http", 
+    "markcount": 1, 
+    "references": [], 
+    "marks": [
+      {
+        "suspicious_features": "GET method with no useragent header", 
+        "type": "generic", 
+        "suspicious_request": "GET http://google.com/"
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "Performs some HTTP requests", 
+    "ttp": {}, 
+    "name": "network_http", 
+    "markcount": 1, 
+    "references": [], 
+    "marks": [
+      {
+        "category": "request", 
+        "type": "ioc", 
+        "ioc": "GET http://google.com/", 
+        "description": null
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "Allocates read-write-execute memory (usually to unpack itself)", 
+    "ttp": {}, 
+    "name": "allocates_rwx", 
+    "markcount": 327, 
+    "references": [], 
+    "marks": [
+      {
+        "pid": 2560, 
+        "cid": 212, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x003f0000", 
+            "allocation_type": 4096, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 213, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 1691648, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01f50000", 
+            "allocation_type": 4096, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 214, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x005b0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 215, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x005c0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 216, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 31457280, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x020f0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 217, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 131072, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x005d0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 218, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x005f0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 223, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x00600000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 224, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x00610000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 225, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x00620000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 226, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01e50000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 227, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01e60000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 228, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01e70000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 229, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01e80000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 230, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01e90000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 241, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01eb0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 242, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01ec0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.96925, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 255, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01ec0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.98425, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 256, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01ed0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.98425, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 259, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01ec0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.98425, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 260, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01ed0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.98425, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 261, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01ee0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.98425, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 262, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01ef0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483086.98425, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 266, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01f00000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 267, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x041c0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 268, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x041d0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 269, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x041e0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 270, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x041f0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 425, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x041f0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 426, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04200000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 432, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x041f0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 433, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04200000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 442, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x041f0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 443, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04200000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 449, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x041f0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 450, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 8192, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04200000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 458, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x041f0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 459, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04200000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 460, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04210000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 461, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04220000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 462, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04230000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.76625, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 481, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04240000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.98425, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 482, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04250000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.98425, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 483, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2560, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x04260000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.98425, 
+          "tid": 2564, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2652, 
+        "cid": 212, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2652, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x003f0000", 
+            "allocation_type": 4096, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.516375, 
+          "tid": 2656, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2652, 
+        "cid": 213, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2652, 
+            "region_size": 1691648, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x01ec0000", 
+            "allocation_type": 4096, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.516375, 
+          "tid": 2656, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2652, 
+        "cid": 214, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2652, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x005b0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.516375, 
+          "tid": 2656, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2652, 
+        "cid": 215, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2652, 
+            "region_size": 4096, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x005c0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.516375, 
+          "tid": 2656, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2652, 
+        "cid": 216, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2652, 
+            "region_size": 31457280, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x020d0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.516375, 
+          "tid": 2656, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2652, 
+        "cid": 217, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "NtAllocateVirtualMemory", 
+          "return_value": 0, 
+          "arguments": {
+            "process_identifier": 2652, 
+            "region_size": 131072, 
+            "stack_dep_bypass": 0, 
+            "stack_pivoted": 0, 
+            "heap_dep_bypass": 0, 
+            "protection": 64, 
+            "base_address": "0x005d0000", 
+            "allocation_type": 12288, 
+            "process_handle": "0xffffffff"
+          }, 
+          "time": 1558483087.516375, 
+          "tid": 2656, 
+          "flags": {
+            "protection": "PAGE_EXECUTE_READWRITE", 
+            "allocation_type": "MEM_COMMIT|MEM_RESERVE"
+          }
+        }, 
+        "type": "call"
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "Creates a service", 
+    "ttp": {
+      "T1031": {
+        "short": "Modify Existing Service", 
+        "long": "Windows service configuration information, including the file path to the service's executable or recovery programs/commands, is stored in the Registry. Service configurations can be modified using utilities such as sc.exe and Reg."
+      }
+    }, 
+    "name": "creates_service", 
+    "markcount": 1, 
+    "references": [], 
+    "marks": [
+      {
+        "pid": 2560, 
+        "cid": 412, 
+        "call": {
+          "category": "services", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "CreateServiceW", 
+          "return_value": 6645584, 
+          "arguments": {
+            "service_start_name": "", 
+            "start_type": 2, 
+            "password": "", 
+            "display_name": "UIgAUMzM", 
+            "filepath": "C:\\ProgramData\\JcgAIEgc\\uOYocMso.exe", 
+            "service_name": "UIgAUMzM", 
+            "filepath_r": "C:\\ProgramData\\JcgAIEgc\\uOYocMso.exe", 
+            "desired_access": 983103, 
+            "service_handle": "0x00656750", 
+            "error_control": 0, 
+            "service_type": 16, 
+            "service_manager_handle": "0x006567c8"
+          }, 
+          "time": 1558483087.45325, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "Drops a binary and executes it", 
+    "ttp": {
+      "T1129": {
+        "short": "Execution through Module Load", 
+        "long": "The Windows module loader can be instructed to load DLLs from arbitrary local paths and arbitrary Universal Naming Convention (UNC) network paths. This functionality resides in NTDLL.dll and is part of the Windows Native API which is called from functions like CreateProcess(), LoadLibrary(), etc. of the Win32 API."
+      }
+    }, 
+    "name": "dropper", 
+    "markcount": 8, 
+    "references": [], 
+    "marks": [
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\eeokscog\\posooYos.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\ProgramData\\DUUAMwkg\\BeUQQcIE.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\uAgQwAUw.bat", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\YygokMcY.bat", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\CcYUMEMc.bat", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\SAUoIwsA.bat", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\YUMQIYMY.bat", 
+        "description": null
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "Drops an executable to the user AppData folder", 
+    "ttp": {
+      "T1129": {
+        "short": "Execution through Module Load", 
+        "long": "The Windows module loader can be instructed to load DLLs from arbitrary local paths and arbitrary Universal Naming Convention (UNC) network paths. This functionality resides in NTDLL.dll and is part of the Windows Native API which is called from functions like CreateProcess(), LoadLibrary(), etc. of the Win32 API."
+      }
+    }, 
+    "name": "exe_appdata", 
+    "markcount": 30, 
+    "references": [], 
+    "marks": [
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\tMUy.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\FoUm.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\PUIk.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\tYki.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\gEYu.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\qAYm.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\vkIo.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\mMYq.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\aoQs.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\tEEi.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\Gwgg.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\RkAQ.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ikEa.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\YskU.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\VkMk.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\aEwC.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\gcUw.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\FMck.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\nMkg.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\JAMG.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\wYAC.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\Kowa.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\gksC.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\CwQa.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\xUAW.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\FEQg.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\oIgG.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\jIQo.exe", 
+        "description": null
+      }, 
+      {
+        "category": "file", 
+        "type": "ioc", 
+        "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\lYEc.exe", 
+        "description": null
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "A process created a hidden window", 
+    "ttp": {
+      "T1143": {
+        "short": "Hidden Window", 
+        "long": "The configurations for how applications run on macOS and OS X are listed in property list (plist) files. One of the tags in these files can be apple.awt.UIElement, which allows for Java applications to prevent the application's icon from appearing in the Dock. A common use for this is when applications run in the system tray, but don't also want to show up in the Dock. However, adversaries can abuse this feature and hide their running window  ."
+      }
+    }, 
+    "name": "stealth_window", 
+    "markcount": 8, 
+    "references": [], 
+    "marks": [
+      {
+        "pid": 2560, 
+        "cid": 469, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "CreateProcessInternalW", 
+          "return_value": 1, 
+          "arguments": {
+            "thread_identifier": 2812, 
+            "thread_handle": "0x00000100", 
+            "process_identifier": 2808, 
+            "current_directory": "C:\\Users\\Administrator\\AppData\\Local\\Temp", 
+            "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\uAgQwAUw.bat", 
+            "track": 1, 
+            "command_line": "\"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391\"", 
+            "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\uAgQwAUw.bat", 
+            "stack_pivoted": 0, 
+            "creation_flags": 134217728, 
+            "inherit_handles": 0, 
+            "process_handle": "0x00000108"
+          }, 
+          "time": 1558483087.81325, 
+          "tid": 2564, 
+          "flags": {
+            "creation_flags": "CREATE_NO_WINDOW"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 487, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "CreateProcessInternalW", 
+          "return_value": 1, 
+          "arguments": {
+            "thread_identifier": 2088, 
+            "thread_handle": "0x00000110", 
+            "process_identifier": 2112, 
+            "current_directory": "", 
+            "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\YygokMcY.bat", 
+            "track": 1, 
+            "command_line": "\"\"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\YygokMcY.bat\" \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\"\"", 
+            "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\YygokMcY.bat", 
+            "stack_pivoted": 0, 
+            "creation_flags": 134217728, 
+            "inherit_handles": 0, 
+            "process_handle": "0x0000010c"
+          }, 
+          "time": 1558483088.03125, 
+          "tid": 2564, 
+          "flags": {
+            "creation_flags": "CREATE_NO_WINDOW"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 3048, 
+        "cid": 399, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "CreateProcessInternalW", 
+          "return_value": 1, 
+          "arguments": {
+            "thread_identifier": 2348, 
+            "thread_handle": "0x00000124", 
+            "process_identifier": 2368, 
+            "current_directory": "C:\\Users\\Administrator\\AppData\\Local\\Temp", 
+            "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\CcYUMEMc.bat", 
+            "track": 1, 
+            "command_line": "\"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391\"", 
+            "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\CcYUMEMc.bat", 
+            "stack_pivoted": 0, 
+            "creation_flags": 134217728, 
+            "inherit_handles": 0, 
+            "process_handle": "0x00000128"
+          }, 
+          "time": 1558483088.48375, 
+          "tid": 3052, 
+          "flags": {
+            "creation_flags": "CREATE_NO_WINDOW"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 3048, 
+        "cid": 417, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "CreateProcessInternalW", 
+          "return_value": 1, 
+          "arguments": {
+            "thread_identifier": 2964, 
+            "thread_handle": "0x00000130", 
+            "process_identifier": 2956, 
+            "current_directory": "", 
+            "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\PeUwoMwM.bat", 
+            "track": 1, 
+            "command_line": "\"\"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\PeUwoMwM.bat\" \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\"\"", 
+            "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\PeUwoMwM.bat", 
+            "stack_pivoted": 0, 
+            "creation_flags": 134217728, 
+            "inherit_handles": 0, 
+            "process_handle": "0x0000012c"
+          }, 
+          "time": 1558483088.93775, 
+          "tid": 3052, 
+          "flags": {
+            "creation_flags": "CREATE_NO_WINDOW"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2704, 
+        "cid": 396, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "CreateProcessInternalW", 
+          "return_value": 1, 
+          "arguments": {
+            "thread_identifier": 2936, 
+            "thread_handle": "0x00000124", 
+            "process_identifier": 1368, 
+            "current_directory": "C:\\Users\\Administrator\\AppData\\Local\\Temp", 
+            "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\SAUoIwsA.bat", 
+            "track": 1, 
+            "command_line": "\"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391\"", 
+            "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\SAUoIwsA.bat", 
+            "stack_pivoted": 0, 
+            "creation_flags": 134217728, 
+            "inherit_handles": 0, 
+            "process_handle": "0x00000128"
+          }, 
+          "time": 1558483089.28125, 
+          "tid": 2728, 
+          "flags": {
+            "creation_flags": "CREATE_NO_WINDOW"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2704, 
+        "cid": 414, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "CreateProcessInternalW", 
+          "return_value": 1, 
+          "arguments": {
+            "thread_identifier": 2556, 
+            "thread_handle": "0x00000130", 
+            "process_identifier": 2552, 
+            "current_directory": "", 
+            "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\kSIUwkcE.bat", 
+            "track": 1, 
+            "command_line": "\"\"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\kSIUwkcE.bat\" \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\"\"", 
+            "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\kSIUwkcE.bat", 
+            "stack_pivoted": 0, 
+            "creation_flags": 134217728, 
+            "inherit_handles": 0, 
+            "process_handle": "0x0000012c"
+          }, 
+          "time": 1558483089.60925, 
+          "tid": 2728, 
+          "flags": {
+            "creation_flags": "CREATE_NO_WINDOW"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2636, 
+        "cid": 399, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "CreateProcessInternalW", 
+          "return_value": 1, 
+          "arguments": {
+            "thread_identifier": 2968, 
+            "thread_handle": "0x00000124", 
+            "process_identifier": 2952, 
+            "current_directory": "C:\\Users\\Administrator\\AppData\\Local\\Temp", 
+            "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\YUMQIYMY.bat", 
+            "track": 1, 
+            "command_line": "\"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391\"", 
+            "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\YUMQIYMY.bat", 
+            "stack_pivoted": 0, 
+            "creation_flags": 134217728, 
+            "inherit_handles": 0, 
+            "process_handle": "0x00000128"
+          }, 
+          "time": 1558483090.10875, 
+          "tid": 2488, 
+          "flags": {
+            "creation_flags": "CREATE_NO_WINDOW"
+          }
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2636, 
+        "cid": 417, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "CreateProcessInternalW", 
+          "return_value": 1, 
+          "arguments": {
+            "thread_identifier": 2376, 
+            "thread_handle": "0x00000130", 
+            "process_identifier": 3056, 
+            "current_directory": "", 
+            "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\hyckYQMo.bat", 
+            "track": 1, 
+            "command_line": "\"\"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\hyckYQMo.bat\" \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\"\"", 
+            "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\hyckYQMo.bat", 
+            "stack_pivoted": 0, 
+            "creation_flags": 134217728, 
+            "inherit_handles": 0, 
+            "process_handle": "0x0000012c"
+          }, 
+          "time": 1558483090.35875, 
+          "tid": 2488, 
+          "flags": {
+            "creation_flags": "CREATE_NO_WINDOW"
+          }
+        }, 
+        "type": "call"
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "Searches running processes potentially to identify processes for sandbox evasion, code injection or memory dumping", 
+    "ttp": {
+      "T1057": {
+        "short": "Process Discovery", 
+        "long": "Adversaries may attempt to get information about running processes on a system. Information obtained could be used to gain an understanding of common software running on systems within the network."
+      }
+    }, 
+    "name": "injection_process_search", 
+    "markcount": 40, 
+    "references": [], 
+    "marks": [
+      {
+        "pid": 2560, 
+        "cid": 272, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32FirstW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "[System Process]", 
+            "process_identifier": 0
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 273, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "System", 
+            "process_identifier": 4
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 274, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "smss.exe", 
+            "process_identifier": 268
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 275, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "csrss.exe", 
+            "process_identifier": 344
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 276, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "wininit.exe", 
+            "process_identifier": 376
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 277, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "csrss.exe", 
+            "process_identifier": 396
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 278, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "winlogon.exe", 
+            "process_identifier": 432
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 279, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "services.exe", 
+            "process_identifier": 476
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 280, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "lsass.exe", 
+            "process_identifier": 492
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 281, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "lsm.exe", 
+            "process_identifier": 500
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 282, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "svchost.exe", 
+            "process_identifier": 608
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 283, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "svchost.exe", 
+            "process_identifier": 684
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 284, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "svchost.exe", 
+            "process_identifier": 764
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 285, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "svchost.exe", 
+            "process_identifier": 816
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 286, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "svchost.exe", 
+            "process_identifier": 860
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 287, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "svchost.exe", 
+            "process_identifier": 892
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 288, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "svchost.exe", 
+            "process_identifier": 292
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 289, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "spoolsv.exe", 
+            "process_identifier": 332
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 290, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "svchost.exe", 
+            "process_identifier": 1052
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 291, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "taskhost.exe", 
+            "process_identifier": 1180
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 292, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "userinit.exe", 
+            "process_identifier": 1256
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 293, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "dwm.exe", 
+            "process_identifier": 1300
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 294, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "explorer.exe", 
+            "process_identifier": 1360
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 295, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "svchost.exe", 
+            "process_identifier": 1756
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 296, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "python.exe", 
+            "process_identifier": 1872
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 297, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "conhost.exe", 
+            "process_identifier": 1900
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 298, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "SearchIndexer.exe", 
+            "process_identifier": 1836
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 299, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "SearchProtocolHost.exe", 
+            "process_identifier": 672
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 300, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "SearchFilterHost.exe", 
+            "process_identifier": 776
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 301, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "SearchProtocolHost.exe", 
+            "process_identifier": 1948
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 302, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "mobsync.exe", 
+            "process_identifier": 2116
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 303, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "python.exe", 
+            "process_identifier": 2264
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 304, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "wsqmcons.exe", 
+            "process_identifier": 2308
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 305, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "sdclt.exe", 
+            "process_identifier": 2352
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 306, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "taskhost.exe", 
+            "process_identifier": 2404
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 307, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe", 
+            "process_identifier": 2560
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 308, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "schtasks.exe", 
+            "process_identifier": 2616
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 309, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e0", 
+            "process_name": "conhost.exe", 
+            "process_identifier": 2624
+          }, 
+          "time": 1558483087.18825, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 2560, 
+        "cid": 368, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000ec", 
+            "process_name": "posooYos.exe", 
+            "process_identifier": 2652
+          }, 
+          "time": 1558483087.29725, 
+          "tid": 2564, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }, 
+      {
+        "pid": 3048, 
+        "cid": 332, 
+        "call": {
+          "category": "process", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "Process32NextW", 
+          "return_value": 1, 
+          "arguments": {
+            "snapshot_handle": "0x000000e4", 
+            "process_name": "BeUQQcIE.exe", 
+            "process_identifier": 2692
+          }, 
+          "time": 1558483088.39075, 
+          "tid": 3052, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "The binary likely contains encrypted or compressed data indicative of a packer", 
+    "ttp": {
+      "T1045": {
+        "short": "Software Packing", 
+        "long": "Software packing is a method of compressing or encrypting an executable. Packing an executable changes the file signature in an attempt to avoid signature-based detection. Most decompression techniques decompress the executable code in memory."
+      }
+    }, 
+    "name": "packer_entropy", 
+    "markcount": 2, 
+    "references": [
+      "http://www.forensickb.com/2013/03/file-entropy-explained.html", 
+      "http://virii.es/U/Using%20Entropy%20Analysis%20to%20Find%20Encrypted%20and%20Packed%20Malware.pdf"
+    ], 
+    "marks": [
+      {
+        "type": "generic", 
+        "section": {
+          "size_of_data": "0x001a0c00", 
+          "virtual_address": "0x00001000", 
+          "entropy": 7.227448472154554, 
+          "name": ".text", 
+          "virtual_size": "0x001a0b93"
+        }, 
+        "entropy": 7.227448472154554, 
+        "description": "A section with a high entropy has been found"
+      }, 
+      {
+        "type": "generic", 
+        "entropy": 0.9973078073586599, 
+        "description": "Overall entropy of this PE file is high"
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "Expresses interest in specific running processes", 
+    "ttp": {
+      "T1057": {
+        "short": "Process Discovery", 
+        "long": "Adversaries may attempt to get information about running processes on a system. Information obtained could be used to gain an understanding of common software running on systems within the network."
+      }
+    }, 
+    "name": "process_interest", 
+    "markcount": 2, 
+    "references": [], 
+    "marks": [
+      {
+        "category": "process", 
+        "type": "ioc", 
+        "ioc": "beuqqcie.exe", 
+        "description": null
+      }, 
+      {
+        "category": "process", 
+        "type": "ioc", 
+        "ioc": "posooyos.exe", 
+        "description": null
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "Uses Windows utilities for basic Windows functionality", 
+    "ttp": {
+      "T1053": {
+        "short": "Scheduled Task", 
+        "long": "Utilities such as at and schtasks, along with the Windows Task Scheduler, can be used to schedule programs or scripts to be executed at a date and time. A task can also be scheduled on a remote system, provided the proper authentication is met to use RPC and file and printer sharing is turned on. Scheduling a task on a remote system typically required being a member of the Administrators group on the the remote system."
+      }
+    }, 
+    "name": "uses_windows_utilities", 
+    "markcount": 3, 
+    "references": [
+      "http://blog.jpcert.or.jp/2016/01/windows-commands-abused-by-attackers.html"
+    ], 
+    "marks": [
+      {
+        "category": "cmdline", 
+        "type": "ioc", 
+        "ioc": "reg add HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced /f /v HideFileExt /t REG_DWORD /d 1", 
+        "description": null
+      }, 
+      {
+        "category": "cmdline", 
+        "type": "ioc", 
+        "ioc": "reg add HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced /f /v Hidden /t REG_DWORD /d 2", 
+        "description": null
+      }, 
+      {
+        "category": "cmdline", 
+        "type": "ioc", 
+        "ioc": "reg add HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System /v EnableLUA /d 0 /t REG_DWORD /f", 
+        "description": null
+      }
+    ], 
+    "severity": 2
+  }, 
+  {
+    "families": [], 
+    "description": "Communicates with host for which no DNS query was performed", 
+    "ttp": {}, 
+    "name": "nolookup_communication", 
+    "markcount": 1, 
+    "references": [], 
+    "marks": [
+      {
+        "host": "200.87.164.69", 
+        "type": "generic"
+      }
+    ], 
+    "severity": 3
+  }, 
+  {
+    "families": [], 
+    "description": "Checks for the presence of known windows from debuggers and forensic tools", 
+    "ttp": {
+      "T1057": {
+        "short": "Process Discovery", 
+        "long": "Adversaries may attempt to get information about running processes on a system. Information obtained could be used to gain an understanding of common software running on systems within the network."
+      }
+    }, 
+    "name": "antidbg_windows", 
+    "markcount": 1, 
+    "references": [], 
+    "marks": [
+      {
+        "pid": 2652, 
+        "cid": 32216, 
+        "call": {
+          "category": "ui", 
+          "status": 1, 
+          "stacktrace": [], 
+          "api": "FindWindowA", 
+          "return_value": 65832, 
+          "arguments": {
+            "class_name": "ConsoleWindowClass", 
+            "window_name": ""
+          }, 
+          "time": 1558483102.703375, 
+          "tid": 2968, 
+          "flags": {}
+        }, 
+        "type": "call"
+      }
+    ], 
+    "severity": 3
+  }, 
+  {
+    "families": [], 
+    "description": "Installs itself for autorun at Windows startup", 
+    "ttp": {
+      "T1060": {
+        "short": "Registry Run Keys / Startup Folder", 
+        "long": "Adding an entry to the \"run keys\" in the Registry or startup folder will cause the program referenced to be executed when a user logs in.  These programs will be executed under the context of the user and will have the account's associated permissions level."
+      }, 
+      "T1053": {
+        "short": "Scheduled Task", 
+        "long": "Utilities such as at and schtasks, along with the Windows Task Scheduler, can be used to schedule programs or scripts to be executed at a date and time. A task can also be scheduled on a remote system, provided the proper authentication is met to use RPC and file and printer sharing is turned on. Scheduling a task on a remote system typically required being a member of the Administrators group on the the remote system."
+      }
+    }, 
+    "name": "persistence_autorun", 
+    "markcount": 5, 
+    "references": [], 
+    "marks": [
+      {
+        "type": "generic", 
+        "reg_key": "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Run\\posooYos.exe", 
+        "reg_value": "C:\\Users\\Administrator\\eeokscog\\posooYos.exe"
+      }, 
+      {
+        "type": "generic", 
+        "reg_key": "HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Run\\BeUQQcIE.exe", 
+        "reg_value": "C:\\ProgramData\\DUUAMwkg\\BeUQQcIE.exe"
+      }, 
+      {
+        "service_name": "UIgAUMzM", 
+        "type": "generic", 
+        "service_path": "C:\\ProgramData\\JcgAIEgc\\uOYocMso.exe"
+      }, 
+      {
+        "type": "generic", 
+        "reg_key": "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Run\\posooYos.exe", 
+        "reg_value": "C:\\Users\\Administrator\\eeokscog\\posooYos.exe"
+      }, 
+      {
+        "type": "generic", 
+        "reg_key": "HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Run\\BeUQQcIE.exe", 
+        "reg_value": "C:\\ProgramData\\DUUAMwkg\\BeUQQcIE.exe"
+      }
+    ], 
+    "severity": 3
+  }, 
+  {
+    "families": [], 
+    "description": "Attempts to modify Explorer settings to prevent file extensions from being displayed", 
+    "ttp": {
+      "T1158": {
+        "short": "Hidden Files and Directories", 
+        "long": "To prevent normal users from accidentally changing special files on a system, most operating systems have the concept of a \u2018hidden\u2019 file. These files don\u2019t show up when a user browses the file system with a GUI or when using normal commands on the command line. Users must explicitly ask to show the hidden files either via a series of Graphical User Interface (GUI) prompts or with command line switches (dir /a for Windows and ls \u2013a for Linux and macOS)."
+      }, 
+      "T1054": {
+        "short": "Indicator Blocking", 
+        "long": "An adversary may attempt to block indicators or events typically captured by sensors from being gathered and analyzed. This could include modifying sensor settings stored in configuration files and/or Registry keys to disable or maliciously redirect event telemetry."
+      }
+    }, 
+    "name": "stealth_hidden_extension", 
+    "markcount": 1, 
+    "references": [], 
+    "marks": [
+      {
+        "category": "registry", 
+        "type": "ioc", 
+        "ioc": "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\HideFileExt", 
+        "description": null
+      }
+    ], 
+    "severity": 3
+  }, 
+  {
+    "families": [], 
+    "description": "Attempts to modify Explorer settings to prevent hidden files from being displayed", 
+    "ttp": {
+      "T1158": {
+        "short": "Hidden Files and Directories", 
+        "long": "To prevent normal users from accidentally changing special files on a system, most operating systems have the concept of a \u2018hidden\u2019 file. These files don\u2019t show up when a user browses the file system with a GUI or when using normal commands on the command line. Users must explicitly ask to show the hidden files either via a series of Graphical User Interface (GUI) prompts or with command line switches (dir /a for Windows and ls \u2013a for Linux and macOS)."
+      }, 
+      "T1054": {
+        "short": "Indicator Blocking", 
+        "long": "An adversary may attempt to block indicators or events typically captured by sensors from being gathered and analyzed. This could include modifying sensor settings stored in configuration files and/or Registry keys to disable or maliciously redirect event telemetry."
+      }
+    }, 
+    "name": "stealth_hiddenfile", 
+    "markcount": 1, 
+    "references": [], 
+    "marks": [
+      {
+        "category": "registry", 
+        "type": "ioc", 
+        "ioc": "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\Hidden", 
+        "description": null
+      }
+    ], 
+    "severity": 3
+  }, 
+  {
+    "families": [], 
+    "description": "Disables Windows Security features", 
+    "ttp": {
+      "T1089": {
+        "short": "Disabling Security Tools", 
+        "long": "Adversaries may disable security tools to avoid possible detection of their tools and activities. This can take the form of killing security software or event logging processes, deleting Registry keys so that tools do not start at run time, or other methods to interfere with security scanning or event reporting."
+      }, 
+      "T1112": {
+        "short": "Modify Registry", 
+        "long": "Adversaries may interact with the Windows Registry to hide configuration information within Registry keys, remove information as part of cleaning up, or as part of other techniques to aid in Persistence and Execution."
+      }
+    }, 
+    "name": "disables_security", 
+    "markcount": 1, 
+    "references": [], 
+    "marks": [
+      {
+        "type": "generic", 
+        "description": "attempts to disable user access control", 
+        "registry": "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\EnableLUA"
+      }
+    ], 
+    "severity": 4
+  }, 
+  {
+    "families": [], 
+    "description": "Connects to an IP address that is no longer responding to requests (legitimate services will remain up-and-running usually)", 
+    "ttp": {}, 
+    "name": "dead_host", 
+    "markcount": 1, 
+    "references": [], 
+    "marks": [
+      {
+        "category": "dead_host", 
+        "type": "ioc", 
+        "ioc": "200.87.164.69:9999", 
+        "description": null
+      }
+    ], 
+    "severity": 5
+  }
+]
\ No newline at end of file
diff --git a/tests/test_reporting.py b/tests/test_reporting.py
index c25b4a9434..3378d56c44 100644
--- a/tests/test_reporting.py
+++ b/tests/test_reporting.py
@@ -2,6 +2,7 @@
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
+import json
 import mock
 import os.path
 import pytest
@@ -20,6 +21,7 @@
 from cuckoo.reporting.mongodb import MongoDB
 from cuckoo.reporting.singlefile import SingleFile
 
+
 def test_init():
     p = Report()
     p.set_options({
@@ -114,6 +116,92 @@ def test_empty_mattermost():
     task(1, {}, conf, {})
     assert len(responses.calls) == 2
 
+@responses.activate
+def test_min_malscore_misp_low():
+    """Try to send event with low malscore."""
+    set_cwd(tempfile.mkdtemp())
+    conf = {
+        "misp": {
+            "enabled": True,
+            "url": "https://misphost",
+            "apikey": "A"*32,
+            "mode": "",
+            "min_malscore": 5
+        }
+    }
+
+    with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps:
+        rsps.add(
+            responses.GET, "https://misphost/servers/getPyMISPVersion.json",
+            json={
+                "version": "2.4.103"
+            }
+        )
+
+        rsps.add(
+            responses.GET, "https://misphost/attributes/describeTypes.json",
+            json={
+                "result": {
+                    "categories": None,
+                    "types": None,
+                    "category_type_mappings": None,
+                    "sane_defaults": True,
+                },
+            },
+        )
+        rsps.add(
+            responses.POST, "https://misphost/events",
+            json={
+                "response": None,
+            },
+        )
+
+        task(2, {}, conf, {"info": {"score": 2}})
+        assert len(rsps.calls) == 0
+
+@responses.activate
+def test_min_malscore_misp():
+    """Try to send event with low malscore."""
+    set_cwd(tempfile.mkdtemp())
+    conf = {
+        "misp": {
+            "enabled": True,
+            "url": "https://misphost",
+            "apikey": "A"*32,
+            "mode": "",
+            "min_malscore": 5
+        }
+    }
+
+    with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps:
+        rsps.add(
+            responses.GET, "https://misphost/servers/getPyMISPVersion.json",
+            json={
+                "version": "2.4.103"
+            }
+        )
+
+        rsps.add(
+            responses.GET, "https://misphost/attributes/describeTypes.json",
+            json={
+                "result": {
+                    "categories": None,
+                    "types": None,
+                    "category_type_mappings": None,
+                    "sane_defaults": True,
+                },
+            },
+        )
+        rsps.add(
+            responses.POST, "https://misphost/events",
+            json={
+                "response": None,
+            },
+        )
+
+        task(2, {}, conf, {"info": {"score": 6}})
+        assert len(rsps.calls) == 3
+
 @responses.activate
 def test_empty_misp():
     """Merely connect to MISP and create the new event."""
@@ -182,27 +270,28 @@ def test_misp_signatures():
     r.misp = mock.MagicMock()
     r.misp.add_internal_comment.return_value = None
 
-    r.signature({
-            "signatures": [
-                {
-                    "description": "Very signature",
-                    "ttp": {
-                        "T1045": {
-                            "short": "Short description",
-                            "long": "A longer description"
-                        }
-                    }
-                }
-            ]
-    }, "event")
+    with open("tests/files/reportsignatures.json", "rb") as fp:
+        signatures = json.load(fp)
+
+    r.signature({"signatures": signatures}, "event")
 
-    assert r.misp.add_internal_comment.call_count == 2
+    assert r.misp.add_internal_comment.call_count == 36
     r.misp.add_internal_comment.assert_has_calls([
-        mock.call("event", "Very signature - (T1045)"),
-        mock.call("event", "TTP: T1045, short: Short description")
-    ])
+        mock.call("event", "Creates a service - (T1031, CreateServiceW)"),
+        mock.call("event", "Searches running processes potentially to identify"
+                           " processes for sandbox evasion, code injection or"
+                           " memory dumping -"
+                           " (T1057, Process32FirstW, Process32NextW)"),
+        mock.call("event", "TTP: T1054, short: Indicator Blocking"),
+        mock.call("event", "Disables Windows Security features -"
+                           " (T1089, T1112, attempts to disable user access"
+                           " control)"),
+        mock.call("event", "Communicates with host for which no DNS query was"
+                           " performed - (200.87.164.69)")
+    ], any_order=True)
 
 def test_misp_all_urls():
+    set_cwd(tempfile.mkdtemp())
     r = MISP()
     r.misp = mock.MagicMock()
     r.misp.add_url.return_value = None
@@ -232,6 +321,7 @@ def test_misp_all_urls():
     )
 
 def test_misp_domain_ipaddr():
+    set_cwd(tempfile.mkdtemp())
     r = MISP()
     r.misp = mock.MagicMock()
     r.misp.add_domains_ips.return_value = None
@@ -245,9 +335,6 @@ def test_misp_domain_ipaddr():
                     "ip": "1.2.3.4",
                 },
                 {
-                    # TODO Now that we have global whitelisting, this
-                    # custom-made support for the MISP reporting module should
-                    # probably be removed.
                     "domain": "time.windows.com",
                     "ip": "1.2.3.4",
                 },
@@ -302,9 +389,9 @@ def test_misp_family():
 
     assert r.misp.add_detection_name.call_count == 3
     r.misp.add_detection_name.assert_has_calls([
-        mock.call("event", "3x4mpl3", "Sandbox detection"),
-        mock.call("event", "3x4mpl3_2", "Sandbox detection"),
-        mock.call("event", "3x4mpl3_3", "Sandbox detection")
+        mock.call("event", "3x4mpl3", "External analysis"),
+        mock.call("event", "3x4mpl3_2", "External analysis"),
+        mock.call("event", "3x4mpl3_3", "External analysis")
     ])
 
     assert r.misp.add_url.call_count == 2

From 148c92e3dc6d741a06055cb5af58215252382b79 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Tue, 28 May 2019 20:45:04 +0200
Subject: [PATCH 104/138] Update apps tests for new whitelist

---
 tests/test_apps.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_apps.py b/tests/test_apps.py
index aaf9389760..4ab746bee7 100644
--- a/tests/test_apps.py
+++ b/tests/test_apps.py
@@ -717,9 +717,9 @@ def test_new_directory(self):
         # TODO Move this to its own 2.0.3 -> 2.0.4 migration handler.
         assert os.path.exists(cwd("stuff"))
         assert os.path.exists(cwd("whitelist"))
-        assert open(cwd("whitelist", "domain.txt"), "rb").read().strip() == (
-            "# You can add whitelisted domains here."
-        )
+
+        wl = open(cwd("whitelist", "domain.txt"), "rb").read().split("\n")
+        assert wl[0] == "# You can add whitelisted domains here."
         assert os.path.exists(cwd("yara", "dumpmem"))
         assert not os.path.exists(cwd("yara", "index_binaries.yar"))
 

From 3d6d977228594143e6f08949f3e727565ad8ef20 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 5 Jun 2019 18:36:43 +0200
Subject: [PATCH 105/138] Store Cuckoo stderr when using supervisord

---
 cuckoo/private/cwd/supervisord.jinja2 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cuckoo/private/cwd/supervisord.jinja2 b/cuckoo/private/cwd/supervisord.jinja2
index bdd59b2c49..f722564cb4 100644
--- a/cuckoo/private/cwd/supervisord.jinja2
+++ b/cuckoo/private/cwd/supervisord.jinja2
@@ -17,6 +17,7 @@ command = {{ cuckoo_path }} -d -m 10000
 user = {{ username }}
 startsecs = 30
 autorestart = true
+stderr_logfile = {{ cwd("supervisord", "cuckoostderr.log") }}
 
 [program:cuckoo-process]
 command = {{ cuckoo_path }} process p%(process_num)d

From d40eae5335788e5ee45f024187a6f12caaa33277 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 5 Jun 2019 19:20:57 +0200
Subject: [PATCH 106/138] Init logging and create cuckootmp for wsgi apps

---
 cuckoo/apps/api.py     | 3 +++
 cuckoo/web/web/wsgi.py | 5 ++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/cuckoo/apps/api.py b/cuckoo/apps/api.py
index 3ddb6fb6e0..ce383d2d58 100644
--- a/cuckoo/apps/api.py
+++ b/cuckoo/apps/api.py
@@ -714,5 +714,8 @@ def cuckoo_api(hostname, port, debug):
     app.run(host=hostname, port=port, debug=debug)
 
 if os.environ.get("CUCKOO_APP") == "api":
+    from cuckoo.core.startup import ensure_tmpdir, init_console_logging
     decide_cwd(exists=True)
     Database().connect()
+    init_console_logging()
+    ensure_tmpdir()
diff --git a/cuckoo/web/web/wsgi.py b/cuckoo/web/web/wsgi.py
index 5174683407..8d08a7c343 100644
--- a/cuckoo/web/web/wsgi.py
+++ b/cuckoo/web/web/wsgi.py
@@ -1,5 +1,5 @@
 # Copyright (C) 2010-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2016 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -24,6 +24,7 @@
 
 import cuckoo
 
+from cuckoo.core.startup import ensure_tmpdir, init_console_logging
 from cuckoo.misc import decide_cwd
 
 if os.environ.get("CUCKOO_APP") == "web":
@@ -33,6 +34,8 @@
     sys.path.insert(0, ".")
 
     cuckoo.core.database.Database().connect()
+    init_console_logging()
+    ensure_tmpdir()
 
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "web.settings")
 

From dc7ad12d65bd41de4cc84fe963f3fdb8ccf6f3ed Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 6 Jun 2019 10:33:53 +0200
Subject: [PATCH 107/138] Don't stop analysis if machinery does not support RDP
 and it is enabled

---
 cuckoo/core/scheduler.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py
index 40d99b580e..b7723777c9 100644
--- a/cuckoo/core/scheduler.py
+++ b/cuckoo/core/scheduler.py
@@ -1,5 +1,5 @@
 # Copyright (C) 2012-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2018 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -479,9 +479,10 @@ def launch_analysis(self):
             try:
                 machinery.enable_remote_control(self.machine.label)
             except NotImplementedError:
-                raise CuckooMachineError(
-                    "Remote control support has not been implemented "
-                    "for this machinery."
+                log.error(
+                    "Remote control support has not been implemented for the "
+                    "configured machinery module: %s",
+                    config("cuckoo:cuckoo:machinery")
                 )
 
         try:
@@ -516,9 +517,10 @@ def launch_analysis(self):
                     )
                     self.db.set_machine_rcparams(self.machine.label, params)
                 except NotImplementedError:
-                    raise CuckooMachineError(
-                        "Remote control support has not been implemented "
-                        "for this machinery."
+                    log.error(
+                        "Remote control support has not been implemented for the "
+                        "configured machinery module: %s",
+                        config("cuckoo:cuckoo:machinery")
                     )
 
             # Enable network routing.
@@ -654,9 +656,10 @@ def launch_analysis(self):
                 try:
                     machinery.disable_remote_control(self.machine.label)
                 except NotImplementedError:
-                    raise CuckooMachineError(
-                        "Remote control support has not been implemented "
-                        "for this machinery."
+                    log.error(
+                        "Remote control support has not been implemented for the "
+                        "configured machinery module: %s",
+                        config("cuckoo:cuckoo:machinery")
                     )
 
             # Mark the machine in the database as stopped. Unless this machine

From ed034205883255c0d7732be8ad004d125eed4972 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 6 Jun 2019 14:26:56 +0200
Subject: [PATCH 108/138] Improve shutdown/cleanup routine

---
 cuckoo/core/plugins.py   |  5 +++++
 cuckoo/core/scheduler.py |  3 ++-
 cuckoo/main.py           | 27 ++++++++++++++++++++-------
 3 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/cuckoo/core/plugins.py b/cuckoo/core/plugins.py
index ae58ec9fd0..83496641cb 100644
--- a/cuckoo/core/plugins.py
+++ b/cuckoo/core/plugins.py
@@ -158,6 +158,7 @@ def default(*args, **kwargs):
         self.enabled = enabled
 
     def stop(self):
+        stopped = []
         for module in self.enabled:
             try:
                 module.stop()
@@ -172,6 +173,10 @@ def stop(self):
             else:
                 log.debug("Stopped auxiliary module: %s",
                           module.__class__.__name__)
+            stopped.append(module)
+
+        for s in stopped:
+            self.enabled.remove(s)
 
 class RunProcessing(object):
     """Analysis Results Processing Engine.
diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py
index b7723777c9..58ccef461d 100644
--- a/cuckoo/core/scheduler.py
+++ b/cuckoo/core/scheduler.py
@@ -591,8 +591,8 @@ def launch_analysis(self):
         finally:
             # Stop Auxiliary modules.
             if not self.stopped_aux:
-                self.aux.stop()
                 self.stopped_aux = True
+                self.aux.stop()
 
             # Take a memory dump of the machine before shutting it off.
             if self.cfg.cuckoo.memory_dump or self.task.memory:
@@ -813,6 +813,7 @@ def cleanup(self):
             self.unroute_network()
 
         if not self.stopped_aux:
+            self.stopped_aux = True
             self.aux.stop()
 
     def force_stop(self):
diff --git a/cuckoo/main.py b/cuckoo/main.py
index 6aeaa27555..b48154be51 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -8,6 +8,7 @@
 import os
 import shutil
 import subprocess
+import signal
 import sys
 
 import cuckoo
@@ -215,13 +216,8 @@ def cuckoo_main(max_analysis_count=0):
     @param max_analysis_count: kill cuckoo after this number of analyses
     """
     rs, sched = None, None
-    try:
-        rs = ResultServer()
-        sched = Scheduler(max_analysis_count)
-        sched.start()
-    except KeyboardInterrupt:
-        log.info("CTRL+C detected! Stopping.. This can take a few seconds")
-    finally:
+
+    def stop():
         if sched:
             sched.running = False
         if rs:
@@ -231,6 +227,23 @@ def cuckoo_main(max_analysis_count=0):
         if sched:
             sched.stop()
 
+    def handle_sigterm(sig, f):
+        stop()
+
+    # Handle a SIGTERM, to reduce the chance of Cuckoo exiting without
+    # cleaning
+    signal.signal(signal.SIGTERM, handle_sigterm)
+
+    try:
+        rs = ResultServer()
+        sched = Scheduler(max_analysis_count)
+        sched.start()
+    except KeyboardInterrupt:
+        log.info("CTRL+C detected! Stopping.. This can take a few seconds")
+    finally:
+        if Pidfile("cuckoo").exists():
+            stop()
+
 @click.group(invoke_without_command=True)
 @click.option("-d", "--debug", is_flag=True, help="Enable verbose logging")
 @click.option("-q", "--quiet", is_flag=True, help="Only log warnings and critical messages")

From e78e46e81e4c1e27f9acd62d1647d942c5a88669 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 6 Jun 2019 16:42:51 +0200
Subject: [PATCH 109/138] Make used results port leading if none is provided in
 machine conf Also use chosen port in rooter/per-analysis routing. Since the
 new resultserver, the user can let Cuckoo choose what the port must be.

---
 cuckoo/apps/apps.py      |  2 +-
 cuckoo/core/guest.py     | 13 +++++++++----
 cuckoo/core/scheduler.py | 17 +++++++++++------
 3 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/cuckoo/apps/apps.py b/cuckoo/apps/apps.py
index ae5e6425fb..b16d721d4b 100644
--- a/cuckoo/apps/apps.py
+++ b/cuckoo/apps/apps.py
@@ -444,7 +444,7 @@ def cuckoo_machine(vmname, action, ip, platform, options, tags,
             resultserver_port = int(resultserver_port)
         else:
             resultserver_ip = cfg["cuckoo"]["resultserver"]["ip"]
-            resultserver_port = cfg["cuckoo"]["resultserver"]["port"]
+            resultserver_port = 0
 
         machines.append(vmname)
         cfg[machinery][vmname] = {
diff --git a/cuckoo/core/guest.py b/cuckoo/core/guest.py
index ae9f95ba90..a75f5a3950 100644
--- a/cuckoo/core/guest.py
+++ b/cuckoo/core/guest.py
@@ -532,13 +532,18 @@ def wait_for_completion(self):
             self.old.wait_for_completion()
             return
 
+        count = 0
         end = time.time() + self.timeout
 
         while db.guest_get_status(self.task_id) == "running" and self.do_run:
-            log.debug(
-                "%s: analysis #%s still processing", self.vmid, self.task_id
-            )
+            if count >= 5:
+                log.debug(
+                    "%s: analysis #%s still processing", self.vmid,
+                    self.task_id
+                )
+                count = 0
 
+            count += 1
             time.sleep(1)
 
             # If the analysis hits the critical timeout, just return straight
@@ -553,7 +558,7 @@ def wait_for_completion(self):
                 # this might fail due to timeouts or just temporary network
                 # issues thus we don't want to abort the analysis just yet and
                 # wait for things to recover
-                log.info(
+                log.warning(
                     "Virtual Machine /status failed. This can indicate the "
                     "guest losing network connectivity"
                 )
diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py
index 58ccef461d..edd6214e41 100644
--- a/cuckoo/core/scheduler.py
+++ b/cuckoo/core/scheduler.py
@@ -65,6 +65,7 @@ def __init__(self, task_id, error_queue):
         self.rt_table = None
         self.unrouted_network = False
         self.stopped_aux = False
+        self.rs_port = config("cuckoo:resultserver:port")
 
     def init(self):
         """Initialize the analysis."""
@@ -210,7 +211,7 @@ def build_options(self):
 
         options["id"] = self.task.id
         options["ip"] = self.machine.resultserver_ip
-        options["port"] = self.machine.resultserver_port
+        options["port"] = self.rs_port
         options["category"] = self.task.category
         options["target"] = self.task.target
         options["package"] = self.task.package
@@ -302,7 +303,7 @@ def route_network(self):
             rooter(
                 "drop_enable", self.machine.ip,
                 config("cuckoo:resultserver:ip"),
-                str(config("cuckoo:resultserver:port"))
+                str(self.rs_port)
             )
 
         if self.route == "inetsim":
@@ -311,7 +312,7 @@ def route_network(self):
                 "inetsim_enable", self.machine.ip,
                 config("routing:inetsim:server"),
                 config("%s:%s:interface" % (machinery, machinery)),
-                str(config("cuckoo:resultserver:port")),
+                str(self.rs_port),
                 config("routing:inetsim:ports") or ""
             )
 
@@ -354,7 +355,7 @@ def unroute_network(self):
             rooter(
                 "drop_disable", self.machine.ip,
                 config("cuckoo:resultserver:ip"),
-                str(config("cuckoo:resultserver:port"))
+                str(self.rs_port)
             )
 
         if self.route == "inetsim":
@@ -363,7 +364,7 @@ def unroute_network(self):
                 "inetsim_disable", self.machine.ip,
                 config("routing:inetsim:server"),
                 config("%s:%s:interface" % (machinery, machinery)),
-                str(config("cuckoo:resultserver:port")),
+                str(self.rs_port),
                 config("routing:inetsim:ports") or ""
             )
 
@@ -450,6 +451,8 @@ def launch_analysis(self):
             })
             return False
 
+        self.rs_port = self.machine.resultserver_port or ResultServer().port
+
         # At this point we can tell the ResultServer about it.
         try:
             ResultServer().add_task(self.task, self.machine)
@@ -819,7 +822,9 @@ def cleanup(self):
     def force_stop(self):
         # Make the guest manager stop the status checking loop and return
         # to the main analysis manager routine.
-        self.db.guest_set_status(self.task.id, "stopping")
+        if self.db.guest_get_status(self.task.id):
+            self.db.guest_set_status(self.task.id, "stopping")
+
         self.guest_manager.stop()
         log.debug("Force stopping task #%s", self.task.id)
 

From 4b94ab8709fe294696252fa08e95d3d29048a640 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 6 Jun 2019 17:17:44 +0200
Subject: [PATCH 110/138] Warn properly if cwd file is missing

---
 cuckoo/main.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/cuckoo/main.py b/cuckoo/main.py
index b48154be51..59907c7697 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -139,13 +139,16 @@ def cuckoo_init(level, ctx, cfg=None):
 
     # Determine if this is a proper CWD.
     if not os.path.exists(cwd(".cwd")):
-        sys.exit(
-            "No proper Cuckoo Working Directory was identified, did you pass "
-            "along the correct directory? For new installations please use a "
-            "non-existant directory to build up the CWD! You can craft a CWD "
-            "manually, but keep in mind that the CWD layout may change along "
-            "with Cuckoo releases (and don't forget to fill out '$CWD/.cwd')!"
-        )
+        sys.stderr.write(red(
+            "\nNo proper Cuckoo Working Directory was identified, did you "
+            "pass along the correct directory?\n"
+            "The '.cwd' file is missing in the specified directory. "
+            "For new installations please use a non-existant directory to "
+            "build up the CWD! You can craft a CWD manually, but keep in mind "
+            "that the CWD layout may change along with Cuckoo releases "
+            "(and don't forget to fill out '$CWD/.cwd')!\n"
+        ))
+        sys.exit(1)
 
     init_console_logging(level)
 

From f4b96a8acacae43cffa7f206da85fc29d8a5a5c8 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 6 Jun 2019 21:44:09 +0200
Subject: [PATCH 111/138] Don't silently drop analysis options if they are
 unknown

---
 cuckoo/core/submit.py | 10 ++++++++++
 tests/test_submit.py  | 16 ++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/cuckoo/core/submit.py b/cuckoo/core/submit.py
index 1ecaa9dc85..589563b76e 100644
--- a/cuckoo/core/submit.py
+++ b/cuckoo/core/submit.py
@@ -93,18 +93,28 @@ def translate_options_to(self, options):
 
         if not int(options.get("human", "1")):
             ret["simulated-human-interaction"] = False
+            options.pop("human")
 
         if options.get("free") == "yes":
             ret["enable-injection"] = False
+            options.pop("free")
 
         if options.get("procmemdump") == "yes":
             ret["process-memory-dump"] = True
+            options.pop("procmemdump")
 
         if options.get("remotecontrol") == "yes":
             ret["remote-control"] = True
+            options.pop("remotecontrol")
 
         if options.get("route"):
             ret["network-routing"] = options["route"]
+            options.pop("route")
+
+        # Propagate any additional manually set key/value pairs.
+        for key, value in options.items():
+            if key not in self.known_web_options:
+                ret[key] = value
 
         return ret
 
diff --git a/tests/test_submit.py b/tests/test_submit.py
index e27db869f8..dade71be12 100644
--- a/tests/test_submit.py
+++ b/tests/test_submit.py
@@ -418,6 +418,14 @@ def test_option_translations_from():
         "remotecontrol": "yes",
     }
 
+    assert sm.translate_options_from({}, {
+        "simulated-human-interaction": False,
+        "function": "DoStuff",
+        "json.calls": "0"
+    }) == {
+        "human": 0, "function": "DoStuff", "json.calls": "0"
+    }
+
 def test_option_translations_to():
     sm = SubmitManager()
 
@@ -440,3 +448,11 @@ def test_option_translations_to():
     }) == {
         "remote-control": True,
     }
+
+    assert sm.translate_options_to({
+        "human": "0", "function": "DoStuff", "json.calls": "0"
+    }) == {
+        "simulated-human-interaction": False,
+        "function": "DoStuff",
+        "json.calls": "0"
+    }

From b65f3ea9b6baeadcf48a25799e86d15f13eed4b1 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Thu, 6 Jun 2019 21:55:30 +0200
Subject: [PATCH 112/138] Update docs config files to latest version

---
 docs/book/_files/conf/auxiliary.conf  | 23 +++++++++++++++++++++++
 docs/book/_files/conf/cuckoo.conf     | 12 ++++++++++++
 docs/book/_files/conf/kvm.conf        |  3 +++
 docs/book/_files/conf/processing.conf |  7 ++++++-
 docs/book/_files/conf/reporting.conf  | 10 ++++++++++
 docs/book/_files/conf/routing.conf    |  5 +++++
 6 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/docs/book/_files/conf/auxiliary.conf b/docs/book/_files/conf/auxiliary.conf
index 39cf076538..d1c8adf62a 100644
--- a/docs/book/_files/conf/auxiliary.conf
+++ b/docs/book/_files/conf/auxiliary.conf
@@ -42,6 +42,29 @@ script = stuff/mitm.py
 # be set to bin/cert.p12.
 certificate = bin/cert.p12
 
+[replay]
+# Enable PCAP replay capabilities.
+enabled = yes
+
+# Specify the path to your local installation of mitmdump. Make sure this
+# path is correct. Note that this should be mitmproxy 3.0.5 or higher,
+# installed in a separate virtualenv (or similar).
+mitmdump = /usr/local/bin/mitmdump
+
+# Listen port base. Each virtual machine will use its own port to be
+# able to make a good distinction between the various running analyses.
+# Generally port 51000 should be fine, in this case port 51001, 51002, etc
+# will also be used - again, one port per analyses.
+port_base = 51000
+
+# Path to the certificate to be used by mitmdump. This file will be
+# automatically generated for you if you run mitmdump once. It's just that
+# you have to copy it from ~/.mitmproxy/mitmproxy-ca-cert.p12 to somewhere
+# in the analyzer/windows/ directory. Recommended is to write the certificate
+# to analyzer/windows/bin/cert.p12, in that case the following option should
+# be set to bin/cert.p12.
+certificate = bin/cert.p12
+
 [services]
 # Provide extra services accessible through the network of the analysis VM
 # provided in separate, standalone, Virtual Machines [yes/no].
diff --git a/docs/book/_files/conf/cuckoo.conf b/docs/book/_files/conf/cuckoo.conf
index 201f3b4ca3..630fc8e2db 100644
--- a/docs/book/_files/conf/cuckoo.conf
+++ b/docs/book/_files/conf/cuckoo.conf
@@ -4,12 +4,24 @@
 # one available.
 version_check = yes
 
+# Cuckoo will stop at startup if the version check reports vulnerabilities in
+# one of Cuckoo's dependencies. This setting ignores the vulnerabilities
+# and starts anyway
+ignore_vulnerabilities = no
+
 # The authentication token that is required to access the Cuckoo API, using
 # HTTP Bearer authentication. This will protect the API instance against
 # unauthorized access and CSRF attacks. It is strongly recommended to set this
 # to a secure value.
 api_token =
 
+# The Web secret is used as a very basic, but successful way to provide basic
+# authentication to the Cuckoo Web Interface. This is a shared secret amongst
+# all users of this Cuckoo instance and will "protect" usage from users outside
+# of this instance. Therefore, if you'd like to share this Cuckoo instance with
+# the outside world, then don't use the Web secret functionality.
+web_secret = 
+
 # If turned on, Cuckoo will delete the original file after its analysis
 # has been completed.
 delete_original = no
diff --git a/docs/book/_files/conf/kvm.conf b/docs/book/_files/conf/kvm.conf
index 9ea38f6576..c1dbcaedac 100644
--- a/docs/book/_files/conf/kvm.conf
+++ b/docs/book/_files/conf/kvm.conf
@@ -1,4 +1,7 @@
 [kvm]
+# Specify a libvirt URI connection string
+dsn = qemu:///system
+
 # Specify a comma-separated list of available machines to be used. For each
 # specified ID you have to define a dedicated section containing the details
 # on the respective machine. (E.g. cuckoo1,cuckoo2,cuckoo3)
diff --git a/docs/book/_files/conf/processing.conf b/docs/book/_files/conf/processing.conf
index 6a3f6b5911..4eb4d27f6d 100644
--- a/docs/book/_files/conf/processing.conf
+++ b/docs/book/_files/conf/processing.conf
@@ -170,4 +170,9 @@ scan = no
 force = no
 # URL to your IRMA installation
 # For example : https://your.irma.host
-url =
+url = 
+# Probes to use on your IRMA instance
+# If not specified, will default to using all available probes
+# Expects comma separated list
+# For example : ClamAV,F-Secure,Avast,ESET,eScan,Avira,Sophos,McAfee,Kaspersky,GData,Comodo,Bitdefender
+probes =
diff --git a/docs/book/_files/conf/reporting.conf b/docs/book/_files/conf/reporting.conf
index 0faaa0c69c..763731a151 100644
--- a/docs/book/_files/conf/reporting.conf
+++ b/docs/book/_files/conf/reporting.conf
@@ -31,6 +31,16 @@ apikey =
 # separated by whitespace. Available modes: maldoc ipaddr hashes url.
 mode = maldoc ipaddr hashes url
 
+distribution = 0
+analysis = 0
+threat_level = 4
+
+# The minimum Cuckoo score for a MISP event to be created
+min_malscore = 0
+
+tag = Cuckoo
+upload_sample = no
+
 [mongodb]
 enabled = no
 host = 127.0.0.1
diff --git a/docs/book/_files/conf/routing.conf b/docs/book/_files/conf/routing.conf
index 310e4d518b..3cd9e753de 100644
--- a/docs/book/_files/conf/routing.conf
+++ b/docs/book/_files/conf/routing.conf
@@ -49,6 +49,11 @@ drop = no
 enabled = no
 server = 192.168.56.1
 
+# Redirect TCP ports (should we also support UDP?). If specified, this should
+# represent whitespace-separated src:dst pairs. E.g., "80:8080 443:8080" will
+# redirect all 80/443 traffic to 8080 on the specified InetSim host.
+ports = 
+
 [tor]
 # Route a VM through Tor, requires a local setup of Tor (please refer to our
 # documentation).

From dd6f8d3b777f268719d8bcd10ea14561d8e1a699 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 7 Jun 2019 00:51:32 +0200
Subject: [PATCH 113/138] Update test for vuln check

---
 tests/test_api.py  |  2 +-
 tests/test_apps.py | 23 ++++++++++++++++-------
 tests/test_init.py |  3 ++-
 tests/test_web.py  | 14 +++++++++++++-
 4 files changed, 32 insertions(+), 10 deletions(-)

diff --git a/tests/test_api.py b/tests/test_api.py
index 797b10783a..7cb8a6f2c5 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -157,7 +157,7 @@ def test_create_submit_opts(self):
         assert t.memory is True
         assert t.enforce_timeout is True
         assert t.options == {
-            "free": "yes",
+            "free": "yes", "procmemdump": "no"
         }
 
     def test_create_submit_urls(self):
diff --git a/tests/test_apps.py b/tests/test_apps.py
index 4ab746bee7..526701acd7 100644
--- a/tests/test_apps.py
+++ b/tests/test_apps.py
@@ -45,10 +45,12 @@ def setup(self):
         set_cwd(tempfile.mkdtemp())
         cuckoo_create()
 
+    @mock.patch("cuckoo.machinery.virtualbox.VirtualBox.version")
     @mock.patch("cuckoo.main.load_signatures")
     @mock.patch("cuckoo.main.cuckoo_main")
-    def test_main(self, p, q):
+    def test_main(self, p, q, vb):
         p.side_effect = SystemExit(0)
+        vb.return_value = "9999"
 
         # Ensure that the "latest" binary value makes sense so that the
         # "run community command" exception is not thrown.
@@ -56,9 +58,11 @@ def test_main(self, p, q):
         main.main(("--cwd", cwd(), "-d", "--nolog"), standalone_mode=False)
         q.assert_called_once()
 
+    @mock.patch("cuckoo.machinery.virtualbox.VirtualBox.version")
     @mock.patch("cuckoo.main.load_signatures")
     @mock.patch("cuckoo.main.log")
-    def test_main_exception(self, p, q):
+    def test_main_exception(self, p, q, vb):
+        vb.return_value = "9999"
         q.side_effect = Exception("this is a test")
         with pytest.raises(SystemExit):
             main.main(
@@ -758,8 +762,9 @@ class context(object):
             log = False
         return context
 
+    @mock.patch("cuckoo.main.check_version")
     @mock.patch("cuckoo.main.green")
-    def test_default_cwd(self, p):
+    def test_default_cwd(self, p, cv):
         set_cwd(tempfile.mkdtemp())
         cuckoo_create()
         with chdir(cwd()):
@@ -767,32 +772,36 @@ def test_default_cwd(self, p):
             cuckoo_init(logging.INFO, self.ctx)
             p.assert_called_once_with("cuckoo community")
 
+    @mock.patch("cuckoo.main.check_version")
     @mock.patch("cuckoo.main.green")
-    def test_hardcoded_cwd(self, p):
+    def test_hardcoded_cwd(self, p, cv):
         set_cwd(tempfile.mkdtemp())
         cuckoo_create()
         decide_cwd(cwd())
         cuckoo_init(logging.INFO, self.ctx)
         p.assert_called_once_with("cuckoo --cwd %s community" % cwd())
 
+    @mock.patch("cuckoo.main.check_version")
     @mock.patch("cuckoo.main.green")
-    def test_hardcoded_cwd_with_space(self, p):
+    def test_hardcoded_cwd_with_space(self, p, cv):
         set_cwd(tempfile.mkdtemp("foo bar"))
         cuckoo_create()
         decide_cwd(cwd())
         cuckoo_init(logging.INFO, self.ctx)
         p.assert_called_once_with('cuckoo --cwd "%s" community' % cwd())
 
+    @mock.patch("cuckoo.main.check_version")
     @mock.patch("cuckoo.main.green")
-    def test_hardcoded_cwd_with_quote(self, p):
+    def test_hardcoded_cwd_with_quote(self, p, cv):
         set_cwd(tempfile.mkdtemp("foo ' bar"))
         cuckoo_create()
         decide_cwd(cwd())
         cuckoo_init(logging.INFO, self.ctx)
         p.assert_called_once_with('cuckoo --cwd "%s" community' % cwd())
 
+    @mock.patch("cuckoo.main.check_version")
     @mock.patch("cuckoo.main.green")
-    def test_has_signatures(self, p):
+    def test_has_signatures(self, p, cv):
         set_cwd(tempfile.mkdtemp())
         sys.modules.pop("signatures", None)
         sys.modules.pop("signatures.android", None)
diff --git a/tests/test_init.py b/tests/test_init.py
index d8272bdfaf..a3aa9a897d 100644
--- a/tests/test_init.py
+++ b/tests/test_init.py
@@ -94,7 +94,8 @@ def test_cuckoo_init_main_nosigs(self, p):
         assert os.path.exists(os.path.join(cwd(), "stuff", "mitm.py"))
         p.assert_not_called()
 
-    def test_cuckoo_init_no_resultserver(self):
+    @mock.patch("cuckoo.main.check_version")
+    def test_cuckoo_init_no_resultserver(self, cv):
         """Test that 'cuckoo init' doesn't launch the ResultServer."""
         with pytest.raises(SystemExit):
             main.main(
diff --git a/tests/test_web.py b/tests/test_web.py
index d625d771c1..40e56d5d16 100644
--- a/tests/test_web.py
+++ b/tests/test_web.py
@@ -1274,6 +1274,11 @@ def test_status(self, p, client):
                     "enabled": True,
                 },
             },
+            "cuckoo": {
+                "cuckoo": {
+                    "ignore_vulnerabilities": True,
+                },
+            }
         })
         db.connect()
         r = client.get("/cuckoo/api/status/")
@@ -1282,7 +1287,14 @@ def test_status(self, p, client):
 
     def test_api_status200(self, client):
         set_cwd(tempfile.mkdtemp())
-        cuckoo_create()
+
+        cuckoo_create(cfg={
+            "cuckoo": {
+                "cuckoo": {
+                    "ignore_vulnerabilities": True,
+                },
+            },
+        })
         Database().connect()
         r = client.get("/cuckoo/api/status")
         assert r.status_code == 200

From d78da2b56a6de0b32fef8604a91d35229a292e85 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 7 Jun 2019 00:52:30 +0200
Subject: [PATCH 114/138] Add ignore vuln flag

---
 .travis.yml            | 2 +-
 appveyor.yml           | 4 ++--
 cuckoo/core/startup.py | 6 ++++--
 cuckoo/main.py         | 6 ++++--
 4 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 42e5c33057..69fd5aa0cf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -89,7 +89,7 @@ script:
   - cuckoo community
 
   # Check the code integrity of the Signatures by running Cuckoo.
-  - cuckoo -d 2>&1|grep "Unable to bind ResultServer"
+  - cuckoo -d --ignore-vuln 2>&1|grep "Unable to bind ResultServer"
 
   # Run tests from the "tests" directory.
   - py.test --cov=cuckoo
diff --git a/appveyor.yml b/appveyor.yml
index 21e919aa07..dfa37be4f3 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -47,9 +47,9 @@ before_test:
   - "psql cuckootestimport <tests/files/sql/11pg.sql >NUL"
 
 test_script:
-  - "cuckoo.exe -d"
+  - "cuckoo.exe -d --ignore-vuln"
   - "cuckoo.exe community"
-  - "cuckoo.exe -d || dir >NUL"
+  - "cuckoo.exe -d --ignore-vuln || dir >NUL"
   - "pytest.exe --cov=cuckoo"
 
 after_test:
diff --git a/cuckoo/core/startup.py b/cuckoo/core/startup.py
index 4f121b9a59..f4066128b7 100644
--- a/cuckoo/core/startup.py
+++ b/cuckoo/core/startup.py
@@ -96,11 +96,13 @@ def check_configs():
             )
     return True
 
-def check_version():
+def check_version(ignore_vuln=False):
     """Check version of Cuckoo."""
     if not config("cuckoo:cuckoo:version_check"):
         return
 
+    ignore_vuln = ignore_vuln or config("cuckoo:cuckoo:ignore_vulnerabilities")
+
     import pkg_resources
 
     print(" Checking for updates...")
@@ -196,7 +198,7 @@ def check_version():
     for warning in warnings:
         print("--> %s\n" % color(warning, 4))
 
-    if warnings and not config("cuckoo:cuckoo:ignore_vulnerabilities"):
+    if warnings and not ignore_vuln:
         print(
             "This check can be disabled by enabling "
             "'ignore_vulnerabilities' in cuckoo.conf under the "
diff --git a/cuckoo/main.py b/cuckoo/main.py
index 59907c7697..1f9c074ce3 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -167,7 +167,7 @@ def cuckoo_init(level, ctx, cfg=None):
     pidfile.create()
 
     check_configs()
-    check_version()
+    check_version(ctx.ignore_vuln)
 
     ctx.log and init_logging(level)
 
@@ -252,10 +252,11 @@ def handle_sigterm(sig, f):
 @click.option("-q", "--quiet", is_flag=True, help="Only log warnings and critical messages")
 @click.option("--nolog", is_flag=True, help="Don't log to file.")
 @click.option("-m", "--maxcount", default=0, help="Maximum number of analyses to process")
+@click.option("--ignore-vuln", is_flag=True, help="Ignore vulnerability warnings and start")
 @click.option("--user", help="Drop privileges to this user")
 @click.option("--cwd", help="Cuckoo Working Directory")
 @click.pass_context
-def main(ctx, debug, quiet, nolog, maxcount, user, cwd):
+def main(ctx, debug, quiet, nolog, maxcount, ignore_vuln, user, cwd):
     """Invoke the Cuckoo daemon or one of its subcommands.
 
     To be able to use different Cuckoo configurations on the same machine with
@@ -277,6 +278,7 @@ def main(ctx, debug, quiet, nolog, maxcount, user, cwd):
     ctx.user = user
 
     ctx.log = not nolog
+    ctx.ignore_vuln = ignore_vuln
 
     if quiet:
         level = logging.WARN

From 2d50423cb2a7b9bdb003397fd05d3baf02d3d48e Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 7 Jun 2019 16:16:56 +0200
Subject: [PATCH 115/138] Disable vuln check in tests

---
 tests/test_apps.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/tests/test_apps.py b/tests/test_apps.py
index 526701acd7..552df7b6f2 100644
--- a/tests/test_apps.py
+++ b/tests/test_apps.py
@@ -760,11 +760,11 @@ class TestCommunitySuggestion(object):
     def ctx(self):
         class context(object):
             log = False
+            ignore_vuln = True
         return context
 
-    @mock.patch("cuckoo.main.check_version")
     @mock.patch("cuckoo.main.green")
-    def test_default_cwd(self, p, cv):
+    def test_default_cwd(self, p):
         set_cwd(tempfile.mkdtemp())
         cuckoo_create()
         with chdir(cwd()):
@@ -772,36 +772,32 @@ def test_default_cwd(self, p, cv):
             cuckoo_init(logging.INFO, self.ctx)
             p.assert_called_once_with("cuckoo community")
 
-    @mock.patch("cuckoo.main.check_version")
     @mock.patch("cuckoo.main.green")
-    def test_hardcoded_cwd(self, p, cv):
+    def test_hardcoded_cwd(self, p):
         set_cwd(tempfile.mkdtemp())
         cuckoo_create()
         decide_cwd(cwd())
         cuckoo_init(logging.INFO, self.ctx)
         p.assert_called_once_with("cuckoo --cwd %s community" % cwd())
 
-    @mock.patch("cuckoo.main.check_version")
     @mock.patch("cuckoo.main.green")
-    def test_hardcoded_cwd_with_space(self, p, cv):
+    def test_hardcoded_cwd_with_space(self, p):
         set_cwd(tempfile.mkdtemp("foo bar"))
         cuckoo_create()
         decide_cwd(cwd())
         cuckoo_init(logging.INFO, self.ctx)
         p.assert_called_once_with('cuckoo --cwd "%s" community' % cwd())
 
-    @mock.patch("cuckoo.main.check_version")
     @mock.patch("cuckoo.main.green")
-    def test_hardcoded_cwd_with_quote(self, p, cv):
+    def test_hardcoded_cwd_with_quote(self, p):
         set_cwd(tempfile.mkdtemp("foo ' bar"))
         cuckoo_create()
         decide_cwd(cwd())
         cuckoo_init(logging.INFO, self.ctx)
         p.assert_called_once_with('cuckoo --cwd "%s" community' % cwd())
 
-    @mock.patch("cuckoo.main.check_version")
     @mock.patch("cuckoo.main.green")
-    def test_has_signatures(self, p, cv):
+    def test_has_signatures(self, p):
         set_cwd(tempfile.mkdtemp())
         sys.modules.pop("signatures", None)
         sys.modules.pop("signatures.android", None)

From e5ba10e9260c207174ee8c3eab45bca07234dc52 Mon Sep 17 00:00:00 2001
From: Malware Utkonos <utkonos@malwarolo.gy>
Date: Sun, 20 Jan 2019 19:55:00 -0500
Subject: [PATCH 116/138] Make it clear that ip forwarding is always required.

---
 docs/book/installation/host/routing.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/book/installation/host/routing.rst b/docs/book/installation/host/routing.rst
index 9745dfecb0..faa614ef06 100644
--- a/docs/book/installation/host/routing.rst
+++ b/docs/book/installation/host/routing.rst
@@ -120,6 +120,11 @@ Rooter and choosing a network routing option for your analysis**.
 Documentation on starting the ``Cuckoo Rooter`` may be found in the
 :ref:`cuckoo_rooter_usage` document.
 
+Both global routing and per-analysis routing require ip forwarding to be enabled:
+
+    $ echo 1 | sudo tee -a /proc/sys/net/ipv4/ip_forward
+    $ sudo sysctl -w net.ipv4.ip_forward=1
+
 .. _routing_iproute2:
 
 Configuring iproute2

From 74d1c4bd8ab923e653e719b2e31a9207cc787874 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 7 Jun 2019 17:08:31 +0200
Subject: [PATCH 117/138] Update hashes

---
 cuckoo/private/cwd/hashes.txt | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/cuckoo/private/cwd/hashes.txt b/cuckoo/private/cwd/hashes.txt
index 8f36b32968..b6b526bf78 100644
--- a/cuckoo/private/cwd/hashes.txt
+++ b/cuckoo/private/cwd/hashes.txt
@@ -292,13 +292,6 @@ c8492c74db400e6300194c1bafd3088a102bdc8e analyzer/windows/modules/auxiliary/huma
 e86627abeb5ecc0112438ad179e9d0487870785a analyzer/windows/modules/packages/ie.py
 
 # TBD
-cb3a77d8dd7edf46de54545ca7b0c5b201f85917 analyzer/windows/bin/execsc.exe
-93727e778dadc13d83cea61a9ea88bf6b5906686 analyzer/windows/modules/auxiliary/human.py
-24cbd18428df8dbc6b9ccd7896d066c492f6d381 analyzer/windows/modules/packages/ps1.py
-6e6680e26bf1cf41909a4efcbd86917cf4b14603 analyzer/windows/modules/packages/pub.py
-d8fce614d615f6bdb3117e92bfa6e4ae2b48ea52 analyzer/windows/modules/packages/vbs.py
-f81e71f788149039f31c9b0b5e2891733e51b5d7 whitelist/ip.txt
-4575c7d09b316cad3a71059a27b78cbcb9c1f6b4 stuff/ttp_descriptions.json
 74c4c577a61f96571ac47e86c83fc0ece9d5f0ad agent/agent.py
 4d567f35bd79192d8f279f474816b9686e71896b analyzer/darwin/lib/api/screenshot.py
 633ab6bd08eb393ca630b59ce0ee5374862c6558 analyzer/darwin/lib/common/hashing.py
@@ -306,15 +299,30 @@ d5e3410184765dba6af2895b0dd528558f51ef8d analyzer/darwin/modules/packages/zip.py
 6db386e3ebea277637397a42616b2232e2d6b771 analyzer/linux/lib/common/abstracts.py
 0c77d682544f214e3314350fe7767e40ecf5b174 analyzer/linux/lib/common/hashing.py
 11d0d726c6c17e8abce33b07df9cb498e021da1c analyzer/linux/modules/auxiliary/stap.py
-faf94dddbe6fc6a262c56735e7c437f326fffe59 analyzer/windows/analyzer.py
+588d4f71a8e6d33628b94d49815f7f27fe621900 analyzer/windows/analyzer.py
+cb3a77d8dd7edf46de54545ca7b0c5b201f85917 analyzer/windows/bin/execsc.exe
 a3847083dc4ee78e186359fb03071489ebfd5932 analyzer/windows/lib/api/process.py
 ba6b59b09ef3a157f6081cd1e0f12168cd20538d analyzer/windows/lib/common/abstracts.py
 105aac03a5a5ddf1eaf9262389a593d1aaebd0fe analyzer/windows/lib/common/hashing.py
-9b0df7467fa48ea6451475c93c27682ae492c33c analyzer/windows/lib/core/pipe.py
+fda170e233e90bd5bf1dbb071c2a35b7661a6b5e analyzer/windows/lib/common/results.py
+761c8dd5207399c08f9a9e86847824ffa449caf1 analyzer/windows/lib/core/pipe.py
+3c1782e4264af5f4ac774d580036c829b660f572 analyzer/windows/lib/core/startup.py
 8f531c7997a8e36e16f1511322adb718630476cd analyzer/windows/modules/auxiliary/disguise.py
+93727e778dadc13d83cea61a9ea88bf6b5906686 analyzer/windows/modules/auxiliary/human.py
 90bfc348b008e717b5a44cb4ae91b8260682bf84 analyzer/windows/modules/auxiliary/reboot.py
 44187be47fd7ddb3b7bed7ab596510168eec1294 analyzer/windows/modules/auxiliary/recentfiles.py
+cff8089fb31140efc903f7d87a955b6b35e8b54f analyzer/windows/modules/auxiliary/screenshots.py
 348e720075790c862be1e67894ed4ab30ed4f7ce analyzer/windows/modules/auxiliary/zer0m0n.py
 e13518903a2fcaec3d0b140d3164c426f07ab647 analyzer/windows/modules/packages/ie.py
+24cbd18428df8dbc6b9ccd7896d066c492f6d381 analyzer/windows/modules/packages/ps1.py
+6e6680e26bf1cf41909a4efcbd86917cf4b14603 analyzer/windows/modules/packages/pub.py
+d8fce614d615f6bdb3117e92bfa6e4ae2b48ea52 analyzer/windows/modules/packages/vbs.py
 09702bc15041a80f399f0c143cc3ec29196e4962 analyzer/windows/modules/packages/zip.py
 c278d582c26003467794ee4d0ab7ae337034c2ea monitor/latest
+4575c7d09b316cad3a71059a27b78cbcb9c1f6b4 stuff/ttp_descriptions.json
+9870109fa8a09c2f1b871ea178bdd3d40390baab web/local_settings.py
+f81e71f788149039f31c9b0b5e2891733e51b5d7 whitelist/ip.txt
+cc78a9c7ecdd5a3862b39ad7e6676723e72eb2ba whitelist/mispdomain.txt
+8f6442b91064e46ab3454d6bc15a4cf1f3949a0f whitelist/misphash.txt
+1f0ec663731206a9bf9363293421c68855aed772 whitelist/mispip.txt
+e57ba6930af466d1a56aa22f791048020fadef88 whitelist/mispurl.txt

From ff1722069d4c0f1d7061f39aadfd7317bba534fc Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 10 Jun 2019 23:30:17 +0200
Subject: [PATCH 118/138] Add Cuckoo rooter cleanup on sigint and sigterm

---
 cuckoo/apps/__init__.py |   2 +-
 cuckoo/apps/rooter.py   | 177 +++++++++++++++++++++++++---------------
 cuckoo/main.py          |   3 +-
 tests/test_rooter.py    |  11 ++-
 4 files changed, 124 insertions(+), 69 deletions(-)

diff --git a/cuckoo/apps/__init__.py b/cuckoo/apps/__init__.py
index 355348e273..a35e690552 100644
--- a/cuckoo/apps/__init__.py
+++ b/cuckoo/apps/__init__.py
@@ -12,4 +12,4 @@
 from .distributed import cuckoo_distributed, cuckoo_distributed_instance
 from .dnsserve import cuckoo_dnsserve
 from .import_ import import_cuckoo
-from .rooter import cuckoo_rooter
+from .rooter import cuckoo_rooter, cleanup_rooter
diff --git a/cuckoo/apps/rooter.py b/cuckoo/apps/rooter.py
index a09bb1230b..f4d8908e66 100644
--- a/cuckoo/apps/rooter.py
+++ b/cuckoo/apps/rooter.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 Cuckoo Foundation.
+# Copyright (C) 2015-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -7,6 +7,7 @@
 import logging
 import os.path
 import re
+import signal
 import socket
 import stat
 import subprocess
@@ -18,6 +19,8 @@
 class s(object):
     service = None
     iptables = None
+    iptables_save = None
+    iptables_restore = None
     ip = None
 
 log = logging.getLogger(__name__)
@@ -29,6 +32,35 @@ def run(*args):
     stdout, stderr = p.communicate()
     return stdout, stderr
 
+def run_iptables(*args):
+    iptables_args = [s.iptables]
+    iptables_args.extend(list(args))
+    iptables_args.extend(["-m", "comment", "--comment", "cuckoo-rooter"])
+    return run(*iptables_args)
+
+def cleanup_rooter():
+    """Filter out all Cuckoo rooter entries from iptables-save and
+    restore the resulting ruleset."""
+    try:
+        stdout, stderr = run(s.iptables_save)
+    except OSError as e:
+        log.error(
+            "Failed to clean Cuckoo rooter rules. Is iptables-save "
+            "available? %s", e
+        )
+        return
+
+    if not stdout:
+        return
+
+    cleaned = []
+    for l in stdout.split("\n"):
+        if l and "cuckoo-rooter" not in l:
+            cleaned.append(l)
+
+    p = subprocess.Popen([s.iptables_restore], stdin=subprocess.PIPE)
+    p.communicate(input="\n".join(cleaned))
+
 def version():
     return {
         "version": __version__,
@@ -78,20 +110,20 @@ def vpn_disable(name):
 
 def forward_drop():
     """Disable any and all forwarding unless explicitly said so."""
-    run(s.iptables, "-P", "FORWARD", "DROP")
+    run_iptables("-P", "FORWARD", "DROP")
 
 def state_enable():
     """Enable stateful connection tracking."""
-    run(
-        s.iptables, "-A", "INPUT", "-m", "state",
-        "--state", "ESTABLISHED,RELATED", "-j", "ACCEPT"
+    run_iptables(
+        "-A", "INPUT", "-m", "state", "--state", "ESTABLISHED,RELATED",
+        "-j", "ACCEPT"
     )
 
 def state_disable():
     """Disable stateful connection tracking."""
     while True:
-        _, err = run(
-            s.iptables, "-D", "INPUT", "-m", "state",
+        _, err = run_iptables(
+            "-D", "INPUT", "-m", "state",
             "--state", "ESTABLISHED,RELATED", "-j", "ACCEPT"
         )
         if err:
@@ -99,14 +131,15 @@ def state_disable():
 
 def enable_nat(interface):
     """Enable NAT on this interface."""
-    run(s.iptables, "-t", "nat", "-A", "POSTROUTING",
-        "-o", interface, "-j", "MASQUERADE")
+    run_iptables(
+        "-t", "nat", "-A", "POSTROUTING", "-o", interface, "-j", "MASQUERADE"
+    )
 
 def disable_nat(interface):
     """Disable NAT on this interface."""
     while True:
-        _, err = run(
-            s.iptables, "-t", "nat", "-D", "POSTROUTING",
+        _, err = run_iptables(
+            "-t", "nat", "-D", "POSTROUTING",
             "-o", interface, "-j", "MASQUERADE"
         )
         if err:
@@ -133,14 +166,14 @@ def flush_rttable(rt_table):
 
 def dns_forward(action, vm_ip, dns_ip, dns_port="53"):
     """Route DNS requests from the VM to a custom DNS on a separate network."""
-    run(
-        s.iptables, "-t", "nat", action, "PREROUTING", "-p", "tcp",
+    run_iptables(
+        "-t", "nat", action, "PREROUTING", "-p", "tcp",
         "--dport", "53", "--source", vm_ip, "-j", "DNAT",
         "--to-destination", "%s:%s" % (dns_ip, dns_port)
     )
 
-    run(
-        s.iptables, "-t", "nat", action, "PREROUTING", "-p", "udp",
+    run_iptables(
+        "-t", "nat", action, "PREROUTING", "-p", "udp",
         "--dport", "53", "--source", vm_ip, "-j", "DNAT",
         "--to-destination", "%s:%s" % (dns_ip, dns_port)
     )
@@ -151,29 +184,29 @@ def forward_enable(src, dst, ipaddr):
     # Delete libvirt's default FORWARD REJECT rules. e.g.:
     # -A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
     # -A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-    run(s.iptables, "-D", "FORWARD", "-i", src, "-j", "REJECT")
-    run(s.iptables, "-D", "FORWARD", "-o", src, "-j", "REJECT")
+    run_iptables("-D", "FORWARD", "-i", src, "-j", "REJECT")
+    run_iptables("-D", "FORWARD", "-o", src, "-j", "REJECT")
 
-    run(
-        s.iptables, "-A", "FORWARD", "-i", src, "-o", dst,
+    run_iptables(
+        "-A", "FORWARD", "-i", src, "-o", dst,
         "--source", ipaddr, "-j", "ACCEPT"
     )
 
-    run(
-        s.iptables, "-A", "FORWARD", "-i", dst, "-o", src,
+    run_iptables(
+        "-A", "FORWARD", "-i", dst, "-o", src,
         "--destination", ipaddr, "-j", "ACCEPT"
     )
 
 def forward_disable(src, dst, ipaddr):
     """Disable forwarding of a specific IP address from one interface into
     another."""
-    run(
-        s.iptables, "-D", "FORWARD", "-i", src, "-o", dst,
+    run_iptables(
+        "-D", "FORWARD", "-i", src, "-o", dst,
         "--source", ipaddr, "-j", "ACCEPT"
     )
 
-    run(
-        s.iptables, "-D", "FORWARD", "-i", dst, "-o", src,
+    run_iptables(
+        "-D", "FORWARD", "-i", dst, "-o", src,
         "--destination", ipaddr, "-j", "ACCEPT"
     )
 
@@ -202,8 +235,8 @@ def inetsim_redirect_port(action, srcip, dstip, ports):
         if not srcport.isdigit() or not dstport.isdigit():
             log.debug("Invalid inetsim ports entry: %s", entry)
             continue
-        run(
-            s.iptables, "-t", "nat", action, "PREROUTING", "--source", srcip,
+        run_iptables(
+            "-t", "nat", action, "PREROUTING", "--source", srcip,
             "-p", "tcp", "--syn", "--dport", srcport,
             "-j", "DNAT", "--to-destination", "%s:%s" % (dstip, dstport)
         )
@@ -213,86 +246,88 @@ def inetsim_enable(ipaddr, inetsim_ip, machinery_iface, resultserver_port,
     """Enable hijacking of all traffic and send it to InetSim."""
     inetsim_redirect_port("-A", ipaddr, inetsim_ip, ports)
 
-    run(
-        s.iptables, "-t", "nat", "-A", "PREROUTING", "--source", ipaddr,
+    run_iptables(
+        "-t", "nat", "-A", "PREROUTING", "--source", ipaddr,
         "-p", "tcp", "--syn", "!", "--dport", resultserver_port,
         "-j", "DNAT", "--to-destination", inetsim_ip
     )
 
-    run(
-        s.iptables, "-t", "nat", "-A", "PREROUTING", "--source", ipaddr,
+    run_iptables(
+        "-t", "nat", "-A", "PREROUTING", "--source", ipaddr,
         "-p", "udp", "-j", "DNAT", "--to-destination", inetsim_ip
     )
 
-    run(
-        s.iptables, "-A", "OUTPUT", "-m", "conntrack", "--ctstate",
+    run_iptables(
+        "-A", "OUTPUT", "-m", "conntrack", "--ctstate",
         "INVALID", "-j", "DROP"
     )
 
-    run(
-        s.iptables, "-A", "OUTPUT", "-m", "state", "--state",
+    run_iptables(
+        "-A", "OUTPUT", "-m", "state", "--state",
         "INVALID", "-j", "DROP"
     )
 
     dns_forward("-A", ipaddr, inetsim_ip)
     forward_enable(machinery_iface, machinery_iface, ipaddr)
 
-    run(s.iptables, "-t", "nat", "-A", "POSTROUTING", "--source", ipaddr,
+    run_iptables(
+        "-t", "nat", "-A", "POSTROUTING", "--source", ipaddr,
         "-o", machinery_iface, "--destination", inetsim_ip, "-j", "MASQUERADE")
 
-    run(s.iptables, "-A", "OUTPUT", "-s", ipaddr, "-j", "DROP")
+    run_iptables("-A", "OUTPUT", "-s", ipaddr, "-j", "DROP")
 
 def inetsim_disable(ipaddr, inetsim_ip, machinery_iface, resultserver_port,
                     ports):
     """Enable hijacking of all traffic and send it to InetSim."""
     inetsim_redirect_port("-D", ipaddr, inetsim_ip, ports)
 
-    run(
-        s.iptables, "-D", "PREROUTING", "-t", "nat", "--source", ipaddr,
+    run_iptables(
+        "-D", "PREROUTING", "-t", "nat", "--source", ipaddr,
         "-p", "tcp", "--syn", "!", "--dport", resultserver_port, "-j", "DNAT",
         "--to-destination", inetsim_ip
     )
-    run(
-        s.iptables, "-t", "nat", "-D", "PREROUTING", "--source", ipaddr,
+    run_iptables(
+        "-t", "nat", "-D", "PREROUTING", "--source", ipaddr,
         "-p", "udp", "-j", "DNAT", "--to-destination", inetsim_ip
     )
 
-    run(
-        s.iptables, "-D", "OUTPUT", "-m", "conntrack", "--ctstate",
+    run_iptables(
+        "-D", "OUTPUT", "-m", "conntrack", "--ctstate",
         "INVALID", "-j", "DROP"
     )
 
-    run(
-        s.iptables, "-D", "OUTPUT", "-m", "state", "--state",
+    run_iptables(
+        "-D", "OUTPUT", "-m", "state", "--state",
         "INVALID", "-j", "DROP"
     )
 
     dns_forward("-D", ipaddr, inetsim_ip)
     forward_disable(machinery_iface, machinery_iface, ipaddr)
 
-    run(s.iptables, "-t", "nat", "-D", "POSTROUTING", "--source", ipaddr,
+    run_iptables(
+        "-t", "nat", "-D", "POSTROUTING", "--source", ipaddr,
         "-o", machinery_iface, "--destination", inetsim_ip, "-j", "MASQUERADE")
 
-    run(s.iptables, "-D", "OUTPUT", "-s", ipaddr, "-j", "DROP")
+    run_iptables("-D", "OUTPUT", "-s", ipaddr, "-j", "DROP")
 
 def tor_toggle(action, vm_ip, resultserver_ip, dns_port, proxy_port):
     """Toggle Tor iptables routing rules."""
     dns_forward(action, vm_ip, resultserver_ip, dns_port)
 
-    run(
-        s.iptables, "-t", "nat", action, "PREROUTING", "-p", "tcp",
+    run_iptables(
+        "-t", "nat", action, "PREROUTING", "-p", "tcp",
         "--source", vm_ip, "!", "--destination", resultserver_ip,
         "-j", "DNAT", "--to-destination",
         "%s:%s" % (resultserver_ip, proxy_port)
     )
 
-    run(
-        s.iptables, "-t", "nat", action, "PREROUTING", "-p", "udp",
+    run_iptables(
+        "-t", "nat", action, "PREROUTING", "-p", "udp",
         "--source", vm_ip, "!", "--destination", resultserver_ip,
         "-j", "DNAT", "--to-destination",
         "%s:%s" % (resultserver_ip, proxy_port)
     )
-    run(s.iptables, action, "OUTPUT", "-s", vm_ip, "-j", "DROP")
+    run_iptables(action, "OUTPUT", "-s", vm_ip, "-j", "DROP")
 
 def tor_enable(vm_ip, resultserver_ip, dns_port, proxy_port):
     """Enable hijacking of all traffic and send it to TOR."""
@@ -304,25 +339,20 @@ def tor_disable(vm_ip, resultserver_ip, dns_port, proxy_port):
 
 def drop_toggle(action, vm_ip, resultserver_ip, resultserver_port, agent_port):
     """Toggle iptables to allow internal Cuckoo traffic."""
-    run(
-        s.iptables, action, "INPUT", "--source", vm_ip, "-p", "tcp",
+    run_iptables(
+        action, "INPUT", "--source", vm_ip, "-p", "tcp",
         "--destination", resultserver_ip, "--dport", "%s" % resultserver_port,
         "-j", "ACCEPT"
     )
 
-    run(
-        s.iptables, action, "OUTPUT", "--source", resultserver_ip,
+    run_iptables(
+        action, "OUTPUT", "--source", resultserver_ip,
         "-p", "tcp", "--destination", vm_ip, "--dport", "%s" % agent_port,
         "-j", "ACCEPT"
     )
 
-    run(
-        s.iptables, action, "INPUT", "--source", vm_ip, "-j", "DROP"
-    )
-
-    run(
-        s.iptables, action, "OUTPUT", "--source", vm_ip, "-j", "DROP"
-    )
+    run_iptables(action, "INPUT", "--source", vm_ip, "-j", "DROP")
+    run_iptables(action, "OUTPUT", "--source", vm_ip, "-j", "DROP")
 
 def drop_enable(vm_ip, resultserver_ip, resultserver_port, agent_port=8000):
     """Enable complete dropping of all non-Cuckoo traffic by default."""
@@ -415,14 +445,33 @@ def cuckoo_rooter(socket_path, group, service, iptables, ip):
     # Initialize global variables.
     s.service = service
     s.iptables = iptables
+    s.iptables_save = "/sbin/iptables-save"
+    s.iptables_restore = "/sbin/iptables-restore"
     s.ip = ip
 
-    while True:
+    # Simple object to allow a signal handler to stop the rooter loop
+    class Run(object):
+        def __init__(self):
+            self.run = True
+    do = Run()
+
+    def handle_sigterm(sig, f):
+        do.run = False
+        server.shutdown(socket.SHUT_RDWR)
+        server.close()
+        cleanup_rooter()
+
+    signal.signal(signal.SIGTERM, handle_sigterm)
+
+    while do.run:
         try:
             command, addr = server.recvfrom(4096)
         except socket.error as e:
             if e.errno == errno.EINTR:
                 continue
+            elif e.errno == errno.EBADF and not do.run:
+                continue
+
             raise e
 
         try:
diff --git a/cuckoo/main.py b/cuckoo/main.py
index 1f9c074ce3..d6bc38a055 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -17,7 +17,7 @@
     fetch_community, submit_tasks, process_tasks, process_task_range,
     cuckoo_rooter, cuckoo_api, cuckoo_distributed, cuckoo_distributed_instance,
     cuckoo_clean, cuckoo_dnsserve, cuckoo_machine, import_cuckoo,
-    migrate_database, migrate_cwd
+    migrate_database, migrate_cwd, cleanup_rooter
 )
 from cuckoo.common.config import read_kv_conf
 from cuckoo.common.exceptions import CuckooCriticalError
@@ -487,6 +487,7 @@ def rooter(ctx, socket, group, service, iptables, ip, sudo):
             cuckoo_rooter(socket, group, service, iptables, ip)
         except KeyboardInterrupt:
             print(red("Aborting the Cuckoo Rooter.."))
+            cleanup_rooter()
 
 @main.command()
 @click.option("-H", "--host", default="localhost", help="Host to bind the API server on")
diff --git a/tests/test_rooter.py b/tests/test_rooter.py
index 2950bdf4d2..ea21cc90ab 100644
--- a/tests/test_rooter.py
+++ b/tests/test_rooter.py
@@ -76,13 +76,17 @@ def test_vpn_disable():
 def test_forward_drop():
     with mock.patch("cuckoo.apps.rooter.run") as p:
         r.forward_drop()
-    p.assert_called_once_with(None, "-P", "FORWARD", "DROP")
+    p.assert_called_once_with(
+        None, "-P", "FORWARD", "DROP", "-m", "comment", "--comment",
+        "cuckoo-rooter"
+    )
 
 def test_enable_nat():
     with mock.patch("cuckoo.apps.rooter.run") as p:
         r.enable_nat("foo")
     p.assert_called_once_with(
-        None, "-t", "nat", "-A", "POSTROUTING", "-o", "foo", "-j", "MASQUERADE"
+        None, "-t", "nat", "-A", "POSTROUTING", "-o", "foo", "-j",
+        "MASQUERADE", "-m", "comment", "--comment", "cuckoo-rooter"
     )
 
 @mock.patch("cuckoo.apps.rooter.run")
@@ -94,7 +98,8 @@ def test_disable_nat(p):
     assert p.call_count == 2
     assert p.call_list[0] == p.call_list[1]
     p.assert_any_call(
-        None, "-t", "nat", "-D", "POSTROUTING", "-o", "foo", "-j", "MASQUERADE"
+        None, "-t", "nat", "-D", "POSTROUTING", "-o", "foo", "-j",
+        "MASQUERADE",  "-m", "comment", "--comment", "cuckoo-rooter"
     )
 
 # TODO init_rttable

From 7c6c2f659111a0ac366346acfd50a3d637f59e43 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Tue, 11 Jun 2019 13:30:39 +0200
Subject: [PATCH 119/138] Update rooter test to include cleanup

---
 tests/test_apps.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_apps.py b/tests/test_apps.py
index 552df7b6f2..a43b1a427e 100644
--- a/tests/test_apps.py
+++ b/tests/test_apps.py
@@ -77,12 +77,14 @@ def test_api(self):
             p.assert_called_once_with("localhost", 8090, False)
 
     if is_linux():
+        @mock.patch("cuckoo.main.cleanup_rooter")
         @mock.patch("cuckoo.main.cuckoo_rooter")
-        def test_rooter_abort(self, p, capsys):
+        def test_rooter_abort(self, p, mr, capsys):
             p.side_effect = KeyboardInterrupt
             main.main(("--cwd", cwd(), "rooter"), standalone_mode=False)
             out, _ = capsys.readouterr()
             assert "Aborting the Cuckoo Rooter" in out
+            mr.assert_called_once()
 
     def test_community(self):
         with mock.patch("cuckoo.main.fetch_community") as p:

From 5b619d0a750608e95e2338c3fd21a93f50dd0eca Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 12 Jun 2019 00:09:25 +0200
Subject: [PATCH 120/138] Add dir cleanup to processing tests

---
 conftest.py              |   2 +-
 tests/test_processing.py | 267 ++++++++++++++++++++++++++++-----------
 2 files changed, 192 insertions(+), 77 deletions(-)

diff --git a/conftest.py b/conftest.py
index 85057555dd..38ca57ad17 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017 Cuckoo Foundation.
+# Copyright (C) 2017-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
diff --git a/tests/test_processing.py b/tests/test_processing.py
index 8414686368..09d738087c 100644
--- a/tests/test_processing.py
+++ b/tests/test_processing.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2018 Cuckoo Foundation.
+# Copyright (C) 2016-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -52,6 +52,22 @@
 db = Database()
 
 class TestProcessing(object):
+
+    def setup(self):
+        self.tmpdirs = []
+
+    def teardown(self):
+        for path in self.tmpdirs:
+            try:
+                shutil.rmtree(path)
+            except:
+                pass
+
+    def mkdtemp(self):
+        path = tempfile.mkdtemp()
+        self.tmpdirs.append(path)
+        return path
+
     def test_init(self):
         p = Processing()
         p.set_options({
@@ -61,7 +77,7 @@ def test_init(self):
         assert p.options.foo == "bar"
 
     def test_debug(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
         init_console_logging()
 
@@ -124,7 +140,7 @@ def test_static_none(self):
         assert s.run() is None
 
     def test_archive_pdf(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
 
         s = Static()
         s.set_task({
@@ -141,7 +157,7 @@ def test_archive_pdf(self):
         assert "%48%65" in s.run()["pdf"][0]["javascript"][0]["orig_code"]
 
     def test_pdf(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
 
         s = Static()
         s.set_task({
@@ -157,7 +173,7 @@ def test_pdf(self):
         assert "var x = unescape" in r["javascript"][0]["orig_code"]
 
     def test_pdf_stringjs(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
 
         s = Static()
         s.set_task({
@@ -173,7 +189,7 @@ def test_pdf_stringjs(self):
         assert "app.alert({" in r["javascript"][0]["orig_code"]
 
     def test_pdf_ignorefake(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
 
         s = Static()
         s.set_task({
@@ -191,7 +207,7 @@ def test_pdf_ignorefake(self):
 
     @mock.patch("cuckoo.processing.static.dispatch")
     def test_pdf_workercrash(self, md):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         md.return_value = None
 
         s = Static()
@@ -208,7 +224,7 @@ def test_pdf_workercrash(self, md):
         assert r["pdf"] == []
 
     def test_phishing0_pdf(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
 
         s = Static()
         s.set_task({
@@ -224,7 +240,7 @@ def test_phishing0_pdf(self):
 
     @mock.patch("cuckoo.processing.static.dispatch")
     def test_pdf_mock(self, p):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
 
         s = Static()
         s.set_task({
@@ -243,7 +259,7 @@ def test_pdf_mock(self, p):
         )
 
     def test_pdf_metadata(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
 
         s = Static()
         s.set_task({
@@ -273,7 +289,7 @@ def test_pdf_metadata(self):
         }
 
     def test_pdf_attach(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
 
         s = Static()
         s.set_task({
@@ -316,7 +332,7 @@ def test_pdf_parse_string(self):
         assert p("\xfe\xff\x00h\x00t\x00t\x00p\x00:\x00/\x00/") == "http://"
 
     def test_office(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
         init_yara()
 
@@ -472,7 +488,7 @@ def test_strings(self):
     def test_screenshot_tesseract(self, p):
         s = Screenshots()
         # Use an empty directory so no actual screenshot analysis is done.
-        s.shots_path = tempfile.mkdtemp()
+        s.shots_path = self.mkdtemp()
         s.set_options({
             "tesseract": None,
         })
@@ -528,7 +544,7 @@ def test_screenshot_truncated(self, p):
         assert s.run() == []
 
     def test_targetinfo(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
         init_yara()
 
@@ -625,7 +641,7 @@ def test_ignore_notesseract(self, p, q):
         os.unlink(shotpath)
 
     def test_virustotal_nokey(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create(cfg={
             "processing": {
                 "virustotal": {
@@ -638,7 +654,7 @@ def test_virustotal_nokey(self):
         e.match("API key not configured")
 
     def test_virustotal_invalidcategory(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
         with pytest.raises(CuckooProcessingError) as e:
             v = VirusTotal()
@@ -650,9 +666,25 @@ def test_virustotal_invalidcategory(self):
 
 @pytest.mark.skipif(not HAVE_VOLATILITY, reason="No Volatility installed")
 class TestVolatility(object):
+
+    def setup(self):
+        self.tmpdirs = []
+
+    def teardown(self):
+        for path in self.tmpdirs:
+            try:
+                shutil.rmtree(path)
+            except:
+                pass
+
+    def mkdtemp(self):
+        path = tempfile.mkdtemp()
+        self.tmpdirs.append(path)
+        return path
+
     @mock.patch("cuckoo.processing.memory.log")
     def test_no_mempath(self, p):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         m = Memory()
         m.memory_path = None
         assert m.run() is None
@@ -661,7 +693,7 @@ def test_no_mempath(self, p):
 
     @mock.patch("cuckoo.processing.memory.log")
     def test_invalid_mempath(self, p):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         m = Memory()
         m.memory_path = "notafile"
         assert m.run() is None
@@ -670,7 +702,7 @@ def test_invalid_mempath(self, p):
 
     @mock.patch("cuckoo.processing.memory.log")
     def test_empty_mempath(self, p):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         m = Memory()
         m.memory_path = Files.temp_put("")
         assert m.run() is None
@@ -679,7 +711,7 @@ def test_empty_mempath(self, p):
 
     @mock.patch("cuckoo.processing.memory.VolatilityManager")
     def test_global_osprofile(self, p):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create(cfg={
             "memory": {
                 "basic": {
@@ -696,7 +728,7 @@ def test_global_osprofile(self, p):
 
     @mock.patch("cuckoo.processing.memory.VolatilityManager")
     def test_vm_osprofile(self, p):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create(cfg={
             "memory": {
                 "basic": {
@@ -737,7 +769,7 @@ def test_wrong_profile(self, p, q):
 
     @mock.patch("volatility.utils.load_as")
     def test_plugin_enabled(self, p):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create(cfg={
             "memory": {
                 "pslist": {
@@ -775,8 +807,23 @@ def test_s(self):
         assert obj_s(vol_obj.NoneObject()) is None
 
 class TestProcessingMachineInfo(object):
+    def setup(self):
+        self.tmpdirs = []
+
+    def teardown(self):
+        for path in self.tmpdirs:
+            try:
+                shutil.rmtree(path)
+            except:
+                pass
+
+    def mkdtemp(self):
+        path = tempfile.mkdtemp()
+        self.tmpdirs.append(path)
+        return path
+
     def test_machine_info_empty(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         rp = RunProcessing({
             "id": 1,
         })
@@ -784,7 +831,7 @@ def test_machine_info_empty(self):
         assert rp.machine == {}
 
     def test_machine_info_cuckoo1(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
 
         rp = RunProcessing({
@@ -800,7 +847,7 @@ def test_machine_info_cuckoo1(self):
         assert rp.machine["ip"] == "192.168.56.101"
 
     def test_machine_info_cuckoo2(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
 
         rp = RunProcessing({
@@ -816,6 +863,21 @@ def test_machine_info_cuckoo2(self):
         }
 
 class TestBehavior(object):
+    def setup(self):
+        self.tmpdirs = []
+
+    def teardown(self):
+        for path in self.tmpdirs:
+            try:
+                shutil.rmtree(path)
+            except:
+                pass
+
+    def mkdtemp(self):
+        path = tempfile.mkdtemp()
+        self.tmpdirs.append(path)
+        return path
+
     def test_process_tree_regular(self):
         pt = ProcessTree(None)
 
@@ -888,7 +950,7 @@ def test_process_tree_pid_reuse(self):
         assert not obj[2]["children"]
 
     def test_bson_limit(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
 
         ba = BehaviorAnalysis()
@@ -914,7 +976,7 @@ def test_bson_limit(self):
         ]
 
     def test_extract_scripts(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
         init_yara()
 
@@ -968,7 +1030,7 @@ def test_extract_scripts(self):
         assert open(out[1]["raw"], "rb").read() == 'echo "Recursive"'
 
     def test_stap_log(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
         init_yara()
 
@@ -1066,6 +1128,7 @@ def test_stap_log(self):
         }
 
 class TestPcap(object):
+
     @classmethod
     def setup_class(cls):
         set_cwd(tempfile.mkdtemp())
@@ -1223,15 +1286,30 @@ def test_network_dns(self):
         assert expected_types == types
 
 class TestPcapAdditional(object):
+    def setup(self):
+        self.tmpdirs = []
+
+    def teardown(self):
+        for path in self.tmpdirs:
+            try:
+                shutil.rmtree(path)
+            except:
+                pass
+
+    def mkdtemp(self):
+        path = tempfile.mkdtemp()
+        self.tmpdirs.append(path)
+        return path
+
     @mock.patch("cuckoo.processing.network.resolve")
     def test_resolve_dns(self, p):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
         p.return_value = "1.2.3.4"
         assert Pcap(None, {})._dns_gethostbyname("google.com") != ""
 
     def test_icmp_ignore_resultserver(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create()
         p = Pcap(None, {})
         pkt = dpkt.icmp.ICMP.Echo()
@@ -1249,7 +1327,7 @@ def test_icmp_ignore_resultserver(self):
         assert len(p.icmp_requests) == 1
 
     def test_no_sorted_pcap(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create(cfg={
             "cuckoo": {
                 "processing": {
@@ -1269,7 +1347,7 @@ def test_no_sorted_pcap(self):
         assert not os.path.exists(cwd("dump_sorted.pcap", analysis=1))
 
     def test_yes_sorted_pcap(self):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create({
             "cuckoo": {
                 "network": {
@@ -1304,7 +1382,7 @@ def test_duplicate_dns_requests(self):
 
     @mock.patch("cuckoo.processing.network.log")
     def test_empty_pcap(self, p):
-        set_cwd(tempfile.mkdtemp())
+        set_cwd(self.mkdtemp())
         cuckoo_create(cfg={
             "cuckoo": {
                 "processing": {
@@ -1325,9 +1403,24 @@ def test_empty_pcap(self, p):
         p.warning.assert_not_called()
 
 class TestPcap2(object):
+    def setup(self):
+        self.tmpdirs = []
+
+    def teardown(self):
+        for path in self.tmpdirs:
+            try:
+                shutil.rmtree(path)
+            except:
+                pass
+
+    def mkdtemp(self):
+        path = tempfile.mkdtemp()
+        self.tmpdirs.append(path)
+        return path
+
     def test_smtp_ex(self):
         obj = Pcap2(
-            "tests/files/pcap/smtp.pcap", None, tempfile.mkdtemp()
+            "tests/files/pcap/smtp.pcap", None, self.mkdtemp()
         ).run()
 
         assert len(obj["smtp_ex"]) == 1
@@ -1346,7 +1439,7 @@ def test_smtp_ex(self):
 
     def test_http_status(self):
         obj = Pcap2(
-            "tests/files/pcap/status-code.pcap", None, tempfile.mkdtemp()
+            "tests/files/pcap/status-code.pcap", None, self.mkdtemp()
         ).run()
         assert len(obj["http_ex"]) == 1
         assert not obj["https_ex"]
@@ -1354,7 +1447,7 @@ def test_http_status(self):
 
     def test_http_nostatus(self):
         obj = Pcap2(
-            "tests/files/pcap/not-http.pcap", None, tempfile.mkdtemp()
+            "tests/files/pcap/not-http.pcap", None, self.mkdtemp()
         ).run()
         assert len(obj["http_ex"]) == 1
 
@@ -1439,45 +1532,67 @@ def create():
         s.process_pcap_binary = create
         s.run()
 
-def test_static_extracted():
-    set_cwd(tempfile.mkdtemp())
-    cuckoo_create(cfg={
-        "processing": {
-            "analysisinfo": {
-                "enabled": False,
+class TestExtracted(object):
+    def setup(self):
+        self.tmpdirs = []
+
+    def teardown(self):
+        for path in self.tmpdirs:
+            try:
+                shutil.rmtree(path)
+            except:
+                pass
+
+    def mkdtemp(self):
+        path = tempfile.mkdtemp()
+        self.tmpdirs.append(path)
+        return path
+
+    def test_static_extracted(self):
+        set_cwd(self.mkdtemp())
+        cuckoo_create(cfg={
+            "processing": {
+                "analysisinfo": {
+                    "enabled": False,
+                },
+                "debug": {
+                    "enabled": False,
+                }
             },
-            "debug": {
-                "enabled": False,
+        })
+        mkdir(cwd(analysis=1))
+        shutil.copy("tests/files/createproc1.docm", cwd("binary", analysis=1))
+
+        open(cwd("yara", "office", "ole.yar"), "wb").write("""
+            rule OleInside {
+                strings:
+                    $s1 = "Win32_Process"
+                condition:
+                    filename matches /word\/vbaProject.bin/ and $s1
             }
-        },
-    })
-    mkdir(cwd(analysis=1))
-    shutil.copy("tests/files/createproc1.docm", cwd("binary", analysis=1))
-
-    open(cwd("yara", "office", "ole.yar"), "wb").write("""
-        rule OleInside {
-            strings:
-                $s1 = "Win32_Process"
-            condition:
-                filename matches /word\/vbaProject.bin/ and $s1
-        }
-    """)
-    init_yara()
-
-    class OleInsideExtractor(Extractor):
-        def handle_yara(self, filepath, match):
-            return (
-                match.category == "office" and
-                match.yara[0].name == "OleInside"
-            )
-
-    ExtractManager._instances = {}
-    ExtractManager.extractors = OleInsideExtractor,
-
-    results = RunProcessing(Dictionary({
-        "id": 1,
-        "category": "file",
-        "target": "tests/files/createproc1.docm",
-    })).run()
-
-    assert len(results["extracted"]) == 1
+        """)
+        init_yara()
+
+        class OleInsideExtractor(Extractor):
+            def handle_yara(self, filepath, match):
+                return (
+                    match.category == "office" and
+                    match.yara[0].name == "OleInside"
+                )
+
+        class X(object):
+            @staticmethod
+            def p():
+                return [Extracted, Static]
+
+        with mock.patch("cuckoo.processing.plugins", new_callable=X.p) as p:
+            ExtractManager._instances = {}
+            ExtractManager.extractors = OleInsideExtractor,
+
+            results = RunProcessing(Dictionary({
+                "id": 1,
+                "category": "file",
+                "target": "tests/files/createproc1.docm",
+            })).run()
+
+            assert len(results["extracted"]) == 1

From edd432713934686748298d3c6ff7807d5a9efc87 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 12 Jun 2019 13:32:15 +0200
Subject: [PATCH 121/138] Update version and hashes

---
 cuckoo/data/signatures/windows/creates_exe.py | 58 ++++++++++++++-----
 cuckoo/misc.py                                |  2 +-
 cuckoo/private/cwd/hashes.txt                 |  8 ++-
 3 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/cuckoo/data/signatures/windows/creates_exe.py b/cuckoo/data/signatures/windows/creates_exe.py
index 13f8da47cf..3bd0f5a06d 100644
--- a/cuckoo/data/signatures/windows/creates_exe.py
+++ b/cuckoo/data/signatures/windows/creates_exe.py
@@ -1,27 +1,57 @@
-# Copyright (C) 2010-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2016 Cuckoo Foundation.
+# Copyright (C) 2010-2015 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
-from cuckoo.common.abstracts import Signature
+from lib.cuckoo.common.abstracts import Signature
+
+try:
+    import re2 as re
+except ImportError:
+    import re
 
 class CreatesExe(Signature):
     name = "creates_exe"
-    description = "Creates a Windows executable on the filesystem"
+    description = "Creates executable files on the filesystem"
     severity = 2
     categories = ["generic"]
     authors = ["Cuckoo Developers"]
     minimum = "2.0"
+    ttp = ["T1129"]
+
+    pattern = (
+        ".*\\.(bat|cmd|com|cpl|dll|exe|js|jse|lnk|msi|msh|msh1|msh2|mshxml|"
+        "msh1xml|msh2xml|ocx|pif|psc1|psc2|ps1|ps1xml|ps2|ps2xml|reg|scf|scr|"
+        "vb|vbe|vbs|ws|wsc|wse|wsh)$"
+    )
+
+    def on_complete(self):
+        for filepath in self.check_file(pattern=self.pattern, actions=["file_written"], regex=True, all=True):
+            self.mark_ioc("file", filepath)
+
+        return self.has_marks()
+
+class CreatesUserFolderEXE(Signature):
+    name = "creates_user_folder_exe"
+    description = "Creates an executable file in a user folder"
+    severity = 3
+    families = ["persistance"]
+    authors = ["Kevin Ross"]
+    minimum = "2.0"
+    ttp = ["T1129"]
 
-    # This is a signature template. It should be used as a skeleton for
-    # creating custom signatures, therefore is disabled by default.
-    # It doesn't verify whether a .exe is actually being created, but
-    # it matches files being opened with any access type, including
-    # read and attributes lookup.
-    enabled = False
+    directories_re = [
+        "^[a-zA-Z]:\\\\Users\\\\[^\\\\]+\\\\AppData\\\\.*",
+        "^[a-zA-Z]:\\\\Documents\\ and\\ Settings\\\\[^\\\\]+\\\\Local\\ Settings\\\\.*",
+    ]
 
     def on_complete(self):
-        match = self.check_file(pattern=".*\\.exe$", regex=True)
-        if match:
-            self.mark_ioc("file", match)
-            return True
+        for dropped in self.get_results("dropped", []):
+            if "filepath" in dropped:
+                droppedtype = dropped["type"]
+                filepath = dropped["filepath"]
+                if "MS-DOS executable" in droppedtype:
+                    for directory in self.directories_re:
+                        if re.match(directory, filepath):
+                            self.mark_ioc("file", filepath)
+
+        return self.has_marks()
diff --git a/cuckoo/misc.py b/cuckoo/misc.py
index 6e82f9f1c3..fab1815f37 100644
--- a/cuckoo/misc.py
+++ b/cuckoo/misc.py
@@ -33,7 +33,7 @@
 
 # Normalized Cuckoo version (i.e., "2.0.5.3" in setup is "2.0.5" here). This
 # because we use StrictVersion() later on which doesn't accept "2.0.5.3".
-version = "2.0.6"
+version = "2.0.7"
 
 def set_cwd(path, raw=None):
     global _root, _raw
diff --git a/cuckoo/private/cwd/hashes.txt b/cuckoo/private/cwd/hashes.txt
index b6b526bf78..2b93618c87 100644
--- a/cuckoo/private/cwd/hashes.txt
+++ b/cuckoo/private/cwd/hashes.txt
@@ -291,7 +291,7 @@ ab9af787dd901cf5b4ffb1192da7d096ee1de9ad analyzer/windows/lib/core/packages.py
 c8492c74db400e6300194c1bafd3088a102bdc8e analyzer/windows/modules/auxiliary/human.py
 e86627abeb5ecc0112438ad179e9d0487870785a analyzer/windows/modules/packages/ie.py
 
-# TBD
+# 2.0.7 release
 74c4c577a61f96571ac47e86c83fc0ece9d5f0ad agent/agent.py
 4d567f35bd79192d8f279f474816b9686e71896b analyzer/darwin/lib/api/screenshot.py
 633ab6bd08eb393ca630b59ce0ee5374862c6558 analyzer/darwin/lib/common/hashing.py
@@ -318,7 +318,13 @@ e13518903a2fcaec3d0b140d3164c426f07ab647 analyzer/windows/modules/packages/ie.py
 6e6680e26bf1cf41909a4efcbd86917cf4b14603 analyzer/windows/modules/packages/pub.py
 d8fce614d615f6bdb3117e92bfa6e4ae2b48ea52 analyzer/windows/modules/packages/vbs.py
 09702bc15041a80f399f0c143cc3ec29196e4962 analyzer/windows/modules/packages/zip.py
+53fddb538c34040d226441bd59d58fc901dcadc7 monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/inject-x64.exe
+630515538733f299523c7fa0fc2ae87781d87fe5 monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/inject-x86.exe
+dc1af7e611fb53d6b82f919041f7470301160483 monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/is32bit.exe
+8078749c13ff8c022f5c0cca3b865e943f21b929 monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/monitor-x64.dll
+b0ceb903d39fe7778e0ad173ef0e02ad0bc7440b monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/monitor-x86.dll
 c278d582c26003467794ee4d0ab7ae337034c2ea monitor/latest
+ec5c6c6446d37ffb254429df4442788f6a79757b signatures/windows/creates_exe.py
 4575c7d09b316cad3a71059a27b78cbcb9c1f6b4 stuff/ttp_descriptions.json
 9870109fa8a09c2f1b871ea178bdd3d40390baab web/local_settings.py
 f81e71f788149039f31c9b0b5e2891733e51b5d7 whitelist/ip.txt

From bc2da822f54ef3b5e12d2841e41c598bb09202b0 Mon Sep 17 00:00:00 2001
From: Jurriaan Bremer <jbr@hatching.io>
Date: Thu, 13 Jun 2019 18:09:53 +0200
Subject: [PATCH 122/138] version 2.0.7

---
 cuckoo/compat/config.py | 4 ++--
 docs/book/conf.py       | 4 ++--
 setup.py                | 4 ++--
 tests/test_config.py    | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/cuckoo/compat/config.py b/cuckoo/compat/config.py
index cb90d9b881..73a5a2c60f 100644
--- a/cuckoo/compat/config.py
+++ b/cuckoo/compat/config.py
@@ -710,7 +710,7 @@ def _205_206(c):
     c["virtualbox"]["controlports"] = "5000-5050"
     return c
 
-def _206_210(c):
+def _206_207(c):
     c["auxiliary"]["replay"]["certificate"] = "bin/cert.p12"
     # We'd like to provide a secure default, but let's not inconvenience
     # upgrading users. TODO Might need to revisited once we write back config.
@@ -738,7 +738,7 @@ def _206_210(c):
     "2.0.3": ("2.0.4", _203_204),
     "2.0.4": ("2.0.5", _204_205),
     "2.0.5": ("2.0.6", _205_206),
-    "2.0.6": ("2.1.0", _206_210),
+    "2.0.6": ("2.0.7", _206_207),
 
     # We're also capable of migrating away from 2.0-dev which basically means
     # that we might have to a partial migration from either 2.0-rc2 or 2.0-rc1.
diff --git a/docs/book/conf.py b/docs/book/conf.py
index 7945cefb7a..8238778c3d 100644
--- a/docs/book/conf.py
+++ b/docs/book/conf.py
@@ -47,9 +47,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.0.6'
+version = '2.0.7'
 # The full version, including alpha/beta/rc tags.
-release = '2.0.6'
+release = '2.0.7'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/setup.py b/setup.py
index 29b9744f82..6d7d60312f 100755
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (C) 2016-2018 Cuckoo Foundation.
+# Copyright (C) 2016-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - https://cuckoosandbox.org/.
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -149,7 +149,7 @@ def do_setup(**kwargs):
 
 do_setup(
     name="Cuckoo",
-    version="2.0.7a1",
+    version="2.0.7",
     author="Stichting Cuckoo Foundation",
     author_email="cuckoo@cuckoofoundation.org",
     packages=[
diff --git a/tests/test_config.py b/tests/test_config.py
index b52f4638e8..5d72eeffad 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -1160,7 +1160,7 @@ def test_migration_205_206():
     assert cfg["virtualbox"]["controlports"] == "5000-5050"
     assert cfg["routing"]["inetsim"]["ports"] is None
 
-def test_migration_206_210():
+def test_migration_206_207():
     set_cwd(tempfile.mkdtemp())
     Folders.create(cwd(), "conf")
 
@@ -1179,7 +1179,7 @@ def test_migration_206_210():
 interface = virbr0
     """)
     cfg = Config.from_confdir(cwd("conf"), loose=True)
-    cfg = migrate(cfg, "2.0.6", "2.1.0")
+    cfg = migrate(cfg, "2.0.6", "2.0.7")
 
     assert cfg["auxiliary"]["replay"]["certificate"] == "bin/cert.p12"
     assert cfg["cuckoo"]["cuckoo"]["api_token"] is None

From 6f89e8a7bdb5699e93a30e6e5a84dfb023bb0720 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 14 Jun 2019 13:24:56 +0200
Subject: [PATCH 123/138] Remove shipped signature Signature used to be emtpy
 and disabled. It will be copied to CWD upon its creation. This causes a
 disabled signature to always be there and never be updates, since the file
 will already exist.

---
 cuckoo/data/signatures/windows/creates_exe.py | 57 -------------------
 cuckoo/main.py                                |  2 +-
 2 files changed, 1 insertion(+), 58 deletions(-)
 delete mode 100644 cuckoo/data/signatures/windows/creates_exe.py

diff --git a/cuckoo/data/signatures/windows/creates_exe.py b/cuckoo/data/signatures/windows/creates_exe.py
deleted file mode 100644
index 3bd0f5a06d..0000000000
--- a/cuckoo/data/signatures/windows/creates_exe.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Copyright (C) 2010-2015 Cuckoo Foundation.
-# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
-# See the file 'docs/LICENSE' for copying permission.
-
-from lib.cuckoo.common.abstracts import Signature
-
-try:
-    import re2 as re
-except ImportError:
-    import re
-
-class CreatesExe(Signature):
-    name = "creates_exe"
-    description = "Creates executable files on the filesystem"
-    severity = 2
-    categories = ["generic"]
-    authors = ["Cuckoo Developers"]
-    minimum = "2.0"
-    ttp = ["T1129"]
-
-    pattern = (
-        ".*\\.(bat|cmd|com|cpl|dll|exe|js|jse|lnk|msi|msh|msh1|msh2|mshxml|"
-        "msh1xml|msh2xml|ocx|pif|psc1|psc2|ps1|ps1xml|ps2|ps2xml|reg|scf|scr|"
-        "vb|vbe|vbs|ws|wsc|wse|wsh)$"
-    )
-
-    def on_complete(self):
-        for filepath in self.check_file(pattern=self.pattern, actions=["file_written"], regex=True, all=True):
-            self.mark_ioc("file", filepath)
-
-        return self.has_marks()
-
-class CreatesUserFolderEXE(Signature):
-    name = "creates_user_folder_exe"
-    description = "Creates an executable file in a user folder"
-    severity = 3
-    families = ["persistance"]
-    authors = ["Kevin Ross"]
-    minimum = "2.0"
-    ttp = ["T1129"]
-
-    directories_re = [
-        "^[a-zA-Z]:\\\\Users\\\\[^\\\\]+\\\\AppData\\\\.*",
-        "^[a-zA-Z]:\\\\Documents\\ and\\ Settings\\\\[^\\\\]+\\\\Local\\ Settings\\\\.*",
-    ]
-
-    def on_complete(self):
-        for dropped in self.get_results("dropped", []):
-            if "filepath" in dropped:
-                droppedtype = dropped["type"]
-                filepath = dropped["filepath"]
-                if "MS-DOS executable" in droppedtype:
-                    for directory in self.directories_re:
-                        if re.match(directory, filepath):
-                            self.mark_ioc("file", filepath)
-
-        return self.has_marks()
diff --git a/cuckoo/main.py b/cuckoo/main.py
index d6bc38a055..a887522146 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -209,7 +209,7 @@ def cuckoo_init(level, ctx, cfg=None):
             "pretty important!"
         )
         log.warning(
-            "You'll be able to fetch all the latest Cuckoo Signaturs, Yara "
+            "You'll be able to fetch all the latest Cuckoo Signatures, Yara "
             "rules, and more goodies by running the following command:"
         )
         log.info("$ %s", green(format_command("community")))

From 3fd9c0b7dcfd983a169b1197ca97bb4f3b28872e Mon Sep 17 00:00:00 2001
From: Jurriaan Bremer <jbr@hatching.io>
Date: Thu, 13 Jun 2019 18:09:53 +0200
Subject: [PATCH 124/138] version 2.0.7

---
 cuckoo/compat/config.py | 4 ++--
 docs/book/conf.py       | 4 ++--
 setup.py                | 4 ++--
 tests/test_config.py    | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/cuckoo/compat/config.py b/cuckoo/compat/config.py
index cb90d9b881..73a5a2c60f 100644
--- a/cuckoo/compat/config.py
+++ b/cuckoo/compat/config.py
@@ -710,7 +710,7 @@ def _205_206(c):
     c["virtualbox"]["controlports"] = "5000-5050"
     return c
 
-def _206_210(c):
+def _206_207(c):
     c["auxiliary"]["replay"]["certificate"] = "bin/cert.p12"
     # We'd like to provide a secure default, but let's not inconvenience
     # upgrading users. TODO Might need to revisited once we write back config.
@@ -738,7 +738,7 @@ def _206_210(c):
     "2.0.3": ("2.0.4", _203_204),
     "2.0.4": ("2.0.5", _204_205),
     "2.0.5": ("2.0.6", _205_206),
-    "2.0.6": ("2.1.0", _206_210),
+    "2.0.6": ("2.0.7", _206_207),
 
     # We're also capable of migrating away from 2.0-dev which basically means
     # that we might have to a partial migration from either 2.0-rc2 or 2.0-rc1.
diff --git a/docs/book/conf.py b/docs/book/conf.py
index 7945cefb7a..8238778c3d 100644
--- a/docs/book/conf.py
+++ b/docs/book/conf.py
@@ -47,9 +47,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.0.6'
+version = '2.0.7'
 # The full version, including alpha/beta/rc tags.
-release = '2.0.6'
+release = '2.0.7'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/setup.py b/setup.py
index 29b9744f82..6d7d60312f 100755
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (C) 2016-2018 Cuckoo Foundation.
+# Copyright (C) 2016-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - https://cuckoosandbox.org/.
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -149,7 +149,7 @@ def do_setup(**kwargs):
 
 do_setup(
     name="Cuckoo",
-    version="2.0.7a1",
+    version="2.0.7",
     author="Stichting Cuckoo Foundation",
     author_email="cuckoo@cuckoofoundation.org",
     packages=[
diff --git a/tests/test_config.py b/tests/test_config.py
index b52f4638e8..5d72eeffad 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -1160,7 +1160,7 @@ def test_migration_205_206():
     assert cfg["virtualbox"]["controlports"] == "5000-5050"
     assert cfg["routing"]["inetsim"]["ports"] is None
 
-def test_migration_206_210():
+def test_migration_206_207():
     set_cwd(tempfile.mkdtemp())
     Folders.create(cwd(), "conf")
 
@@ -1179,7 +1179,7 @@ def test_migration_206_210():
 interface = virbr0
     """)
     cfg = Config.from_confdir(cwd("conf"), loose=True)
-    cfg = migrate(cfg, "2.0.6", "2.1.0")
+    cfg = migrate(cfg, "2.0.6", "2.0.7")
 
     assert cfg["auxiliary"]["replay"]["certificate"] == "bin/cert.p12"
     assert cfg["cuckoo"]["cuckoo"]["api_token"] is None

From 9abb5caea59d2fd1a8b30ce0ccd209ec724acbda Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 14 Jun 2019 14:30:12 +0200
Subject: [PATCH 125/138] Update config and startup tests

---
 cuckoo/compat/config.py | 7 +++++++
 tests/test_config.py    | 9 +++++++++
 tests/test_startup.py   | 1 -
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/cuckoo/compat/config.py b/cuckoo/compat/config.py
index 73a5a2c60f..a8ccce8e80 100644
--- a/cuckoo/compat/config.py
+++ b/cuckoo/compat/config.py
@@ -718,6 +718,13 @@ def _206_207(c):
     c["cuckoo"]["cuckoo"]["web_secret"] = None
     c["kvm"]["kvm"]["dsn"] = "qemu:///system"
     c["processing"]["irma"]["probes"] = None
+    c["reporting"]["misp"]["distribution"] = 0
+    c["reporting"]["misp"]["analysis"] = 0
+    c["reporting"]["misp"]["threat_level"] = 4
+    c["reporting"]["misp"]["min_malscore"] = 0
+    c["reporting"]["misp"]["tag"] = "Cuckoo"
+    c["reporting"]["misp"]["upload_sample"] = False
+
     return c
 
 migrations = {
diff --git a/tests/test_config.py b/tests/test_config.py
index 5d72eeffad..36846cb9f1 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -1177,6 +1177,9 @@ def test_migration_206_207():
 [kvm]
 machines = cuckoo1
 interface = virbr0
+    """)
+    Files.create(cwd("conf"), "reporting.conf", """
+[misp]
     """)
     cfg = Config.from_confdir(cwd("conf"), loose=True)
     cfg = migrate(cfg, "2.0.6", "2.0.7")
@@ -1186,6 +1189,12 @@ def test_migration_206_207():
     assert cfg["cuckoo"]["cuckoo"]["web_secret"] is None
     assert cfg["processing"]["irma"]["probes"] is None
     assert cfg["kvm"]["kvm"]["dsn"] == "qemu:///system"
+    assert cfg["reporting"]["misp"]["distribution"] == 0
+    assert cfg["reporting"]["misp"]["analysis"] == 0
+    assert cfg["reporting"]["misp"]["threat_level"] == 4
+    assert cfg["reporting"]["misp"]["min_malscore"] == 0
+    assert cfg["reporting"]["misp"]["tag"] == "Cuckoo"
+    assert cfg["reporting"]["misp"]["upload_sample"] is False
 
 
 class FullMigration(object):
diff --git a/tests/test_startup.py b/tests/test_startup.py
index a14fb87471..419792fcb7 100644
--- a/tests/test_startup.py
+++ b/tests/test_startup.py
@@ -88,7 +88,6 @@ def log(fmt, *args):
     logs = "\n".join(logs)
     assert "KVM" in logs
     assert "Xen" in logs
-    assert "CreatesExe" in logs
     assert "SystemMetrics" in logs
 
 @mock.patch("cuckoo.core.startup.cuckoo")

From fd3adafba4435f7d2476529239b368d103d31bed Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 14 Jun 2019 14:31:48 +0200
Subject: [PATCH 126/138] Read mysql dump per command The MySQLdb driver seems
 to have issues with compound statements. Importing an SQL file counts as
 this. Reading per command solves this.

---
 tests/files/sql/060my.sql | 12 ++++++------
 tests/files/sql/11my.sql  | 11 +++++------
 tests/test_database.py    | 14 ++++++++++++--
 3 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/tests/files/sql/060my.sql b/tests/files/sql/060my.sql
index 92fdc33800..01c01962f0 100644
--- a/tests/files/sql/060my.sql
+++ b/tests/files/sql/060my.sql
@@ -1,9 +1,3 @@
--- MySQL dump 10.13  Distrib 5.5.53, for debian-linux-gnu (x86_64)
---
--- Host: localhost    Database: cuckoo
--- ------------------------------------------------------
--- Server version	5.5.53-0ubuntu0.14.04.1
-
 /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
 /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
 /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
@@ -191,3 +185,9 @@ UNLOCK TABLES;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
 -- Dump completed on 2016-11-17  2:47:52
+
+-- MySQL dump 10.13  Distrib 5.5.53, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: cuckoo
+-- ------------------------------------------------------
+-- Server version	5.5.53-0ubuntu0.14.04.1
\ No newline at end of file
diff --git a/tests/files/sql/11my.sql b/tests/files/sql/11my.sql
index b567515bae..8bc76d3fdc 100644
--- a/tests/files/sql/11my.sql
+++ b/tests/files/sql/11my.sql
@@ -1,9 +1,3 @@
--- MySQL dump 10.14  Distrib 5.5.47-MariaDB, for debian-linux-gnu (x86_64)
---
--- Host: localhost    Database: cuckoo11
--- ------------------------------------------------------
--- Server version	5.5.47-MariaDB-1ubuntu0.14.04.1
-
 /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
 /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
 /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
@@ -290,3 +284,8 @@ UNLOCK TABLES;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
 -- Dump completed on 2017-02-07 12:41:40
+-- MySQL dump 10.14  Distrib 5.5.47-MariaDB, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: cuckoo11
+-- ------------------------------------------------------
+-- Server version	5.5.47-MariaDB-1ubuntu0.14.04.1
\ No newline at end of file
diff --git a/tests/test_database.py b/tests/test_database.py
index 03803ce7cf..a77a66b718 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -273,6 +273,14 @@ class TestPostgreSQL(DatabaseEngine):
 class TestMySQL(DatabaseEngine):
     URI = "mysql://cuckoo:cuckoo@localhost/cuckootest"
 
+def parse_mysql_dump(dump):
+    for c in dump.split(";"):
+        c = c.strip()
+
+        if c and not c.startswith("--"):
+            c = "%s;" % c
+            yield c
+
 @pytest.mark.skipif("sys.platform != 'linux2'")
 class DatabaseMigrationEngine(object):
     """Test database migration(s)."""
@@ -430,7 +438,8 @@ class TestDatabaseMigration060MySQL(DatabaseMigration060):
 
     @staticmethod
     def execute_script(cls, script):
-        cls.s.execute(script)
+        for command in parse_mysql_dump(script):
+            cls.s.execute(command)
 
     @staticmethod
     def migrate(cls):
@@ -510,7 +519,8 @@ class TestDatabaseMigration11MySQL(DatabaseMigration11):
 
     @staticmethod
     def execute_script(cls, script):
-        cls.s.execute(script)
+        for command in parse_mysql_dump(script):
+            cls.s.execute(command)
 
 @mock.patch("cuckoo.core.database.create_engine")
 @mock.patch("cuckoo.core.database.sessionmaker")

From f437a9be7faf42d61ea26037f2ed942e01895748 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Fri, 14 Jun 2019 14:59:05 +0200
Subject: [PATCH 127/138] Use xenial on travis ci

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 69fd5aa0cf..41cea05ee6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -98,6 +98,7 @@ after_success:
   - coveralls
   - codecov
 
+dist: xenial
 addons:
   apt:
     packages:

From 3e7b27c42f41f35eb7943862b60f63fbc2040893 Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Mon, 17 Jun 2019 17:39:19 +0200
Subject: [PATCH 128/138] Disable file download auth by default

---
 cuckoo/private/cwd/hashes.txt | 2 +-
 cuckoo/web/web/settings.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cuckoo/private/cwd/hashes.txt b/cuckoo/private/cwd/hashes.txt
index 2b93618c87..98cdd52375 100644
--- a/cuckoo/private/cwd/hashes.txt
+++ b/cuckoo/private/cwd/hashes.txt
@@ -324,7 +324,7 @@ dc1af7e611fb53d6b82f919041f7470301160483 monitor/2deb9ccd75d5a7a3fe05b2625b03a86
 8078749c13ff8c022f5c0cca3b865e943f21b929 monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/monitor-x64.dll
 b0ceb903d39fe7778e0ad173ef0e02ad0bc7440b monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/monitor-x86.dll
 c278d582c26003467794ee4d0ab7ae337034c2ea monitor/latest
-ec5c6c6446d37ffb254429df4442788f6a79757b signatures/windows/creates_exe.py
+0000000000000000000000000000000000000000 signatures/windows/creates_exe.py
 4575c7d09b316cad3a71059a27b78cbcb9c1f6b4 stuff/ttp_descriptions.json
 9870109fa8a09c2f1b871ea178bdd3d40390baab web/local_settings.py
 f81e71f788149039f31c9b0b5e2891733e51b5d7 whitelist/ip.txt
diff --git a/cuckoo/web/web/settings.py b/cuckoo/web/web/settings.py
index c48ac96f7c..7de3d650d2 100644
--- a/cuckoo/web/web/settings.py
+++ b/cuckoo/web/web/settings.py
@@ -110,7 +110,7 @@
     "django.middleware.csrf.CsrfViewMiddleware",
     # Cuckoo Authentication & headers.
     "web.middle.CuckooAuthentication",
-    "web.middle.CuckooFileDownloadAuthentication",
+    # "web.middle.CuckooFileDownloadAuthentication",
     "web.middle.CuckooHeaders",
     # Our custom exception handler.
     "web.errors.ExceptionMiddleware"

From 85da862c75908f87dbe54a32e3d613b08e5870aa Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 19 Jun 2019 17:32:33 +0200
Subject: [PATCH 129/138] Ensure tmpdir is created

---
 cuckoo/main.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cuckoo/main.py b/cuckoo/main.py
index a887522146..14500cfbe0 100644
--- a/cuckoo/main.py
+++ b/cuckoo/main.py
@@ -530,6 +530,10 @@ def api(ctx, host, port, uwsgi, nginx):
 
     init_console_logging(level=ctx.parent.level)
     Database().connect()
+
+    if not ensure_tmpdir():
+        sys.exit(1)
+
     cuckoo_api(host, port, ctx.parent.level == logging.DEBUG)
 
 @main.command()
@@ -638,6 +642,9 @@ def web(ctx, args, host, port, uwsgi, nginx):
     init_console_logging(level=ctx.parent.level)
     Database().connect()
 
+    if not ensure_tmpdir():
+        sys.exit(1)
+
     try:
         execute_from_command_line(
             ("cuckoo", "runserver", "%s:%d" % (host, port))

From 13cbe0d9e457be3673304533043e992ead1ea9b2 Mon Sep 17 00:00:00 2001
From: Jurriaan Bremer <jbr@hatching.io>
Date: Wed, 19 Jun 2019 19:27:09 +0200
Subject: [PATCH 130/138] latest minor tweaks for 2.0.7 release

---
 cuckoo/core/resultserver.py   | 4 ++--
 cuckoo/private/cwd/hashes.txt | 3 ---
 docs/book/usage/api.rst       | 4 ++--
 setup.py                      | 2 +-
 4 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index c473e2d38c..96117eff27 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -1,5 +1,5 @@
 # Copyright (C) 2012-2013 Claudio Guarnieri.
-# Copyright (C) 2014-2018 Cuckoo Foundation.
+# Copyright (C) 2014-2019 Cuckoo Foundation.
 # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
 # See the file 'docs/LICENSE' for copying permission.
 
@@ -37,7 +37,7 @@
 RESULT_UPLOADABLE = ("files", "shots", "buffer",  "extracted", "memory")
 RESULT_DIRECTORIES = RESULT_UPLOADABLE + ("reports", "logs")
 
-# Prevent malicious clients from using potentially dangerious filenames
+# Prevent malicious clients from using potentially dangerous filenames
 # E.g. C API confusion by using null, or using the colon on NTFS (Alternate
 # Data Streams); XXX: just replace illegal chars?
 BANNED_PATH_CHARS = b'\x00:'
diff --git a/cuckoo/private/cwd/hashes.txt b/cuckoo/private/cwd/hashes.txt
index 98cdd52375..6a72fb6be7 100644
--- a/cuckoo/private/cwd/hashes.txt
+++ b/cuckoo/private/cwd/hashes.txt
@@ -8,7 +8,6 @@ f128dbf6ac5c90dbed99f9626d58b63b133b7f8d signatures/extractor/__init__.py
 335390b78fa1c1ffdf530874819ffcb16dab2174 signatures/linux/__init__.py
 1de5e62f213fa0e67af9c1a8ac886a43a08787a2 signatures/network/__init__.py
 069e496f3820f73fbd36f7f195dad989e4e0d863 signatures/windows/__init__.py
-68f1176dee909f2743bab5ee569cc82ab1e3ef5a signatures/windows/creates_exe.py
 
 # 2.0.0 release
 ae4f2d8ddc2e6ff4b9c255cbb93cc0adecabf9e0 __init__.py
@@ -171,7 +170,6 @@ fe60ee51bc7cf8760fb37b73f4526126748d286b signatures/cross/__init__.py
 1e8e816f5d82dbfc4431df7dfaccce9f4621220f signatures/darwin/__init__.py
 dbae884db020dd533231d689b265eb008de23b3d signatures/network/__init__.py
 f4b225e5c18010fd6e2c4adfe2e70c90854784bf signatures/windows/__init__.py
-7092cf73c9588bd00198a698847663d1f274f3bc signatures/windows/creates_exe.py
 2b9a6b1a397977c31430b66543f4dfac02af48f2 signatures/windows/generic_metrics.py
 da39a3ee5e6b4b0d3255bfef95601890afd80709 storage/analyses/.gitignore
 da39a3ee5e6b4b0d3255bfef95601890afd80709 storage/baseline/.gitignore
@@ -324,7 +322,6 @@ dc1af7e611fb53d6b82f919041f7470301160483 monitor/2deb9ccd75d5a7a3fe05b2625b03a86
 8078749c13ff8c022f5c0cca3b865e943f21b929 monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/monitor-x64.dll
 b0ceb903d39fe7778e0ad173ef0e02ad0bc7440b monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/monitor-x86.dll
 c278d582c26003467794ee4d0ab7ae337034c2ea monitor/latest
-0000000000000000000000000000000000000000 signatures/windows/creates_exe.py
 4575c7d09b316cad3a71059a27b78cbcb9c1f6b4 stuff/ttp_descriptions.json
 9870109fa8a09c2f1b871ea178bdd3d40390baab web/local_settings.py
 f81e71f788149039f31c9b0b5e2891733e51b5d7 whitelist/ip.txt
diff --git a/docs/book/usage/api.rst b/docs/book/usage/api.rst
index f0c40d7b0b..2710bf8e17 100644
--- a/docs/book/usage/api.rst
+++ b/docs/book/usage/api.rst
@@ -175,7 +175,7 @@ each one. For details click on the resource name.
 +-------------------------------------+------------------------------------------------------------------------------------------------------------------+
 | ``GET`` :ref:`files_get`            | Returns the content of the binary with the specified SHA256 hash.                                                |
 +-------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| ``GET`` :ref:`pcap_get`             | Returns the content of the PCAP associated with the given task.                                                  |
+| ``GET`` :ref:`api_pcap_get`         | Returns the content of the PCAP associated with the given task.                                                  |
 +-------------------------------------+------------------------------------------------------------------------------------------------------------------+
 | ``GET`` :ref:`machines_list`        | Returns the list of analysis machines available to Cuckoo.                                                       |
 +-------------------------------------+------------------------------------------------------------------------------------------------------------------+
@@ -912,7 +912,7 @@ Returns details on the file matching either the specified MD5 hash, SHA256 hash
 * ``200`` - no error
 * ``404`` - file not found
 
-.. _pcap_get:
+.. _api_pcap_get:
 
 /pcap/get
 ---------
diff --git a/setup.py b/setup.py
index 6d7d60312f..4cb727e5f5 100755
--- a/setup.py
+++ b/setup.py
@@ -213,7 +213,7 @@ def do_setup(**kwargs):
         "python-dateutil==2.4.2",
         "python-magic==0.4.12",
         "roach>=0.1.2, <0.2",
-        "sflock>=0.3.8, <0.4",
+        "sflock>=0.3.10, <0.4",
         "sqlalchemy==1.3.3",
         "unicorn==1.0.1",
         "wakeonlan==0.2.2",

From 2a39660c80c5e3ce5219c0edf85f4522782c7f4e Mon Sep 17 00:00:00 2001
From: doomedraven <doomedraven@users.noreply.github.com>
Date: Tue, 30 Jul 2019 11:25:46 +0200
Subject: [PATCH 131/138] no WLS

---
 docs/book/installation/index.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/book/installation/index.rst b/docs/book/installation/index.rst
index 4453636e52..d5051d334f 100644
--- a/docs/book/installation/index.rst
+++ b/docs/book/installation/index.rst
@@ -10,7 +10,8 @@ Cuckoo has proved to work smoothly on *Mac OS X* and *Microsoft Windows 7* as
 host as well. The recommended and tested setup for guests are *Windows XP* and
 *64-bit Windows 7* for Windows analysis, *Mac OS X Yosemite* for Mac OS X
 analysis, and Debian for Linux Analysis, although Cuckoo should work with
-other releases of guest Operating Systems as well.
+other releases of guest Operating Systems as well. Don't use 
+**Windows linux subsystem (WLS)** as system to run Cuckoo Sandbox.
 
 .. note::
 

From c3af2c87443f11bec81c646c01311b2c45db0ac3 Mon Sep 17 00:00:00 2001
From: doomedraven <doomedraven@users.noreply.github.com>
Date: Wed, 18 Sep 2019 09:40:40 +0200
Subject: [PATCH 132/138] Update physical.conf

---
 cuckoo/private/cwd/conf/physical.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cuckoo/private/cwd/conf/physical.conf b/cuckoo/private/cwd/conf/physical.conf
index a2e0ecc3ff..cb3b76420f 100644
--- a/cuckoo/private/cwd/conf/physical.conf
+++ b/cuckoo/private/cwd/conf/physical.conf
@@ -9,6 +9,7 @@ user = {{ physical.physical.user }}
 password = {{ physical.physical.password }}
 
 # Default network interface.
+# verify that the interface exist with command: ip addr
 interface = {{ physical.physical.interface }}
 
 [fog]

From 959d39dfe9608c0ffd50a7d9fd122cdaad2808e7 Mon Sep 17 00:00:00 2001
From: flavio <flavio.ranchon@gmail.com>
Date: Wed, 26 Feb 2020 21:19:22 +0100
Subject: [PATCH 133/138] Add function documentation

---
 cuckoo/data/analyzer/linux/lib/api/process.py      | 4 ++++
 cuckoo/data/analyzer/linux/lib/common/abstracts.py | 3 +--
 cuckoo/data/analyzer/linux/lib/core/config.py      | 7 ++++---
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/cuckoo/data/analyzer/linux/lib/api/process.py b/cuckoo/data/analyzer/linux/lib/api/process.py
index 4807de8cf2..7aaae353ba 100644
--- a/cuckoo/data/analyzer/linux/lib/api/process.py
+++ b/cuckoo/data/analyzer/linux/lib/api/process.py
@@ -38,6 +38,10 @@ def get_proc_status(self):
         return {}
 
     def execute(self, cmd):
+        """Start a subprocess.
+        @param cmd: process path
+        @return: subprocess status
+        """
         self.proc = proc = subprocess.Popen(cmd)
         self.pid = proc.pid
         return True
diff --git a/cuckoo/data/analyzer/linux/lib/common/abstracts.py b/cuckoo/data/analyzer/linux/lib/common/abstracts.py
index 3bf8fd0c96..9507ff816a 100644
--- a/cuckoo/data/analyzer/linux/lib/common/abstracts.py
+++ b/cuckoo/data/analyzer/linux/lib/common/abstracts.py
@@ -32,8 +32,7 @@ def check(self):
 
     def execute(self, cmd):
         """Start an executable for analysis.
-        @param path: executable path
-        @param args: executable arguments
+        @param cmd: executable path
         @return: process pid
         """
         p = Process()
diff --git a/cuckoo/data/analyzer/linux/lib/core/config.py b/cuckoo/data/analyzer/linux/lib/core/config.py
index 98eb7106d5..4a70b227c3 100644
--- a/cuckoo/data/analyzer/linux/lib/core/config.py
+++ b/cuckoo/data/analyzer/linux/lib/core/config.py
@@ -25,9 +25,10 @@ def __init__(self, cfg):
                 setattr(self, name, value)
 
     def get(self, name, default=None):
-        if hasattr(self, name):
-            return getattr(self, name)
-        return default
+        """Get attribute.
+        @return: attribute.
+        """
+        return getattr(self, name, default)
 
     def get_options(self):
         """Get analysis options.

From d20a0c08c4188d120535bab2b6ac9cd8c61dab8f Mon Sep 17 00:00:00 2001
From: Ricardo van Zutphen <ricardo@hatching.io>
Date: Wed, 4 Mar 2020 13:50:31 +0100
Subject: [PATCH 134/138] Update README.rst

---
 README.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.rst b/README.rst
index f28b809e98..da13d90b79 100644
--- a/README.rst
+++ b/README.rst
@@ -12,7 +12,6 @@ environment.
 
 If you want to contribute to development, report a bug, make a feature request
 or ask a question, please first take a look at our `community guidelines`_.
-For development, please also take a look at the `contribution requirements`_.
 Make sure you check our existing Issues and Pull Requests and that you join
 our `IRC or Slack channel <https://cuckoosandbox.org/discussion>`_.
 

From 546d46af7d0e6351e11a06e85450e5f0b935a6f1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 23 Apr 2020 10:06:00 +0000
Subject: [PATCH 135/138] Bump handlebars from 4.0.5 to 4.7.6 in
 /cuckoo/web/src

Bumps [handlebars](https://github.com/wycats/handlebars.js) from 4.0.5 to 4.7.6.
- [Release notes](https://github.com/wycats/handlebars.js/releases)
- [Changelog](https://github.com/handlebars-lang/handlebars.js/blob/master/release-notes.md)
- [Commits](https://github.com/wycats/handlebars.js/compare/v4.0.5...v4.7.6)

Signed-off-by: dependabot[bot] <support@github.com>
---
 cuckoo/web/src/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cuckoo/web/src/package.json b/cuckoo/web/src/package.json
index a582e5a0d5..6c391e7638 100644
--- a/cuckoo/web/src/package.json
+++ b/cuckoo/web/src/package.json
@@ -37,7 +37,7 @@
     "gulp-util": "3.0.7",
     "gulp-watch": "4.3.10",
     "gulp-wrap": "0.13.0",
-    "handlebars": "4.0.5",
+    "handlebars": "4.7.6",
     "vinyl-buffer": "1.0.0",
     "vinyl-source-stream": "1.1.0"
   }

From 1962aa3fa874bf47b7931626460c2c276640c725 Mon Sep 17 00:00:00 2001
From: Nex <nex@nex.sx>
Date: Tue, 2 Jun 2020 09:30:18 +0200
Subject: [PATCH 136/138] Using safelist/blocklist

---
 cuckoo/apps/apps.py                           | 10 +--
 cuckoo/auxiliary/sniffer.py                   |  8 +--
 cuckoo/common/config.py                       |  2 +-
 cuckoo/common/objects.py                      |  4 +-
 cuckoo/common/safelist.py                     | 70 +++++++++++++++++++
 cuckoo/common/utils.py                        |  4 +-
 cuckoo/common/virustotal.py                   |  4 +-
 cuckoo/common/whitelist.py                    | 70 -------------------
 cuckoo/compat/config.py                       |  8 +--
 cuckoo/core/feedback.py                       |  6 +-
 cuckoo/core/resultserver.py                   |  2 +-
 .../data/{whitelist => safelist}/domain.txt   |  2 +-
 cuckoo/data/safelist/ip.txt                   |  1 +
 .../{whitelist => safelist}/mispdomain.txt    |  0
 .../data/{whitelist => safelist}/misphash.txt |  0
 .../data/{whitelist => safelist}/mispip.txt   |  0
 .../data/{whitelist => safelist}/mispurl.txt  |  0
 cuckoo/data/stuff/ttp_descriptions.json       | 14 ++--
 cuckoo/data/whitelist/ip.txt                  |  1 -
 cuckoo/private/cwd/conf/processing.conf       |  6 +-
 cuckoo/private/cwd/hashes.txt                 | 10 +--
 cuckoo/processing/memory.py                   |  2 +-
 cuckoo/processing/network.py                  | 34 ++++-----
 cuckoo/processing/suricata.py                 |  4 +-
 cuckoo/reporting/misp.py                      | 26 +++----
 docs/CHANGELOG                                |  2 +-
 docs/book/_files/conf/processing.conf         |  6 +-
 setup.py                                      |  2 +-
 tests/files/conf/20c2_plain/processing.conf   |  6 +-
 tests/test_apps.py                            |  8 +--
 tests/test_config.py                          | 10 +--
 tests/test_utils.py                           | 10 +--
 32 files changed, 166 insertions(+), 166 deletions(-)
 create mode 100644 cuckoo/common/safelist.py
 delete mode 100644 cuckoo/common/whitelist.py
 rename cuckoo/data/{whitelist => safelist}/domain.txt (94%)
 create mode 100644 cuckoo/data/safelist/ip.txt
 rename cuckoo/data/{whitelist => safelist}/mispdomain.txt (100%)
 rename cuckoo/data/{whitelist => safelist}/misphash.txt (100%)
 rename cuckoo/data/{whitelist => safelist}/mispip.txt (100%)
 rename cuckoo/data/{whitelist => safelist}/mispurl.txt (100%)
 delete mode 100644 cuckoo/data/whitelist/ip.txt

diff --git a/cuckoo/apps/apps.py b/cuckoo/apps/apps.py
index b16d721d4b..545cf00260 100644
--- a/cuckoo/apps/apps.py
+++ b/cuckoo/apps/apps.py
@@ -504,15 +504,15 @@ def migrate_cwd():
     mkdir(cwd("stuff"))
     mkdir(cwd("yara", "office"))
 
-    # Create the new $CWD/whitelist/ directory.
-    if not os.path.exists(cwd("whitelist")):
+    # Create the new $CWD/safelist/ directory.
+    if not os.path.exists(cwd("safelist")):
         shutil.copytree(
-            cwd("..", "data", "whitelist", private=True), cwd("whitelist")
+            cwd("..", "data", "safelist", private=True), cwd("safelist")
         )
     else:
-        data_wl = cwd("..", "data", "whitelist", private=True)
+        data_wl = cwd("..", "data", "safelist", private=True)
         for wl_file in os.listdir(data_wl):
-            cwd_wl = cwd("whitelist", wl_file)
+            cwd_wl = cwd("safelist", wl_file)
             if not os.path.isfile(cwd_wl):
                 shutil.copy(os.path.join(data_wl, wl_file), cwd_wl)
 
diff --git a/cuckoo/auxiliary/sniffer.py b/cuckoo/auxiliary/sniffer.py
index 3242cdfa63..3962269b35 100644
--- a/cuckoo/auxiliary/sniffer.py
+++ b/cuckoo/auxiliary/sniffer.py
@@ -107,11 +107,11 @@ def _check_output(self, out, err):
                 "standard output, got: %r." % out
             )
 
-        err_whitelist_start = (
+        err_safelist_start = (
             "tcpdump: listening on ",
         )
 
-        err_whitelist_ends = (
+        err_safelist_ends = (
             "packet captured",
             "packets captured",
             "packet received by filter",
@@ -124,10 +124,10 @@ def _check_output(self, out, err):
         )
 
         for line in err.split("\n"):
-            if not line or line.startswith(err_whitelist_start):
+            if not line or line.startswith(err_safelist_start):
                 continue
 
-            if line.endswith(err_whitelist_ends):
+            if line.endswith(err_safelist_ends):
                 continue
 
             raise CuckooOperationalError(
diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py
index 6ff03c39b9..7728d513d6 100644
--- a/cuckoo/common/config.py
+++ b/cuckoo/common/config.py
@@ -579,7 +579,7 @@ class Config(object):
             },
             "network": {
                 "enabled": Boolean(True),
-                "whitelist_dns": Boolean(False),
+                "safelist_dns": Boolean(False),
                 "allowed_dns": String(),
             },
             "procmemory": {
diff --git a/cuckoo/common/objects.py b/cuckoo/common/objects.py
index 11cb2c9322..f8b40d880b 100644
--- a/cuckoo/common/objects.py
+++ b/cuckoo/common/objects.py
@@ -16,7 +16,7 @@
 import tempfile
 import zipfile
 
-from cuckoo.common.whitelist import is_whitelisted_domain
+from cuckoo.common.safelist import is_safelisted_domain
 
 try:
     import pydeep
@@ -383,7 +383,7 @@ def get_urls(self):
         # http://stackoverflow.com/a/454589
         urls, f = set(), open(self.file_path, "rb")
         for url in re.findall(URL_REGEX, self.mmap(f.fileno())):
-            if not is_whitelisted_domain(url[1]):
+            if not is_safelisted_domain(url[1]):
                 urls.add("".join(url))
         return list(urls)
 
diff --git a/cuckoo/common/safelist.py b/cuckoo/common/safelist.py
new file mode 100644
index 0000000000..a9eaef08b6
--- /dev/null
+++ b/cuckoo/common/safelist.py
@@ -0,0 +1,70 @@
+# Copyright (C) 2015-2019 Cuckoo Foundation.
+# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
+# See the file 'docs/LICENSE' for copying permission.
+
+import os.path
+
+from cuckoo.misc import cwd
+
+domains = set()
+ips = set()
+mispdomains = set()
+mispips = set()
+mispurls = set()
+misphashes = set()
+
+def _load_safelist(wlset, wl_file):
+    wl_path = cwd("safelist", wl_file)
+
+    if not os.path.isfile(wl_path):
+        wl_path = cwd("..", "data", "safelist", wl_file, private=True)
+
+    with open(wl_path, "rb") as fp:
+        safelist = fp.read()
+
+    for entry in safelist.split("\n"):
+        entry = entry.strip()
+        if entry and not entry.startswith("#"):
+            wlset.add(entry)
+
+def is_safelisted_domain(domain):
+    if not domains:
+        # Initialize the domain safelist.
+        _load_safelist(domains, "domain.txt")
+
+    return domain in domains
+
+def is_safelisted_ip(ip):
+    if not ips:
+        # Initialize the ip safelist.
+        _load_safelist(ips, "ip.txt")
+
+    return ip in ips
+
+def is_safelisted_mispdomain(domain):
+    if not mispdomains:
+        # Initialize the misp domain safelist.
+        _load_safelist(mispdomains, "mispdomain.txt")
+
+    return domain in mispdomains
+
+def is_safelisted_mispip(ip):
+    if not mispips:
+        # Initialize the misp ip safelist.
+        _load_safelist(mispips, "mispip.txt")
+
+    return ip in mispips
+
+def is_safelisted_mispurl(url):
+    if not mispurls:
+        # Initialize the misp url safelist.
+        _load_safelist(mispurls, "mispurl.txt")
+
+    return url in mispurls
+
+def is_safelisted_misphash(hash):
+    if not misphashes:
+        # Initialize the misp hash safelist.
+        _load_safelist(misphashes, "misphash.txt")
+
+    return hash in misphashes
diff --git a/cuckoo/common/utils.py b/cuckoo/common/utils.py
index e587712998..2e9e4c8c9d 100644
--- a/cuckoo/common/utils.py
+++ b/cuckoo/common/utils.py
@@ -263,7 +263,7 @@ def get_os_release():
     ))
     return msg
 
-_jsbeautify_blacklist = [
+_jsbeautify_blocklist = [
     "",
     "error: Unknown p.a.c.k.e.r. encoding.\n",
 ]
@@ -280,7 +280,7 @@ def jsbeautify(javascript):
         except Exception as e:
             log.exception("Unable to beautify javascript: %s", e)
 
-        if sys.stdout.getvalue() not in _jsbeautify_blacklist:
+        if sys.stdout.getvalue() not in _jsbeautify_blocklist:
             log.warning(
                 "jsbeautifier returned error: %s", sys.stdout.getvalue()
             )
diff --git a/cuckoo/common/virustotal.py b/cuckoo/common/virustotal.py
index d5d08cead7..8be92b5b22 100644
--- a/cuckoo/common/virustotal.py
+++ b/cuckoo/common/virustotal.py
@@ -29,7 +29,7 @@ class VirusTotalAPI(object):
     URL_SCAN = "https://www.virustotal.com/vtapi/v2/url/scan"
     HASH_DOWNLOAD = "https://www.virustotal.com/vtapi/v2/file/download"
 
-    VARIANT_BLACKLIST = [
+    VARIANT_BLOCKLIST = [
         "generic", "malware", "trojan", "agent", "win32", "multi", "w32",
         "trojanclicker", "trojware", "win", "a variant of win32", "trj",
         "susp", "dangerousobject", "backdoor", "clicker", "variant", "heur",
@@ -198,7 +198,7 @@ def normalize(self, variant):
             if len(word) < 4:
                 continue
 
-            if word.lower() in self.VARIANT_BLACKLIST:
+            if word.lower() in self.VARIANT_BLOCKLIST:
                 continue
 
             # Random hashes that are specific to this file.
diff --git a/cuckoo/common/whitelist.py b/cuckoo/common/whitelist.py
deleted file mode 100644
index d8d802550b..0000000000
--- a/cuckoo/common/whitelist.py
+++ /dev/null
@@ -1,70 +0,0 @@
-# Copyright (C) 2015-2019 Cuckoo Foundation.
-# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
-# See the file 'docs/LICENSE' for copying permission.
-
-import os.path
-
-from cuckoo.misc import cwd
-
-domains = set()
-ips = set()
-mispdomains = set()
-mispips = set()
-mispurls = set()
-misphashes = set()
-
-def _load_whitelist(wlset, wl_file):
-    wl_path = cwd("whitelist", wl_file)
-
-    if not os.path.isfile(wl_path):
-        wl_path = cwd("..", "data", "whitelist", wl_file, private=True)
-
-    with open(wl_path, "rb") as fp:
-        whitelist = fp.read()
-
-    for entry in whitelist.split("\n"):
-        entry = entry.strip()
-        if entry and not entry.startswith("#"):
-            wlset.add(entry)
-
-def is_whitelisted_domain(domain):
-    if not domains:
-        # Initialize the domain whitelist.
-        _load_whitelist(domains, "domain.txt")
-
-    return domain in domains
-
-def is_whitelisted_ip(ip):
-    if not ips:
-        # Initialize the ip whitelist.
-        _load_whitelist(ips, "ip.txt")
-
-    return ip in ips
-
-def is_whitelisted_mispdomain(domain):
-    if not mispdomains:
-        # Initialize the misp domain whitelist.
-        _load_whitelist(mispdomains, "mispdomain.txt")
-
-    return domain in mispdomains
-
-def is_whitelisted_mispip(ip):
-    if not mispips:
-        # Initialize the misp ip whitelist.
-        _load_whitelist(mispips, "mispip.txt")
-
-    return ip in mispips
-
-def is_whitelisted_mispurl(url):
-    if not mispurls:
-        # Initialize the misp url whitelist.
-        _load_whitelist(mispurls, "mispurl.txt")
-
-    return url in mispurls
-
-def is_whitelisted_misphash(hash):
-    if not misphashes:
-        # Initialize the misp hash whitelist.
-        _load_whitelist(misphashes, "misphash.txt")
-
-    return hash in misphashes
diff --git a/cuckoo/compat/config.py b/cuckoo/compat/config.py
index a8ccce8e80..8e95e1f18e 100644
--- a/cuckoo/compat/config.py
+++ b/cuckoo/compat/config.py
@@ -489,7 +489,7 @@ def _20c1_20c2(c):
         "apikey": None,
         "maxioc": 100,
     }
-    c["processing"]["network"]["whitelist-dns"] = False
+    c["processing"]["network"]["safelist-dns"] = False
     c["processing"]["network"]["allowed-dns"] = None
     c["processing"]["procmemory"]["extract_img"] = True
     c["processing"]["procmemory"]["dump_delete"] = False
@@ -554,9 +554,9 @@ def _20c2_200(c):
         "company": None,
         "email": None,
     }
-    c["processing"]["network"]["whitelist_dns"] = cast(
-        "processing:network:whitelist_dns",
-        c["processing"]["network"].pop("whitelist-dns", None)
+    c["processing"]["network"]["safelist_dns"] = cast(
+        "processing:network:safelist_dns",
+        c["processing"]["network"].pop("safelist-dns", None)
     )
     c["processing"]["network"]["allowed_dns"] = cast(
         "processing:network:allowed_dns",
diff --git a/cuckoo/core/feedback.py b/cuckoo/core/feedback.py
index 2e7ec64c75..758f62c8fb 100644
--- a/cuckoo/core/feedback.py
+++ b/cuckoo/core/feedback.py
@@ -20,7 +20,7 @@
 class CuckooFeedback(object):
     """Contact Cuckoo HQ with feedback & optional analysis dump."""
     endpoint = "https://feedback.cuckoosandbox.org/api/submit/"
-    exc_whitelist = (
+    exc_safelist = (
         CuckooFeedbackError,
     )
 
@@ -41,8 +41,8 @@ def send_exception(self, exception, request):
             automated=True, message="Exception encountered: %s" % exception
         )
 
-        if isinstance(exception, self.exc_whitelist):
-            log.debug("A whitelisted exception occurred: %s", exception)
+        if isinstance(exception, self.exc_safelist):
+            log.debug("A safelisted exception occurred: %s", exception)
             return
 
         # Ignore 404 exceptions regarding ".map" development files.
diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py
index 96117eff27..e307124394 100644
--- a/cuckoo/core/resultserver.py
+++ b/cuckoo/core/resultserver.py
@@ -33,7 +33,7 @@
 BUFSIZE = 16 * 1024
 
 # Directories in which analysis-related files will be stored; also acts as
-# whitelist
+# safelist
 RESULT_UPLOADABLE = ("files", "shots", "buffer",  "extracted", "memory")
 RESULT_DIRECTORIES = RESULT_UPLOADABLE + ("reports", "logs")
 
diff --git a/cuckoo/data/whitelist/domain.txt b/cuckoo/data/safelist/domain.txt
similarity index 94%
rename from cuckoo/data/whitelist/domain.txt
rename to cuckoo/data/safelist/domain.txt
index b97ef46b7e..6bdaedfffd 100644
--- a/cuckoo/data/whitelist/domain.txt
+++ b/cuckoo/data/safelist/domain.txt
@@ -1,4 +1,4 @@
-# You can add whitelisted domains here.
+# You can add safelisted domains here.
 java.com
 www.msn.com
 www.bing.com
diff --git a/cuckoo/data/safelist/ip.txt b/cuckoo/data/safelist/ip.txt
new file mode 100644
index 0000000000..aeeba83834
--- /dev/null
+++ b/cuckoo/data/safelist/ip.txt
@@ -0,0 +1 @@
+# You can add safelisted IPs here.
diff --git a/cuckoo/data/whitelist/mispdomain.txt b/cuckoo/data/safelist/mispdomain.txt
similarity index 100%
rename from cuckoo/data/whitelist/mispdomain.txt
rename to cuckoo/data/safelist/mispdomain.txt
diff --git a/cuckoo/data/whitelist/misphash.txt b/cuckoo/data/safelist/misphash.txt
similarity index 100%
rename from cuckoo/data/whitelist/misphash.txt
rename to cuckoo/data/safelist/misphash.txt
diff --git a/cuckoo/data/whitelist/mispip.txt b/cuckoo/data/safelist/mispip.txt
similarity index 100%
rename from cuckoo/data/whitelist/mispip.txt
rename to cuckoo/data/safelist/mispip.txt
diff --git a/cuckoo/data/whitelist/mispurl.txt b/cuckoo/data/safelist/mispurl.txt
similarity index 100%
rename from cuckoo/data/whitelist/mispurl.txt
rename to cuckoo/data/safelist/mispurl.txt
diff --git a/cuckoo/data/stuff/ttp_descriptions.json b/cuckoo/data/stuff/ttp_descriptions.json
index 7a1336c92f..05ab152b2b 100644
--- a/cuckoo/data/stuff/ttp_descriptions.json
+++ b/cuckoo/data/stuff/ttp_descriptions.json
@@ -32,7 +32,7 @@
         "short": "Fallback Channels"
     },
     "T1009": {
-        "long": "Some security tools inspect files with static signatures to determine if they are known malicious. Adversaries may add data to files to increase the size beyond what security tools are capable of handling or to change the file hash to avoid hash-based blacklists.",
+        "long": "Some security tools inspect files with static signatures to determine if they are known malicious. Adversaries may add data to files to increase the size beyond what security tools are capable of handling or to change the file hash to avoid hash-based blocklists.",
         "short": "Binary Padding"
     },
     "T1010": {
@@ -336,7 +336,7 @@
         "short": "Windows Management Instrumentation Event Subscription"
     },
     "T1085": {
-        "long": "The rundll32.exe program can be called to execute an arbitrary binary. Adversaries may take advantage of this functionality to proxy execution of code to avoid triggering security tools that may not monitor execution of the rundll32.exe process because of whitelists or false positives from Windows using rundll32.exe for normal operations.",
+        "long": "The rundll32.exe program can be called to execute an arbitrary binary. Adversaries may take advantage of this functionality to proxy execution of code to avoid triggering security tools that may not monitor execution of the rundll32.exe process because of safelists or false positives from Windows using rundll32.exe for normal operations.",
         "short": "Rundll32"
     },
     "T1086": {
@@ -504,7 +504,7 @@
         "short": "Network Share Connection Removal"
     },
     "T1127": {
-        "long": "There are many utilities used for software development related tasks that can be used to execute code in various forms to assist in development, debugging, and reverse engineering. These utilities may often be signed with legitimate certificates that allow them to execute on a system and proxy execution of malicious code through a trusted process that effectively bypasses application whitelisting defensive solutions.",
+        "long": "There are many utilities used for software development related tasks that can be used to execute code in various forms to assist in development, debugging, and reverse engineering. These utilities may often be signed with legitimate certificates that allow them to execute on a system and proxy execution of malicious code through a trusted process that effectively bypasses application safelisting defensive solutions.",
         "short": "Trusted Developer Utilities"
     },
     "T1128": {
@@ -592,7 +592,7 @@
         "short": "HISTCONTROL"
     },
     "T1149": {
-        "long": "As of OS X 10.8, mach-O binaries introduced a new header called LC_MAIN that points to the binary\u2019s entry point for execution. Previously, there were two headers to achieve this same effect: LC_THREAD and LC_UNIXTHREAD  . The entry point for a binary can be hijacked so that initial execution flows to a malicious addition (either another section or a code cave) and then goes back to the initial entry point so that the victim doesn\u2019t know anything was different  . By modifying a binary in this way, application whitelisting can be bypassed because the file name or application path is still the same.",
+        "long": "As of OS X 10.8, mach-O binaries introduced a new header called LC_MAIN that points to the binary\u2019s entry point for execution. Previously, there were two headers to achieve this same effect: LC_THREAD and LC_UNIXTHREAD  . The entry point for a binary can be hijacked so that initial execution flows to a malicious addition (either another section or a code cave) and then goes back to the initial entry point so that the victim doesn\u2019t know anything was different  . By modifying a binary in this way, application safelisting can be bypassed because the file name or application path is still the same.",
         "short": "LC_MAIN Hijacking"
     },
     "T1150": {
@@ -860,7 +860,7 @@
         "short": "Kernel Modules and Extensions"
     },
     "T1216": {
-        "long": "Scripts signed with trusted certificates can be used to proxy execution of malicious files. This behavior may bypass signature validation restrictions and application whitelisting solutions that do not account for use of these scripts.",
+        "long": "Scripts signed with trusted certificates can be used to proxy execution of malicious files. This behavior may bypass signature validation restrictions and application safelisting solutions that do not account for use of these scripts.",
         "short": "Signed Script Proxy Execution"
     },
     "T1217": {
@@ -868,11 +868,11 @@
         "short": "Browser Bookmark Discovery"
     },
     "T1218": {
-        "long": "Binaries signed with trusted digital certificates can execute on Windows systems protected by digital signature validation. Several Microsoft signed binaries that are default on Windows installations can be used to proxy execution of other files. This behavior may be abused by adversaries to execute malicious files that could bypass application whitelisting and signature validation on systems. This technique accounts for proxy execution methods that are not already accounted for within the existing techniques.",
+        "long": "Binaries signed with trusted digital certificates can execute on Windows systems protected by digital signature validation. Several Microsoft signed binaries that are default on Windows installations can be used to proxy execution of other files. This behavior may be abused by adversaries to execute malicious files that could bypass application safelisting and signature validation on systems. This technique accounts for proxy execution methods that are not already accounted for within the existing techniques.",
         "short": "Signed Binary Proxy Execution"
     },
     "T1219": {
-        "long": "An adversary may use legitimate desktop support and remote access software, such as Team Viewer, Go2Assist, LogMein, AmmyyAdmin, etc, to establish an interactive command and control channel to target systems within networks. These services are commonly used as legitimate technical support software, and may be whitelisted within a target environment. Remote access tools like VNC, Ammy, and Teamviewer are used frequently when compared with other legitimate software commonly used by adversaries.",
+        "long": "An adversary may use legitimate desktop support and remote access software, such as Team Viewer, Go2Assist, LogMein, AmmyyAdmin, etc, to establish an interactive command and control channel to target systems within networks. These services are commonly used as legitimate technical support software, and may be safelisted within a target environment. Remote access tools like VNC, Ammy, and Teamviewer are used frequently when compared with other legitimate software commonly used by adversaries.",
         "short": "Remote Access Tools"
     },
     "T1220": {
diff --git a/cuckoo/data/whitelist/ip.txt b/cuckoo/data/whitelist/ip.txt
deleted file mode 100644
index dfdbe87ff0..0000000000
--- a/cuckoo/data/whitelist/ip.txt
+++ /dev/null
@@ -1 +0,0 @@
-# You can add whitelisted IPs here.
diff --git a/cuckoo/private/cwd/conf/processing.conf b/cuckoo/private/cwd/conf/processing.conf
index d755c26f57..2cd142bed9 100644
--- a/cuckoo/private/cwd/conf/processing.conf
+++ b/cuckoo/private/cwd/conf/processing.conf
@@ -61,10 +61,10 @@ maxioc = {{ processing.misp.maxioc }}
 [network]
 enabled = {{ processing.network.enabled }}
 
-# Allow domain whitelisting
-whitelist_dns = {{ processing.network.whitelist_dns }}
+# Allow domain safelisting
+safelist_dns = {{ processing.network.safelist_dns }}
 
-# Allow DNS responses from your configured DNS server for whitelisting to
+# Allow DNS responses from your configured DNS server for safelisting to
 # deactivate when responses come from some other DNS
 # Can be also multiple like : 8.8.8.8,8.8.4.4
 allowed_dns = {{ processing.network.allowed_dns }}
diff --git a/cuckoo/private/cwd/hashes.txt b/cuckoo/private/cwd/hashes.txt
index 6a72fb6be7..cfce710cb9 100644
--- a/cuckoo/private/cwd/hashes.txt
+++ b/cuckoo/private/cwd/hashes.txt
@@ -324,8 +324,8 @@ b0ceb903d39fe7778e0ad173ef0e02ad0bc7440b monitor/2deb9ccd75d5a7a3fe05b2625b03a86
 c278d582c26003467794ee4d0ab7ae337034c2ea monitor/latest
 4575c7d09b316cad3a71059a27b78cbcb9c1f6b4 stuff/ttp_descriptions.json
 9870109fa8a09c2f1b871ea178bdd3d40390baab web/local_settings.py
-f81e71f788149039f31c9b0b5e2891733e51b5d7 whitelist/ip.txt
-cc78a9c7ecdd5a3862b39ad7e6676723e72eb2ba whitelist/mispdomain.txt
-8f6442b91064e46ab3454d6bc15a4cf1f3949a0f whitelist/misphash.txt
-1f0ec663731206a9bf9363293421c68855aed772 whitelist/mispip.txt
-e57ba6930af466d1a56aa22f791048020fadef88 whitelist/mispurl.txt
+f81e71f788149039f31c9b0b5e2891733e51b5d7 safelist/ip.txt
+cc78a9c7ecdd5a3862b39ad7e6676723e72eb2ba safelist/mispdomain.txt
+8f6442b91064e46ab3454d6bc15a4cf1f3949a0f safelist/misphash.txt
+1f0ec663731206a9bf9363293421c68855aed772 safelist/mispip.txt
+e57ba6930af466d1a56aa22f791048020fadef88 safelist/mispurl.txt
diff --git a/cuckoo/processing/memory.py b/cuckoo/processing/memory.py
index b30853749a..d67bfaa302 100644
--- a/cuckoo/processing/memory.py
+++ b/cuckoo/processing/memory.py
@@ -619,7 +619,7 @@ def apihooks(self):
         command = self.plugins["apihooks"](self.config)
         for process, module, hook in command.calculate():
             proc_name = str(process.ImageFileName) if process else ''
-            if command.whitelist(hook.hook_mode | hook.hook_type,
+            if command.safelist(hook.hook_mode | hook.hook_type,
                                  proc_name, hook.VictimModule,
                                  hook.HookModule, hook.Function):
                 continue
diff --git a/cuckoo/processing/network.py b/cuckoo/processing/network.py
index e5c13adca7..6435080c1b 100644
--- a/cuckoo/processing/network.py
+++ b/cuckoo/processing/network.py
@@ -26,7 +26,7 @@
 from cuckoo.common.irc import ircMessage
 from cuckoo.common.objects import File
 from cuckoo.common.utils import convert_to_printable
-from cuckoo.common.whitelist import is_whitelisted_domain, is_whitelisted_ip
+from cuckoo.common.safelist import is_safelisted_domain, is_safelisted_ip
 from cuckoo.misc import mkdir
 
 # Be less verbose about httpreplay logging messages.
@@ -83,19 +83,19 @@ def __init__(self, filepath, options):
         self.irc_requests = []
         # Dictionary containing all the results of this processing.
         self.results = {}
-        # List for holding whitelisted IP-s according to DNS responses
-        self.whitelist_ips = []
-        # state of whitelisting
-        self.whitelist_enabled = self.options.get("whitelist_dns")
+        # List for holding safelisted IP-s according to DNS responses
+        self.safelist_ips = []
+        # state of safelisting
+        self.safelist_enabled = self.options.get("safelist_dns")
         # List of known good DNS servers
         self.known_dns = self._build_known_dns()
         # List of all used DNS servers
         self.dns_servers = []
 
-    def _is_whitelisted(self, conn, hostname):
-        """Check if whitelisting conditions are met"""
-        # Is whitelistng enabled?
-        if not self.whitelist_enabled:
+    def _is_safelisted(self, conn, hostname):
+        """Check if safelisting conditions are met"""
+        # Is safelistng enabled?
+        if not self.safelist_enabled:
             return False
 
         # Is DNS recording coming from allowed NS server.
@@ -107,8 +107,8 @@ def _is_whitelisted(self, conn, hostname):
         else:
             return False
 
-        # Is hostname whitelisted.
-        if not is_whitelisted_domain(hostname):
+        # Is hostname safelisted.
+        if not is_safelisted_domain(hostname):
             return False
 
         return True
@@ -211,7 +211,7 @@ def _add_hosts(self, connection):
                     # We add external IPs to the list, only the first time
                     # we see them and if they're the destination of the
                     # first packet they appear in.
-                    if not self._is_private_ip(ip) and ip not in self.whitelist_ips:
+                    if not self._is_private_ip(ip) and ip not in self.safelist_ips:
                         self.unique_hosts.append(ip)
         except:
             pass
@@ -397,9 +397,9 @@ def _add_dns(self, conn, udpdata):
                 # TODO: add srv handling
                 query["answers"].append(ans)
 
-            if self._is_whitelisted(conn, q_name):
-                log.debug("DNS target {0} whitelisted. Skipping ...".format(q_name))
-                self.whitelist_ips = self.whitelist_ips + _ip
+            if self._is_safelisted(conn, q_name):
+                log.debug("DNS target {0} safelisted. Skipping ...".format(q_name))
+                self.safelist_ips = self.safelist_ips + _ip
                 return True
 
             self._add_domain(query["request"])
@@ -635,7 +635,7 @@ def run(self):
                     offset = file.tell()
                     continue
 
-                if is_whitelisted_ip(connection["dst"]):
+                if is_safelisted_ip(connection["dst"]):
                     continue
 
                 self._add_hosts(connection)
@@ -768,7 +768,7 @@ def run(self):
         for s, ts, protocol, sent, recv in l:
             srcip, srcport, dstip, dstport = s
 
-            if is_whitelisted_ip(dstip):
+            if is_safelisted_ip(dstip):
                 continue
 
             if protocol == "smtp":
diff --git a/cuckoo/processing/suricata.py b/cuckoo/processing/suricata.py
index b0d93871a1..031098fe00 100644
--- a/cuckoo/processing/suricata.py
+++ b/cuckoo/processing/suricata.py
@@ -25,7 +25,7 @@ class Suricata(Processing):
     """Suricata processing module."""
 
     # List of Suricata Signatures IDs that should be ignored.
-    sid_blacklist = [
+    sid_blocklist = [
         # SURICATA FRAG IPv6 Fragmentation overlap
         2200074,
 
@@ -127,7 +127,7 @@ def parse_eve_json(self):
             if event["event_type"] == "alert":
                 alert = event["alert"]
 
-                if alert["signature_id"] in self.sid_blacklist:
+                if alert["signature_id"] in self.sid_blocklist:
                     log.debug(
                         "Ignoring alert with sid=%d, signature=%s",
                         alert["signature_id"], alert["signature"]
diff --git a/cuckoo/reporting/misp.py b/cuckoo/reporting/misp.py
index fb5413b060..53cfdebd2c 100644
--- a/cuckoo/reporting/misp.py
+++ b/cuckoo/reporting/misp.py
@@ -9,9 +9,9 @@
 
 from cuckoo.common.abstracts import Report
 from cuckoo.common.exceptions import CuckooProcessingError
-from cuckoo.common.whitelist import (
-    is_whitelisted_mispdomain, is_whitelisted_mispip, is_whitelisted_mispurl,
-    is_whitelisted_misphash
+from cuckoo.common.safelist import (
+    is_safelisted_mispdomain, is_safelisted_mispip, is_safelisted_mispurl,
+    is_safelisted_misphash
 )
 
 log = logging.getLogger(__name__)
@@ -37,15 +37,15 @@ def all_urls(self, results, event):
         urls = set()
         for protocol in ("http_ex", "https_ex"):
             for entry in results.get("network", {}).get(protocol, []):
-                if is_whitelisted_mispdomain(entry["host"]):
+                if is_safelisted_mispdomain(entry["host"]):
                     continue
-                if is_whitelisted_mispdomain(entry["host"]):
+                if is_safelisted_mispdomain(entry["host"]):
                     continue
 
                 url = "%s://%s%s" % (
                       entry["protocol"], entry["host"], entry["uri"])
 
-                if not is_whitelisted_mispurl(url):
+                if not is_safelisted_mispurl(url):
                     urls.add(url)
 
         self.misp.add_url(event, sorted(list(urls)))
@@ -53,10 +53,10 @@ def all_urls(self, results, event):
     def domain_ipaddr(self, results, event):
         domains, ips = {}, set()
         for domain in results.get("network", {}).get("domains", []):
-            if is_whitelisted_mispip(domain["ip"]):
+            if is_safelisted_mispip(domain["ip"]):
                 continue
 
-            if is_whitelisted_mispdomain(domain["domain"]):
+            if is_safelisted_mispdomain(domain["domain"]):
                 continue
 
             domains[domain["domain"]] = domain["ip"]
@@ -64,7 +64,7 @@ def domain_ipaddr(self, results, event):
 
         ipaddrs = set()
         for ipaddr in results.get("network", {}).get("hosts", []):
-            if ipaddr not in ips and not is_whitelisted_mispip(ipaddr):
+            if ipaddr not in ips and not is_safelisted_mispip(ipaddr):
                 ipaddrs.add(ipaddr)
 
         self.misp.add_domains_ips(event, domains)
@@ -145,11 +145,11 @@ def run(self, results):
 
         if results.get("target", {}).get("category") == "file":
             f = results.get("target", {}).get("file", {})
-            hash_whitelisted = is_whitelisted_misphash(f["md5"]) or \
-                               is_whitelisted_misphash(f["sha1"]) or \
-                               is_whitelisted_misphash(f["sha256"])
+            hash_safelisted = is_safelisted_misphash(f["md5"]) or \
+                               is_safelisted_misphash(f["sha1"]) or \
+                               is_safelisted_misphash(f["sha256"])
 
-            if hash_whitelisted:
+            if hash_safelisted:
                 return
 
         if score < self.options.get("min_malscore", 0):
diff --git a/docs/CHANGELOG b/docs/CHANGELOG
index f2d20db8ef..21160ef7e0 100644
--- a/docs/CHANGELOG
+++ b/docs/CHANGELOG
@@ -78,7 +78,7 @@ Cuckoo Sandbox 2.0-rc1 (2016-01-21)
 * Added option to skip calls from JSON report
 * Added option to load the entire process memory dump into IDA Pro
 * Added some process memory dump analysis improvements
-* Added URLs parsing from memory dump and URLs whitelist
+* Added URLs parsing from memory dump and URLs safelist
 * Added tracking and reporting dead IP address/port combinations
 * Added maliciousness scoring system
 * Added option to web interface to submit dropped files for analysis
diff --git a/docs/book/_files/conf/processing.conf b/docs/book/_files/conf/processing.conf
index 4eb4d27f6d..a2bfacf95e 100644
--- a/docs/book/_files/conf/processing.conf
+++ b/docs/book/_files/conf/processing.conf
@@ -61,10 +61,10 @@ maxioc = 100
 [network]
 enabled = yes
 
-# Allow domain whitelisting
-whitelist_dns = no
+# Allow domain safelisting
+safelist_dns = no
 
-# Allow DNS responses from your configured DNS server for whitelisting to
+# Allow DNS responses from your configured DNS server for safelisting to
 # deactivate when responses come from some other DNS
 # Can be also multiple like : 8.8.8.8,8.8.4.4
 allowed_dns = 
diff --git a/setup.py b/setup.py
index 4cb727e5f5..a36dc07d44 100755
--- a/setup.py
+++ b/setup.py
@@ -65,7 +65,7 @@ def githash():
 cwd_private = os.path.join("cuckoo", "private")
 
 hashes_ignore = (
-    "whitelist/domain.txt",
+    "safelist/domain.txt",
 )
 
 def update_hashes():
diff --git a/tests/files/conf/20c2_plain/processing.conf b/tests/files/conf/20c2_plain/processing.conf
index d3376b59a7..339f9da419 100644
--- a/tests/files/conf/20c2_plain/processing.conf
+++ b/tests/files/conf/20c2_plain/processing.conf
@@ -58,10 +58,10 @@ maxioc = 100
 [network]
 enabled = yes
 
-# Allow domain whitelisting
-whitelist-dns = no
+# Allow domain safelisting
+safelist-dns = no
 
-# Allow DNS responses from your configured DNS server for whitelisting to
+# Allow DNS responses from your configured DNS server for safelisting to
 # deactivate when responses come from some other DNS
 # Can be also multiple like : 8.8.8.8,8.8.4.4
 allowed-dns =
diff --git a/tests/test_apps.py b/tests/test_apps.py
index a43b1a427e..4ed607fec8 100644
--- a/tests/test_apps.py
+++ b/tests/test_apps.py
@@ -714,7 +714,7 @@ def test_new_directory(self):
         shutil.rmtree(cwd("yara", "scripts"))
         shutil.rmtree(cwd("yara", "shellcode"))
         shutil.rmtree(cwd("stuff"))
-        shutil.rmtree(cwd("whitelist"))
+        shutil.rmtree(cwd("safelist"))
         open(cwd("yara", "index_binaries.yar"), "wb").write("hello")
         migrate_cwd()
         # TODO Move this to its own 2.0.2 -> 2.0.3 migration handler.
@@ -722,10 +722,10 @@ def test_new_directory(self):
         assert os.path.exists(cwd("yara", "shellcode", ".gitignore"))
         # TODO Move this to its own 2.0.3 -> 2.0.4 migration handler.
         assert os.path.exists(cwd("stuff"))
-        assert os.path.exists(cwd("whitelist"))
+        assert os.path.exists(cwd("safelist"))
 
-        wl = open(cwd("whitelist", "domain.txt"), "rb").read().split("\n")
-        assert wl[0] == "# You can add whitelisted domains here."
+        wl = open(cwd("safelist", "domain.txt"), "rb").read().split("\n")
+        assert wl[0] == "# You can add safelisted domains here."
         assert os.path.exists(cwd("yara", "dumpmem"))
         assert not os.path.exists(cwd("yara", "index_binaries.yar"))
 
diff --git a/tests/test_config.py b/tests/test_config.py
index 36846cb9f1..a81c7fd03c 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -882,7 +882,7 @@ def test_migration_20c1_20c2():
     assert cfg["processing"]["misp"]["url"] is None
     assert cfg["processing"]["misp"]["apikey"] is None
     assert cfg["processing"]["misp"]["maxioc"] == 100
-    assert cfg["processing"]["network"]["whitelist-dns"] is False
+    assert cfg["processing"]["network"]["safelist-dns"] is False
     assert cfg["processing"]["network"]["allowed-dns"] is None
     assert cfg["processing"]["procmemory"]["extract_img"] is True
     assert cfg["processing"]["procmemory"]["dump_delete"] is False
@@ -927,7 +927,7 @@ def test_migration_20c2_200():
 """)
     Files.create(cwd("conf"), "processing.conf", """
 [network]
-whitelist-dns = yes
+safelist-dns = yes
 allowed-dns = 8.8.8.8
 [procmemory]
 enabled = yes
@@ -988,7 +988,7 @@ def test_migration_20c2_200():
 """)
     cfg = Config.from_confdir(cwd("conf"), loose=True)
     assert "vpn" in cfg
-    assert "whitelist-dns" in cfg["processing"]["network"]
+    assert "safelist-dns" in cfg["processing"]["network"]
     assert "allowed-dns" in cfg["processing"]["network"]
     cfg = migrate(cfg, "2.0-rc2", "2.0.0")
     assert cfg["auxiliary"]["mitm"]["script"] == "mitm.py"
@@ -1000,9 +1000,9 @@ def test_migration_20c2_200():
     assert cfg["cuckoo"]["feedback"]["email"] is None
     assert cfg["cuckoo"]["processing"]["analysis_size_limit"] == 128*1024*1024
     assert cfg["cuckoo"]["resultserver"]["upload_max_size"] == 128*1024*1024
-    assert "whitelist-dns" not in cfg["processing"]["network"]
+    assert "safelist-dns" not in cfg["processing"]["network"]
     assert "allowed-dns" not in cfg["processing"]["network"]
-    assert cfg["processing"]["network"]["whitelist_dns"] is True
+    assert cfg["processing"]["network"]["safelist_dns"] is True
     assert cfg["processing"]["procmemory"]["extract_dll"] is False
     assert cfg["processing"]["network"]["allowed_dns"] == "8.8.8.8"
     assert cfg["processing"]["virustotal"]["enabled"] is False
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 8d58f84461..6f58a27e11 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -18,7 +18,7 @@
 from cuckoo.common.files import (
     Folders, Files, Storage, temppath, open_exclusive
 )
-from cuckoo.common.whitelist import is_whitelisted_domain
+from cuckoo.common.safelist import is_safelisted_domain
 from cuckoo.common import utils
 from cuckoo.main import cuckoo_create
 from cuckoo.misc import set_cwd, getuser
@@ -435,10 +435,10 @@ def test_list_of():
     assert utils.list_of_ints([1, 2]) is True
     assert utils.list_of_ints([lambda x: x]) is False
 
-def test_is_whitelisted_domain():
-    assert is_whitelisted_domain("java.com") is True
-    assert is_whitelisted_domain("java2.com") is False
-    assert is_whitelisted_domain("crl.microsoft.com") is True
+def test_is_safelisted_domain():
+    assert is_safelisted_domain("java.com") is True
+    assert is_safelisted_domain("java2.com") is False
+    assert is_safelisted_domain("crl.microsoft.com") is True
 
 def test_open_exclusive():
     fpath = os.path.join(tempfile.mkdtemp(), "yeet.exclusive")

From c05116d93b61008edac63efdf6726ab4ffaa06e0 Mon Sep 17 00:00:00 2001
From: Nex <nex@nex.sx>
Date: Sun, 5 Jul 2020 16:00:37 +0200
Subject: [PATCH 137/138] Updated links to documentation

---
 README.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.rst b/README.rst
index da13d90b79..ca644f9dbd 100644
--- a/README.rst
+++ b/README.rst
@@ -16,15 +16,15 @@ Make sure you check our existing Issues and Pull Requests and that you join
 our `IRC or Slack channel <https://cuckoosandbox.org/discussion>`_.
 
 For setup instructions, please refer
-`to <https://cuckoo.sh/docs/installation/host/requirements.html>`_
-`our <https://cuckoo.sh/docs/installation/host/installation.html>`_
-`documentation <https://cuckoo.sh/docs/index.html>`_.
+`to <https://docs.cuckoosandbox.org/en/latest/installation/host/requirements>`_
+`our <https://docs.cuckoosandbox.org/en/latest/installation/host/installation>`_
+`documentation <https://docs.cuckoosandbox.org/en/latest/>`_.
 
 This is a development version, we do not recommend its use in production; the
 latest stable version may be installed through :code:`pip install -U cuckoo`.
 
 You can find the full documentation of the latest stable release
-`here <https://cuckoo.sh/docs>`_.
+`here <https://docs.cuckoosandbox.org/en/latest/>`_.
 
 .. image:: https://travis-ci.org/cuckoosandbox/cuckoo.png?branch=master
    :alt: Linux Build Status
@@ -42,5 +42,5 @@ You can find the full documentation of the latest stable release
    :alt: Codecov Coverage Status
    :target: https://codecov.io/gh/cuckoosandbox/cuckoo
 
-.. _`community guidelines`: https://cuckoo.sh/docs/introduction/community.html
+.. _`community guidelines`: https://docs.cuckoosandbox.org/en/latest/introduction/community.html
 .. _`contribution requirements`: http://www.cuckoofoundation.org/contribute.html

From 50452a39ff7c3e0c4c94d114bc6317101633b958 Mon Sep 17 00:00:00 2001
From: Nex <nex@nex.sx>
Date: Mon, 26 Apr 2021 17:48:32 +0200
Subject: [PATCH 138/138] Added notice on current state

---
 README.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.rst b/README.rst
index ca644f9dbd..35fb0dcfcd 100644
--- a/README.rst
+++ b/README.rst
@@ -2,6 +2,10 @@
    :alt: Cuckoo Sandbox
    :target: https://cuckoosandbox.org/
 
+**PLEASE NOTE: Cuckoo Sandbox 2.x is currently unmaintained. Any open issues
+or pull requests will most likely not be processed, as a current full rewrite
+of Cuckoo is undergoing and will be announced soon.**
+
 `Cuckoo Sandbox <https://cuckoosandbox.org/>`_ is the leading open source
 automated malware analysis system.