diff --git a/.travis.yml b/.travis.yml index f1f40c7168..41cea05ee6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ before_install: sudo apt-get install libfreerdp-dev libjpeg-turbo8-dev # Install guacd. - wget https://www.apache.org/dist/guacamole/0.9.14/source/guacamole-server-0.9.14.tar.gz + wget http://archive.apache.org/dist/guacamole/0.9.14/source/guacamole-server-0.9.14.tar.gz tar xvf guacamole-server-0.9.14.tar.gz && cd guacamole-server-0.9.14 ./configure --with-init-dir=/etc/init.d && make && sudo make install && sudo ldconfig && cd ../ @@ -44,7 +44,7 @@ before_install: sudo mysql cuckootestimport < tests/files/sql/11my.sql psql -U postgres -c "CREATE DATABASE cuckootestimport" psql -U postgres cuckootestimport /dev/null - pip install psycopg2 mysql-python m2crypto==0.24.0 weasyprint + pip install psycopg2 mysql-python m2crypto==0.24.0 weasyprint==0.36 else brew update || brew update brew install libmagic cairo pango mongodb @@ -68,7 +68,7 @@ install: - pip install -U pip setuptools - python setup.py sdist - pip install -e . - - pip install pytest pytest-cov codecov coveralls distorm3 pycrypto + - pip install pytest==4.1.1 pytest-cov codecov coveralls distorm3 pycrypto - pip install flask-testing mock pytest-django pytest-pythonpath responses # Install Volatility. @@ -89,7 +89,7 @@ script: - cuckoo community # Check the code integrity of the Signatures by running Cuckoo. - - cuckoo -d 2>&1|grep "Unable to bind ResultServer" + - cuckoo -d --ignore-vuln 2>&1|grep "Unable to bind ResultServer" # Run tests from the "tests" directory. - py.test --cov=cuckoo @@ -97,3 +97,11 @@ script: after_success: - coveralls - codecov + +dist: xenial +addons: + apt: + packages: + - sqlite3 + sources: + - travis-ci/sqlite3 diff --git a/README.rst b/README.rst index f28b809e98..35fb0dcfcd 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,10 @@ :alt: Cuckoo Sandbox :target: https://cuckoosandbox.org/ +**PLEASE NOTE: Cuckoo Sandbox 2.x is currently unmaintained. Any open issues +or pull requests will most likely not be processed, as a current full rewrite +of Cuckoo is undergoing and will be announced soon.** + `Cuckoo Sandbox `_ is the leading open source automated malware analysis system. @@ -12,20 +16,19 @@ environment. If you want to contribute to development, report a bug, make a feature request or ask a question, please first take a look at our `community guidelines`_. -For development, please also take a look at the `contribution requirements`_. Make sure you check our existing Issues and Pull Requests and that you join our `IRC or Slack channel `_. For setup instructions, please refer -`to `_ -`our `_ -`documentation `_. +`to `_ +`our `_ +`documentation `_. This is a development version, we do not recommend its use in production; the latest stable version may be installed through :code:`pip install -U cuckoo`. You can find the full documentation of the latest stable release -`here `_. +`here `_. .. image:: https://travis-ci.org/cuckoosandbox/cuckoo.png?branch=master :alt: Linux Build Status @@ -43,5 +46,5 @@ You can find the full documentation of the latest stable release :alt: Codecov Coverage Status :target: https://codecov.io/gh/cuckoosandbox/cuckoo -.. _`community guidelines`: https://cuckoo.sh/docs/introduction/community.html +.. _`community guidelines`: https://docs.cuckoosandbox.org/en/latest/introduction/community.html .. _`contribution requirements`: http://www.cuckoofoundation.org/contribute.html diff --git a/appveyor.yml b/appveyor.yml index 50a420393c..dfa37be4f3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -25,8 +25,8 @@ install: - "python.exe setup.py sdist" - "pip.exe install -e ." - "pip.exe install psycopg2 mysqlclient==1.3.9 codecov flask-testing" - - "pip.exe install pytest pytest-cov pytest-django pytest-pythonpath" - - "pip.exe install flask-sqlalchemy==2.1 mock==2.0.0 responses==0.5.1" + - "pip.exe install pytest==4.1.1 pytest-cov pytest-django pytest-pythonpath" + - "pip.exe install flask-sqlalchemy==2.4.0 mock==2.0.0 responses==0.5.1" build: false @@ -47,9 +47,9 @@ before_test: - "psql cuckootestimport NUL" test_script: - - "cuckoo.exe -d" + - "cuckoo.exe -d --ignore-vuln" - "cuckoo.exe community" - - "cuckoo.exe -d || dir >NUL" + - "cuckoo.exe -d --ignore-vuln || dir >NUL" - "pytest.exe --cov=cuckoo" after_test: diff --git a/conftest.py b/conftest.py index b7a3ca691f..38ca57ad17 100644 --- a/conftest.py +++ b/conftest.py @@ -1,12 +1,13 @@ -# Copyright (C) 2017 Cuckoo Foundation. +# Copyright (C) 2017-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. import os import shutil import sys +import tempfile -from cuckoo.misc import is_windows, is_linux, is_macosx +from cuckoo.misc import is_windows, is_linux, is_macosx, getuser, mkdir # Note that collect_ignore is a parameter for pytest so that it knows which # unit tests to skip etc. In other words, perform platform-specific unit tests @@ -35,3 +36,6 @@ sys.path.insert(0, "cuckoo/data/analyzer/darwin") collect_ignore.append("tests/windows") collect_ignore.append("tests/linux") + +# Ensure the Cuckoo TMP dir exists, as some tests rely on it. +mkdir(os.path.join(tempfile.gettempdir(), "cuckoo-tmp-%s" % getuser())) diff --git a/cuckoo/apps/__init__.py b/cuckoo/apps/__init__.py index 355348e273..a35e690552 100644 --- a/cuckoo/apps/__init__.py +++ b/cuckoo/apps/__init__.py @@ -12,4 +12,4 @@ from .distributed import cuckoo_distributed, cuckoo_distributed_instance from .dnsserve import cuckoo_dnsserve from .import_ import import_cuckoo -from .rooter import cuckoo_rooter +from .rooter import cuckoo_rooter, cleanup_rooter diff --git a/cuckoo/apps/api.py b/cuckoo/apps/api.py index 18d66c1f39..ce383d2d58 100644 --- a/cuckoo/apps/api.py +++ b/cuckoo/apps/api.py @@ -38,7 +38,7 @@ def json_error(status_code, message): return r def shutdown_server(): - """Shutdown API werkzeug server""" + """Shutdown API werkzeug server.""" shutdown = request.environ.get("werkzeug.server.shutdown") if shutdown: shutdown() @@ -676,8 +676,8 @@ def vpn_status(): @app.route("/exit") def exit_api(): - """Shuts down the server if in debug mode and - using the werkzeug server""" + """Shut down the server if in debug mode and + using the werkzeug server.""" if not app.debug: return json_error(403, "This call can only be used in debug mode") @@ -714,5 +714,8 @@ def cuckoo_api(hostname, port, debug): app.run(host=hostname, port=port, debug=debug) if os.environ.get("CUCKOO_APP") == "api": + from cuckoo.core.startup import ensure_tmpdir, init_console_logging decide_cwd(exists=True) Database().connect() + init_console_logging() + ensure_tmpdir() diff --git a/cuckoo/apps/apps.py b/cuckoo/apps/apps.py index 75aad720a4..545cf00260 100644 --- a/cuckoo/apps/apps.py +++ b/cuckoo/apps/apps.py @@ -99,7 +99,7 @@ def fetch_community(branch="master", force=False, filepath=None): open(filepath, "wb").write(t.extractfile(member).read()) def enumerate_files(path, pattern): - """Yields all filepaths from a directory.""" + """Yield all filepaths from a directory.""" if os.path.isfile(path): yield path elif os.path.isdir(path): @@ -346,7 +346,7 @@ def process_tasks(instance, maxcount, timeout): def cuckoo_clean(): """Clean up cuckoo setup. - It deletes logs, all stored data from file system and configured + Delete logs, all stored data from file system and configured databases (SQL and MongoDB). """ # Init logging (without writing to file). @@ -444,7 +444,7 @@ def cuckoo_machine(vmname, action, ip, platform, options, tags, resultserver_port = int(resultserver_port) else: resultserver_ip = cfg["cuckoo"]["resultserver"]["ip"] - resultserver_port = cfg["cuckoo"]["resultserver"]["port"] + resultserver_port = 0 machines.append(vmname) cfg[machinery][vmname] = { @@ -504,11 +504,17 @@ def migrate_cwd(): mkdir(cwd("stuff")) mkdir(cwd("yara", "office")) - # Create the new $CWD/whitelist/ directory. - if not os.path.exists(cwd("whitelist")): + # Create the new $CWD/safelist/ directory. + if not os.path.exists(cwd("safelist")): shutil.copytree( - cwd("..", "data", "whitelist", private=True), cwd("whitelist") + cwd("..", "data", "safelist", private=True), cwd("safelist") ) + else: + data_wl = cwd("..", "data", "safelist", private=True) + for wl_file in os.listdir(data_wl): + cwd_wl = cwd("safelist", wl_file) + if not os.path.isfile(cwd_wl): + shutil.copy(os.path.join(data_wl, wl_file), cwd_wl) # Create the new $CWD/yara/dumpmem/ directory. if not os.path.exists(cwd("yara", "dumpmem")): diff --git a/cuckoo/apps/rooter.py b/cuckoo/apps/rooter.py index 838c0b7471..f4d8908e66 100644 --- a/cuckoo/apps/rooter.py +++ b/cuckoo/apps/rooter.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Cuckoo Foundation. +# Copyright (C) 2015-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -7,6 +7,7 @@ import logging import os.path import re +import signal import socket import stat import subprocess @@ -18,6 +19,8 @@ class s(object): service = None iptables = None + iptables_save = None + iptables_restore = None ip = None log = logging.getLogger(__name__) @@ -29,6 +32,35 @@ def run(*args): stdout, stderr = p.communicate() return stdout, stderr +def run_iptables(*args): + iptables_args = [s.iptables] + iptables_args.extend(list(args)) + iptables_args.extend(["-m", "comment", "--comment", "cuckoo-rooter"]) + return run(*iptables_args) + +def cleanup_rooter(): + """Filter out all Cuckoo rooter entries from iptables-save and + restore the resulting ruleset.""" + try: + stdout, stderr = run(s.iptables_save) + except OSError as e: + log.error( + "Failed to clean Cuckoo rooter rules. Is iptables-save " + "available? %s", e + ) + return + + if not stdout: + return + + cleaned = [] + for l in stdout.split("\n"): + if l and "cuckoo-rooter" not in l: + cleaned.append(l) + + p = subprocess.Popen([s.iptables_restore], stdin=subprocess.PIPE) + p.communicate(input="\n".join(cleaned)) + def version(): return { "version": __version__, @@ -59,7 +91,7 @@ def rt_available(rt_table): return False def vpn_status(): - """Gets current VPN status.""" + """Get current VPN status.""" ret = {} for line in run(s.service, "openvpn", "status")[0].split("\n"): x = re.search("'(?P\\w+)'\\ is\\ (?Pnot)?", line) @@ -78,20 +110,20 @@ def vpn_disable(name): def forward_drop(): """Disable any and all forwarding unless explicitly said so.""" - run(s.iptables, "-P", "FORWARD", "DROP") + run_iptables("-P", "FORWARD", "DROP") def state_enable(): """Enable stateful connection tracking.""" - run( - s.iptables, "-A", "INPUT", "-m", "state", - "--state", "ESTABLISHED,RELATED", "-j", "ACCEPT" + run_iptables( + "-A", "INPUT", "-m", "state", "--state", "ESTABLISHED,RELATED", + "-j", "ACCEPT" ) def state_disable(): """Disable stateful connection tracking.""" while True: - _, err = run( - s.iptables, "-D", "INPUT", "-m", "state", + _, err = run_iptables( + "-D", "INPUT", "-m", "state", "--state", "ESTABLISHED,RELATED", "-j", "ACCEPT" ) if err: @@ -99,14 +131,15 @@ def state_disable(): def enable_nat(interface): """Enable NAT on this interface.""" - run(s.iptables, "-t", "nat", "-A", "POSTROUTING", - "-o", interface, "-j", "MASQUERADE") + run_iptables( + "-t", "nat", "-A", "POSTROUTING", "-o", interface, "-j", "MASQUERADE" + ) def disable_nat(interface): """Disable NAT on this interface.""" while True: - _, err = run( - s.iptables, "-t", "nat", "-D", "POSTROUTING", + _, err = run_iptables( + "-t", "nat", "-D", "POSTROUTING", "-o", interface, "-j", "MASQUERADE" ) if err: @@ -125,7 +158,7 @@ def init_rttable(rt_table, interface): run(s.ip, *args) def flush_rttable(rt_table): - """Flushes specified routing table entries.""" + """Flush specified routing table entries.""" if rt_table in ["local", "main", "default"]: return @@ -133,14 +166,14 @@ def flush_rttable(rt_table): def dns_forward(action, vm_ip, dns_ip, dns_port="53"): """Route DNS requests from the VM to a custom DNS on a separate network.""" - run( - s.iptables, "-t", "nat", action, "PREROUTING", "-p", "tcp", + run_iptables( + "-t", "nat", action, "PREROUTING", "-p", "tcp", "--dport", "53", "--source", vm_ip, "-j", "DNAT", "--to-destination", "%s:%s" % (dns_ip, dns_port) ) - run( - s.iptables, "-t", "nat", action, "PREROUTING", "-p", "udp", + run_iptables( + "-t", "nat", action, "PREROUTING", "-p", "udp", "--dport", "53", "--source", vm_ip, "-j", "DNAT", "--to-destination", "%s:%s" % (dns_ip, dns_port) ) @@ -151,29 +184,29 @@ def forward_enable(src, dst, ipaddr): # Delete libvirt's default FORWARD REJECT rules. e.g.: # -A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable # -A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable - run(s.iptables, "-D", "FORWARD", "-i", src, "-j", "REJECT") - run(s.iptables, "-D", "FORWARD", "-o", src, "-j", "REJECT") + run_iptables("-D", "FORWARD", "-i", src, "-j", "REJECT") + run_iptables("-D", "FORWARD", "-o", src, "-j", "REJECT") - run( - s.iptables, "-A", "FORWARD", "-i", src, "-o", dst, + run_iptables( + "-A", "FORWARD", "-i", src, "-o", dst, "--source", ipaddr, "-j", "ACCEPT" ) - run( - s.iptables, "-A", "FORWARD", "-i", dst, "-o", src, + run_iptables( + "-A", "FORWARD", "-i", dst, "-o", src, "--destination", ipaddr, "-j", "ACCEPT" ) def forward_disable(src, dst, ipaddr): """Disable forwarding of a specific IP address from one interface into another.""" - run( - s.iptables, "-D", "FORWARD", "-i", src, "-o", dst, + run_iptables( + "-D", "FORWARD", "-i", src, "-o", dst, "--source", ipaddr, "-j", "ACCEPT" ) - run( - s.iptables, "-D", "FORWARD", "-i", dst, "-o", src, + run_iptables( + "-D", "FORWARD", "-i", dst, "-o", src, "--destination", ipaddr, "-j", "ACCEPT" ) @@ -202,8 +235,8 @@ def inetsim_redirect_port(action, srcip, dstip, ports): if not srcport.isdigit() or not dstport.isdigit(): log.debug("Invalid inetsim ports entry: %s", entry) continue - run( - s.iptables, "-t", "nat", action, "PREROUTING", "--source", srcip, + run_iptables( + "-t", "nat", action, "PREROUTING", "--source", srcip, "-p", "tcp", "--syn", "--dport", srcport, "-j", "DNAT", "--to-destination", "%s:%s" % (dstip, dstport) ) @@ -213,86 +246,88 @@ def inetsim_enable(ipaddr, inetsim_ip, machinery_iface, resultserver_port, """Enable hijacking of all traffic and send it to InetSim.""" inetsim_redirect_port("-A", ipaddr, inetsim_ip, ports) - run( - s.iptables, "-t", "nat", "-A", "PREROUTING", "--source", ipaddr, + run_iptables( + "-t", "nat", "-A", "PREROUTING", "--source", ipaddr, "-p", "tcp", "--syn", "!", "--dport", resultserver_port, "-j", "DNAT", "--to-destination", inetsim_ip ) - run( - s.iptables, "-t", "nat", "-A", "PREROUTING", "--source", ipaddr, + run_iptables( + "-t", "nat", "-A", "PREROUTING", "--source", ipaddr, "-p", "udp", "-j", "DNAT", "--to-destination", inetsim_ip ) - run( - s.iptables, "-A", "OUTPUT", "-m", "conntrack", "--ctstate", + run_iptables( + "-A", "OUTPUT", "-m", "conntrack", "--ctstate", "INVALID", "-j", "DROP" ) - run( - s.iptables, "-A", "OUTPUT", "-m", "state", "--state", + run_iptables( + "-A", "OUTPUT", "-m", "state", "--state", "INVALID", "-j", "DROP" ) dns_forward("-A", ipaddr, inetsim_ip) forward_enable(machinery_iface, machinery_iface, ipaddr) - run(s.iptables, "-t", "nat", "-A", "POSTROUTING", "--source", ipaddr, + run_iptables( + "-t", "nat", "-A", "POSTROUTING", "--source", ipaddr, "-o", machinery_iface, "--destination", inetsim_ip, "-j", "MASQUERADE") - run(s.iptables, "-A", "OUTPUT", "-s", ipaddr, "-j", "DROP") + run_iptables("-A", "OUTPUT", "-s", ipaddr, "-j", "DROP") def inetsim_disable(ipaddr, inetsim_ip, machinery_iface, resultserver_port, ports): """Enable hijacking of all traffic and send it to InetSim.""" inetsim_redirect_port("-D", ipaddr, inetsim_ip, ports) - run( - s.iptables, "-D", "PREROUTING", "-t", "nat", "--source", ipaddr, + run_iptables( + "-D", "PREROUTING", "-t", "nat", "--source", ipaddr, "-p", "tcp", "--syn", "!", "--dport", resultserver_port, "-j", "DNAT", "--to-destination", inetsim_ip ) - run( - s.iptables, "-t", "nat", "-D", "PREROUTING", "--source", ipaddr, + run_iptables( + "-t", "nat", "-D", "PREROUTING", "--source", ipaddr, "-p", "udp", "-j", "DNAT", "--to-destination", inetsim_ip ) - run( - s.iptables, "-D", "OUTPUT", "-m", "conntrack", "--ctstate", + run_iptables( + "-D", "OUTPUT", "-m", "conntrack", "--ctstate", "INVALID", "-j", "DROP" ) - run( - s.iptables, "-D", "OUTPUT", "-m", "state", "--state", + run_iptables( + "-D", "OUTPUT", "-m", "state", "--state", "INVALID", "-j", "DROP" ) dns_forward("-D", ipaddr, inetsim_ip) forward_disable(machinery_iface, machinery_iface, ipaddr) - run(s.iptables, "-t", "nat", "-D", "POSTROUTING", "--source", ipaddr, + run_iptables( + "-t", "nat", "-D", "POSTROUTING", "--source", ipaddr, "-o", machinery_iface, "--destination", inetsim_ip, "-j", "MASQUERADE") - run(s.iptables, "-D", "OUTPUT", "-s", ipaddr, "-j", "DROP") + run_iptables("-D", "OUTPUT", "-s", ipaddr, "-j", "DROP") def tor_toggle(action, vm_ip, resultserver_ip, dns_port, proxy_port): """Toggle Tor iptables routing rules.""" dns_forward(action, vm_ip, resultserver_ip, dns_port) - run( - s.iptables, "-t", "nat", action, "PREROUTING", "-p", "tcp", + run_iptables( + "-t", "nat", action, "PREROUTING", "-p", "tcp", "--source", vm_ip, "!", "--destination", resultserver_ip, "-j", "DNAT", "--to-destination", "%s:%s" % (resultserver_ip, proxy_port) ) - run( - s.iptables, "-t", "nat", action, "PREROUTING", "-p", "udp", + run_iptables( + "-t", "nat", action, "PREROUTING", "-p", "udp", "--source", vm_ip, "!", "--destination", resultserver_ip, "-j", "DNAT", "--to-destination", "%s:%s" % (resultserver_ip, proxy_port) ) - run(s.iptables, action, "OUTPUT", "-s", vm_ip, "-j", "DROP") + run_iptables(action, "OUTPUT", "-s", vm_ip, "-j", "DROP") def tor_enable(vm_ip, resultserver_ip, dns_port, proxy_port): """Enable hijacking of all traffic and send it to TOR.""" @@ -304,25 +339,20 @@ def tor_disable(vm_ip, resultserver_ip, dns_port, proxy_port): def drop_toggle(action, vm_ip, resultserver_ip, resultserver_port, agent_port): """Toggle iptables to allow internal Cuckoo traffic.""" - run( - s.iptables, action, "INPUT", "--source", vm_ip, "-p", "tcp", + run_iptables( + action, "INPUT", "--source", vm_ip, "-p", "tcp", "--destination", resultserver_ip, "--dport", "%s" % resultserver_port, "-j", "ACCEPT" ) - run( - s.iptables, action, "OUTPUT", "--source", resultserver_ip, + run_iptables( + action, "OUTPUT", "--source", resultserver_ip, "-p", "tcp", "--destination", vm_ip, "--dport", "%s" % agent_port, "-j", "ACCEPT" ) - run( - s.iptables, action, "INPUT", "--source", vm_ip, "-j", "DROP" - ) - - run( - s.iptables, action, "OUTPUT", "--source", vm_ip, "-j", "DROP" - ) + run_iptables(action, "INPUT", "--source", vm_ip, "-j", "DROP") + run_iptables(action, "OUTPUT", "--source", vm_ip, "-j", "DROP") def drop_enable(vm_ip, resultserver_ip, resultserver_port, agent_port=8000): """Enable complete dropping of all non-Cuckoo traffic by default.""" @@ -415,14 +445,33 @@ def cuckoo_rooter(socket_path, group, service, iptables, ip): # Initialize global variables. s.service = service s.iptables = iptables + s.iptables_save = "/sbin/iptables-save" + s.iptables_restore = "/sbin/iptables-restore" s.ip = ip - while True: + # Simple object to allow a signal handler to stop the rooter loop + class Run(object): + def __init__(self): + self.run = True + do = Run() + + def handle_sigterm(sig, f): + do.run = False + server.shutdown(socket.SHUT_RDWR) + server.close() + cleanup_rooter() + + signal.signal(signal.SIGTERM, handle_sigterm) + + while do.run: try: command, addr = server.recvfrom(4096) except socket.error as e: if e.errno == errno.EINTR: continue + elif e.errno == errno.EBADF and not do.run: + continue + raise e try: diff --git a/cuckoo/auxiliary/replay.py b/cuckoo/auxiliary/replay.py index dc41deea35..984dd11a5e 100644 --- a/cuckoo/auxiliary/replay.py +++ b/cuckoo/auxiliary/replay.py @@ -24,7 +24,7 @@ def __init__(self): self.port = None def pcap2mitm(self, pcappath, tlsmaster): - """Used to translate a .pcap into a .mitm file.""" + """Translate a .pcap into a .mitm file.""" mitmpath = tempfile.mktemp(suffix=".mitm") with open(mitmpath, "wb") as f: httpreplay.utils.pcap2mitm(pcappath, f, tlsmaster, True) diff --git a/cuckoo/auxiliary/services.py b/cuckoo/auxiliary/services.py index e79f5ea2f8..55517e0e84 100644 --- a/cuckoo/auxiliary/services.py +++ b/cuckoo/auxiliary/services.py @@ -14,7 +14,7 @@ db = Database() class Services(Auxiliary): - """Allows one or more additional VMs to be run next to an analysis. Either + """Allow one or more additional VMs to be run next to an analysis. Either as global services (which are generally never rebooted) or on a per-analysis basis.""" diff --git a/cuckoo/auxiliary/sniffer.py b/cuckoo/auxiliary/sniffer.py index 3242cdfa63..3962269b35 100644 --- a/cuckoo/auxiliary/sniffer.py +++ b/cuckoo/auxiliary/sniffer.py @@ -107,11 +107,11 @@ def _check_output(self, out, err): "standard output, got: %r." % out ) - err_whitelist_start = ( + err_safelist_start = ( "tcpdump: listening on ", ) - err_whitelist_ends = ( + err_safelist_ends = ( "packet captured", "packets captured", "packet received by filter", @@ -124,10 +124,10 @@ def _check_output(self, out, err): ) for line in err.split("\n"): - if not line or line.startswith(err_whitelist_start): + if not line or line.startswith(err_safelist_start): continue - if line.endswith(err_whitelist_ends): + if line.endswith(err_safelist_ends): continue raise CuckooOperationalError( diff --git a/cuckoo/common/abstracts.py b/cuckoo/common/abstracts.py index f6b92b9710..1b163b8885 100644 --- a/cuckoo/common/abstracts.py +++ b/cuckoo/common/abstracts.py @@ -1,5 +1,5 @@ # Copyright (C) 2012-2013 Claudio Guarnieri. -# Copyright (C) 2014-2018 Cuckoo Foundation. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -166,7 +166,7 @@ def init_once(cls): pass def pcap_path(self, task_id): - """Returns the .pcap path for this task id.""" + """Return the .pcap path for this task id.""" return cwd("storage", "analyses", "%s" % task_id, "dump.pcap") def set_options(self, options): @@ -228,7 +228,7 @@ def _initialize(self, module_name): ) def _initialize_check(self): - """Runs checks against virtualization software when a machine manager + """Run checks against virtualization software when a machine manager is initialized. @note: in machine manager modules you may override or superclass his method. @@ -271,7 +271,7 @@ def machines(self): return self.db.list_machines() def availables(self): - """How many machines are free. + """Return how many machines are free. @return: free machines count. """ return self.db.count_machines_available() @@ -297,13 +297,13 @@ def release(self, label=None): self.db.unlock_machine(label) def running(self): - """Returns running virtual machines. + """Return running virtual machines. @return: running virtual machines list. """ return self.db.list_machines(locked=True) def shutdown(self): - """Shutdown the machine manager. Kills all alive machines. + """Shutdown the machine manager and kill all alive machines. @raise CuckooMachineError: if unable to stop machine. """ if len(self.running()) > 0: @@ -339,13 +339,13 @@ def stop(self, label=None): raise NotImplementedError def _list(self): - """Lists virtual machines configured. + """List virtual machines configured. @raise NotImplementedError: this method is abstract. """ raise NotImplementedError def dump_memory(self, label, path): - """Takes a memory dump of a machine. + """Take a memory dump of a machine. @param path: path to where to store the memory dump. """ raise NotImplementedError @@ -372,7 +372,7 @@ def get_remote_control_params(self, label): raise NotImplementedError def _wait_status(self, label, *states): - """Waits for a vm status. + """Wait for a vm status. @param label: virtual machine name. @param state: virtual machine status, accepts multiple states as list. @raise CuckooMachineError: if default waiting timeout expire. @@ -396,6 +396,11 @@ def _wait_status(self, label, *states): waitme += 1 current = self._status(label) + @staticmethod + def version(): + """Return the version of the virtualization software""" + return None + class LibVirtMachinery(Machinery): """Libvirt based machine manager. @@ -428,7 +433,7 @@ def initialize(self, module): super(LibVirtMachinery, self).initialize(module) def _initialize_check(self): - """Runs all checks when a machine manager is initialized. + """Run all checks when a machine manager is initialized. @raise CuckooMachineError: if libvirt version is not supported. """ # Version checks. @@ -444,7 +449,7 @@ def _initialize_check(self): super(LibVirtMachinery, self)._initialize_check() def start(self, label, task): - """Starts a virtual machine. + """Start a virtual machine. @param label: virtual machine name. @param task: task object. @raise CuckooMachineError: if unable to start virtual machine. @@ -497,7 +502,7 @@ def start(self, label, task): self._wait_status(label, self.RUNNING) def stop(self, label): - """Stops a virtual machine. Kill them all. + """Stop a virtual machine. Kill them all. @param label: virtual machine name. @raise CuckooMachineError: if unable to stop virtual machine. """ @@ -531,7 +536,7 @@ def shutdown(self): self.vms = None def dump_memory(self, label, path): - """Takes a memory dump. + """Take a memory dump. @param path: path to where to store the memory dump. """ log.debug("Dumping memory for machine %s", label) @@ -550,7 +555,7 @@ def dump_memory(self, label, path): self._disconnect(conn) def _status(self, label): - """Gets current status of a vm. + """Get current status of a vm. @param label: virtual machine name. @return: status string. """ @@ -595,7 +600,7 @@ def _status(self, label): "{0}".format(label)) def _connect(self): - """Connects to libvirt subsystem. + """Connect to libvirt subsystem. @raise CuckooMachineError: when unable to connect to libvirt. """ # Check if a connection string is available. @@ -609,7 +614,7 @@ def _connect(self): raise CuckooMachineError("Cannot connect to libvirt") def _disconnect(self, conn): - """Disconnects to libvirt subsystem. + """Disconnect from libvirt subsystem. @raise CuckooMachineError: if cannot disconnect from libvirt. """ try: @@ -841,11 +846,6 @@ class Signature(object): filter_apinames = [] filter_categories = [] - # If no on_call() handler is present and this field has been set, then - # dispatch on a per-API basis to the accompanying API. That is, rather - # than calling the generic on_call(), call, e.g., on_call_CreateFile(). - on_call_dispatch = False - def __init__(self, caller): """ @param caller: calling object. Stores results in caller.results @@ -865,7 +865,7 @@ def init_once(cls): pass def _check_value(self, pattern, subject, regex=False, all=False): - """Checks a pattern against a given subject. + """Check a pattern against a given subject. @param pattern: string or expression to check for. @param subject: target of the check. @param regex: boolean representing if the pattern is a regular @@ -990,7 +990,7 @@ def get_keys(self, pid=None, actions=None): def check_file(self, pattern, regex=False, actions=None, pid=None, all=False): - """Checks for a file being opened. + """Check for a file being opened. @param pattern: string or expression to check for. @param regex: boolean representing if the pattern is a regular expression or not and therefore should be compiled. @@ -1013,7 +1013,7 @@ def check_file(self, pattern, regex=False, actions=None, pid=None, def check_dll_loaded(self, pattern, regex=False, actions=None, pid=None, all=False): - """Checks for DLLs being loaded. + """Check for DLLs being loaded. @param pattern: string or expression to check for. @param regex: boolean representing if the pattern is a regular expression or not and therefore should be compiled. @@ -1027,7 +1027,7 @@ def check_dll_loaded(self, pattern, regex=False, actions=None, pid=None, all=all) def check_command_line(self, pattern, regex=False, all=False): - """Checks for a command line being opened. + """Check for a command line being opened. @param pattern: string or expression to check for. @param regex: boolean representing if the pattern is a regular expression or not and therefore should be compiled. @@ -1040,7 +1040,7 @@ def check_command_line(self, pattern, regex=False, all=False): def check_key(self, pattern, regex=False, actions=None, pid=None, all=False): - """Checks for a registry key being accessed. + """Check for a registry key being accessed. @param pattern: string or expression to check for. @param regex: boolean representing if the pattern is a regular expression or not and therefore should be compiled. @@ -1068,7 +1068,7 @@ def get_mutexes(self, pid=None): return self.get_summary_generic(pid, ["mutex"]) def check_mutex(self, pattern, regex=False, all=False): - """Checks for a mutex being opened. + """Check for a mutex being opened. @param pattern: string or expression to check for. @param regex: boolean representing if the pattern is a regular expression or not and therefore should be compiled. @@ -1080,11 +1080,11 @@ def check_mutex(self, pattern, regex=False, all=False): all=all) def get_command_lines(self): - """Retrieves all command lines used.""" + """Retrieve all command lines used.""" return self.get_summary("command_line") def get_wmi_queries(self): - """Retrieves all executed WMI queries.""" + """Retrieve all executed WMI queries.""" return self.get_summary("wmi_query") def get_net_generic(self, subtype): @@ -1095,68 +1095,68 @@ def get_net_generic(self, subtype): return self.get_results("network", {}).get(subtype, []) def get_net_hosts(self): - """Returns a list of all hosts.""" + """Return a list of all hosts.""" return self.get_net_generic("hosts") def get_net_domains(self): - """Returns a list of all domains.""" + """Return a list of all domains.""" return self.get_net_generic("domains") def get_net_http(self): - """Returns a list of all http data.""" + """Return a list of all http data.""" return self.get_net_generic("http") def get_net_http_ex(self): - """Returns a list of all http data.""" + """Return a list of all http data.""" return \ self.get_net_generic("http_ex") + self.get_net_generic("https_ex") def get_net_udp(self): - """Returns a list of all udp data.""" + """Return a list of all udp data.""" return self.get_net_generic("udp") def get_net_icmp(self): - """Returns a list of all icmp data.""" + """Return a list of all icmp data.""" return self.get_net_generic("icmp") def get_net_irc(self): - """Returns a list of all irc data.""" + """Return a list of all irc data.""" return self.get_net_generic("irc") def get_net_smtp(self): - """Returns a list of all smtp data.""" + """Return a list of all smtp data.""" return self.get_net_generic("smtp") def get_net_smtp_ex(self): - """"Returns a list of all smtp data""" + """"Return a list of all smtp data""" return self.get_net_generic("smtp_ex") def get_virustotal(self): - """Returns the information retrieved from virustotal.""" + """Return the information retrieved from virustotal.""" return self.get_results("virustotal", {}) def get_volatility(self, module=None): - """Returns the data that belongs to the given module.""" + """Return the data that belongs to the given module.""" volatility = self.get_results("memory", {}) return volatility if module is None else volatility.get(module, {}) def get_apkinfo(self, section=None, default={}): - """Returns the apkinfo results for this analysis.""" + """Return the apkinfo results for this analysis.""" apkinfo = self.get_results("apkinfo", {}) return apkinfo if section is None else apkinfo.get(section, default) def get_droidmon(self, section=None, default={}): - """Returns the droidmon results for this analysis.""" + """Return the droidmon results for this analysis.""" droidmon = self.get_results("droidmon", {}) return droidmon if section is None else droidmon.get(section, default) def get_googleplay(self, section=None, default={}): - """Returns the Google Play results for this analysis.""" + """Return the Google Play results for this analysis.""" googleplay = self.get_results("googleplay", {}) return googleplay if section is None else googleplay.get(section, default) def check_ip(self, pattern, regex=False, all=False): - """Checks for an IP address being contacted. + """Check for an IP address being contacted. @param pattern: string or expression to check for. @param regex: boolean representing if the pattern is a regular expression or not and therefore should be compiled. @@ -1168,7 +1168,7 @@ def check_ip(self, pattern, regex=False, all=False): all=all) def check_domain(self, pattern, regex=False, all=False): - """Checks for a domain being contacted. + """Check for a domain being contacted. @param pattern: string or expression to check for. @param regex: boolean representing if the pattern is a regular expression or not and therefore should be compiled. @@ -1184,7 +1184,7 @@ def check_domain(self, pattern, regex=False, all=False): all=all) def check_url(self, pattern, regex=False, all=False): - """Checks for a URL being contacted. + """Check for a URL being contacted. @param pattern: string or expression to check for. @param regex: boolean representing if the pattern is a regular expression or not and therefore should be compiled. @@ -1274,7 +1274,7 @@ def mark(self, **kwargs): self.marks.append(mark) def has_marks(self, count=None): - """Returns true if this signature has one or more marks.""" + """Return true if this signature has one or more marks.""" if count is not None: return len(self.marks) >= count return not not self.marks @@ -1288,10 +1288,6 @@ def on_call(self, call, process): @param call: logged API call. @param process: proc object. """ - # Dispatch this call to a per-API specific handler. - if self.on_call_dispatch: - return getattr(self, "on_call_%s" % call["api"])(call, process) - raise NotImplementedError def on_signature(self, signature): @@ -1431,17 +1427,29 @@ def run(self): behavior[self.key].""" raise NotImplementedError + class ProtocolHandler(object): """Abstract class for protocol handlers coming out of the analysis.""" - def __init__(self, handler, version=None): - self.handler = handler + def __init__(self, task_id, ctx, version=None): + self.task_id = task_id + self.handler = ctx + self.fd = None self.version = version - def init(self): - pass + def __enter__(self): + self.init() + + def __exit__(self, type, value, traceback): + self.close() def close(self): - pass + if self.fd: + self.fd.close() + self.fd = None + + def handle(self): + raise NotImplementedError + class Extractor(object): """One piece in a series of recursive extractors & unpackers.""" diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py index 01f135280a..7728d513d6 100644 --- a/cuckoo/common/config.py +++ b/cuckoo/common/config.py @@ -1,5 +1,5 @@ # Copyright (C) 2012-2013 Claudio Guarnieri. -# Copyright (C) 2014-2018 Cuckoo Foundation. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -148,7 +148,7 @@ def parse(self, value): log.error("Incorrect UUID %s", value) def check(self, value): - """Checks if the value is of type UUID.""" + """Check if the value is of type UUID.""" try: click.UUID(value) return True @@ -207,6 +207,7 @@ class Config(object): "cuckoo": { "cuckoo": { "version_check": Boolean(True), + "ignore_vulnerabilities": Boolean(False, required=False), "delete_original": Boolean(False), "delete_bin_copy": Boolean(False), "machinery": String("virtualbox"), @@ -242,7 +243,8 @@ class Config(object): "resultserver": { "ip": String("192.168.56.1"), "port": Int(2042), - "force_port": Boolean(False), + "force_port": Boolean(False, False), # Unused + "pool_size": Int(0, False), "upload_max_size": Int(128 * 1024 * 1024), }, "processing": { @@ -577,7 +579,7 @@ class Config(object): }, "network": { "enabled": Boolean(True), - "whitelist_dns": Boolean(False), + "safelist_dns": Boolean(False), "allowed_dns": String(), }, "procmemory": { @@ -750,6 +752,12 @@ class Config(object): "url": String(), "apikey": String(sanitize=True), "mode": String("maldoc ipaddr hashes url"), + "distribution": Int(0, required=False), + "analysis": Int(0, required=False), + "threat_level": Int(4, required=False), + "min_malscore": Int(0, required=False), + "tag": String("Cuckoo", required=False), + "upload_sample": Boolean(False, required=False), }, "mongodb": { "enabled": Boolean(False), @@ -1041,7 +1049,7 @@ def get(self, section): @staticmethod def from_confdir(dirpath, loose=False, sanitize=False): - """Reads all the configuration from a configuration directory. If + """Read all the configuration from a configuration directory. If `sanitize` is set, then black out sensitive fields.""" ret = {} for filename in os.listdir(dirpath): @@ -1182,7 +1190,7 @@ def cast(s, value): return type_.parse(value) def read_kv_conf(filepath): - """Reads a flat Cuckoo key/value configuration file.""" + """Read a flat Cuckoo key/value configuration file.""" ret = {} for line in open(filepath, "rb"): line = line.strip() diff --git a/cuckoo/common/dns.py b/cuckoo/common/dns.py index 56f6009b78..a13bce6081 100644 --- a/cuckoo/common/dns.py +++ b/cuckoo/common/dns.py @@ -36,9 +36,9 @@ def set_timeout_value(value): # standard gethostbyname in thread # http://code.activestate.com/recipes/473878/ def with_timeout(func, args=(), kwargs={}): - """This function will spawn a thread and run the given function - using the args, kwargs and return the given default value if the - timeout_duration is exceeded. + """Spawn a thread and run the given function using the args, + kwargs and return the given default value if the timeout_duration + is exceeded. """ class ResultThread(threading.Thread): daemon = True diff --git a/cuckoo/common/files.py b/cuckoo/common/files.py index 525382b818..f67de03f15 100644 --- a/cuckoo/common/files.py +++ b/cuckoo/common/files.py @@ -7,13 +7,14 @@ import tempfile import ntpath import shutil +import errno from cuckoo.common.config import config from cuckoo.common.exceptions import CuckooOperationalError from cuckoo.misc import getuser def temppath(): - """Returns the true temporary directory.""" + """Return the true temporary directory.""" tmppath = config("cuckoo:cuckoo:tmppath") # Backwards compatibility with older configuration. @@ -24,6 +25,16 @@ def temppath(): return tmppath +def open_exclusive(path, mode='wb', bufsize=-1): + """Open a file with O_EXCL, failing if it already exists + [In Python 3, use open with x]""" + fd = os.open(path, os.O_CREAT|os.O_EXCL|os.O_WRONLY, 0644) + try: + return os.fdopen(fd, mode, bufsize) + except: + os.close(fd) + raise + class Storage(object): @staticmethod def get_filename_from_path(path): @@ -37,7 +48,7 @@ def get_filename_from_path(path): class Folders(Storage): @staticmethod def create(root=".", folders=None): - """Creates a directory or multiple directories. + """Create a directory or multiple directories. @param root: root path. @param folders: folders list to be created. @raise CuckooOperationalError: if fails to create folder. @@ -56,7 +67,10 @@ def create(root=".", folders=None): if not os.path.isdir(folder_path): try: os.makedirs(folder_path) - except OSError: + except OSError as e: + if e.errno == errno.EEXIST: + # Race condition, ignore + continue raise CuckooOperationalError( "Unable to create folder: %s" % folder_path ) @@ -149,7 +163,7 @@ def copy(path_target, path_dest): @staticmethod def hash_file(method, filepath): - """Calculates an hash on a file by path. + """Calculate a hash on a file by path. @param method: callable hashing method @param path: file path @return: computed hash string diff --git a/cuckoo/common/netlog.py b/cuckoo/common/netlog.py index 54ff29a74c..69f70cbf38 100644 --- a/cuckoo/common/netlog.py +++ b/cuckoo/common/netlog.py @@ -1,5 +1,5 @@ # Copyright (C) 2010-2013 Claudio Guarnieri. -# Copyright (C) 2014-2016 Cuckoo Foundation. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -18,9 +18,9 @@ elif hasattr(bson, "loads"): bson_decode = lambda d: bson.loads(d) -from cuckoo.common.abstracts import ProtocolHandler from cuckoo.common.files import Storage from cuckoo.common.exceptions import CuckooResultError +from cuckoo.misc import cwd log = logging.getLogger(__name__) @@ -54,8 +54,8 @@ def default_converter_64bit(v): return v.decode("latin-1") return v -class BsonParser(ProtocolHandler): - """Receives and interprets .bson logs from the monitor. +class BsonParser(object): + """Interprets .bson logs from the monitor. The monitor provides us with "info" messages that explain how the function arguments will come through later on. This class remembers these info @@ -76,15 +76,15 @@ class BsonParser(ProtocolHandler): "x": pointer_converter_32bit, } - def init(self): - self.fd = self.handler - + def __init__(self, fd, task_id=None): + self.fd = fd self.infomap = {} self.flags_value = {} self.flags_bitmask = {} self.pid = None self.is_64bit = False self.buffer_sha1 = None + self.task_id = task_id def resolve_flags(self, apiname, argdict, flags): # Resolve 1:1 values. @@ -117,9 +117,9 @@ def resolve_flags(self, apiname, argdict, flags): flags[argument] = "|".join(flags[argument]) def determine_unserializers(self, arginfo): - """Determines which unserializers (or converters) have to be used in - order to parse the various arguments for this function call. Keeps in - mind whether the current bson is 32-bit or 64-bit.""" + """Determine which unserializers (or converters) have to be used in + order to parse the various arguments for this function call. Maintains + whether the current bson is 32-bit or 64-bit.""" argnames, converters = [], [] for argument in arginfo: @@ -205,17 +205,11 @@ def __iter__(self): if sha1 != self.buffer_sha1: log.warning("Incorrect sha1 passed along for a buffer.") - # If the parent is netlogs ResultHandler then we actually dump - # it - this should only be the case during the analysis, any - # after processing will then be ignored. - from cuckoo.core.resultserver import ResultHandler - - if isinstance(self.fd, ResultHandler): - filepath = os.path.join( - self.fd.storagepath, "buffer", self.buffer_sha1 - ) - with open(filepath, "wb") as f: - f.write(buf) + filepath = cwd( + "buffer", self.buffer_sha1, analysis=self.task_id + ) + with open(filepath, "wb") as f: + f.write(buf) continue diff --git a/cuckoo/common/objects.py b/cuckoo/common/objects.py index 11cb2c9322..f8b40d880b 100644 --- a/cuckoo/common/objects.py +++ b/cuckoo/common/objects.py @@ -16,7 +16,7 @@ import tempfile import zipfile -from cuckoo.common.whitelist import is_whitelisted_domain +from cuckoo.common.safelist import is_safelisted_domain try: import pydeep @@ -383,7 +383,7 @@ def get_urls(self): # http://stackoverflow.com/a/454589 urls, f = set(), open(self.file_path, "rb") for url in re.findall(URL_REGEX, self.mmap(f.fileno())): - if not is_whitelisted_domain(url[1]): + if not is_safelisted_domain(url[1]): urls.add("".join(url)) return list(urls) diff --git a/cuckoo/common/safelist.py b/cuckoo/common/safelist.py new file mode 100644 index 0000000000..a9eaef08b6 --- /dev/null +++ b/cuckoo/common/safelist.py @@ -0,0 +1,70 @@ +# Copyright (C) 2015-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os.path + +from cuckoo.misc import cwd + +domains = set() +ips = set() +mispdomains = set() +mispips = set() +mispurls = set() +misphashes = set() + +def _load_safelist(wlset, wl_file): + wl_path = cwd("safelist", wl_file) + + if not os.path.isfile(wl_path): + wl_path = cwd("..", "data", "safelist", wl_file, private=True) + + with open(wl_path, "rb") as fp: + safelist = fp.read() + + for entry in safelist.split("\n"): + entry = entry.strip() + if entry and not entry.startswith("#"): + wlset.add(entry) + +def is_safelisted_domain(domain): + if not domains: + # Initialize the domain safelist. + _load_safelist(domains, "domain.txt") + + return domain in domains + +def is_safelisted_ip(ip): + if not ips: + # Initialize the ip safelist. + _load_safelist(ips, "ip.txt") + + return ip in ips + +def is_safelisted_mispdomain(domain): + if not mispdomains: + # Initialize the misp domain safelist. + _load_safelist(mispdomains, "mispdomain.txt") + + return domain in mispdomains + +def is_safelisted_mispip(ip): + if not mispips: + # Initialize the misp ip safelist. + _load_safelist(mispips, "mispip.txt") + + return ip in mispips + +def is_safelisted_mispurl(url): + if not mispurls: + # Initialize the misp url safelist. + _load_safelist(mispurls, "mispurl.txt") + + return url in mispurls + +def is_safelisted_misphash(hash): + if not misphashes: + # Initialize the misp hash safelist. + _load_safelist(misphashes, "misphash.txt") + + return hash in misphashes diff --git a/cuckoo/common/scripting.py b/cuckoo/common/scripting.py index ae1a73f96b..0f12c3dd0b 100644 --- a/cuckoo/common/scripting.py +++ b/cuckoo/common/scripting.py @@ -105,7 +105,7 @@ def get_script(self): return " ".join(self.args.get("command", [])) def ps1_cmdarg(s, minimum=1): - """Creates an exactly matching PowerShell command line argument regex, + """Create an exactly matching PowerShell command line argument regex, instead of a regex that matches anything with the same characters.""" return "".join( "([%s%s^]" % (ch.lower(), ch.upper()) for ch in s diff --git a/cuckoo/common/utils.py b/cuckoo/common/utils.py index fd8e6390a9..2e9e4c8c9d 100644 --- a/cuckoo/common/utils.py +++ b/cuckoo/common/utils.py @@ -1,9 +1,8 @@ # Copyright (C) 2012-2013 Claudio Guarnieri. -# Copyright (C) 2014-2018 Cuckoo Foundation. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. -import pkg_resources import base64 import bs4 import chardet @@ -12,6 +11,7 @@ import jsbeautifier import json import logging +import operator import os import platform import re @@ -21,7 +21,7 @@ import warnings import xmlrpclib -from distutils.version import StrictVersion +from distutils.version import StrictVersion, LooseVersion from cuckoo.common.constants import GITHUB_URL, ISSUES_PAGE_URL from cuckoo.misc import cwd, version @@ -35,7 +35,7 @@ ) def convert_char(c): - """Escapes characters. + """Escape characters. @param c: dirty char. @return: sanitized char. """ @@ -75,14 +75,14 @@ def constant_time_compare(a, b): return result == 0 def validate_hash(h): - """Validates a hash by length and contents.""" + """Validate a hash by length and contents.""" if len(h) not in (32, 40, 64, 128): return False return bool(re.match("[0-9a-fA-F]*$", h)) def validate_url(url, allow_invalid=False): - """Validates an URL using Django's built-in URL validator""" + """Validate an URL using Django's built-in URL validator""" from django.core.validators import URLValidator val = URLValidator(schemes=["http", "https"]) @@ -230,7 +230,7 @@ def guid_name(guid): return GUIDS.get(guid) def exception_message(): - """Creates a message describing an unhandled exception.""" + """Create a message describing an unhandled exception.""" def get_os_release(): """Returns detailed OS release.""" if platform.linux_distribution()[0]: @@ -240,6 +240,8 @@ def get_os_release(): else: return "Unknown" + import pkg_resources + msg = ( "Oops! Cuckoo failed in an unhandled exception!\nSometimes bugs are " "already fixed in the development release, it is therefore " @@ -261,7 +263,7 @@ def get_os_release(): )) return msg -_jsbeautify_blacklist = [ +_jsbeautify_blocklist = [ "", "error: Unknown p.a.c.k.e.r. encoding.\n", ] @@ -269,7 +271,7 @@ def get_os_release(): _jsbeautify_lock = threading.Lock() def jsbeautify(javascript): - """Beautifies Javascript through jsbeautifier and ignore some messages.""" + """Beautify Javascript through jsbeautifier and ignore some messages.""" with _jsbeautify_lock: origout, sys.stdout = sys.stdout, io.StringIO() @@ -278,7 +280,7 @@ def jsbeautify(javascript): except Exception as e: log.exception("Unable to beautify javascript: %s", e) - if sys.stdout.getvalue() not in _jsbeautify_blacklist: + if sys.stdout.getvalue() not in _jsbeautify_blocklist: log.warning( "jsbeautifier returned error: %s", sys.stdout.getvalue() ) @@ -287,7 +289,7 @@ def jsbeautify(javascript): return javascript def htmlprettify(html): - """Beautifies HTML through BeautifulSoup4.""" + """Beautify HTML through BeautifulSoup4.""" # The following ignores the following bs4 warning: # UserWarning: "." looks like a filename, not markup. with warnings.catch_warnings(): @@ -295,7 +297,7 @@ def htmlprettify(html): return bs4.BeautifulSoup(html, "html.parser").prettify() def json_default(obj): - """JSON serializer for objects not serializable by default json code""" + """JSON serialize objects not serializable by default json code""" if hasattr(obj, "to_dict"): return obj.to_dict() @@ -328,7 +330,7 @@ def parse_bool(value): return bool(int(value)) def supported_version(version, minimum, maximum): - """Checks if a version number is supported as per the minimum and maximum + """Check if a version number is supported as per the minimum and maximum version numbers.""" if minimum and StrictVersion(version) < StrictVersion(minimum): return False @@ -351,3 +353,16 @@ def list_of_ints(l): def list_of_strings(l): return list_of(l, basestring) + +def cmp_version(first, second, op): + op_lookup = { + ">": operator.gt, + "<": operator.lt, + ">=": operator.ge, + "<=": operator.le, + "!=": operator.ne, + "==": operator.eq + } + op = op_lookup.get(op) + + return op(LooseVersion(first), LooseVersion(second)) diff --git a/cuckoo/common/virustotal.py b/cuckoo/common/virustotal.py index d5d08cead7..8be92b5b22 100644 --- a/cuckoo/common/virustotal.py +++ b/cuckoo/common/virustotal.py @@ -29,7 +29,7 @@ class VirusTotalAPI(object): URL_SCAN = "https://www.virustotal.com/vtapi/v2/url/scan" HASH_DOWNLOAD = "https://www.virustotal.com/vtapi/v2/file/download" - VARIANT_BLACKLIST = [ + VARIANT_BLOCKLIST = [ "generic", "malware", "trojan", "agent", "win32", "multi", "w32", "trojanclicker", "trojware", "win", "a variant of win32", "trj", "susp", "dangerousobject", "backdoor", "clicker", "variant", "heur", @@ -198,7 +198,7 @@ def normalize(self, variant): if len(word) < 4: continue - if word.lower() in self.VARIANT_BLACKLIST: + if word.lower() in self.VARIANT_BLOCKLIST: continue # Random hashes that are specific to this file. diff --git a/cuckoo/common/whitelist.py b/cuckoo/common/whitelist.py deleted file mode 100644 index 3ecba46f4b..0000000000 --- a/cuckoo/common/whitelist.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (C) 2015-2019 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os.path - -from cuckoo.misc import cwd - -domains = set() -ips = set() - -def _load_whitelist(wlset, wl_file): - for b in (True, False): - private = {"private": True} if b else {} - wl_path = cwd("whitelist", wl_file, **private) - if not os.path.isfile(wl_path): - continue - - with open(wl_path, "rb") as fp: - whitelist = fp.read() - - for entry in whitelist.split("\n"): - entry = entry.strip() - if entry and not entry.startswith("#"): - wlset.add(entry) - -def is_whitelisted_domain(domain): - if not domains: - # Initialize the domain whitelist. - _load_whitelist(domains, "domain.txt") - - return domain in domains - -def is_whitelisted_ip(ip): - if not ips: - # Initialize the ip whitelist. - _load_whitelist(ips, "ip.txt") - - return ip in ips diff --git a/cuckoo/compat/config.py b/cuckoo/compat/config.py index b690a22bf2..8e95e1f18e 100644 --- a/cuckoo/compat/config.py +++ b/cuckoo/compat/config.py @@ -481,7 +481,6 @@ def _20c1_20c2(c): } c["cuckoo"]["routing"]["rt_table"] = "main" c["cuckoo"]["routing"]["auto_rt"] = True - c["cuckoo"]["resultserver"]["force_port"] = False if c["cuckoo"]["timeouts"]["critical"] == 600: c["cuckoo"]["timeouts"]["critical"] = 60 c["processing"]["misp"] = { @@ -490,7 +489,7 @@ def _20c1_20c2(c): "apikey": None, "maxioc": 100, } - c["processing"]["network"]["whitelist-dns"] = False + c["processing"]["network"]["safelist-dns"] = False c["processing"]["network"]["allowed-dns"] = None c["processing"]["procmemory"]["extract_img"] = True c["processing"]["procmemory"]["dump_delete"] = False @@ -555,9 +554,9 @@ def _20c2_200(c): "company": None, "email": None, } - c["processing"]["network"]["whitelist_dns"] = cast( - "processing:network:whitelist_dns", - c["processing"]["network"].pop("whitelist-dns", None) + c["processing"]["network"]["safelist_dns"] = cast( + "processing:network:safelist_dns", + c["processing"]["network"].pop("safelist-dns", None) ) c["processing"]["network"]["allowed_dns"] = cast( "processing:network:allowed_dns", @@ -711,7 +710,7 @@ def _205_206(c): c["virtualbox"]["controlports"] = "5000-5050" return c -def _206_210(c): +def _206_207(c): c["auxiliary"]["replay"]["certificate"] = "bin/cert.p12" # We'd like to provide a secure default, but let's not inconvenience # upgrading users. TODO Might need to revisited once we write back config. @@ -719,6 +718,13 @@ def _206_210(c): c["cuckoo"]["cuckoo"]["web_secret"] = None c["kvm"]["kvm"]["dsn"] = "qemu:///system" c["processing"]["irma"]["probes"] = None + c["reporting"]["misp"]["distribution"] = 0 + c["reporting"]["misp"]["analysis"] = 0 + c["reporting"]["misp"]["threat_level"] = 4 + c["reporting"]["misp"]["min_malscore"] = 0 + c["reporting"]["misp"]["tag"] = "Cuckoo" + c["reporting"]["misp"]["upload_sample"] = False + return c migrations = { @@ -739,7 +745,7 @@ def _206_210(c): "2.0.3": ("2.0.4", _203_204), "2.0.4": ("2.0.5", _204_205), "2.0.5": ("2.0.6", _205_206), - "2.0.6": ("2.1.0", _206_210), + "2.0.6": ("2.0.7", _206_207), # We're also capable of migrating away from 2.0-dev which basically means # that we might have to a partial migration from either 2.0-rc2 or 2.0-rc1. diff --git a/cuckoo/core/database.py b/cuckoo/core/database.py index 07ef8c4524..d4994fb9f5 100644 --- a/cuckoo/core/database.py +++ b/cuckoo/core/database.py @@ -1,5 +1,5 @@ # Copyright (C) 2012-2013 Claudio Guarnieri. -# Copyright (C) 2014-2018 Cuckoo Foundation. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -16,7 +16,7 @@ from cuckoo.common.exceptions import CuckooOperationalError from cuckoo.common.exceptions import CuckooDependencyError from cuckoo.common.objects import File, URL, Dictionary -from cuckoo.common.utils import Singleton, classlock, json_encode +from cuckoo.common.utils import Singleton, classlock, json_encode, parse_bool from cuckoo.misc import cwd, format_command from sqlalchemy import create_engine, Column, not_, func @@ -120,7 +120,7 @@ def rcparams(self, value): self._rcparams = value def to_dict(self): - """Converts object to dict. + """Convert object to dict. @return: dict """ d = {} @@ -136,7 +136,7 @@ def to_dict(self): return d def to_json(self): - """Converts object to JSON. + """Convert object to JSON. @return: JSON data """ return json.dumps(self.to_dict()) @@ -198,7 +198,7 @@ def __repr__(self): return "".format(self.id, self.name) def to_dict(self): - """Converts object to dict. + """Convert object to dict. @return: dict """ d = {} @@ -211,7 +211,7 @@ def to_dict(self): return d def to_json(self): - """Converts object to JSON. + """Convert object to JSON. @return: JSON data """ return json.dumps(self.to_dict()) @@ -256,7 +256,7 @@ def __repr__(self): return "".format(self.id, self.sha256) def to_dict(self): - """Converts object to dict. + """Convert object to dict. @return: dict """ d = {} @@ -265,7 +265,7 @@ def to_dict(self): return d def to_json(self): - """Converts object to JSON. + """Convert object to JSON. @return: JSON data """ return json.dumps(self.to_dict()) @@ -291,7 +291,7 @@ class Error(Base): task_id = Column(Integer, ForeignKey("tasks.id"), nullable=False) def to_dict(self): - """Converts object to dict. + """Convert object to dict. @return: dict """ d = {} @@ -300,7 +300,7 @@ def to_dict(self): return d def to_json(self): - """Converts object to JSON. + """Convert object to JSON. @return: JSON data """ return json.dumps(self.to_dict()) @@ -371,7 +371,7 @@ def options(self, value): self._options = value def to_dict(self, dt=False): - """Converts object to dict. + """Convert object to dict. @param dt: encode datetime objects @return: dict """ @@ -398,7 +398,7 @@ def to_dict(self, dt=False): return d def to_json(self): - """Converts object to JSON. + """Convert object to JSON. @return: JSON data """ return json_encode(self.to_dict()) @@ -465,7 +465,7 @@ def connect(self, schema_check=None, dsn=None, create=True): self._create_tables() def _create_tables(self): - """Creates all the database tables etc.""" + """Create all database tables etc.""" try: Base.metadata.create_all(self.engine) except SQLAlchemyError as e: @@ -509,7 +509,7 @@ def __del__(self): self.engine.dispose() def _connect_database(self, connection_string): - """Connect to a Database. + """Connect to a database. @param connection_string: Connection string specifying the database """ try: @@ -576,7 +576,7 @@ def clean_machines(self): session.query(Machine).delete() session.commit() except SQLAlchemyError as e: - log.debug("Database error cleaning machines: {0}".format(e)) + log.exception("Database error cleaning machines: {0}".format(e)) session.rollback() finally: session.close() @@ -622,7 +622,7 @@ def add_machine(self, name, label, ip, platform, options, tags, interface, try: session.commit() except SQLAlchemyError as e: - log.debug("Database error adding machine: {0}".format(e)) + log.exception("Database error adding machine: {0}".format(e)) session.rollback() finally: session.close() @@ -649,7 +649,7 @@ def set_status(self, task_id, status): session.commit() except SQLAlchemyError as e: - log.debug("Database error setting status: {0}".format(e)) + log.exception("Database error setting status: {0}".format(e)) session.rollback() finally: session.close() @@ -670,14 +670,14 @@ def set_route(self, task_id, route): row.route = route session.commit() except SQLAlchemyError as e: - log.debug("Database error setting route: {0}".format(e)) + log.exception("Database error setting route: {0}".format(e)) session.rollback() finally: session.close() @classlock def fetch(self, machine=None, service=True): - """Fetches a task waiting to be processed and locks it for running. + """Fetch a task waiting to be processed and lock it for running. @return: None or task """ session = self.Session() @@ -697,14 +697,14 @@ def fetch(self, machine=None, service=True): return row except SQLAlchemyError as e: - log.debug("Database error fetching task: {0}".format(e)) + log.exception("Database error fetching task: {0}".format(e)) session.rollback() finally: session.close() @classlock def guest_start(self, task_id, name, label, manager): - """Logs guest start. + """Log guest start. @param task_id: task identifier @param name: vm name @param label: vm label @@ -720,7 +720,7 @@ def guest_start(self, task_id, name, label, manager): session.refresh(guest) return guest.id except SQLAlchemyError as e: - log.debug("Database error logging guest start: {0}".format(e)) + log.exception("Database error logging guest start: {0}".format(e)) session.rollback() return None finally: @@ -728,7 +728,7 @@ def guest_start(self, task_id, name, label, manager): @classlock def guest_get_status(self, task_id): - """Logs guest start. + """Log guest start. @param task_id: task id @return: guest status """ @@ -737,7 +737,7 @@ def guest_get_status(self, task_id): guest = session.query(Guest).filter_by(task_id=task_id).first() return guest.status if guest else None except SQLAlchemyError as e: - log.debug("Database error logging guest start: {0}".format(e)) + log.exception("Database error logging guest start: {0}".format(e)) session.rollback() return finally: @@ -745,7 +745,7 @@ def guest_get_status(self, task_id): @classlock def guest_set_status(self, task_id, status): - """Logs guest start. + """Log guest start. @param task_id: task identifier @param status: status """ @@ -756,7 +756,7 @@ def guest_set_status(self, task_id, status): session.commit() session.refresh(guest) except SQLAlchemyError as e: - log.debug("Database error logging guest start: {0}".format(e)) + log.exception("Database error logging guest start: {0}".format(e)) session.rollback() return None finally: @@ -764,14 +764,14 @@ def guest_set_status(self, task_id, status): @classlock def guest_remove(self, guest_id): - """Removes a guest start entry.""" + """Remove a guest start entry.""" session = self.Session() try: guest = session.query(Guest).get(guest_id) session.delete(guest) session.commit() except SQLAlchemyError as e: - log.debug("Database error logging guest remove: {0}".format(e)) + log.exception("Database error logging guest remove: {0}".format(e)) session.rollback() return None finally: @@ -779,7 +779,7 @@ def guest_remove(self, guest_id): @classlock def guest_stop(self, guest_id): - """Logs guest stop. + """Log guest stop. @param guest_id: guest log entry id """ session = self.Session() @@ -789,7 +789,7 @@ def guest_stop(self, guest_id): guest.shutdown_on = datetime.datetime.now() session.commit() except SQLAlchemyError as e: - log.debug("Database error logging guest stop: {0}".format(e)) + log.exception("Database error logging guest stop: {0}".format(e)) session.rollback() except TypeError: log.warning("Data inconsistency in guests table detected, it might be a crash leftover. Continue") @@ -799,7 +799,7 @@ def guest_stop(self, guest_id): @classlock def list_machines(self, locked=False): - """Lists virtual machines. + """List virtual machines. @return: list of virtual machines """ session = self.Session() @@ -810,14 +810,14 @@ def list_machines(self, locked=False): machines = session.query(Machine).options(joinedload("tags")).all() return machines except SQLAlchemyError as e: - log.debug("Database error listing machines: {0}".format(e)) + log.exception("Database error listing machines: {0}".format(e)) return [] finally: session.close() @classlock def lock_machine(self, label=None, platform=None, tags=None): - """Places a lock on a free virtual machine. + """Place a lock on a free virtual machine. @param label: optional virtual machine label @param platform: optional virtual machine platform @param tags: optional tags required (list) @@ -853,7 +853,7 @@ def lock_machine(self, label=None, platform=None, tags=None): # Get the first free machine. machine = machines.filter_by(locked=False).first() except SQLAlchemyError as e: - log.debug("Database error locking machine: {0}".format(e)) + log.exception("Database error locking machine: {0}".format(e)) session.close() return None @@ -864,7 +864,7 @@ def lock_machine(self, label=None, platform=None, tags=None): session.commit() session.refresh(machine) except SQLAlchemyError as e: - log.debug("Database error locking machine: {0}".format(e)) + log.exception("Database error locking machine: {0}".format(e)) session.rollback() return None finally: @@ -876,7 +876,7 @@ def lock_machine(self, label=None, platform=None, tags=None): @classlock def unlock_machine(self, label): - """Remove lock form a virtual machine. + """Remove a lock from a virtual machine. @param label: virtual machine label @return: unlocked machine """ @@ -884,7 +884,7 @@ def unlock_machine(self, label): try: machine = session.query(Machine).filter_by(label=label).first() except SQLAlchemyError as e: - log.debug("Database error unlocking machine: {0}".format(e)) + log.exception("Database error unlocking machine: {0}".format(e)) session.close() return None @@ -895,7 +895,7 @@ def unlock_machine(self, label): session.commit() session.refresh(machine) except SQLAlchemyError as e: - log.debug("Database error locking machine: {0}".format(e)) + log.exception("Database error locking machine: {0}".format(e)) session.rollback() return None finally: @@ -905,7 +905,7 @@ def unlock_machine(self, label): @classlock def count_machines_available(self): - """How many virtual machines are ready for analysis. + """Return number of virtual machines ready for analysis. @return: free virtual machines count """ session = self.Session() @@ -913,14 +913,14 @@ def count_machines_available(self): machines_count = session.query(Machine).filter_by(locked=False).count() return machines_count except SQLAlchemyError as e: - log.debug("Database error counting machines: {0}".format(e)) + log.exception("Database error counting machines: {0}".format(e)) return 0 finally: session.close() @classlock def get_available_machines(self): - """ Which machines are available + """Return machines that are available. @return: free virtual machines """ session = self.Session() @@ -928,7 +928,9 @@ def get_available_machines(self): machines = session.query(Machine).options(joinedload("tags")).filter_by(locked=False).all() return machines except SQLAlchemyError as e: - log.debug("Database error getting available machines: {0}".format(e)) + log.exception( + "Database error getting available machines: {0}".format(e) + ) return [] finally: session.close() @@ -943,7 +945,9 @@ def set_machine_status(self, label, status): try: machine = session.query(Machine).filter_by(label=label).first() except SQLAlchemyError as e: - log.debug("Database error setting machine status: {0}".format(e)) + log.exception( + "Database error setting machine status: {0}".format(e) + ) session.close() return @@ -954,7 +958,7 @@ def set_machine_status(self, label, status): session.commit() session.refresh(machine) except SQLAlchemyError as e: - log.debug("Database error setting machine status: %s", e) + log.exception("Database error setting machine status: %s", e) session.rollback() finally: session.close() @@ -971,7 +975,7 @@ def set_machine_rcparams(self, label, rcparams): try: machine = session.query(Machine).filter_by(label=label).first() except SQLAlchemyError as e: - log.debug("Database error setting machine rcparams: %s", e) + log.exception("Database error setting machine rcparams: %s", e) session.close() return @@ -981,7 +985,7 @@ def set_machine_rcparams(self, label, rcparams): session.commit() session.refresh(machine) except SQLAlchemyError as e: - log.debug("Database error setting machine rcparams: %s", e) + log.exception("Database error setting machine rcparams: %s", e) session.rollback() finally: session.close() @@ -1000,7 +1004,7 @@ def add_error(self, message, task_id, action=None): try: session.commit() except SQLAlchemyError as e: - log.debug("Database error adding error log: {0}".format(e)) + log.exception("Database error adding error log: {0}".format(e)) session.rollback() finally: session.close() @@ -1036,6 +1040,16 @@ def add(self, obj, timeout=0, package="", options="", priority=1, if not priority: priority = 1 + try: + memory = parse_bool(memory) + except ValueError: + memory = False + + try: + enforce_timeout = parse_bool(enforce_timeout) + except ValueError: + enforce_timeout = False + if isinstance(obj, File): sample = Sample(md5=obj.get_md5(), crc32=obj.get_crc32(), @@ -1054,11 +1068,13 @@ def add(self, obj, timeout=0, package="", options="", priority=1, try: sample = session.query(Sample).filter_by(md5=obj.get_md5()).first() except SQLAlchemyError as e: - log.debug("Error querying sample for hash: {0}".format(e)) + log.exception( + "Error querying sample for hash: {0}".format(e) + ) session.close() return None except SQLAlchemyError as e: - log.debug("Database error adding task: {0}".format(e)) + log.exception("Database error adding task: {0}".format(e)) session.close() return None @@ -1113,7 +1129,7 @@ def add(self, obj, timeout=0, package="", options="", priority=1, session.commit() task_id = task.id except SQLAlchemyError as e: - log.debug("Database error adding task: {0}".format(e)) + log.exception("Database error adding task: {0}".format(e)) session.rollback() return None finally: @@ -1281,7 +1297,7 @@ def add_submit(self, tmp_path, submit_type, data): session.refresh(submit) submit_id = submit.id except SQLAlchemyError as e: - log.debug("Database error adding submit entry: %s", e) + log.exception("Database error adding submit entry: %s", e) session.rollback() finally: session.close() @@ -1296,7 +1312,7 @@ def view_submit(self, submit_id, tasks=False): q = q.options(joinedload("tasks")) submit = q.get(submit_id) except SQLAlchemyError as e: - log.debug("Database error viewing submit: %s", e) + log.exception("Database error viewing submit: %s", e) return finally: session.close() @@ -1325,7 +1341,7 @@ def reschedule(self, task_id, priority=None): try: session.commit() except SQLAlchemyError as e: - log.debug("Database error rescheduling task: {0}".format(e)) + log.exception("Database error rescheduling task: {0}".format(e)) session.rollback() return False finally: @@ -1390,13 +1406,13 @@ def list_tasks(self, limit=None, details=True, category=None, owner=None, tasks = search.limit(limit).offset(offset).all() return tasks except SQLAlchemyError as e: - log.debug("Database error listing tasks: {0}".format(e)) + log.exception("Database error listing tasks: {0}".format(e)) return [] finally: session.close() def minmax_tasks(self): - """Find tasks minimum and maximum + """Find tasks minimum and maximum. @return: unix timestamps of minimum and maximum """ session = self.Session() @@ -1409,14 +1425,14 @@ def minmax_tasks(self): return int(_min[0].strftime("%s")), int(_max[0].strftime("%s")) except SQLAlchemyError as e: - log.debug("Database error counting tasks: {0}".format(e)) + log.exception("Database error counting tasks: {0}".format(e)) return finally: session.close() @classlock def count_tasks(self, status=None): - """Count tasks in the database + """Count tasks in the database. @param status: apply a filter according to the task status @return: number of tasks found """ @@ -1428,7 +1444,7 @@ def count_tasks(self, status=None): tasks_count = session.query(Task).count() return tasks_count except SQLAlchemyError as e: - log.debug("Database error counting tasks: {0}".format(e)) + log.exception("Database error counting tasks: {0}".format(e)) return 0 finally: session.close() @@ -1450,7 +1466,7 @@ def view_task(self, task_id, details=True): else: task = session.query(Task).get(task_id) except SQLAlchemyError as e: - log.debug("Database error viewing task: {0}".format(e)) + log.exception("Database error viewing task: {0}".format(e)) return None else: if task: @@ -1473,7 +1489,7 @@ def view_tasks(self, task_ids): joinedload("tags") ).filter(Task.id.in_(task_ids)).order_by(Task.id).all() except SQLAlchemyError as e: - log.debug("Database error viewing tasks: {0}".format(e)) + log.exception("Database error viewing tasks: {0}".format(e)) return [] else: for task in tasks: @@ -1494,7 +1510,7 @@ def delete_task(self, task_id): session.delete(task) session.commit() except SQLAlchemyError as e: - log.debug("Database error deleting task: {0}".format(e)) + log.exception("Database error deleting task: {0}".format(e)) session.rollback() return False finally: @@ -1503,7 +1519,7 @@ def delete_task(self, task_id): @classlock def view_sample(self, sample_id): - """Retrieve information on a sample given a sample id. + """Retrieve information on a sample given a sample ID. @param sample_id: ID of the sample to query. @return: details on the sample used in sample: sample_id. """ @@ -1513,7 +1529,7 @@ def view_sample(self, sample_id): except AttributeError: return None except SQLAlchemyError as e: - log.debug("Database error viewing task: {0}".format(e)) + log.exception("Database error viewing task: {0}".format(e)) return None else: if sample: @@ -1536,7 +1552,7 @@ def find_sample(self, md5=None, sha256=None): elif sha256: sample = session.query(Sample).filter_by(sha256=sha256).first() except SQLAlchemyError as e: - log.debug("Database error searching sample: {0}".format(e)) + log.exception("Database error searching sample: {0}".format(e)) return None else: if sample: @@ -1547,12 +1563,12 @@ def find_sample(self, md5=None, sha256=None): @classlock def count_samples(self): - """Counts the amount of samples in the database.""" + """Count number of samples in the database.""" session = self.Session() try: sample_count = session.query(Sample).count() except SQLAlchemyError as e: - log.debug("Database error counting samples: {0}".format(e)) + log.exception("Database error counting samples: {0}".format(e)) return 0 finally: session.close() @@ -1568,7 +1584,7 @@ def view_machine(self, name): try: machine = session.query(Machine).options(joinedload("tags")).filter_by(name=name).first() except SQLAlchemyError as e: - log.debug("Database error viewing machine: {0}".format(e)) + log.exception("Database error viewing machine: {0}".format(e)) return None else: if machine: @@ -1587,7 +1603,9 @@ def view_machine_by_label(self, label): try: machine = session.query(Machine).options(joinedload("tags")).filter_by(label=label).first() except SQLAlchemyError as e: - log.debug("Database error viewing machine by label: {0}".format(e)) + log.exception( + "Database error viewing machine by label: {0}".format(e) + ) return None else: if machine: @@ -1607,7 +1625,7 @@ def view_errors(self, task_id): q = session.query(Error).filter_by(task_id=task_id) errors = q.order_by(Error.id).all() except SQLAlchemyError as e: - log.debug("Database error viewing errors: {0}".format(e)) + log.exception("Database error viewing errors: {0}".format(e)) return [] finally: session.close() @@ -1646,6 +1664,6 @@ def processing_get_task(self, instance): if task.processing == instance: return task.id except SQLAlchemyError as e: - log.debug("Database error getting new processing tasks: %s", e) + log.exception("Database error getting new processing tasks: %s", e) finally: session.close() diff --git a/cuckoo/core/feedback.py b/cuckoo/core/feedback.py index 2735fc7d19..758f62c8fb 100644 --- a/cuckoo/core/feedback.py +++ b/cuckoo/core/feedback.py @@ -18,9 +18,9 @@ log = logging.getLogger(__name__) class CuckooFeedback(object): - """Contacts Cuckoo HQ with feedback & optional analysis dump.""" + """Contact Cuckoo HQ with feedback & optional analysis dump.""" endpoint = "https://feedback.cuckoosandbox.org/api/submit/" - exc_whitelist = ( + exc_safelist = ( CuckooFeedbackError, ) @@ -41,8 +41,8 @@ def send_exception(self, exception, request): automated=True, message="Exception encountered: %s" % exception ) - if isinstance(exception, self.exc_whitelist): - log.debug("A whitelisted exception occurred: %s", exception) + if isinstance(exception, self.exc_safelist): + log.debug("A safelisted exception occurred: %s", exception) return # Ignore 404 exceptions regarding ".map" development files. @@ -200,7 +200,7 @@ def include_report_web(self, task_id): return self.include_report(report) def gather_export_files(self, dirpath): - """Returns a list of all files of interest from an analysis.""" + """Return a list of all files of interest from an analysis.""" ret = [] for name in self.export_files: if isinstance(name, basestring): diff --git a/cuckoo/core/guest.py b/cuckoo/core/guest.py index 2803f93188..a75f5a3950 100644 --- a/cuckoo/core/guest.py +++ b/cuckoo/core/guest.py @@ -30,7 +30,7 @@ db = Database() def analyzer_zipfile(platform, monitor): - """Creates the Zip file that is sent to the Guest.""" + """Create the zip file that is sent to the Guest.""" t = time.time() zip_data = io.BytesIO() @@ -106,6 +106,7 @@ def __init__(self, vm_id, ip, platform, task_id): # TODO, pull options parameter into __init__ so we can do this here self.timeout = None self.server = None + self.do_run = True def wait(self, status): """Waiting for status. @@ -117,7 +118,7 @@ def wait(self, status): end = time.time() + self.timeout self.server._set_timeout(self.timeout) - while db.guest_get_status(self.task_id) == "starting": + while db.guest_get_status(self.task_id) == "starting" and self.do_run: # Check if we've passed the timeout. if time.time() > end: raise CuckooGuestCriticalTimeout( @@ -179,6 +180,8 @@ def start_analysis(self, options, monitor): # availability of the agent and verify that it's ready to receive # data. self.wait(CUCKOO_GUEST_INIT) + if not self.do_run: + return # Invoke the upload of the analyzer to the guest. self.upload_analyzer(monitor) @@ -233,7 +236,7 @@ def wait_for_completion(self): end = time.time() + self.timeout self.server._set_timeout(self.timeout) - while db.guest_get_status(self.task_id) == "running": + while db.guest_get_status(self.task_id) == "running" and self.do_run: time.sleep(1) # If the analysis hits the critical timeout, just return straight @@ -286,11 +289,17 @@ def __init__(self, vmid, ipaddr, platform, task_id, analysis_manager): self.environ = {} self.options = {} + self.do_run = True @property def aux(self): return self.analysis_manager.aux + def stop(self): + self.do_run = False + if self.is_old: + self.old.do_run = False + def get(self, method, *args, **kwargs): """Simple wrapper around requests.get().""" do_raise = kwargs.pop("do_raise", True) @@ -304,7 +313,7 @@ def get(self, method, *args, **kwargs): except requests.ConnectionError: raise CuckooGuestError( "Cuckoo Agent failed without error status, please try " - "upgrading to the latest version of agent.py (>= 0.8) and " + "upgrading to the latest version of agent.py (>= 0.10) and " "notify us if the issue persists." ) @@ -323,7 +332,7 @@ def post(self, method, *args, **kwargs): except requests.ConnectionError: raise CuckooGuestError( "Cuckoo Agent failed without error status, please try " - "upgrading to the latest version of agent.py (>= 0.8) and " + "upgrading to the latest version of agent.py (>= 0.10) and " "notify us if the issue persists." ) @@ -334,7 +343,7 @@ def wait_available(self): """Wait until the Virtual Machine is available for usage.""" end = time.time() + self.timeout - while db.guest_get_status(self.task_id) == "starting": + while db.guest_get_status(self.task_id) == "starting" and self.do_run: try: socket.create_connection((self.ipaddr, self.port), 1).close() break @@ -416,14 +425,16 @@ def start_analysis(self, options, monitor): @param options: the task options @param monitor: identifier of the monitor to be used. """ - log.info("Starting analysis on guest (id=%s, ip=%s)", - self.vmid, self.ipaddr) + log.info("Starting analysis #%s on guest (id=%s, ip=%s)", + self.task_id, self.vmid, self.ipaddr) self.options = options self.timeout = options["timeout"] + config("cuckoo:timeouts:critical") # Wait for the agent to come alive. self.wait_available() + if not self.do_run: + return # Could be beautified a bit, but basically we have to perform the # same check here as we did in wait_available(). @@ -521,11 +532,18 @@ def wait_for_completion(self): self.old.wait_for_completion() return + count = 0 end = time.time() + self.timeout - while db.guest_get_status(self.task_id) == "running": - log.debug("%s: analysis still processing", self.vmid) + while db.guest_get_status(self.task_id) == "running" and self.do_run: + if count >= 5: + log.debug( + "%s: analysis #%s still processing", self.vmid, + self.task_id + ) + count = 0 + count += 1 time.sleep(1) # If the analysis hits the critical timeout, just return straight @@ -536,11 +554,17 @@ def wait_for_completion(self): try: status = self.get("/status", timeout=5).json() + except CuckooGuestError: + # this might fail due to timeouts or just temporary network + # issues thus we don't want to abort the analysis just yet and + # wait for things to recover + log.warning( + "Virtual Machine /status failed. This can indicate the " + "guest losing network connectivity" + ) + continue except Exception as e: - log.info("Virtual Machine /status failed (%r)", e) - # this might fail due to timeouts or just temporary network issues - # thus we don't want to abort the analysis just yet and wait for things to - # recover + log.error("Virtual machine /status failed. %s", e) continue if status["status"] == "complete": @@ -548,8 +572,8 @@ def wait_for_completion(self): return elif status["status"] == "exception": log.warning( - "%s: analysis caught an exception\n%s", - self.vmid, status["description"] + "%s: analysis #%s caught an exception\n%s", + self.vmid, self.task_id, status["description"] ) return diff --git a/cuckoo/core/init.py b/cuckoo/core/init.py index 53ba3b8c8b..0fdb1b9da0 100644 --- a/cuckoo/core/init.py +++ b/cuckoo/core/init.py @@ -11,7 +11,7 @@ from cuckoo.misc import cwd def write_supervisor_conf(username): - """Writes supervisord.conf configuration file if it does not exist yet.""" + """Write supervisord.conf configuration file if it does not exist yet.""" # TODO Handle updates? if os.path.exists(cwd("supervisord.conf")): return diff --git a/cuckoo/core/log.py b/cuckoo/core/log.py index 7c3287c4db..64437844d0 100644 --- a/cuckoo/core/log.py +++ b/cuckoo/core/log.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2017 Cuckoo Foundation. +# Copyright (C) 2016-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -6,22 +6,32 @@ import json import logging import logging.handlers -import thread +import os +import threading import time +import gevent.thread + from cuckoo.common.colors import red, yellow, cyan from cuckoo.core.database import Database from cuckoo.misc import cwd +_task_threads = {} _tasks = {} _loggers = {} +_tasks_lock = threading.Lock() + # Current GMT+x. if time.localtime().tm_isdst: tz = time.altzone / -3600. else: tz = time.timezone / -3600. +# The greenlet library (used by Gevent) also creates some state per thread, +# so we can (ab)use this for both multi-threading and Gevent code +task_key = gevent.thread.get_ident + class DatabaseHandler(logging.Handler): """Logging to database handler. Used to log errors related to tasks in database. @@ -41,12 +51,11 @@ class TaskHandler(logging.Handler): """ def emit(self, record): - task_id = _tasks.get(thread.get_ident()) - if not task_id: + task = _tasks.get(task_key()) + if not task: return - with open(cwd("cuckoo.log", analysis=task_id), "a+b") as f: - f.write("%s\n" % self.format(record)) + task[1].write("%s\n" % self.format(record)) class ConsoleHandler(logging.StreamHandler): """Logging to console handler.""" @@ -72,9 +81,8 @@ class JsonFormatter(logging.Formatter): def format(self, record): action = record.__dict__.get("action") status = record.__dict__.get("status") - task_id = _tasks.get( - thread.get_ident(), record.__dict__.get("task_id") - ) + task = _tasks.get(task_key()) + task_id = task[0] if task else record.__dict__.get("task_id") d = { "action": action, "task_id": task_id, @@ -96,11 +104,39 @@ def filter(self, record): def task_log_start(task_id): """Associate a thread with a task.""" - _tasks[thread.get_ident()] = task_id + _tasks_lock.acquire() + try: + if task_id not in _task_threads: + task_path = cwd(analysis=task_id) + if not os.path.exists(task_path): + return + + _task_threads[task_id] = [] + fp = open(os.path.join(task_path, "cuckoo.log"), "a+b") + _tasks[task_key()] = (task_id, fp) + else: + existing_key = _task_threads[task_id][0] + _tasks[task_key()] = _tasks[existing_key] + + _task_threads[task_id].append(task_key()) + finally: + _tasks_lock.release() def task_log_stop(task_id): """Disassociate a thread from a task.""" - _tasks.pop(thread.get_ident(), None) + _tasks_lock.acquire() + try: + thread_key = task_key() + if thread_key not in _tasks: + return + + _, fp = _tasks.pop(thread_key) + _task_threads[task_id].remove(thread_key) + if not _task_threads[task_id]: + fp.close() + _task_threads.pop(task_id) + finally: + _tasks_lock.release() def init_logger(name, level=None): formatter = logging.Formatter( diff --git a/cuckoo/core/plugins.py b/cuckoo/core/plugins.py index 0869947c76..83496641cb 100644 --- a/cuckoo/core/plugins.py +++ b/cuckoo/core/plugins.py @@ -9,10 +9,11 @@ import logging import os import pkgutil +import sys import cuckoo -from cuckoo.common.abstracts import Configuration +from cuckoo.common.abstracts import Configuration, Signature from cuckoo.common.config import config2 from cuckoo.common.exceptions import ( CuckooConfigurationError, CuckooProcessingError, CuckooReportError, @@ -157,6 +158,7 @@ def default(*args, **kwargs): self.enabled = enabled def stop(self): + stopped = [] for module in self.enabled: try: module.stop() @@ -171,6 +173,10 @@ def stop(self): else: log.debug("Stopped auxiliary module: %s", module.__class__.__name__) + stopped.append(module) + + for s in stopped: + self.enabled.remove(s) class RunProcessing(object): """Analysis Results Processing Engine. @@ -337,9 +343,41 @@ def __init__(self, results): if self.should_enable_signature(signature): self.signatures.append(signature(self)) - # Signatures to call per API name. + # Cache of signatures to call per API name. self.api_sigs = {} + # Prebuild a list of signatures that *may* be interested + self.call_always = set() + self.call_for_api = {} + self.call_for_cat = {} + for sig in self.signatures: + # Direct dispatch per API call + for n in dir(sig): + if n.startswith("on_call_"): + self.call_for_api.setdefault(n[8:], set()).add(sig) + if not self._on_call_defined(sig): + # Not implemented... + continue + if not sig.filter_apinames and not sig.filter_categories: + self.call_always.add(sig) + continue + for api in sig.filter_apinames: + self.call_for_api.setdefault(api, set()).add(sig) + for cat in sig.filter_categories: + self.call_for_cat.setdefault(cat, set()).add(sig) + + def _on_call_defined(self, sig): + """Test if on_call is defined. This is not pretty, but it allows + on_call to be defined in `abstracts` for documentation purposes. + """ + + # In Python 3, we can just use a simple check + if sys.version_info[0] >= 3: + return sig.on_call is not Signature.on_call + + # Check where the method was defined + return sig.on_call.__func__.__module__ != Signature.on_call.__func__.__module__ + @classmethod def init_once(cls): cls.available_signatures = [] @@ -425,8 +463,6 @@ def call_signature(self, signature, handler, *args, **kwargs): signature.matched = True for sig in self.signatures: self.call_signature(sig, sig.on_signature, signature) - except NotImplementedError: - return False except: task_id = self.results.get("info", {}).get("id") log.exception( @@ -436,37 +472,27 @@ def call_signature(self, signature, handler, *args, **kwargs): ) return True - def init_api_sigs(self, apiname, category): - """Initialize a list of signatures for which we should trigger its - on_call method for this particular API name and category.""" - self.api_sigs[apiname] = [] - - for sig in self.signatures: - if sig.filter_apinames and apiname not in sig.filter_apinames: - continue - - if sig.filter_categories and category not in sig.filter_categories: - continue - - self.api_sigs[apiname].append(sig) - def yield_calls(self, proc): """Yield calls of interest to each interested signature.""" for idx, call in enumerate(proc.get("calls", [])): - - # Initialize a list of signatures to call for this API call. - if call["api"] not in self.api_sigs: - self.init_api_sigs(call["api"], call.get("category")) - - # See the following SO answer on why we're using reversed() here. - # http://stackoverflow.com/a/10665800 - for sig in reversed(self.api_sigs[call["api"]]): + api = call.get("api") + sigs = self.api_sigs.get(api) + if sigs is None: + # Build interested signatures + cat = call.get("category") + sigs = self.call_always.union( + self.call_for_api.get(api, set()), + self.call_for_cat.get(cat, set()) + ) + self.api_sigs[api] = sigs + name = "on_call_" + api + for sig in sigs: sig.cid, sig.call = idx, call - if self.call_signature(sig, sig.on_call, call, proc) is False: - self.api_sigs[call["api"]].remove(sig) + func = getattr(sig, name, sig.on_call) + self.call_signature(sig, func, call, proc) def process_yara_matches(self): - """Yields any Yara matches to each signature.""" + """Yield any Yara matches to each signature.""" def loop_yara(category, filepath, matches): for match in matches: match = YaraMatch(match, category) @@ -650,7 +676,7 @@ def process(self, module): ) def run(self): - """Generates all reports. + """Generate all reports. @raise CuckooReportError: if a report module fails. """ # In every reporting module you can specify a numeric value that diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py index a76533a205..e307124394 100644 --- a/cuckoo/core/resultserver.py +++ b/cuckoo/core/resultserver.py @@ -1,460 +1,425 @@ # Copyright (C) 2012-2013 Claudio Guarnieri. -# Copyright (C) 2014-2018 Cuckoo Foundation. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. +from __future__ import print_function + import errno +import gevent.pool +import gevent.server +import gevent.socket import json +import logging import os import socket -import select -import logging -import datetime -import SocketServer import threading from cuckoo.common.abstracts import ProtocolHandler from cuckoo.common.config import config -from cuckoo.common.exceptions import CuckooOperationalError from cuckoo.common.exceptions import CuckooCriticalError -from cuckoo.common.exceptions import CuckooResultError -from cuckoo.common.files import Folders -from cuckoo.common.netlog import BsonParser +from cuckoo.common.exceptions import CuckooOperationalError +from cuckoo.common.files import open_exclusive from cuckoo.common.utils import Singleton from cuckoo.core.log import task_log_start, task_log_stop from cuckoo.misc import cwd log = logging.getLogger(__name__) -BUFSIZE = 1024 * 1024 - -class Disconnect(Exception): - pass - -class ResultServer(SocketServer.ThreadingTCPServer, object): - """Result server. Singleton! - - This class handles results coming back from the analysis machines. - """ - - __metaclass__ = Singleton - - allow_reuse_address = True - daemon_threads = True - - def __init__(self, *args, **kwargs): - self.analysistasks = {} - self.analysishandlers = {} - - self.ip = config("cuckoo:resultserver:ip") - self.port = config("cuckoo:resultserver:port") - while True: - try: - server_addr = self.ip, self.port - SocketServer.ThreadingTCPServer.__init__( - self, server_addr, ResultHandler, *args, **kwargs - ) - except Exception as e: - if e.errno == errno.EADDRINUSE: - if config("cuckoo:resultserver:force_port"): - raise CuckooCriticalError( - "Cannot bind ResultServer on port %d, " - "bailing." % self.port - ) - else: - log.warning("Cannot bind ResultServer on port %s, " - "trying another port.", self.port) - self.port += 1 - elif e.errno == errno.EADDRNOTAVAIL: - raise CuckooCriticalError( - "Unable to bind ResultServer on %s:%s %s. This " - "usually happens when you start Cuckoo without " - "bringing up the virtual interface associated with " - "the ResultServer IP address. Please refer to " - "https://cuckoo.sh/docs/faq/#troubles-problem" - " for more information." % (self.ip, self.port, e) - ) - else: - raise CuckooCriticalError( - "Unable to bind ResultServer on %s:%s: %s" % - (self.ip, self.port, e) - ) - else: - log.debug( - "ResultServer running on %s:%s.", self.ip, self.port - ) - self.servethread = threading.Thread(target=self.serve_forever) - self.servethread.setDaemon(True) - self.servethread.start() - break - - def serve_forever(self, poll_interval=0.5): +# Maximum line length to read for netlog messages, to avoid memory exhaustion +MAX_NETLOG_LINE = 4 * 1024 + +# Maximum number of bytes to buffer for a single connection +BUFSIZE = 16 * 1024 + +# Directories in which analysis-related files will be stored; also acts as +# safelist +RESULT_UPLOADABLE = ("files", "shots", "buffer", "extracted", "memory") +RESULT_DIRECTORIES = RESULT_UPLOADABLE + ("reports", "logs") + +# Prevent malicious clients from using potentially dangerous filenames +# E.g. C API confusion by using null, or using the colon on NTFS (Alternate +# Data Streams); XXX: just replace illegal chars? +BANNED_PATH_CHARS = b'\x00:' + +def netlog_sanitize_fname(path): + """Validate agent-provided path for result files""" + path = path.replace("\\", "/") + dir_part, name = os.path.split(path) + if dir_part not in RESULT_UPLOADABLE: + raise CuckooOperationalError("Netlog client requested banned path: %r" + % path) + if any(c in BANNED_PATH_CHARS for c in name): + for c in BANNED_PATH_CHARS: + path = path.replace(c, "X") + + return path + +class HandlerContext(object): + """Holds context for protocol handlers. + + Can safely be cancelled from another thread, though in practice this will + not occur often -- usually the connection between VM and the ResultServer + will be reset during shutdown.""" + def __init__(self, task_id, storagepath, sock): + self.task_id = task_id + self.command = None + + # The path where artifacts will be stored + self.storagepath = storagepath + self.sock = sock + self.buf = "" + + def __repr__(self): + return "" % self.command + + def cancel(self): + """Cancel this context; gevent might complain about this with an + exception later on.""" try: - super(ResultServer, self).serve_forever(poll_interval) - except AttributeError as e: - if "NoneType" not in e.message or "select" not in e.message: - raise - - def add_task(self, task, machine): - """Register a task/machine with the ResultServer.""" - self.analysistasks[machine.ip] = task, machine - self.analysishandlers[task.id] = [] - - def del_task(self, task, machine): - """Delete ResultServer state and wait for pending RequestHandlers.""" - x = self.analysistasks.pop(machine.ip, None) - if not x: - log.warning("ResultServer did not have %s in its task info.", - machine.ip) - handlers = self.analysishandlers.pop(task.id, None) - for h in handlers: - h.end_request.set() - h.done_event.wait() - - def register_handler(self, handler): - """Register a RequestHandler so that we can later wait for it.""" - task, machine = self.get_ctx_for_ip(handler.client_address[0]) - if not task or not machine: - return False - - self.analysishandlers[task.id].append(handler) - - def get_ctx_for_ip(self, ip): - """Return state for this IP's task.""" - x = self.analysistasks.get(ip) - if not x: - log.debug("ResultServer unable to map ip to context: %s.", ip) - return None, None - - return x - - def build_storage_path(self, ip): - """Initialize analysis storage folder.""" - task, machine = self.get_ctx_for_ip(ip) - if not task or not machine: - return - - return cwd("storage", "analyses", "%s" % task.id) - -class ResultHandler(SocketServer.BaseRequestHandler): - """Result handler. - - This handler speaks our analysis log network protocol. - """ - - def setup(self): - self.rawlogfd = None - self.protocol = None - self.startbuf = "" - self.end_request = threading.Event() - self.done_event = threading.Event() - self.server.register_handler(self) - - if hasattr(select, "poll"): - self.poll = select.poll() - self.poll.register(self.request, select.POLLIN) - else: - self.poll = None - - def finish(self): - self.done_event.set() - - if self.protocol: - self.protocol.close() - if self.rawlogfd: - self.rawlogfd.close() - - def wait_sock_or_end(self): - while True: - if self.end_request.isSet(): - return False - - if self.poll: - if self.poll.poll(1000): - return True - else: - rs, _, _ = select.select([self.request], [], [], 1) - if rs: - return True - - def seek(self, pos): - pass - - def read(self, length): - buf = "" - while len(buf) < length: - if not self.wait_sock_or_end(): - raise Disconnect() - tmp = self.request.recv(length-len(buf)) - if not tmp: - raise Disconnect() - buf += tmp - - if isinstance(self.protocol, BsonParser): - if self.rawlogfd: - self.rawlogfd.write(buf) - else: - self.startbuf += buf - - if len(self.startbuf) > 0x10000: - raise CuckooResultError( - "Somebody is knowingly overflowing the startbuf " - "buffer, possibly to use excessive amounts of memory." - ) - - return buf - - def read_any(self): - if not self.wait_sock_or_end(): - raise Disconnect() - tmp = self.request.recv(BUFSIZE) - if not tmp: - raise Disconnect() - return tmp + self.sock.shutdown(socket.SHUT_RD) + except socket.error: + pass - def read_newline(self, strip=False): - buf = "" - while "\n" not in buf: - buf += self.read(1) + def read(self): + try: + return self.sock.recv(16384) + except socket.error as e: + if e.errno == errno.EBADF: + return "" - if strip: - buf = buf.strip() + if e.errno != errno.ECONNRESET: + raise + log.debug("Task #%s had connection reset for %r", self.task_id, + self) + return "" + def drain_buffer(self): + """Drain buffer and end buffering""" + buf, self.buf = self.buf, None return buf - def negotiate_protocol(self): - protocol = self.read_newline(strip=True) - - # Command with version number. - if " " in protocol: - command, version = protocol.split() - version = int(version) - else: - command, version = protocol, None - - if command == "BSON": - self.protocol = BsonParser(self, version) - elif command == "FILE": - self.protocol = FileUpload(self, version) - elif command == "LOG": - self.protocol = LogHandler(self, version) - else: - raise CuckooOperationalError( - "Netlog failure, unknown protocol requested." - ) - - self.protocol.init() - - def handle(self): - ip, port = self.client_address - - self.storagepath = self.server.build_storage_path(ip) - if not self.storagepath: - return - - task, _ = self.server.get_ctx_for_ip(ip) - task_log_start(task.id) - - # Create all missing folders for this analysis. - self.create_folders() - - try: - # Initialize the protocol handler class for this connection. - self.negotiate_protocol() - - for event in self.protocol: - if isinstance(self.protocol, BsonParser) and event["type"] == "process": - self.open_process_log(event) - except CuckooResultError as e: - log.warning( - "ResultServer connection stopping because of " - "CuckooResultError: %s.", e - ) - except (Disconnect, socket.error): - pass - except: - log.exception("FIXME - exception in resultserver connection %s", - self.client_address) - - task_log_stop(task.id) - - def open_process_log(self, event): - pid = event["pid"] - ppid = event["ppid"] - procname = event["process_name"] - - if self.rawlogfd: - log.debug( - "ResultServer got a new process message but already " - "has pid %d ppid %s procname %s.", pid, ppid, procname - ) - raise CuckooResultError( - "ResultServer connection state inconsistent." - ) - - if not isinstance(pid, (int, long)): - raise CuckooResultError( - "An invalid process identifier has been provided, this " - "could be a potential security hazard." - ) - - # Only report this process when we're tracking it. - if event["track"]: - log.debug( - "New process (pid=%s, ppid=%s, name=%s)", - pid, ppid, procname.encode("utf8") - ) - - filepath = os.path.join(self.storagepath, "logs", "%s.bson" % pid) - self.rawlogfd = open(filepath, "wb") - self.rawlogfd.write(self.startbuf) - - def create_folders(self): - folders = "shots", "files", "logs", "buffer", "extracted" + def read_newline(self): + """Read until the next newline character, but never more than + `MAX_NETLOG_LINE`.""" + while True: + pos = self.buf.find("\n") + if pos < 0: + if len(self.buf) >= MAX_NETLOG_LINE: + raise CuckooOperationalError("Received overly long line") + buf = self.read() + if buf == "": + raise EOFError + self.buf += buf + continue + line, self.buf = self.buf[:pos], self.buf[pos + 1:] + return line + + def copy_to_fd(self, fd, max_size=None): + if max_size: + fd = WriteLimiter(fd, max_size) + fd.write(self.drain_buffer()) + while True: + buf = self.read() + if buf == "": + break + fd.write(buf) + fd.flush() + +class WriteLimiter(object): + def __init__(self, fd, remain): + self.fd = fd + self.remain = remain + self.warned = False + + def write(self, buf): + size = len(buf) + write = min(size, self.remain) + if write: + self.fd.write(buf[:write]) + self.remain -= write + if size and size != write: + if not self.warned: + log.warning("Uploaded file length larger than upload_max_size, " + "stopping upload.") + self.fd.write("... (truncated)") + self.warned = True - try: - Folders.create(self.storagepath, folders) - except CuckooOperationalError as e: - log.error("Issue creating analyses folders: %s", e) - return False + def flush(self): + self.fd.flush() class FileUpload(ProtocolHandler): - RESTRICTED_DIRECTORIES = "reports/", - lock = threading.Lock() - def init(self): self.upload_max_size = config("cuckoo:resultserver:upload_max_size") self.storagepath = self.handler.storagepath self.fd = None - self.filelog = os.path.join(self.handler.storagepath, "files.json") - def __iter__(self): + def handle(self): # Read until newline for file path, e.g., # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin + self.handler.sock.settimeout(30) + dump_path = netlog_sanitize_fname(self.handler.read_newline()) - dump_path = self.handler.read_newline(strip=True).replace("\\", "/") - - if self.version >= 2: - filepath = self.handler.read_newline(strip=True) - pids = map(int, self.handler.read_newline(strip=True).split()) + if self.version and self.version >= 2: + # NB: filepath is only used as metadata + filepath = self.handler.read_newline() + pids = map(int, self.handler.read_newline().split()) else: filepath, pids = None, [] - log.debug("File upload request for %s", dump_path) + log.debug("Task #%s: File upload for %r", self.task_id, dump_path) + file_path = os.path.join(self.storagepath, dump_path.decode("utf-8")) - dir_part, filename = os.path.split(dump_path) + try: + self.fd = open_exclusive(file_path) + except OSError as e: + if e.errno == errno.EEXIST: + raise CuckooOperationalError("Analyzer for task #%s tried to " + "overwrite an existing file" % + self.task_id) + raise + + # Append-writes are atomic + with open(self.filelog, "a+b") as f: + print(json.dumps({ + "path": dump_path, + "filepath": filepath, + "pids": pids, + }), file=f) - if "./" in dump_path or not dir_part or dump_path.startswith("/"): - raise CuckooOperationalError( - "FileUpload failure, banned path: %s" % dump_path - ) + self.handler.sock.settimeout(None) + try: + return self.handler.copy_to_fd(self.fd, self.upload_max_size) + finally: + log.debug("Task #%s uploaded file length: %s", self.task_id, + self.fd.tell()) - for restricted in self.RESTRICTED_DIRECTORIES: - if restricted in dir_part: - raise CuckooOperationalError( - "FileUpload failure, banned path." - ) +class LogHandler(ProtocolHandler): + """The live analysis log. Can only be opened once in a single session.""" + def init(self): + self.logpath = os.path.join(self.handler.storagepath, "analysis.log") try: - Folders.create(self.storagepath, dir_part) - except CuckooOperationalError: - log.error("Unable to create folder %s", dir_part) + self.fd = open_exclusive(self.logpath, bufsize=1) + except OSError: + log.error("Task #%s: attempted to reopen live log analysis.log.", + self.task_id) return - file_path = os.path.join(self.storagepath, dump_path) + log.debug("Task #%s: live log analysis.log initialized.", + self.task_id) - if not file_path.startswith(self.storagepath): - raise CuckooOperationalError( - "FileUpload failure, path sanitization failed." - ) + def handle(self): + if self.fd: + return self.handler.copy_to_fd(self.fd) - if os.path.exists(file_path): - log.warning( - "Analyzer tried to overwrite an existing file, " - "closing connection." - ) +class BsonStore(ProtocolHandler): + def init(self): + # We cheat a little bit through the "version" variable, but that's + # acceptable and backwards compatible (for now). Backwards compatible + # in the sense that newer Cuckoo Monitor binaries work with older + # versions of Cuckoo, the other way around doesn't apply here. + if self.version is None: + log.warning("Agent is sending BSON files without PID parameter, " + "you should probably update it") + self.fd = None return - self.fd = open(file_path, "wb") - chunk = self.handler.read_any() - while chunk: - self.fd.write(chunk) + self.fd = open(os.path.join(self.handler.storagepath, + "logs", "%d.bson" % self.version), "wb") - if self.fd.tell() >= self.upload_max_size: - log.warning( - "Uploaded file length larger than upload_max_size, " - "stopping upload." - ) - self.fd.write("... (truncated)") - break + def handle(self): + """Read a BSON stream, attempting at least basic validation, and + log failures.""" + log.debug("Task #%s is sending a BSON stream", self.task_id) + if self.fd: + return self.handler.copy_to_fd(self.fd) - try: - chunk = self.handler.read_any() - except: - break +class GeventResultServerWorker(gevent.server.StreamServer): + """The new ResultServer, providing a huge performance boost as well as + implementing a new dropped file storage format avoiding small fd limits. - self.lock.acquire() + The old ResultServer would start a new thread per socket, greatly impacting + the overall performance of Cuckoo Sandbox. The new ResultServer uses + so-called Greenlets, low overhead green-threads by Gevent, imposing much + less kernel overhead. - with open(self.filelog, "a+b") as f: - f.write("%s\n" % json.dumps({ - "path": dump_path, - "filepath": filepath, - "pids": pids, - })) + Furthermore, instead of writing each dropped file to its own location (in + $CWD/storage/analyses//files/_filename.ext) it's + capable of storing all dropped files in a streamable container format. This + is one of various steps to start being able to use less fd's in Cuckoo. + """ + commands = { + "BSON": BsonStore, + "FILE": FileUpload, + "LOG": LogHandler, + } + task_mgmt_lock = threading.Lock() - self.lock.release() + def __init__(self, *args, **kwargs): + super(GeventResultServerWorker, self).__init__(*args, **kwargs) - log.debug("Uploaded file length: %s", self.fd.tell()) - return - yield + # Store IP address to task_id mapping + self.tasks = {} - def close(self): - if self.fd: - self.fd.close() + # Store running handlers for task_id + self.handlers = {} -class LogHandler(ProtocolHandler): - def init(self): - self.logpath = os.path.join(self.handler.storagepath, "analysis.log") - self.fd = self._open() - log.debug("LogHandler for live analysis.log initialized.") + def do_run(self): + self.serve_forever() - def __iter__(self): - if not self.fd: - return + def add_task(self, task_id, ipaddr): + with self.task_mgmt_lock: + self.tasks[ipaddr] = task_id + log.debug("Now tracking machine %s for task #%s", ipaddr, task_id) - while True: + def del_task(self, task_id, ipaddr): + """Delete ResultServer state and abort pending RequestHandlers. Since + we're about to shutdown the VM, any remaining open connections can + be considered a bug from the VM side, since all connections should + have been closed after the analyzer signalled completion.""" + with self.task_mgmt_lock: + if self.tasks.pop(ipaddr, None) is None: + log.warning( + "ResultServer did not have a task with ID %s and IP %s", + task_id, ipaddr + ) + else: + log.debug( + "Stopped tracking machine %s for task #%s", + ipaddr, task_id + ) + ctxs = self.handlers.pop(task_id, set()) + for ctx in ctxs: + log.debug("Cancel %s for task %r", ctx, task_id) + ctx.cancel() + + def handle(self, sock, addr): + """Handle the incoming connection. + Gevent will close the socket when the function returns.""" + ipaddr = addr[0] + + with self.task_mgmt_lock: + task_id = self.tasks.get(ipaddr) + if not task_id: + log.warning("ResultServer did not have a task for IP %s", + ipaddr) + return + + storagepath = cwd(analysis=task_id) + ctx = HandlerContext(task_id, storagepath, sock) + task_log_start(task_id) + try: try: - buf = self.handler.read_any() - except Disconnect: - break + protocol = self.negotiate_protocol(task_id, ctx) + except EOFError: + return + + # Registering the context allows us to abort the handler by + # shutting down its socket when the task is deleted; this should + # prevent lingering sockets + with self.task_mgmt_lock: + # NOTE: the task may have been cancelled during the negotation + # protocol and a different task for that IP address may have + # been registered + if self.tasks.get(ipaddr) != task_id: + log.warning("Task #%s for IP %s was cancelled during " + "negotiation", task_id, ipaddr) + return + s = self.handlers.setdefault(task_id, set()) + s.add(ctx) - if not buf: - break + try: + with protocol: + protocol.handle() + except CuckooOperationalError as e: + log.error(e) + finally: + with self.task_mgmt_lock: + s.discard(ctx) + ctx.cancel() + if ctx.buf: + # This is usually not a good sign + log.warning("Task #%s with protocol %s has unprocessed " + "data before getting disconnected", + task_id, protocol) + + finally: + task_log_stop(task_id) + + def negotiate_protocol(self, task_id, ctx): + header = ctx.read_newline() + if " " in header: + command, version = header.split() + version = int(version) + else: + command, version = header, None + klass = self.commands.get(command) + if not klass: + log.warning("Task #%s: unknown netlog protocol requested (%r), " + "terminating connection.", task_id, command) + return + ctx.command = command + return klass(task_id, ctx, version) + +class ResultServer(object): + """Manager for the ResultServer worker and task state.""" + __metaclass__ = Singleton - self.fd.write(buf) - self.fd.flush() + def __init__(self): + ip = config("cuckoo:resultserver:ip") + port = config("cuckoo:resultserver:port") + pool_size = config('cuckoo:resultserver:pool_size') - return - yield + sock = gevent.socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - def close(self): - if self.fd: - self.fd.close() + try: + sock.bind((ip, port)) + except (OSError, socket.error) as e: + if e.errno == errno.EADDRINUSE: + raise CuckooCriticalError( + "Cannot bind ResultServer on port %d " + "because it was in use, bailing." % port + ) + elif e.errno == errno.EADDRNOTAVAIL: + raise CuckooCriticalError( + "Unable to bind ResultServer on %s:%s %s. This " + "usually happens when you start Cuckoo without " + "bringing up the virtual interface associated with " + "the ResultServer IP address. Please refer to " + "https://cuckoo.sh/docs/faq/#troubles-problem " + "for more information." % (ip, port, e) + ) + else: + raise CuckooCriticalError( + "Unable to bind ResultServer on %s:%s: %s" % + (ip, port, e) + ) + + # We allow user to specify port 0 to get a random port, report it back + # here + _, self.port = sock.getsockname() + sock.listen(128) - def _open(self): - if not os.path.exists(self.logpath): - return open(self.logpath, "wb") + self.thread = threading.Thread(target=self.create_server, + args=(sock, pool_size)) + self.thread.daemon = True + self.thread.start() - log.debug("Log analysis.log already existing, appending data.") - fd = open(self.logpath, "ab") + def add_task(self, task, machine): + """Register a task/machine with the ResultServer.""" + self.instance.add_task(task.id, machine.ip) - # add a fake log entry, saying this had to be re-opened - # use the same format as the default logger, in case anyone wants to parse this - # 2015-02-23 12:05:05,092 [lib.api.process] DEBUG: Using QueueUserAPC injection. - now = datetime.datetime.now() - print >>fd, "\n%s,%03.0f [lib.core.resultserver] WARNING: This log file was re-opened, log entries will be appended." % ( - now.strftime("%Y-%m-%d %H:%M:%S"), now.microsecond / 1000.0 - ) + def del_task(self, task, machine): + """Delete running task and cancel existing handlers.""" + self.instance.del_task(task.id, machine.ip) - return fd + def create_server(self, sock, pool_size): + if pool_size: + pool = gevent.pool.Pool(pool_size) + else: + pool = 'default' + self.instance = GeventResultServerWorker(sock, spawn=pool) + self.instance.do_run() diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py index c13c02e542..edd6214e41 100644 --- a/cuckoo/core/scheduler.py +++ b/cuckoo/core/scheduler.py @@ -1,5 +1,5 @@ # Copyright (C) 2012-2013 Claudio Guarnieri. -# Copyright (C) 2014-2018 Cuckoo Foundation. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -25,7 +25,7 @@ from cuckoo.core.plugins import RunAuxiliary, RunProcessing from cuckoo.core.plugins import RunSignatures, RunReporting from cuckoo.core.log import task_log_start, task_log_stop, logger -from cuckoo.core.resultserver import ResultServer +from cuckoo.core.resultserver import ResultServer, RESULT_DIRECTORIES from cuckoo.core.rooter import rooter from cuckoo.misc import cwd @@ -37,6 +37,7 @@ active_analysis_count = 0 + class AnalysisManager(threading.Thread): """Analysis Manager. @@ -62,6 +63,9 @@ def __init__(self, task_id, error_queue): self.route = None self.interface = None self.rt_table = None + self.unrouted_network = False + self.stopped_aux = False + self.rs_port = config("cuckoo:resultserver:port") def init(self): """Initialize the analysis.""" @@ -76,8 +80,10 @@ def init(self): # If we're not able to create the analysis storage folder, we have to # abort the analysis. + # Also create all directories that the ResultServer can use for file + # uploads. try: - Folders.create(self.storage) + Folders.create(self.storage, RESULT_DIRECTORIES) except CuckooOperationalError: log.error("Unable to create analysis folder %s", self.storage) return False @@ -138,7 +144,7 @@ def init(self): return True def store_task_info(self): - """grab latest task from db (if available) and update self.task""" + """Grab latest task from db (if available) and update self.task""" dbtask = self.db.view_task(self.task.id) self.task = dbtask.to_dict() @@ -205,7 +211,7 @@ def build_options(self): options["id"] = self.task.id options["ip"] = self.machine.resultserver_ip - options["port"] = self.machine.resultserver_port + options["port"] = self.rs_port options["category"] = self.task.category options["target"] = self.task.target options["package"] = self.task.package @@ -297,7 +303,7 @@ def route_network(self): rooter( "drop_enable", self.machine.ip, config("cuckoo:resultserver:ip"), - str(config("cuckoo:resultserver:port")) + str(self.rs_port) ) if self.route == "inetsim": @@ -306,7 +312,7 @@ def route_network(self): "inetsim_enable", self.machine.ip, config("routing:inetsim:server"), config("%s:%s:interface" % (machinery, machinery)), - str(config("cuckoo:resultserver:port")), + str(self.rs_port), config("routing:inetsim:ports") or "" ) @@ -349,7 +355,7 @@ def unroute_network(self): rooter( "drop_disable", self.machine.ip, config("cuckoo:resultserver:ip"), - str(config("cuckoo:resultserver:port")) + str(self.rs_port) ) if self.route == "inetsim": @@ -358,7 +364,7 @@ def unroute_network(self): "inetsim_disable", self.machine.ip, config("routing:inetsim:server"), config("%s:%s:interface" % (machinery, machinery)), - str(config("cuckoo:resultserver:port")), + str(self.rs_port), config("routing:inetsim:ports") or "" ) @@ -370,6 +376,8 @@ def unroute_network(self): str(config("routing:tor:proxyport")) ) + self.unrouted_network = True + def wait_finish(self): """Some VMs don't have an actual agent. Mainly those that are used as assistance for an analysis through the services auxiliary module. This @@ -443,6 +451,8 @@ def launch_analysis(self): }) return False + self.rs_port = self.machine.resultserver_port or ResultServer().port + # At this point we can tell the ResultServer about it. try: ResultServer().add_task(self.task, self.machine) @@ -472,9 +482,10 @@ def launch_analysis(self): try: machinery.enable_remote_control(self.machine.label) except NotImplementedError: - raise CuckooMachineError( - "Remote control support has not been implemented " - "for this machinery." + log.error( + "Remote control support has not been implemented for the " + "configured machinery module: %s", + config("cuckoo:cuckoo:machinery") ) try: @@ -509,9 +520,10 @@ def launch_analysis(self): ) self.db.set_machine_rcparams(self.machine.label, params) except NotImplementedError: - raise CuckooMachineError( - "Remote control support has not been implemented " - "for this machinery." + log.error( + "Remote control support has not been implemented for the " + "configured machinery module: %s", + config("cuckoo:cuckoo:machinery") ) # Enable network routing. @@ -581,7 +593,9 @@ def launch_analysis(self): }) finally: # Stop Auxiliary modules. - self.aux.stop() + if not self.stopped_aux: + self.stopped_aux = True + self.aux.stop() # Take a memory dump of the machine before shutting it off. if self.cfg.cuckoo.memory_dump or self.task.memory: @@ -645,9 +659,10 @@ def launch_analysis(self): try: machinery.disable_remote_control(self.machine.label) except NotImplementedError: - raise CuckooMachineError( - "Remote control support has not been implemented " - "for this machinery." + log.error( + "Remote control support has not been implemented for the " + "configured machinery module: %s", + config("cuckoo:cuckoo:machinery") ) # Mark the machine in the database as stopped. Unless this machine @@ -660,7 +675,8 @@ def launch_analysis(self): ResultServer().del_task(self.task, self.machine) # Drop the network routing rules if any. - self.unroute_network() + if not self.unrouted_network: + self.unroute_network() try: # Release the analysis machine. But only if the machine has @@ -793,6 +809,25 @@ def run(self): task_log_stop(self.task.id) active_analysis_count -= 1 + def cleanup(self): + # In case the analysis manager crashes, the network cleanup + # should still be performed. + if not self.unrouted_network: + self.unroute_network() + + if not self.stopped_aux: + self.stopped_aux = True + self.aux.stop() + + def force_stop(self): + # Make the guest manager stop the status checking loop and return + # to the main analysis manager routine. + if self.db.guest_get_status(self.task.id): + self.db.guest_set_status(self.task.id, "stopping") + + self.guest_manager.stop() + log.debug("Force stopping task #%s", self.task.id) + class Scheduler(object): """Tasks Scheduler. @@ -809,6 +844,7 @@ def __init__(self, maxcount=None): self.db = Database() self.maxcount = maxcount self.total_analysis_count = 0 + self.analysis_managers = set() def initialize(self): """Initialize the machine manager.""" @@ -897,9 +933,38 @@ def initialize(self): def stop(self): """Stop scheduler.""" self.running = False + + # Force stop all analysis managers. + for am in self.analysis_managers: + try: + am.force_stop() + except Exception as e: + log.exception("Error force stopping analysis manager: %s", e) + # Shutdown machine manager (used to kill machines that still alive). machinery.shutdown() + # Remove network rules if any are present and stop auxiliary modules + for am in self.analysis_managers: + try: + am.cleanup() + except Exception as e: + log.exception( + "Error while cleaning up analysis manager: %s", e + ) + + def _cleanup_managers(self): + cleaned = set() + for am in self.analysis_managers: + if not am.isAlive(): + try: + am.cleanup() + except Exception as e: + log.exception("Error in analysis manager cleanup: %s", e) + + cleaned.add(am) + return cleaned + def start(self): """Start scheduler.""" self.initialize() @@ -917,6 +982,10 @@ def start(self): while self.running: time.sleep(1) + # Run cleanup on finished analysis managers and untrack them + for am in self._cleanup_managers(): + self.analysis_managers.discard(am) + # Wait until the machine lock is not locked. This is only the case # when all machines are fully running, rather that about to start # or still busy starting. This way we won't have race conditions @@ -1024,6 +1093,7 @@ def start(self): analysis = AnalysisManager(task.id, errors) analysis.daemon = True analysis.start() + self.analysis_managers.add(analysis) # Deal with errors. try: diff --git a/cuckoo/core/startup.py b/cuckoo/core/startup.py index 7cbc0bc338..f4066128b7 100644 --- a/cuckoo/core/startup.py +++ b/cuckoo/core/startup.py @@ -3,22 +3,25 @@ # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. +import errno import logging import logging.handlers import os import requests import socket +import sys import yara -from distutils.version import StrictVersion +from distutils.version import StrictVersion, LooseVersion import cuckoo -from cuckoo.common.colors import red, green, yellow +from cuckoo.common.colors import red, green, yellow, bold, color from cuckoo.common.config import Config, config, config2 from cuckoo.common.exceptions import CuckooStartupError, CuckooFeedbackError from cuckoo.common.files import temppath from cuckoo.common.objects import File +from cuckoo.common.utils import cmp_version from cuckoo.core.database import ( Database, TASK_RUNNING, TASK_FAILED_ANALYSIS, TASK_PENDING ) @@ -27,7 +30,7 @@ from cuckoo.core.log import init_logger from cuckoo.core.plugins import RunSignatures from cuckoo.core.rooter import rooter -from cuckoo.misc import cwd, version, getuser, mkdir +from cuckoo.misc import cwd, version, mkdir log = logging.getLogger(__name__) @@ -48,7 +51,7 @@ def check_specific_config(filename): ) def check_configs(): - """Checks if config files exist. + """Check if config files exist. @raise CuckooStartupError: if config files do not exist. """ configs = ( @@ -93,11 +96,15 @@ def check_configs(): ) return True -def check_version(): - """Checks version of Cuckoo.""" +def check_version(ignore_vuln=False): + """Check version of Cuckoo.""" if not config("cuckoo:cuckoo:version_check"): return + ignore_vuln = ignore_vuln or config("cuckoo:cuckoo:ignore_vulnerabilities") + + import pkg_resources + print(" Checking for updates...") try: @@ -116,6 +123,89 @@ def check_version(): except ValueError: old = True + warnings = [] + for deptype, vulns in r.get("vulnerable", {}).iteritems(): + for dep in vulns: + compare = dep.get("highest") or dep.get("lowest") + + # Check if any of the mentioned Python dependencies are installed + if deptype == "pydep": + try: + v = pkg_resources.get_distribution( + dep["name"]).parsed_version + except (pkg_resources.DistributionNotFound, ValueError): + continue + + # See if the mentioned virtualization software is used + elif deptype == "machinery": + if config("cuckoo:cuckoo:machinery") != dep["name"]: + continue + + # If the version number cannot be determined, raise a warning + # to be sure. Virtualization vulnerabilities can potentially + # cause a lot of damage + v = cuckoo.machinery.plugins[dep["name"]].version() + if not v: + warnings.append( + bold(red( + "Potentially vulnerable %s version installed. " + "Failed to retrieve its version. Update if version" + " is: %s" % (dep["name"], compare)))) + continue + + else: + continue + + warn = False + # If a range is specified, check if the current version falls + # within the range. + if dep.get("highest") and dep.get("lowest"): + lv = LooseVersion(str(v)) + if (lv >= LooseVersion(dep["lowest"]) and + lv <= LooseVersion(dep["highest"])): + warn = True + + # If no range is specified, use the specified operator to see if + # the installed version is + # 'if highest/lowest specified' + elif cmp_version(str(v), compare, dep["op"]): + warn = True + + # Warn the user the dependency must be updated/ + if warn: + info = dep.get("info") + message = "Vulnerable version of %s installed (%s). It is " \ + "highly recommended to update. Please update and " \ + "restart Cuckoo." % (dep["name"], v) + + if deptype == "pydep": + message += " 'pip install %s%s'" % ( + dep["name"], dep["recommended"] + ) + + else: + message += " Recommended version: %s" % dep["recommended"] + + message = bold(red(message)) + + if info: + message += yellow("\nAdditional information: %s" % info) + + warnings.append(message) + + if warnings: + print(color(bold(red("Vulnerable dependencies found\n")), 5)) + for warning in warnings: + print("--> %s\n" % color(warning, 4)) + + if warnings and not ignore_vuln: + print( + "This check can be disabled by enabling " + "'ignore_vulnerabilities' in cuckoo.conf under the " + "[cuckoo] section" + ) + sys.exit(1) + if old: msg = "Cuckoo Sandbox version %s is available now." % r["version"] print(red(" Outdated! ") + msg) @@ -131,14 +221,14 @@ def check_version(): return r def init_logging(level): - """Initializes logging.""" + """Initialize logging.""" logging.getLogger().setLevel(logging.DEBUG) init_logger("cuckoo.log", level) init_logger("cuckoo.json") init_logger("task") def init_console_logging(level=logging.INFO): - """Initializes logging only to console and database.""" + """Initialize logging only to console and database.""" logging.getLogger().setLevel(logging.DEBUG) init_logger("console", level) init_logger("database") @@ -170,7 +260,7 @@ def init_tasks(): db.set_status(task.id, TASK_FAILED_ANALYSIS) def init_modules(): - """Initializes plugins.""" + """Initialize plugins.""" log.debug("Imported modules...") categories = ( @@ -415,7 +505,7 @@ def init_routing(): rooter("init_rttable", rt_table, interface) def ensure_tmpdir(): - """Verifies if the current user can read and create files in the + """Verify if the current user can read and create files in the cuckoo temporary directory (and creates it, if needed).""" try: if not os.path.isdir(temppath()): diff --git a/cuckoo/core/submit.py b/cuckoo/core/submit.py index 22657bf86c..589563b76e 100644 --- a/cuckoo/core/submit.py +++ b/cuckoo/core/submit.py @@ -58,7 +58,7 @@ def _handle_string(self, submit, tmppath, line): ) def translate_options_from(self, entry, options): - """Translates from Web Interface options to Cuckoo database options.""" + """Translate from Web Interface options to Cuckoo database options.""" ret = {} if not options.get("simulated-human-interaction", True): @@ -88,23 +88,33 @@ def translate_options_from(self, entry, options): return ret def translate_options_to(self, options): - """Translates from Cuckoo database options to Web Interface options.""" + """Translate from Cuckoo database options to Web Interface options.""" ret = {} if not int(options.get("human", "1")): ret["simulated-human-interaction"] = False + options.pop("human") if options.get("free") == "yes": ret["enable-injection"] = False + options.pop("free") if options.get("procmemdump") == "yes": ret["process-memory-dump"] = True + options.pop("procmemdump") if options.get("remotecontrol") == "yes": ret["remote-control"] = True + options.pop("remotecontrol") if options.get("route"): ret["network-routing"] = options["route"] + options.pop("route") + + # Propagate any additional manually set key/value pairs. + for key, value in options.items(): + if key not in self.known_web_options: + ret[key] = value return ret @@ -147,7 +157,7 @@ def pre(self, submit_type, data, options=None): def get_files(self, submit_id, password=None, astree=False): """ - Returns files or URLs from a submitted analysis. + Return files or URLs from a submitted analysis. @param password: The password to unlock container archives with @param astree: sflock option; determines the format in which the files are returned @return: A tree of files @@ -195,7 +205,7 @@ def get_files(self, submit_id, password=None, astree=False): return files, submit.data["errors"], submit.data["options"] def submit(self, submit_id, config): - """Reads, interprets, and converts the JSON configuration provided by + """Read, interpret, and convert the JSON configuration provided by the Web Interface into something we insert into the database.""" ret = [] submit = db.view_submit(submit_id) diff --git a/cuckoo/data/agent/agent.py b/cuckoo/data/agent/agent.py index 60df37bd42..1be1c170db 100644 --- a/cuckoo/data/agent/agent.py +++ b/cuckoo/data/agent/agent.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015-2017 Cuckoo Foundation. +# Copyright (C) 2015-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -21,7 +21,7 @@ import SimpleHTTPServer import SocketServer -AGENT_VERSION = "0.9" +AGENT_VERSION = "0.10" AGENT_FEATURES = [ "execpy", "pinning", "logs", "largefile", "unicodepath", ] @@ -36,9 +36,9 @@ def do_GET(self): request.client_ip, request.client_port = self.client_address request.form = {} request.files = {} + request.method = "GET" - if "client_ip" not in state or request.client_ip == state["client_ip"]: - self.httpd.handle(self) + self.httpd.handle(self) def do_POST(self): environ = { @@ -53,6 +53,7 @@ def do_POST(self): request.client_ip, request.client_port = self.client_address request.form = {} request.files = {} + request.method = "POST" # Another pretty fancy workaround. Since we provide backwards # compatibility with the Old Agent we will get an xmlrpc request @@ -68,8 +69,7 @@ def do_POST(self): else: request.form[key] = value.value.decode("utf8") - if "client_ip" not in state or request.client_ip == state["client_ip"]: - self.httpd.handle(self) + self.httpd.handle(self) class MiniHTTPServer(object): def __init__(self): @@ -96,6 +96,12 @@ def register(fn): return register def handle(self, obj): + if "client_ip" in state and request.client_ip != state["client_ip"]: + if request.client_ip != "127.0.0.1": + return + if obj.path != "/status" or request.method != "POST": + return + for route, fn in self.routes[obj.command]: if route.match(obj.path): ret = fn() @@ -166,6 +172,7 @@ class request(object): files = {} client_ip = None client_port = None + method = None environ = { "werkzeug.server.shutdown": lambda: app.shutdown(), } diff --git a/cuckoo/data/analyzer/darwin/lib/api/screenshot.py b/cuckoo/data/analyzer/darwin/lib/api/screenshot.py index ea22a8ab94..613551fd07 100644 --- a/cuckoo/data/analyzer/darwin/lib/api/screenshot.py +++ b/cuckoo/data/analyzer/darwin/lib/api/screenshot.py @@ -38,7 +38,7 @@ def have_pil(self): return HAVE_PIL def equal(self, img1, img2, skip_area=None): - """Compares two screenshots using Root-Mean-Square Difference (RMS). + """Compare two screenshots using Root-Mean-Square Difference (RMS). @param img1: screenshot to compare. @param img2: screenshot to compare. @return: equal status. diff --git a/cuckoo/data/analyzer/darwin/lib/common/hashing.py b/cuckoo/data/analyzer/darwin/lib/common/hashing.py index fac551f2fd..010d4fa11c 100644 --- a/cuckoo/data/analyzer/darwin/lib/common/hashing.py +++ b/cuckoo/data/analyzer/darwin/lib/common/hashing.py @@ -6,7 +6,7 @@ def hash_file(method, path): - """Calculates an hash on a file by path. + """Calculate a hash on a file by path. @param method: callable hashing method @param path: file path @return: computed hash string diff --git a/cuckoo/data/analyzer/darwin/modules/packages/zip.py b/cuckoo/data/analyzer/darwin/modules/packages/zip.py index eaf475c6da..18388c8a03 100644 --- a/cuckoo/data/analyzer/darwin/modules/packages/zip.py +++ b/cuckoo/data/analyzer/darwin/modules/packages/zip.py @@ -90,7 +90,7 @@ def _extract_nested_archives(self, archive, where, password): def _prepare_archive_at_path(filename): - """ Verifies that there's a readable zip archive at the given path. + """ Verify that there's a readable zip archive at the given path. This function returns a new name for the archive (for most cases it's the same as the original one; but if an archive named "foo.zip" contains diff --git a/cuckoo/data/analyzer/linux/lib/api/process.py b/cuckoo/data/analyzer/linux/lib/api/process.py index 4807de8cf2..7aaae353ba 100644 --- a/cuckoo/data/analyzer/linux/lib/api/process.py +++ b/cuckoo/data/analyzer/linux/lib/api/process.py @@ -38,6 +38,10 @@ def get_proc_status(self): return {} def execute(self, cmd): + """Start a subprocess. + @param cmd: process path + @return: subprocess status + """ self.proc = proc = subprocess.Popen(cmd) self.pid = proc.pid return True diff --git a/cuckoo/data/analyzer/linux/lib/common/abstracts.py b/cuckoo/data/analyzer/linux/lib/common/abstracts.py index 015e035dab..9507ff816a 100644 --- a/cuckoo/data/analyzer/linux/lib/common/abstracts.py +++ b/cuckoo/data/analyzer/linux/lib/common/abstracts.py @@ -31,9 +31,8 @@ def check(self): return True def execute(self, cmd): - """Starts an executable for analysis. - @param path: executable path - @param args: executable arguments + """Start an executable for analysis. + @param cmd: executable path @return: process pid """ p = Process() diff --git a/cuckoo/data/analyzer/linux/lib/common/hashing.py b/cuckoo/data/analyzer/linux/lib/common/hashing.py index 78d1d1936e..1ffa54190d 100644 --- a/cuckoo/data/analyzer/linux/lib/common/hashing.py +++ b/cuckoo/data/analyzer/linux/lib/common/hashing.py @@ -10,7 +10,7 @@ def sha256_file(path): return hash_file(hashlib.sha256, path) def hash_file(method, path): - """Calculates an hash on a file by path. + """Calculate a hash on a file by path. @param method: callable hashing method @param path: file path @return: computed hash string diff --git a/cuckoo/data/analyzer/linux/lib/core/config.py b/cuckoo/data/analyzer/linux/lib/core/config.py index 98eb7106d5..4a70b227c3 100644 --- a/cuckoo/data/analyzer/linux/lib/core/config.py +++ b/cuckoo/data/analyzer/linux/lib/core/config.py @@ -25,9 +25,10 @@ def __init__(self, cfg): setattr(self, name, value) def get(self, name, default=None): - if hasattr(self, name): - return getattr(self, name) - return default + """Get attribute. + @return: attribute. + """ + return getattr(self, name, default) def get_options(self): """Get analysis options. diff --git a/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py b/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py index ae2b20b390..3ddbdfeec0 100644 --- a/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py +++ b/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py @@ -14,7 +14,7 @@ log = logging.getLogger(__name__) class STAP(Auxiliary): - """system-wide syscall trace with stap.""" + """System-wide syscall trace with stap.""" priority = -10 # low prio to wrap tightly around the analysis def __init__(self): diff --git a/cuckoo/data/analyzer/windows/analyzer.py b/cuckoo/data/analyzer/windows/analyzer.py index 66d0a6bdb4..de1be36162 100644 --- a/cuckoo/data/analyzer/windows/analyzer.py +++ b/cuckoo/data/analyzer/windows/analyzer.py @@ -31,8 +31,9 @@ from lib.core.ioctl import zer0m0n from lib.core.packages import choose_package from lib.core.pipe import PipeServer, PipeForwarder, PipeDispatcher +from lib.core.pipe import disconnect_pipes from lib.core.privileges import grant_privilege -from lib.core.startup import init_logging, set_clock +from lib.core.startup import init_logging, disconnect_logger, set_clock from modules import auxiliary log = logging.getLogger("analyzer") @@ -46,11 +47,11 @@ def __init__(self): self.dumped = [] def is_protected_filename(self, file_name): - """Do we want to inject into a process with this name?""" + """Return whether or not to inject into a process with this name.""" return file_name.lower() in self.PROTECTED_NAMES def add_pid(self, filepath, pid, verbose=True): - """Tracks a process identifier for this file.""" + """Track a process identifier for this file.""" if not pid or filepath.lower() not in self.files: return @@ -150,7 +151,7 @@ def add_pids(self, pids): self.add_pid(pids) def has_pid(self, pid, notrack=True): - """Is this process identifier being tracked?""" + """Return whether or not this process identifier being tracked.""" if int(pid) in self.pids: return True @@ -435,7 +436,7 @@ def __init__(self): self.reboot = [] def get_pipe_path(self, name): - """Returns \\\\.\\PIPE on Windows XP and \\??\\PIPE elsewhere.""" + """Return \\\\.\\PIPE on Windows XP and \\??\\PIPE elsewhere.""" version = sys.getwindowsversion() if version.major == 5 and version.minor == 1: return "\\\\.\\PIPE\\%s" % name @@ -509,7 +510,7 @@ def prepare(self): self.target = self.config.target def stop(self): - """Allows an auxiliary module to stop the analysis.""" + """Allow an auxiliary module to stop the analysis.""" self.do_run = False def complete(self): @@ -518,11 +519,9 @@ def complete(self): self.command_pipe.stop() self.log_pipe_server.stop() - # Dump all the notified files. - self.files.dump_files() - - # Hell yeah. - log.info("Analysis completed.") + # Cleanly close remaining connections + disconnect_pipes() + disconnect_logger() def run(self): """Run analysis. @@ -790,8 +789,11 @@ def run(self): log.warning("Exception running finish callback of auxiliary " "module %s: %s", aux.__class__.__name__, e) - # Let's invoke the completion procedure. - self.complete() + # Dump all the notified files. + self.files.dump_files() + + # Hell yeah. + log.info("Analysis completed.") return True if __name__ == "__main__": @@ -832,11 +834,24 @@ def run(self): "description": error_exc, } finally: + try: + # Let's invoke the completion procedure. + analyzer.complete() + except Exception as e: + complete_excp = traceback.format_exc() + data["status"] = "exception" + if "description" in data: + data["description"] += "%s\n%s" % ( + data["description"], complete_excp + ) + else: + data["description"] = complete_excp + # Report that we're finished. First try with the XML RPC thing and # if that fails, attempt the new Agent. try: server = xmlrpclib.Server("http://127.0.0.1:8000") server.complete(success, error, "unused_path") - except xmlrpclib.ProtocolError: + except Exception as e: urllib2.urlopen("http://127.0.0.1:8000/status", urllib.urlencode(data)).read() diff --git a/cuckoo/data/analyzer/windows/lib/api/process.py b/cuckoo/data/analyzer/windows/lib/api/process.py index ba8770174d..ae7359795a 100644 --- a/cuckoo/data/analyzer/windows/lib/api/process.py +++ b/cuckoo/data/analyzer/windows/lib/api/process.py @@ -127,7 +127,7 @@ def __init__(self, pid=None, tid=None, process_name=None): @staticmethod def set_config(config): - """Sets the analyzer configuration once.""" + """Set the analyzer configuration once.""" Process.config = config def get_system_info(self): diff --git a/cuckoo/data/analyzer/windows/lib/common/abstracts.py b/cuckoo/data/analyzer/windows/lib/common/abstracts.py index c9ae91a119..e47109f3d6 100644 --- a/cuckoo/data/analyzer/windows/lib/common/abstracts.py +++ b/cuckoo/data/analyzer/windows/lib/common/abstracts.py @@ -107,7 +107,7 @@ def move_curdir(self, filepath): return outpath def init_regkeys(self, regkeys): - """Initializes the registry to avoid annoying popups, configure + """Initialize the registry to avoid annoying popups, configure settings, etc. @param regkeys: the root keys, subkeys, and key/value pairs. """ @@ -130,7 +130,7 @@ def init_regkeys(self, regkeys): def execute(self, path, args, mode=None, maximize=False, env=None, source=None, trigger=None): - """Starts an executable for analysis. + """Start an executable for analysis. @param path: executable path @param args: executable arguments @param mode: monitor mode - which functions to instrument @@ -169,7 +169,7 @@ def execute(self, path, args, mode=None, maximize=False, env=None, return p.pid def package_files(self): - """A list of files to upload to host. + """Return a list of files to upload to host. The list should be a list of tuples (, ). (package_files is a folder that will be created in analysis folder). """ diff --git a/cuckoo/data/analyzer/windows/lib/common/hashing.py b/cuckoo/data/analyzer/windows/lib/common/hashing.py index bbea930962..adbd321349 100644 --- a/cuckoo/data/analyzer/windows/lib/common/hashing.py +++ b/cuckoo/data/analyzer/windows/lib/common/hashing.py @@ -7,7 +7,7 @@ def hash_file(method, path): - """Calculates an hash on a file by path. + """Calculate a hash on a file by path. @param method: callable hashing method @param path: file path @return: computed hash string diff --git a/cuckoo/data/analyzer/windows/lib/common/results.py b/cuckoo/data/analyzer/windows/lib/common/results.py index ba5a602c68..e0455458e6 100644 --- a/cuckoo/data/analyzer/windows/lib/common/results.py +++ b/cuckoo/data/analyzer/windows/lib/common/results.py @@ -76,7 +76,9 @@ def send(self, data, retry=True): def close(self): try: + self.sock.shutdown(socket.SHUT_RDWR) self.sock.close() + self.sock = None except Exception: pass diff --git a/cuckoo/data/analyzer/windows/lib/core/pipe.py b/cuckoo/data/analyzer/windows/lib/core/pipe.py index 88598b2be5..20645530b2 100644 --- a/cuckoo/data/analyzer/windows/lib/core/pipe.py +++ b/cuckoo/data/analyzer/windows/lib/core/pipe.py @@ -6,6 +6,7 @@ import logging import socket import threading +import errno from ctypes import create_string_buffer, c_uint, byref, sizeof @@ -19,10 +20,11 @@ log = logging.getLogger(__name__) BUFSIZE = 0x10000 +open_handles = set() class PipeForwarder(threading.Thread): - """The Pipe Forwarder forwards all data received from a local pipe to - the Cuckoo server through a socket.""" + """Forward all data received from a local pipe to the Cuckoo + server through a socket.""" sockets = {} active = {} @@ -30,6 +32,7 @@ def __init__(self, pipe_handle, destination): threading.Thread.__init__(self) self.pipe_handle = pipe_handle self.destination = destination + self.do_run = True def run(self): buf = create_string_buffer(BUFSIZE) @@ -63,24 +66,31 @@ def run(self): return if pid.value: - if pid.value not in self.sockets: - self.sockets[pid.value] = ( - socket.create_connection(self.destination) - ) + sock = self.sockets.get(pid.value) + if not sock: + sock = socket.create_connection(self.destination) + self.sockets[pid.value] = sock - sock = self.sockets[pid.value] self.active[pid.value] = True else: sock = socket.create_connection(self.destination) - while True: + open_handles.add(sock) + + while self.do_run: success = KERNEL32.ReadFile( self.pipe_handle, byref(buf), sizeof(buf), byref(bytes_read), None ) if success or KERNEL32.GetLastError() == ERROR_MORE_DATA: - sock.sendall(buf.raw[:bytes_read.value]) + try: + sock.sendall(buf.raw[:bytes_read.value]) + except socket.error as e: + if e.errno != errno.EBADF: + log.warning("Failed socket operation: %s", e) + break + # If we get the broken pipe error then this pipe connection has # been terminated for one reason or another. So break from the # loop and make the socket "inactive", that is, another pipe @@ -98,9 +108,12 @@ def run(self): if pid.value: self.active[pid.value] = False + def stop(self): + self.do_run = False + class PipeDispatcher(threading.Thread): - """Receives commands through a local pipe, forwards them to the - dispatcher, and returns the response.""" + """Receive commands through a local pipe, forward them to the + dispatcher, and return the response.""" def __init__(self, pipe_handle, dispatcher): threading.Thread.__init__(self) @@ -145,9 +158,12 @@ def run(self): KERNEL32.CloseHandle(self.pipe_handle) + def stop(self): + self.do_run = False + class PipeServer(threading.Thread): - """The Pipe Server accepts incoming pipe handlers and initializes - them in a new thread.""" + """Accept incoming pipe handlers and initialize them in + a new thread.""" def __init__(self, pipe_handler, pipe_name, message=False, **kwargs): threading.Thread.__init__(self) @@ -156,6 +172,7 @@ def __init__(self, pipe_handler, pipe_name, message=False, **kwargs): self.message = message self.kwargs = kwargs self.do_run = True + self.handlers = set() def run(self): while self.do_run: @@ -182,8 +199,23 @@ def run(self): handler = self.pipe_handler(pipe_handle, **self.kwargs) handler.daemon = True handler.start() + self.handlers.add(handler) else: KERNEL32.CloseHandle(pipe_handle) def stop(self): self.do_run = False + for h in self.handlers: + try: + if h.isAlive(): + h.stop() + except: + pass + +def disconnect_pipes(): + for sock in open_handles: + try: + sock.shutdown(socket.SHUT_RDWR) + sock.close() + except: + log.exception("Could not close socket") diff --git a/cuckoo/data/analyzer/windows/lib/core/startup.py b/cuckoo/data/analyzer/windows/lib/core/startup.py index adc102bdac..d644fd6c3d 100644 --- a/cuckoo/data/analyzer/windows/lib/core/startup.py +++ b/cuckoo/data/analyzer/windows/lib/core/startup.py @@ -10,22 +10,29 @@ from lib.common.results import NetlogHandler log = logging.getLogger() +netlog_handler = None def init_logging(): """Initialize logger.""" formatter = logging.Formatter( "%(asctime)s [%(name)s] %(levelname)s: %(message)s" ) + sh = logging.StreamHandler() sh.setFormatter(formatter) log.addHandler(sh) - nh = NetlogHandler() - nh.setFormatter(formatter) - log.addHandler(nh) + global netlog_handler + netlog_handler = NetlogHandler() + netlog_handler.setFormatter(formatter) + log.addHandler(netlog_handler) log.setLevel(logging.DEBUG) +def disconnect_logger(): + """Cleanly close the logger. Note that LogHandler also implements close.""" + netlog_handler.close() + def set_clock(clock): st = SYSTEMTIME() st.wYear = clock.year diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py b/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py index fa74c71de9..f5fa69a7e8 100644 --- a/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py +++ b/cuckoo/data/analyzer/windows/modules/auxiliary/disguise.py @@ -66,7 +66,7 @@ class Disguise(Auxiliary): ] def change_productid(self): - """Randomizes Windows ProductId. + """Randomize Windows ProductId. The Windows ProductId is occasionally used by malware to detect public setups of Cuckoo, e.g., Malwr.com. """ diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py b/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py index 144f36ac51..54c421d278 100644 --- a/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py +++ b/cuckoo/data/analyzer/windows/modules/auxiliary/reboot.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) class Reboot(Auxiliary): - """Prepares the environment to behave as if the VM has been rebooted.""" + """Prepare the environment to behave as if the VM has been rebooted.""" def start(self): if self.analyzer.config.package != "reboot": diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py b/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py index 25497dd032..50eed421fc 100644 --- a/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py +++ b/cuckoo/data/analyzer/windows/modules/auxiliary/recentfiles.py @@ -17,7 +17,7 @@ log = logging.getLogger(__name__) class RecentFiles(Auxiliary): - """Populates the Desktop with recent files in order to combat recent + """Populate the Desktop with recent files in order to combat recent anti-sandbox measures.""" extensions = [ diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/screenshots.py b/cuckoo/data/analyzer/windows/modules/auxiliary/screenshots.py index 2aae0f8727..ad7f4e4f1d 100644 --- a/cuckoo/data/analyzer/windows/modules/auxiliary/screenshots.py +++ b/cuckoo/data/analyzer/windows/modules/auxiliary/screenshots.py @@ -1,5 +1,5 @@ # Copyright (C) 2012-2013 Claudio Guarnieri. -# Copyright (C) 2014-2017 Cuckoo Foundation. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -74,13 +74,14 @@ def run(self): tmpio.seek(0) # now upload to host from the StringIO - nf = NetlogFile() - nf.init("shots/%04d.jpg" % img_counter) - - for chunk in tmpio: - nf.sock.sendall(chunk) + try: + nf = NetlogFile() + nf.init("shots/%04d.jpg" % img_counter) - nf.close() + for chunk in tmpio: + nf.sock.sendall(chunk) + finally: + nf.close() img_last = img_current diff --git a/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py b/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py index e5b656e75c..3d8e479922 100644 --- a/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py +++ b/cuckoo/data/analyzer/windows/modules/auxiliary/zer0m0n.py @@ -13,7 +13,7 @@ log = logging.getLogger(__name__) class LoadZer0m0n(Auxiliary): - """Loads the zer0m0n kernel driver.""" + """Load the zer0m0n kernel driver.""" def start(self): if self.options.get("analysis") not in ("both", "kernel"): diff --git a/cuckoo/data/analyzer/windows/modules/packages/ie.py b/cuckoo/data/analyzer/windows/modules/packages/ie.py index 8252072a0b..9084c8c588 100644 --- a/cuckoo/data/analyzer/windows/modules/packages/ie.py +++ b/cuckoo/data/analyzer/windows/modules/packages/ie.py @@ -94,6 +94,22 @@ class IE(Package): "CheckExeSignatures": "no", }, ], + [ + HKEY_LOCAL_MACHINE, + "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer", + { + # Disable SmartScreen Windows 8 + "SmartScreenEnabled": "Off" + } + ], + [ + HKEY_CURRENT_USER, + "Software\\Microsoft\\Internet Explorer\\PhishingFilter", + { + # Disable SmartScreen Filter Windows 7 + "EnabledV9": 0 + } + ], ] def setup_proxy(self, proxy_host): diff --git a/cuckoo/data/analyzer/windows/modules/packages/zip.py b/cuckoo/data/analyzer/windows/modules/packages/zip.py index 339cf41833..de33fc355a 100644 --- a/cuckoo/data/analyzer/windows/modules/packages/zip.py +++ b/cuckoo/data/analyzer/windows/modules/packages/zip.py @@ -51,7 +51,7 @@ def extract_zip(self, zip_path, extract_path, password): self.extract_zip(os.path.join(extract_path, name), extract_path, password) def is_overwritten(self, zip_path): - """Checks if the ZIP file contains another file with the same name, so it is going to be overwritten. + """Check if the ZIP file contains another file with the same name, so it is going to be overwritten. @param zip_path: zip file path @return: comparison boolean """ diff --git a/cuckoo/data/monitor/latest b/cuckoo/data/monitor/latest index 85b52a2642..8879054a99 100644 --- a/cuckoo/data/monitor/latest +++ b/cuckoo/data/monitor/latest @@ -1 +1 @@ -e071e63a66e831163a40abc45109fdf71fee829e \ No newline at end of file +2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b diff --git a/cuckoo/private/whitelist/domain.txt b/cuckoo/data/safelist/domain.txt similarity index 69% rename from cuckoo/private/whitelist/domain.txt rename to cuckoo/data/safelist/domain.txt index d5c30dd02a..6bdaedfffd 100644 --- a/cuckoo/private/whitelist/domain.txt +++ b/cuckoo/data/safelist/domain.txt @@ -1,3 +1,4 @@ +# You can add safelisted domains here. java.com www.msn.com www.bing.com @@ -25,3 +26,10 @@ www.msftncsi.com ocsp.msocsp.com ocsp.omniroot.com crl.microsoft.com +www.msftconnecttest.com +v10.vortex-win.data.microsoft.com +settings-win.data.microsoft.com +win10.ipv6.microsoft.com +sls.update.microsoft.com +fs.microsoft.com +ctldl.windowsupdate.com \ No newline at end of file diff --git a/cuckoo/data/safelist/ip.txt b/cuckoo/data/safelist/ip.txt new file mode 100644 index 0000000000..aeeba83834 --- /dev/null +++ b/cuckoo/data/safelist/ip.txt @@ -0,0 +1 @@ +# You can add safelisted IPs here. diff --git a/cuckoo/data/safelist/mispdomain.txt b/cuckoo/data/safelist/mispdomain.txt new file mode 100644 index 0000000000..79c56a4f93 --- /dev/null +++ b/cuckoo/data/safelist/mispdomain.txt @@ -0,0 +1,12 @@ +# Domains that should not be reported to MISP should be added here +www.msftncsi.com +dns.msftncsi.com +teredo.ipv6.microsoft.com +time.windows.com +www.msftconnecttest.com +v10.vortex-win.data.microsoft.com +settings-win.data.microsoft.com +win10.ipv6.microsoft.com +sls.update.microsoft.com +fs.microsoft.com +ctldl.windowsupdate.com diff --git a/cuckoo/data/safelist/misphash.txt b/cuckoo/data/safelist/misphash.txt new file mode 100644 index 0000000000..c4a3cde4e0 --- /dev/null +++ b/cuckoo/data/safelist/misphash.txt @@ -0,0 +1 @@ +# Hashes of file that should not be reported to MISP should be added here \ No newline at end of file diff --git a/cuckoo/data/safelist/mispip.txt b/cuckoo/data/safelist/mispip.txt new file mode 100644 index 0000000000..8837e4c867 --- /dev/null +++ b/cuckoo/data/safelist/mispip.txt @@ -0,0 +1,5 @@ +# IPs that should not be reported to MISP should be added here +13.74.179.117 +40.81.120.221 +40.77.226.249 +8.8.8.8 diff --git a/cuckoo/data/safelist/mispurl.txt b/cuckoo/data/safelist/mispurl.txt new file mode 100644 index 0000000000..2eb812c76d --- /dev/null +++ b/cuckoo/data/safelist/mispurl.txt @@ -0,0 +1 @@ +# URLs that should not be reported to MISP should be added here \ No newline at end of file diff --git a/cuckoo/data/signatures/windows/creates_exe.py b/cuckoo/data/signatures/windows/creates_exe.py deleted file mode 100644 index 13f8da47cf..0000000000 --- a/cuckoo/data/signatures/windows/creates_exe.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (C) 2010-2013 Claudio Guarnieri. -# Copyright (C) 2014-2016 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from cuckoo.common.abstracts import Signature - -class CreatesExe(Signature): - name = "creates_exe" - description = "Creates a Windows executable on the filesystem" - severity = 2 - categories = ["generic"] - authors = ["Cuckoo Developers"] - minimum = "2.0" - - # This is a signature template. It should be used as a skeleton for - # creating custom signatures, therefore is disabled by default. - # It doesn't verify whether a .exe is actually being created, but - # it matches files being opened with any access type, including - # read and attributes lookup. - enabled = False - - def on_complete(self): - match = self.check_file(pattern=".*\\.exe$", regex=True) - if match: - self.mark_ioc("file", match) - return True diff --git a/cuckoo/data/stuff/ttp_descriptions.json b/cuckoo/data/stuff/ttp_descriptions.json index 7a1336c92f..05ab152b2b 100644 --- a/cuckoo/data/stuff/ttp_descriptions.json +++ b/cuckoo/data/stuff/ttp_descriptions.json @@ -32,7 +32,7 @@ "short": "Fallback Channels" }, "T1009": { - "long": "Some security tools inspect files with static signatures to determine if they are known malicious. Adversaries may add data to files to increase the size beyond what security tools are capable of handling or to change the file hash to avoid hash-based blacklists.", + "long": "Some security tools inspect files with static signatures to determine if they are known malicious. Adversaries may add data to files to increase the size beyond what security tools are capable of handling or to change the file hash to avoid hash-based blocklists.", "short": "Binary Padding" }, "T1010": { @@ -336,7 +336,7 @@ "short": "Windows Management Instrumentation Event Subscription" }, "T1085": { - "long": "The rundll32.exe program can be called to execute an arbitrary binary. Adversaries may take advantage of this functionality to proxy execution of code to avoid triggering security tools that may not monitor execution of the rundll32.exe process because of whitelists or false positives from Windows using rundll32.exe for normal operations.", + "long": "The rundll32.exe program can be called to execute an arbitrary binary. Adversaries may take advantage of this functionality to proxy execution of code to avoid triggering security tools that may not monitor execution of the rundll32.exe process because of safelists or false positives from Windows using rundll32.exe for normal operations.", "short": "Rundll32" }, "T1086": { @@ -504,7 +504,7 @@ "short": "Network Share Connection Removal" }, "T1127": { - "long": "There are many utilities used for software development related tasks that can be used to execute code in various forms to assist in development, debugging, and reverse engineering. These utilities may often be signed with legitimate certificates that allow them to execute on a system and proxy execution of malicious code through a trusted process that effectively bypasses application whitelisting defensive solutions.", + "long": "There are many utilities used for software development related tasks that can be used to execute code in various forms to assist in development, debugging, and reverse engineering. These utilities may often be signed with legitimate certificates that allow them to execute on a system and proxy execution of malicious code through a trusted process that effectively bypasses application safelisting defensive solutions.", "short": "Trusted Developer Utilities" }, "T1128": { @@ -592,7 +592,7 @@ "short": "HISTCONTROL" }, "T1149": { - "long": "As of OS X 10.8, mach-O binaries introduced a new header called LC_MAIN that points to the binary\u2019s entry point for execution. Previously, there were two headers to achieve this same effect: LC_THREAD and LC_UNIXTHREAD . The entry point for a binary can be hijacked so that initial execution flows to a malicious addition (either another section or a code cave) and then goes back to the initial entry point so that the victim doesn\u2019t know anything was different . By modifying a binary in this way, application whitelisting can be bypassed because the file name or application path is still the same.", + "long": "As of OS X 10.8, mach-O binaries introduced a new header called LC_MAIN that points to the binary\u2019s entry point for execution. Previously, there were two headers to achieve this same effect: LC_THREAD and LC_UNIXTHREAD . The entry point for a binary can be hijacked so that initial execution flows to a malicious addition (either another section or a code cave) and then goes back to the initial entry point so that the victim doesn\u2019t know anything was different . By modifying a binary in this way, application safelisting can be bypassed because the file name or application path is still the same.", "short": "LC_MAIN Hijacking" }, "T1150": { @@ -860,7 +860,7 @@ "short": "Kernel Modules and Extensions" }, "T1216": { - "long": "Scripts signed with trusted certificates can be used to proxy execution of malicious files. This behavior may bypass signature validation restrictions and application whitelisting solutions that do not account for use of these scripts.", + "long": "Scripts signed with trusted certificates can be used to proxy execution of malicious files. This behavior may bypass signature validation restrictions and application safelisting solutions that do not account for use of these scripts.", "short": "Signed Script Proxy Execution" }, "T1217": { @@ -868,11 +868,11 @@ "short": "Browser Bookmark Discovery" }, "T1218": { - "long": "Binaries signed with trusted digital certificates can execute on Windows systems protected by digital signature validation. Several Microsoft signed binaries that are default on Windows installations can be used to proxy execution of other files. This behavior may be abused by adversaries to execute malicious files that could bypass application whitelisting and signature validation on systems. This technique accounts for proxy execution methods that are not already accounted for within the existing techniques.", + "long": "Binaries signed with trusted digital certificates can execute on Windows systems protected by digital signature validation. Several Microsoft signed binaries that are default on Windows installations can be used to proxy execution of other files. This behavior may be abused by adversaries to execute malicious files that could bypass application safelisting and signature validation on systems. This technique accounts for proxy execution methods that are not already accounted for within the existing techniques.", "short": "Signed Binary Proxy Execution" }, "T1219": { - "long": "An adversary may use legitimate desktop support and remote access software, such as Team Viewer, Go2Assist, LogMein, AmmyyAdmin, etc, to establish an interactive command and control channel to target systems within networks. These services are commonly used as legitimate technical support software, and may be whitelisted within a target environment. Remote access tools like VNC, Ammy, and Teamviewer are used frequently when compared with other legitimate software commonly used by adversaries.", + "long": "An adversary may use legitimate desktop support and remote access software, such as Team Viewer, Go2Assist, LogMein, AmmyyAdmin, etc, to establish an interactive command and control channel to target systems within networks. These services are commonly used as legitimate technical support software, and may be safelisted within a target environment. Remote access tools like VNC, Ammy, and Teamviewer are used frequently when compared with other legitimate software commonly used by adversaries.", "short": "Remote Access Tools" }, "T1220": { diff --git a/cuckoo/data/web/local_settings.py b/cuckoo/data/web/local_settings.py index 7e44d34810..10a9677bb7 100644 --- a/cuckoo/data/web/local_settings.py +++ b/cuckoo/data/web/local_settings.py @@ -6,7 +6,7 @@ import web.errors # Maximum upload size (10GB, so there's basically no limit). -MAX_UPLOAD_SIZE = 10*1024*1024*1024 +MAX_UPLOAD_SIZE = 10 * 1024 * 1024 * 1024 # Override default secret key stored in $CWD/web/.secret_key # Make this unique, and don't share it with anybody. @@ -37,3 +37,10 @@ handler404 = web.errors.handler404 handler500 = web.errors.handler500 + +#A list of strings representing the subnets or ipaddresses that can download +#samples and dropped files +#Values in this list can be ipv4 or ipv6 separated by "," +#(e.g. '127.0.0.0/8,10.0.0.0/8,fd00::/8'). +ALLOWED_FILEDOWNLOAD_SUBNETS = '127.0.0.0/8,10.0.0.0/8,fd00::/8' + diff --git a/cuckoo/data/whitelist/domain.txt b/cuckoo/data/whitelist/domain.txt deleted file mode 100644 index 32a0ad0846..0000000000 --- a/cuckoo/data/whitelist/domain.txt +++ /dev/null @@ -1 +0,0 @@ -# You can add whitelisted domains here. diff --git a/cuckoo/data/whitelist/ip.txt b/cuckoo/data/whitelist/ip.txt deleted file mode 100644 index dfdbe87ff0..0000000000 --- a/cuckoo/data/whitelist/ip.txt +++ /dev/null @@ -1 +0,0 @@ -# You can add whitelisted IPs here. diff --git a/cuckoo/machinery/avd.py b/cuckoo/machinery/avd.py index bfa789464d..387800f630 100644 --- a/cuckoo/machinery/avd.py +++ b/cuckoo/machinery/avd.py @@ -19,7 +19,7 @@ class Avd(Machinery): """Virtualization layer for Android Emulator.""" def _initialize_check(self): - """Runs all checks when a machine manager is initialized. + """Run all checks when a machine manager is initialized. @raise CuckooMachineError: if the android emulator is not found. """ self.emulator_processes = {} @@ -76,7 +76,7 @@ def start(self, label, task): self.start_agent(label) def stop(self, label): - """Stops a virtual machine. + """Stop a virtual machine. @param label: virtual machine name. @raise CuckooMachineError: if unable to stop. """ @@ -84,20 +84,20 @@ def stop(self, label): self.stop_emulator(label) def _list(self): - """Lists virtual machines installed. + """List virtual machines installed. @return: virtual machine names list. """ return self.options.avd.machines def _status(self, label): - """Gets current status of a vm. + """Get current status of a vm. @param label: virtual machine name. @return: status string. """ log.debug("Getting status for %s" % label) def duplicate_reference_machine(self, label): - """Creates a new emulator based on a reference one.""" + """Create a new emulator based on a reference one.""" reference_machine = self.options.avd.reference_machine log.debug("Duplicate Reference Machine '{0}'.".format(reference_machine)) @@ -127,7 +127,7 @@ def duplicate_reference_machine(self, label): # todo:will see def delete_old_emulator(self, label): - """Deletes any trace of an emulator that would have the same name as + """Delete any trace of an emulator that would have the same name as the one of the current emulator.""" old_emulator_config_file = os.path.join(self.options.avd.avd_path, "%s.ini" % label) @@ -142,7 +142,7 @@ def delete_old_emulator(self, label): shutil.rmtree(old_emulator_path) def replace_content_in_file(self, fileName, contentToReplace, replacementContent): - """Replaces the specified motif by a specified value in the specified + """Replace the specified motif by a specified value in the specified file. """ @@ -157,7 +157,7 @@ def replace_content_in_file(self, fileName, contentToReplace, replacementContent fd.writelines(newLines) def start_emulator(self, label, task): - """Starts the emulator.""" + """Start the emulator.""" emulator_port = self.options.get(label)["emulator_port"] cmd = [ @@ -216,7 +216,7 @@ def stop_emulator(self, label): del self.emulator_processes[label] def wait_for_device_ready(self, label): - """Analyzes the emulator and returns when it's ready.""" + """Analyze the emulator and return when it's ready.""" emulator_port = str(self.options.get(label)["emulator_port"]) adb = self.options.avd.adb_path @@ -294,7 +294,7 @@ def start_agent(self, label): time.sleep(10) def check_adb_recognize_emulator(self, label): - """Checks that ADB recognizes the emulator. Returns True if device is + """Check that ADB recognizes the emulator. Return True if device is recognized by ADB, False otherwise. """ log.debug("Checking if ADB recognizes emulator...") @@ -311,7 +311,7 @@ def check_adb_recognize_emulator(self, label): return False def restart_adb_server(self): - """Restarts ADB server. This function is not used because we have to + """Restart ADB server. This function is not used because we have to verify we don't have multiple devices. """ log.debug("Restarting ADB server...") diff --git a/cuckoo/machinery/esx.py b/cuckoo/machinery/esx.py index 68caa84fb3..3ae4c307af 100644 --- a/cuckoo/machinery/esx.py +++ b/cuckoo/machinery/esx.py @@ -18,7 +18,7 @@ class ESX(LibVirtMachinery): """Virtualization layer for ESXi/ESX based on python-libvirt.""" def _initialize_check(self): - """Runs all checks when a machine manager is initialized. + """Run all checks when a machine manager is initialized. @raise CuckooMachineError: if configuration is invalid """ if not self.options.esx.dsn: diff --git a/cuckoo/machinery/physical.py b/cuckoo/machinery/physical.py index ff56937ab6..1fa40c75e1 100644 --- a/cuckoo/machinery/physical.py +++ b/cuckoo/machinery/physical.py @@ -30,7 +30,7 @@ class Physical(Machinery): ERROR = "error" def _initialize_check(self): - """Ensures that credentials have been entered into the config file. + """Ensure that credentials have been entered into the config file. @raise CuckooCriticalError: if no credentials were provided or if one or more physical machines are offline. """ @@ -85,7 +85,7 @@ def start(self, label, task): "%s (STATUS=%s)" % (label, status)) def stop(self, label): - """Stops a physical machine. + """Stop a physical machine. @param label: physical machine name. @raise CuckooMachineError: if unable to stop. """ @@ -119,7 +119,7 @@ def stop(self, label): continue def _list(self): - """Lists physical machines installed. + """List physical machines installed. @return: physical machine names list. """ active_machines = [] @@ -130,7 +130,7 @@ def _list(self): return active_machines def _status(self, label): - """Gets current status of a physical machine. + """Get current status of a physical machine. @param label: physical machine name. @return: status string. """ @@ -239,7 +239,7 @@ def fog_init(self): ) def fog_queue_task(self, hostname): - """Queues a task with FOG to deploy the given machine after reboot.""" + """Queue a task with FOG to deploy the given machine after reboot.""" if hostname in self.fog_machines: macaddr, download = self.fog_machines[hostname] self.fog_query(download) diff --git a/cuckoo/machinery/qemu.py b/cuckoo/machinery/qemu.py index 5efdb853c0..712f37b725 100644 --- a/cuckoo/machinery/qemu.py +++ b/cuckoo/machinery/qemu.py @@ -116,7 +116,7 @@ def __init__(self): self.state = {} def _initialize_check(self): - """Runs all checks when a machine manager is initialized. + """Run all checks when a machine manager is initialized. @raise CuckooMachineError: if QEMU binary is not found. """ # VirtualBox specific checks. @@ -205,7 +205,7 @@ def start(self, label, task): raise CuckooMachineError("QEMU failed starting the machine: %s" % e) def stop(self, label): - """Stops a virtual machine. + """Stop a virtual machine. @param label: virtual machine label. @raise CuckooMachineError: if unable to stop. """ @@ -235,7 +235,7 @@ def stop(self, label): self.state[vm_info.name] = None def _status(self, name): - """Gets current status of a vm. + """Get current status of a vm. @param name: virtual machine name. @return: status string. """ diff --git a/cuckoo/machinery/virtualbox.py b/cuckoo/machinery/virtualbox.py index 9d9303974b..dbd5b4649b 100644 --- a/cuckoo/machinery/virtualbox.py +++ b/cuckoo/machinery/virtualbox.py @@ -1,5 +1,5 @@ # Copyright (C) 2011-2013 Claudio Guarnieri. -# Copyright (C) 2014-2018 Cuckoo Foundation. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -7,6 +7,8 @@ import os import re import subprocess +import thread +import threading import time from cuckoo.common.abstracts import Machinery @@ -19,6 +21,43 @@ log = logging.getLogger(__name__) +class IgnoreLock(object): + """Behaves like a Lock object. Always allows the creating thread to + ignore the lock. The lock is used to prevent Virtualbox start/stop race + conditions. In the event of Cuckoo stopping, the scheduler should always + be able to perform the stopping operation.""" + + def __init__(self): + self.parent = thread.get_ident() + self._takers = [] + self._ev = threading.Event() + + def acquire(self): + th_ident = thread.get_ident() + self._takers.append(th_ident) + + if th_ident == self.parent: + return True + + while self._takers[0] != th_ident: + self._ev.wait(timeout=0.1) + + self._ev.clear() + + return True + + def release(self): + self._takers.remove(thread.get_ident()) + self._ev.set() + + def __enter__(self): + self.acquire() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.release() + +_power_lock = IgnoreLock() + class VirtualBox(Machinery): """Virtualization layer for VirtualBox.""" @@ -30,7 +69,7 @@ class VirtualBox(Machinery): ERROR = "machete" def _initialize_check(self): - """Runs all checks when a machine manager is initialized. + """Run all checks when a machine manager is initialized. @raise CuckooMachineError: if VBoxManage is not found. """ if not self.options.virtualbox.path: @@ -128,10 +167,12 @@ def start(self, label, task): self.options.virtualbox.path, "startvm", label, "--type", self.options.virtualbox.mode ] - _, err = Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - close_fds=True - ).communicate() + + with _power_lock: + _, err = Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + close_fds=True + ).communicate() if err: raise OSError(err) except OSError as e: @@ -182,7 +223,7 @@ def dump_pcap(self, label, task): return def stop(self, label): - """Stops a virtual machine. + """Stop a virtual machine. @param label: virtual machine name. @raise CuckooMachineError: if unable to stop. """ @@ -207,10 +248,12 @@ def stop(self, label): args = [ self.options.virtualbox.path, "controlvm", label, "poweroff" ] - proc = Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - close_fds=True - ) + + with _power_lock: + proc = Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + close_fds=True + ) # Sometimes VBoxManage stucks when stopping vm so we needed # to add a timeout and kill it after that. @@ -224,9 +267,12 @@ def stop(self, label): proc.terminate() if proc.returncode != 0 and stop_me < vm_state_timeout: + _, err = proc.communicate() log.debug( - "VBoxManage exited with error powering off the machine" + "VBoxManage exited with error powering off the " + "machine: %s", err ) + raise OSError(err) except OSError as e: raise CuckooMachineError( "VBoxManage failed powering off the machine: %s" % e @@ -235,7 +281,7 @@ def stop(self, label): self._wait_status(label, self.POWEROFF, self.ABORTED, self.SAVED) def _list(self): - """Lists virtual machines installed. + """List virtual machines installed. @return: virtual machine names list. """ try: @@ -268,8 +314,8 @@ def _list(self): return machines def vminfo(self, label, field): - """Returns False if invoking vboxmanage fails. Otherwise the VM - information value, if any.""" + """Return False if invoking vboxmanage fails. Otherwise return the + VM information value, if any.""" try: args = [ self.options.virtualbox.path, @@ -314,7 +360,7 @@ def vminfo(self, label, field): return line.split("=", 1)[1] def _status(self, label): - """Gets current status of a vm. + """Get current status of a vm. @param label: virtual machine name. @return: status string. """ @@ -332,7 +378,7 @@ def _status(self, label): ) def dump_memory(self, label, path): - """Takes a memory dump. + """Take a memory dump. @param path: path to where to store the memory dump. """ @@ -481,3 +527,29 @@ def _set_flag(self, label, key, val): ) _, _ = proc.communicate() return proc + + @staticmethod + def version(): + """Get the version for the installed Virtualbox""" + try: + proc = Popen( + [config("virtualbox:virtualbox:path"), "--version"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True + ) + output, err = proc.communicate() + except OSError: + return None + + output = output.strip() + version = "" + for c in output: + if not c.isdigit() and c != ".": + break + version += c + + # A 3 digit version number is expected. If it has none or fewer, return + # None because we are unsure what we have. + if len(version.split(".", 2)) < 3: + return None + + return version diff --git a/cuckoo/machinery/vmware.py b/cuckoo/machinery/vmware.py index 498be2a351..a19b2eb521 100644 --- a/cuckoo/machinery/vmware.py +++ b/cuckoo/machinery/vmware.py @@ -44,7 +44,7 @@ def _initialize_check(self): super(VMware, self)._initialize_check() def _check_vmx(self, vmx_path): - """Checks whether a vmx file exists and is valid. + """Check whether a vmx file exists and is valid. @param vmx_path: path to vmx file @raise CuckooMachineError: if file not found or not ending with .vmx """ @@ -56,7 +56,7 @@ def _check_vmx(self, vmx_path): raise CuckooMachineError("Vm file %s not found" % vmx_path) def _check_snapshot(self, vmx_path, snapshot): - """Checks snapshot existance. + """Check snapshot existence. @param vmx_path: path to vmx file @param snapshot: snapshot name @raise CuckooMachineError: if snapshot not found @@ -114,7 +114,7 @@ def start(self, vmx_path, task): "mode: %s" % (vmx_path, mode, e)) def stop(self, vmx_path): - """Stops a virtual machine. + """Stop a virtual machine. @param vmx_path: path to vmx file @raise CuckooMachineError: if unable to stop. """ @@ -135,7 +135,7 @@ def stop(self, vmx_path): vmx_path) def _revert(self, vmx_path, snapshot): - """Revets machine to snapshot. + """Revert machine to snapshot. @param vmx_path: path to vmx file @param snapshot: snapshot name @raise CuckooMachineError: if unable to revert @@ -154,7 +154,7 @@ def _revert(self, vmx_path, snapshot): "machine %s: %s" % (vmx_path, e)) def _is_running(self, vmx_path): - """Checks if virtual machine is running. + """Check if virtual machine is running. @param vmx_path: path to vmx file @return: running status """ diff --git a/cuckoo/machinery/vsphere.py b/cuckoo/machinery/vsphere.py index 4f947937c4..7ada17d6c1 100644 --- a/cuckoo/machinery/vsphere.py +++ b/cuckoo/machinery/vsphere.py @@ -55,7 +55,7 @@ def _initialize(self, module_name): random.seed() def _initialize_check(self): - """Runs checks against virtualization software when a machine manager + """Run checks against virtualization software when a machine manager is initialized. @raise CuckooCriticalError: if a misconfiguration or unsupported state is found. @@ -96,8 +96,7 @@ def _initialize_check(self): # Workaround for PEP-0476 issues in recent Python versions if self.options.vsphere.unverified_ssl: - sslContext = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - sslContext.verify_mode = ssl.CERT_NONE + sslContext = ssl._create_unverified_context() self.connect_opts["sslContext"] = sslContext log.warn("Turning off SSL certificate verification!") diff --git a/cuckoo/machinery/xenserver.py b/cuckoo/machinery/xenserver.py index 8023ba901c..3a5ba2b291 100644 --- a/cuckoo/machinery/xenserver.py +++ b/cuckoo/machinery/xenserver.py @@ -189,7 +189,7 @@ def _snapshot_from_vm_uuid(self, uuid): return machine.snapshot def _is_halted(self, vm): - """Checks if the virtual machine is running. + """Check if the virtual machine is running. @param uuid: vm uuid """ return vm["power_state"] == "Halted" @@ -262,7 +262,7 @@ def _list(self): return vm_list def _status(self, label): - """Gets current status of a vm. + """Get current status of a vm. @param label: virtual machine uuid @return: status string. """ diff --git a/cuckoo/main.py b/cuckoo/main.py index 292a8c5dd7..14500cfbe0 100644 --- a/cuckoo/main.py +++ b/cuckoo/main.py @@ -8,6 +8,7 @@ import os import shutil import subprocess +import signal import sys import cuckoo @@ -16,7 +17,7 @@ fetch_community, submit_tasks, process_tasks, process_task_range, cuckoo_rooter, cuckoo_api, cuckoo_distributed, cuckoo_distributed_instance, cuckoo_clean, cuckoo_dnsserve, cuckoo_machine, import_cuckoo, - migrate_database, migrate_cwd + migrate_database, migrate_cwd, cleanup_rooter ) from cuckoo.common.config import read_kv_conf from cuckoo.common.exceptions import CuckooCriticalError @@ -89,6 +90,41 @@ def _ignore_first_makedirs(dst): open(cwd("cwd", "init-post.jinja2", private=True), "rb").read() ).render() +def cuckoo_resources(): + try: + import resource + except ImportError: + # Only Unix platforms have this option; should not be an issue on + # Windows + return + + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + limit = soft + if soft != hard: + log.debug("Increasing resource limit for number of open files to %s", + hard if hard != resource.RLIM_INFINITY else '[unlimited]') + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + limit = hard + except ValueError: + log.debug( + "Failed to increase open file limit from %s to %s", soft, hard + ) + + if limit != resource.RLIM_INFINITY: + # This can really affect the stability of Cuckoo, so the user should + # really fix it. TODO: find a good minimum. + if limit <= 4096: + log.error("The maximum number of open files is low (%s). If you " + "do not increase it, you may run into errors later " + "on.", hard) + log.error("See also: https://cuckoo.sh/docs/faq/index.html#" + "ioerror-errno-24-too-many-open-files") + + # IDEAS: + # Check if limit is realistic versus the number of VMs + # Same for pool_size + def cuckoo_init(level, ctx, cfg=None): """Initialize Cuckoo configuration. @param quiet: enable quiet mode. @@ -103,16 +139,22 @@ def cuckoo_init(level, ctx, cfg=None): # Determine if this is a proper CWD. if not os.path.exists(cwd(".cwd")): - sys.exit( - "No proper Cuckoo Working Directory was identified, did you pass " - "along the correct directory? For new installations please use a " - "non-existant directory to build up the CWD! You can craft a CWD " - "manually, but keep in mind that the CWD layout may change along " - "with Cuckoo releases (and don't forget to fill out '$CWD/.cwd')!" - ) + sys.stderr.write(red( + "\nNo proper Cuckoo Working Directory was identified, did you " + "pass along the correct directory?\n" + "The '.cwd' file is missing in the specified directory. " + "For new installations please use a non-existant directory to " + "build up the CWD! You can craft a CWD manually, but keep in mind " + "that the CWD layout may change along with Cuckoo releases " + "(and don't forget to fill out '$CWD/.cwd')!\n" + )) + sys.exit(1) init_console_logging(level) + # Make sure user is aware of potential resource limits + cuckoo_resources() + # Only one Cuckoo process should exist per CWD. Run this check before any # files are possibly modified. Note that we mkdir $CWD/pidfiles/ here as # its CWD migration rules only kick in after the pidfile check. @@ -125,7 +167,7 @@ def cuckoo_init(level, ctx, cfg=None): pidfile.create() check_configs() - check_version() + check_version(ctx.ignore_vuln) ctx.log and init_logging(level) @@ -167,7 +209,7 @@ def cuckoo_init(level, ctx, cfg=None): "pretty important!" ) log.warning( - "You'll be able to fetch all the latest Cuckoo Signaturs, Yara " + "You'll be able to fetch all the latest Cuckoo Signatures, Yara " "rules, and more goodies by running the following command:" ) log.info("$ %s", green(format_command("community"))) @@ -176,25 +218,46 @@ def cuckoo_main(max_analysis_count=0): """Cuckoo main loop. @param max_analysis_count: kill cuckoo after this number of analyses """ + rs, sched = None, None + + def stop(): + if sched: + sched.running = False + if rs: + rs.instance.stop() + + Pidfile("cuckoo").remove() + if sched: + sched.stop() + + def handle_sigterm(sig, f): + stop() + + # Handle a SIGTERM, to reduce the chance of Cuckoo exiting without + # cleaning + signal.signal(signal.SIGTERM, handle_sigterm) + try: - ResultServer() + rs = ResultServer() sched = Scheduler(max_analysis_count) sched.start() except KeyboardInterrupt: - sched.stop() - - Pidfile("cuckoo").remove() + log.info("CTRL+C detected! Stopping.. This can take a few seconds") + finally: + if Pidfile("cuckoo").exists(): + stop() @click.group(invoke_without_command=True) @click.option("-d", "--debug", is_flag=True, help="Enable verbose logging") @click.option("-q", "--quiet", is_flag=True, help="Only log warnings and critical messages") @click.option("--nolog", is_flag=True, help="Don't log to file.") @click.option("-m", "--maxcount", default=0, help="Maximum number of analyses to process") +@click.option("--ignore-vuln", is_flag=True, help="Ignore vulnerability warnings and start") @click.option("--user", help="Drop privileges to this user") @click.option("--cwd", help="Cuckoo Working Directory") @click.pass_context -def main(ctx, debug, quiet, nolog, maxcount, user, cwd): - """Invokes the Cuckoo daemon or one of its subcommands. +def main(ctx, debug, quiet, nolog, maxcount, ignore_vuln, user, cwd): + """Invoke the Cuckoo daemon or one of its subcommands. To be able to use different Cuckoo configurations on the same machine with the same Cuckoo installation, we use the so-called Cuckoo Working @@ -215,6 +278,7 @@ def main(ctx, debug, quiet, nolog, maxcount, user, cwd): ctx.user = user ctx.log = not nolog + ctx.ignore_vuln = ignore_vuln if quiet: level = logging.WARN @@ -247,7 +311,7 @@ def main(ctx, debug, quiet, nolog, maxcount, user, cwd): @click.pass_context @click.option("--conf", type=click.Path(exists=True, file_okay=True, readable=True), help="Flat key/value configuration file") def init(ctx, conf): - """Initializes Cuckoo and its configuration.""" + """Initialize Cuckoo and its configuration.""" if conf and os.path.exists(conf): cfg = read_kv_conf(conf) else: @@ -398,7 +462,7 @@ def process(ctx, instance, report, maxcount, timeout): @click.option("--sudo", is_flag=True, help="Request superuser privileges") @click.pass_context def rooter(ctx, socket, group, service, iptables, ip, sudo): - """Instantiates the Cuckoo Rooter.""" + """Instantiate the Cuckoo Rooter.""" init_console_logging(level=ctx.parent.level) if sudo: @@ -423,6 +487,7 @@ def rooter(ctx, socket, group, service, iptables, ip, sudo): cuckoo_rooter(socket, group, service, iptables, ip) except KeyboardInterrupt: print(red("Aborting the Cuckoo Rooter..")) + cleanup_rooter() @main.command() @click.option("-H", "--host", default="localhost", help="Host to bind the API server on") @@ -465,6 +530,10 @@ def api(ctx, host, port, uwsgi, nginx): init_console_logging(level=ctx.parent.level) Database().connect() + + if not ensure_tmpdir(): + sys.exit(1) + cuckoo_api(host, port, ctx.parent.level == logging.DEBUG) @main.command() @@ -573,6 +642,9 @@ def web(ctx, args, host, port, uwsgi, nginx): init_console_logging(level=ctx.parent.level) Database().connect() + if not ensure_tmpdir(): + sys.exit(1) + try: execute_from_command_line( ("cuckoo", "runserver", "%s:%d" % (host, port)) @@ -626,7 +698,7 @@ def migrate(revision): @click.argument("path", type=click.Path(file_okay=False, exists=True)) @click.pass_context def import_(ctx, mode, path): - """Imports an older Cuckoo setup into a new CWD. The old setup should be + """Import an older Cuckoo setup into a new CWD. The old setup should be identified by PATH and the new CWD may be specified with the --cwd parameter, e.g., "cuckoo --cwd /tmp/cwd import old-cuckoo".""" if os.path.exists(os.path.join(path, ".cwd")): diff --git a/cuckoo/misc.py b/cuckoo/misc.py index a137abc1b7..fab1815f37 100644 --- a/cuckoo/misc.py +++ b/cuckoo/misc.py @@ -33,7 +33,7 @@ # Normalized Cuckoo version (i.e., "2.0.5.3" in setup is "2.0.5" here). This # because we use StrictVersion() later on which doesn't accept "2.0.5.3". -version = "2.0.6" +version = "2.0.7" def set_cwd(path, raw=None): global _root, _raw @@ -41,7 +41,7 @@ def set_cwd(path, raw=None): _raw = raw def cwd(*args, **kwargs): - """Returns absolute path to this file in the Cuckoo Working Directory or + """Return absolute path to this file in the Cuckoo Working Directory or optionally - when private=True has been passed along - to our private Cuckoo Working Directory which is not configurable.""" if kwargs.get("private"): @@ -62,7 +62,7 @@ def cwd(*args, **kwargs): return os.path.join(_root, *args) def decide_cwd(cwd=None, exists=False): - """Decides and sets the CWD, optionally checks if it's a valid CWD.""" + """Decide and set the CWD, optionally check if it's a valid CWD.""" if not cwd: cwd = os.environ.get("CUCKOO_CWD") @@ -104,7 +104,7 @@ def getuser(): return "" def load_signatures(): - """Loads additional Signatures from the Cuckoo Working Directory. + """Load additional Signatures from the Cuckoo Working Directory. This method is quite hacky in the sense that it magically imports Signatures from an arbitrary directory - one that doesn't belong to the @@ -188,7 +188,7 @@ def is_macosx(): return sys.platform == "darwin" def Popen(*args, **kwargs): - """Drops the close_fds argument on Windows platforms in certain situations + """Drop the close_fds argument on Windows platforms in certain situations where it'd otherwise cause an exception from the subprocess module.""" if is_windows() and "close_fds" in kwargs: if "stdin" in kwargs or "stdout" in kwargs or "stderr" in kwargs: @@ -197,7 +197,7 @@ def Popen(*args, **kwargs): return subprocess.Popen(*args, **kwargs) def drop_privileges(username): - """Drops privileges to selected user. + """Drop privileges to selected user. @param username: drop privileges to this username """ if not HAVE_PWD: @@ -225,7 +225,7 @@ def __init__(self, name): self.pid = None def create(self): - """Creates pidfile for the current process.""" + """Create pidfile for the current process.""" with open(self.filepath, "wb") as f: f.write(str(os.getpid())) @@ -249,7 +249,7 @@ def read(self): return self.pid def proc_exists(self, pid): - """Returns boolean if the process exists or None when unsupported.""" + """Return boolean of process existence, or None when unsupported.""" if not pid: return False diff --git a/cuckoo/private/cwd/conf/cuckoo.conf b/cuckoo/private/cwd/conf/cuckoo.conf index 554efe3d37..2b8095c246 100644 --- a/cuckoo/private/cwd/conf/cuckoo.conf +++ b/cuckoo/private/cwd/conf/cuckoo.conf @@ -4,6 +4,11 @@ # one available. version_check = {{ cuckoo.cuckoo.version_check }} +# Cuckoo will stop at startup if the version check reports vulnerabilities in +# one of Cuckoo's dependencies. This setting ignores the vulnerabilities +# and starts anyway +ignore_vulnerabilities = {{ cuckoo.cuckoo.ignore_vulnerabilities }} + # The authentication token that is required to access the Cuckoo API, using # HTTP Bearer authentication. This will protect the API instance against # unauthorized access and CSRF attacks. It is strongly recommended to set this @@ -99,14 +104,10 @@ email = {{ cuckoo.feedback.email }} # `resultserver_ip` for all your virtual machines in machinery configuration. ip = {{ cuckoo.resultserver.ip }} -# Specify a port number to bind the result server on. +# Specify a port number to bind the result server on. Set to 0 to use a random +# port. port = {{ cuckoo.resultserver.port }} -# Force the port chosen above, don't try another one (we can select another -# port dynamically if we can not bind this one, but that is not an option -# in some setups) -force_port = {{ cuckoo.resultserver.force_port }} - # Maximum size of uploaded files from VM (screenshots, dropped files, log). # The value is expressed in bytes, by default 128 MB. upload_max_size = {{ cuckoo.resultserver.upload_max_size }} diff --git a/cuckoo/private/cwd/conf/physical.conf b/cuckoo/private/cwd/conf/physical.conf index a2e0ecc3ff..cb3b76420f 100644 --- a/cuckoo/private/cwd/conf/physical.conf +++ b/cuckoo/private/cwd/conf/physical.conf @@ -9,6 +9,7 @@ user = {{ physical.physical.user }} password = {{ physical.physical.password }} # Default network interface. +# verify that the interface exist with command: ip addr interface = {{ physical.physical.interface }} [fog] diff --git a/cuckoo/private/cwd/conf/processing.conf b/cuckoo/private/cwd/conf/processing.conf index d755c26f57..2cd142bed9 100644 --- a/cuckoo/private/cwd/conf/processing.conf +++ b/cuckoo/private/cwd/conf/processing.conf @@ -61,10 +61,10 @@ maxioc = {{ processing.misp.maxioc }} [network] enabled = {{ processing.network.enabled }} -# Allow domain whitelisting -whitelist_dns = {{ processing.network.whitelist_dns }} +# Allow domain safelisting +safelist_dns = {{ processing.network.safelist_dns }} -# Allow DNS responses from your configured DNS server for whitelisting to +# Allow DNS responses from your configured DNS server for safelisting to # deactivate when responses come from some other DNS # Can be also multiple like : 8.8.8.8,8.8.4.4 allowed_dns = {{ processing.network.allowed_dns }} diff --git a/cuckoo/private/cwd/conf/reporting.conf b/cuckoo/private/cwd/conf/reporting.conf index 8742e19c62..752d6ae24f 100644 --- a/cuckoo/private/cwd/conf/reporting.conf +++ b/cuckoo/private/cwd/conf/reporting.conf @@ -31,6 +31,16 @@ apikey = {{ reporting.misp.apikey }} # separated by whitespace. Available modes: maldoc ipaddr hashes url. mode = {{ reporting.misp.mode }} +distribution = {{ reporting.misp.distribution }} +analysis = {{ reporting.misp.analysis }} +threat_level = {{ reporting.misp.threat_level }} + +# The minimum Cuckoo score for a MISP event to be created +min_malscore = {{ reporting.misp.min_malscore }} + +tag = {{ reporting.misp.tag }} +upload_sample = {{ reporting.misp.upload_sample }} + [mongodb] enabled = {{ reporting.mongodb.enabled }} host = {{ reporting.mongodb.host }} diff --git a/cuckoo/private/cwd/hashes.txt b/cuckoo/private/cwd/hashes.txt index 0c5ee3432a..cfce710cb9 100644 --- a/cuckoo/private/cwd/hashes.txt +++ b/cuckoo/private/cwd/hashes.txt @@ -8,7 +8,6 @@ f128dbf6ac5c90dbed99f9626d58b63b133b7f8d signatures/extractor/__init__.py 335390b78fa1c1ffdf530874819ffcb16dab2174 signatures/linux/__init__.py 1de5e62f213fa0e67af9c1a8ac886a43a08787a2 signatures/network/__init__.py 069e496f3820f73fbd36f7f195dad989e4e0d863 signatures/windows/__init__.py -68f1176dee909f2743bab5ee569cc82ab1e3ef5a signatures/windows/creates_exe.py # 2.0.0 release ae4f2d8ddc2e6ff4b9c255cbb93cc0adecabf9e0 __init__.py @@ -171,7 +170,6 @@ fe60ee51bc7cf8760fb37b73f4526126748d286b signatures/cross/__init__.py 1e8e816f5d82dbfc4431df7dfaccce9f4621220f signatures/darwin/__init__.py dbae884db020dd533231d689b265eb008de23b3d signatures/network/__init__.py f4b225e5c18010fd6e2c4adfe2e70c90854784bf signatures/windows/__init__.py -7092cf73c9588bd00198a698847663d1f274f3bc signatures/windows/creates_exe.py 2b9a6b1a397977c31430b66543f4dfac02af48f2 signatures/windows/generic_metrics.py da39a3ee5e6b4b0d3255bfef95601890afd80709 storage/analyses/.gitignore da39a3ee5e6b4b0d3255bfef95601890afd80709 storage/baseline/.gitignore @@ -291,12 +289,43 @@ ab9af787dd901cf5b4ffb1192da7d096ee1de9ad analyzer/windows/lib/core/packages.py c8492c74db400e6300194c1bafd3088a102bdc8e analyzer/windows/modules/auxiliary/human.py e86627abeb5ecc0112438ad179e9d0487870785a analyzer/windows/modules/packages/ie.py -# TBD -b327de7ae427d9e39f43d11f15b4754fc99ed98b agent/agent.py +# 2.0.7 release +74c4c577a61f96571ac47e86c83fc0ece9d5f0ad agent/agent.py +4d567f35bd79192d8f279f474816b9686e71896b analyzer/darwin/lib/api/screenshot.py +633ab6bd08eb393ca630b59ce0ee5374862c6558 analyzer/darwin/lib/common/hashing.py +d5e3410184765dba6af2895b0dd528558f51ef8d analyzer/darwin/modules/packages/zip.py +6db386e3ebea277637397a42616b2232e2d6b771 analyzer/linux/lib/common/abstracts.py +0c77d682544f214e3314350fe7767e40ecf5b174 analyzer/linux/lib/common/hashing.py +11d0d726c6c17e8abce33b07df9cb498e021da1c analyzer/linux/modules/auxiliary/stap.py +588d4f71a8e6d33628b94d49815f7f27fe621900 analyzer/windows/analyzer.py cb3a77d8dd7edf46de54545ca7b0c5b201f85917 analyzer/windows/bin/execsc.exe +a3847083dc4ee78e186359fb03071489ebfd5932 analyzer/windows/lib/api/process.py +ba6b59b09ef3a157f6081cd1e0f12168cd20538d analyzer/windows/lib/common/abstracts.py +105aac03a5a5ddf1eaf9262389a593d1aaebd0fe analyzer/windows/lib/common/hashing.py +fda170e233e90bd5bf1dbb071c2a35b7661a6b5e analyzer/windows/lib/common/results.py +761c8dd5207399c08f9a9e86847824ffa449caf1 analyzer/windows/lib/core/pipe.py +3c1782e4264af5f4ac774d580036c829b660f572 analyzer/windows/lib/core/startup.py +8f531c7997a8e36e16f1511322adb718630476cd analyzer/windows/modules/auxiliary/disguise.py 93727e778dadc13d83cea61a9ea88bf6b5906686 analyzer/windows/modules/auxiliary/human.py +90bfc348b008e717b5a44cb4ae91b8260682bf84 analyzer/windows/modules/auxiliary/reboot.py +44187be47fd7ddb3b7bed7ab596510168eec1294 analyzer/windows/modules/auxiliary/recentfiles.py +cff8089fb31140efc903f7d87a955b6b35e8b54f analyzer/windows/modules/auxiliary/screenshots.py +348e720075790c862be1e67894ed4ab30ed4f7ce analyzer/windows/modules/auxiliary/zer0m0n.py +e13518903a2fcaec3d0b140d3164c426f07ab647 analyzer/windows/modules/packages/ie.py 24cbd18428df8dbc6b9ccd7896d066c492f6d381 analyzer/windows/modules/packages/ps1.py 6e6680e26bf1cf41909a4efcbd86917cf4b14603 analyzer/windows/modules/packages/pub.py d8fce614d615f6bdb3117e92bfa6e4ae2b48ea52 analyzer/windows/modules/packages/vbs.py -f81e71f788149039f31c9b0b5e2891733e51b5d7 whitelist/ip.txt +09702bc15041a80f399f0c143cc3ec29196e4962 analyzer/windows/modules/packages/zip.py +53fddb538c34040d226441bd59d58fc901dcadc7 monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/inject-x64.exe +630515538733f299523c7fa0fc2ae87781d87fe5 monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/inject-x86.exe +dc1af7e611fb53d6b82f919041f7470301160483 monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/is32bit.exe +8078749c13ff8c022f5c0cca3b865e943f21b929 monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/monitor-x64.dll +b0ceb903d39fe7778e0ad173ef0e02ad0bc7440b monitor/2deb9ccd75d5a7a3fe05b2625b03a8639d6ee36b/monitor-x86.dll +c278d582c26003467794ee4d0ab7ae337034c2ea monitor/latest 4575c7d09b316cad3a71059a27b78cbcb9c1f6b4 stuff/ttp_descriptions.json +9870109fa8a09c2f1b871ea178bdd3d40390baab web/local_settings.py +f81e71f788149039f31c9b0b5e2891733e51b5d7 safelist/ip.txt +cc78a9c7ecdd5a3862b39ad7e6676723e72eb2ba safelist/mispdomain.txt +8f6442b91064e46ab3454d6bc15a4cf1f3949a0f safelist/misphash.txt +1f0ec663731206a9bf9363293421c68855aed772 safelist/mispip.txt +e57ba6930af466d1a56aa22f791048020fadef88 safelist/mispurl.txt diff --git a/cuckoo/private/cwd/supervisord.jinja2 b/cuckoo/private/cwd/supervisord.jinja2 index bdd59b2c49..f722564cb4 100644 --- a/cuckoo/private/cwd/supervisord.jinja2 +++ b/cuckoo/private/cwd/supervisord.jinja2 @@ -17,6 +17,7 @@ command = {{ cuckoo_path }} -d -m 10000 user = {{ username }} startsecs = 30 autorestart = true +stderr_logfile = {{ cwd("supervisord", "cuckoostderr.log") }} [program:cuckoo-process] command = {{ cuckoo_path }} process p%(process_num)d diff --git a/cuckoo/processing/apkinfo.py b/cuckoo/processing/apkinfo.py index 7e64cffec4..a5b7f3fd6f 100644 --- a/cuckoo/processing/apkinfo.py +++ b/cuckoo/processing/apkinfo.py @@ -30,7 +30,7 @@ def check_size(self, file_list): return False def _apk_files(self, apk): - """Returns a list of files in the APK.""" + """Return a list of files in the APK.""" ret = [] for fname, filetype in apk.get_files_types().items(): buf = apk.zip.read(fname) diff --git a/cuckoo/processing/baseline.py b/cuckoo/processing/baseline.py index 3f7dc83812..58f0ca9f9a 100644 --- a/cuckoo/processing/baseline.py +++ b/cuckoo/processing/baseline.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) class Baseline(Processing): - """Reduces Baseline results from gathered information.""" + """Reduce Baseline results from gathered information.""" order = 2 def deep_tuple(self, o, bl=None): @@ -39,9 +39,9 @@ def normalize(self, plugin, o): return self.deep_tuple(o, plugins.get(plugin)) def memory(self, baseline, report): - """Finds the differences between the analysis report and the baseline - report. Puts the differences into the baseline part of the report and - also marks the existing rows with a `class_` attribute.""" + """Find the differences between the analysis report and the baseline + report. Put the differences into the baseline part of the report and + mark the existing rows with a `class_` attribute.""" results = {} for plugin in baseline.keys() + report.keys(): diff --git a/cuckoo/processing/behavior.py b/cuckoo/processing/behavior.py index d711cead30..5d07a232b8 100644 --- a/cuckoo/processing/behavior.py +++ b/cuckoo/processing/behavior.py @@ -19,7 +19,7 @@ log = logging.getLogger(__name__) class Summary(BehaviorHandler): - """Generates overview summary information (not split by process).""" + """Generate overview summary information (not split by process).""" key = "summary" event_types = ["generic"] @@ -294,7 +294,7 @@ def run(self): ApiStats(self), # platform specific stuff - WindowsMonitor(self), + WindowsMonitor(self, task_id=self.task["id"]), LinuxSystemTap(self), # Reboot information. diff --git a/cuckoo/processing/irma.py b/cuckoo/processing/irma.py index 927cbe1470..013f1b1b91 100644 --- a/cuckoo/processing/irma.py +++ b/cuckoo/processing/irma.py @@ -14,7 +14,7 @@ log = logging.getLogger(__name__) class Irma(Processing): - """Gets antivirus signatures from IRMA for various results. + """Get antivirus signatures from IRMA for various results. Currently obtains IRMA results for the target sample. """ @@ -97,7 +97,7 @@ def _get_results(self, sha256): ) def run(self): - """Runs IRMA processing + """Run IRMA processing @return: full IRMA report. """ self.key = "irma" @@ -123,13 +123,13 @@ def run(self): self._scan_file(self.file_path, self.force) results = self._get_results(sha256) or {} - """ FIXME! could use a proper fix here - that probably needs changes on IRMA side aswell - -- - related to https://github.com/elastic/elasticsearch/issues/15377 - entropy value is sometimes 0 and sometimes like 0.10191042566270775 - other issue is that results type changes between string and object :/ - """ + # FIXME! could use a proper fix here + # that probably needs changes on IRMA side aswell + # -- + # related to https://github.com/elastic/elasticsearch/issues/15377 + # entropy value is sometimes 0 and sometimes like 0.10191042566270775 + # other issue is that results type changes between string and object :/ + for idx, result in enumerate(results["probe_results"]): if result["name"] == "PE Static Analyzer": log.debug("Ignoring PE results at index {0}".format(idx)) diff --git a/cuckoo/processing/memory.py b/cuckoo/processing/memory.py index 26873ae291..d67bfaa302 100644 --- a/cuckoo/processing/memory.py +++ b/cuckoo/processing/memory.py @@ -90,7 +90,7 @@ def get_dtb(self): return False def init_config(self): - """Creates a volatility configuration.""" + """Create a volatility configuration.""" if self.config is not None and self.addr_space is not None: return @@ -619,7 +619,7 @@ def apihooks(self): command = self.plugins["apihooks"](self.config) for process, module, hook in command.calculate(): proc_name = str(process.ImageFileName) if process else '' - if command.whitelist(hook.hook_mode | hook.hook_type, + if command.safelist(hook.hook_mode | hook.hook_type, proc_name, hook.VictimModule, hook.HookModule, hook.Function): continue diff --git a/cuckoo/processing/network.py b/cuckoo/processing/network.py index 5dc39fe962..6435080c1b 100644 --- a/cuckoo/processing/network.py +++ b/cuckoo/processing/network.py @@ -26,7 +26,7 @@ from cuckoo.common.irc import ircMessage from cuckoo.common.objects import File from cuckoo.common.utils import convert_to_printable -from cuckoo.common.whitelist import is_whitelisted_domain, is_whitelisted_ip +from cuckoo.common.safelist import is_safelisted_domain, is_safelisted_ip from cuckoo.misc import mkdir # Be less verbose about httpreplay logging messages. @@ -42,7 +42,7 @@ class Pcap(object): ssl_ports = 443, def __init__(self, filepath, options): - """Creates a new instance. + """Create a new instance. @param filepath: path to PCAP file @param options: config options """ @@ -83,19 +83,19 @@ def __init__(self, filepath, options): self.irc_requests = [] # Dictionary containing all the results of this processing. self.results = {} - # List for holding whitelisted IP-s according to DNS responses - self.whitelist_ips = [] - # state of whitelisting - self.whitelist_enabled = self.options.get("whitelist_dns") + # List for holding safelisted IP-s according to DNS responses + self.safelist_ips = [] + # state of safelisting + self.safelist_enabled = self.options.get("safelist_dns") # List of known good DNS servers self.known_dns = self._build_known_dns() # List of all used DNS servers self.dns_servers = [] - def _is_whitelisted(self, conn, hostname): - """Checks if whitelisting conditions are met""" - # Is whitelistng enabled? - if not self.whitelist_enabled: + def _is_safelisted(self, conn, hostname): + """Check if safelisting conditions are met""" + # Is safelistng enabled? + if not self.safelist_enabled: return False # Is DNS recording coming from allowed NS server. @@ -107,8 +107,8 @@ def _is_whitelisted(self, conn, hostname): else: return False - # Is hostname whitelisted. - if not is_whitelisted_domain(hostname): + # Is hostname safelisted. + if not is_safelisted_domain(hostname): return False return True @@ -211,13 +211,13 @@ def _add_hosts(self, connection): # We add external IPs to the list, only the first time # we see them and if they're the destination of the # first packet they appear in. - if not self._is_private_ip(ip) and ip not in self.whitelist_ips: + if not self._is_private_ip(ip) and ip not in self.safelist_ips: self.unique_hosts.append(ip) except: pass def _tcp_dissect(self, conn, data): - """Runs all TCP dissectors. + """Run all TCP dissectors. @param conn: connection. @param data: payload data. """ @@ -237,7 +237,7 @@ def _tcp_dissect(self, conn, data): self._https_identify(conn, data) def _udp_dissect(self, conn, data): - """Runs all UDP dissectors. + """Run all UDP dissectors. @param conn: connection. @param data: payload data. """ @@ -247,7 +247,7 @@ def _udp_dissect(self, conn, data): self._add_dns(conn, data) def _check_icmp(self, icmp_data): - """Checks for ICMP traffic. + """Check for ICMP traffic. @param icmp_data: ICMP data flow. """ try: @@ -257,7 +257,7 @@ def _check_icmp(self, icmp_data): return False def _icmp_dissect(self, conn, data): - """Runs all ICMP dissectors. + """Run all ICMP dissectors. @param conn: connection. @param data: payload data. """ @@ -282,7 +282,7 @@ def _icmp_dissect(self, conn, data): self.icmp_requests.append(entry) def _check_dns(self, udpdata): - """Checks for DNS traffic. + """Check for DNS traffic. @param udpdata: UDP data flow. """ try: @@ -293,7 +293,7 @@ def _check_dns(self, udpdata): return True def _add_dns(self, conn, udpdata): - """Adds a DNS data flow. + """Add a DNS data flow. @param udpdata: UDP data flow. """ dns = dpkt.dns.DNS(udpdata) @@ -397,9 +397,9 @@ def _add_dns(self, conn, udpdata): # TODO: add srv handling query["answers"].append(ans) - if self._is_whitelisted(conn, q_name): - log.debug("DNS target {0} whitelisted. Skipping ...".format(q_name)) - self.whitelist_ips = self.whitelist_ips + _ip + if self._is_safelisted(conn, q_name): + log.debug("DNS target {0} safelisted. Skipping ...".format(q_name)) + self.safelist_ips = self.safelist_ips + _ip return True self._add_domain(query["request"]) @@ -443,7 +443,7 @@ def _add_domain(self, domain): "ip": self._dns_gethostbyname(domain)}) def _check_http(self, tcpdata): - """Checks for HTTP traffic. + """Check for HTTP traffic. @param tcpdata: TCP data flow. """ try: @@ -459,7 +459,7 @@ def _check_http(self, tcpdata): return True def _add_http(self, tcpdata, dport): - """Adds an HTTP flow. + """Add an HTTP flow. @param tcpdata: TCP data flow. @param dport: destination port. """ @@ -562,7 +562,7 @@ def _process_smtp(self): def _check_irc(self, tcpdata): """ - Checks for IRC traffic. + Check for IRC traffic. @param tcpdata: tcp data flow """ try: @@ -574,7 +574,7 @@ def _check_irc(self, tcpdata): def _add_irc(self, tcpdata): """ - Adds an IRC communication. + Add an IRC communication. @param tcpdata: TCP data in flow @param dport: destination port """ @@ -635,7 +635,7 @@ def run(self): offset = file.tell() continue - if is_whitelisted_ip(connection["dst"]): + if is_safelisted_ip(connection["dst"]): continue self._add_hosts(connection) @@ -732,7 +732,7 @@ def run(self): return self.results class Pcap2(object): - """Interprets the PCAP file through the httpreplay library which parses + """Interpret the PCAP file through the httpreplay library which parses the various protocols, decrypts and decodes them, and then provides us with the high level representation of it.""" @@ -768,7 +768,7 @@ def run(self): for s, ts, protocol, sent, recv in l: srcip, srcport, dstip, dstport = s - if is_whitelisted_ip(dstip): + if is_safelisted_ip(dstip): continue if protocol == "smtp": @@ -936,7 +936,7 @@ def conn_from_flowtuple(ft): # it for the temp files # this code is mostly taken from some SO post, can't remember the url though def batch_sort(input_iterator, output_path, output_class): - """batch sort helper with temporary files, supports sorting large stuff""" + """Batch sort helper with temporary files, supports sorting large stuff.""" chunks = [] try: while True: @@ -1015,7 +1015,7 @@ def next(self): return Keyed((flowtuple, ts, self.ctr), rpkt) def sort_pcap(inpath, outpath): - """Use SortCap class together with batch_sort to sort a pcap""" + """Use SortCap class together with batch_sort to sort a pcap.""" inc = SortCap(inpath) batch_sort( inc, outpath, lambda path: SortCap(path, linktype=inc.linktype) @@ -1023,7 +1023,7 @@ def sort_pcap(inpath, outpath): return 0 def flowtuple_from_raw(raw, linktype=1): - """Parse a packet from a pcap just enough to gain a flow description tuple""" + """Parse a packet from a pcap just enough to gain a flow description tuple.""" ip = iplayer_from_raw(raw, linktype) if isinstance(ip, dpkt.ip.IP): diff --git a/cuckoo/processing/platform/windows.py b/cuckoo/processing/platform/windows.py index 30aee00e6c..85bcfa9d91 100644 --- a/cuckoo/processing/platform/windows.py +++ b/cuckoo/processing/platform/windows.py @@ -213,6 +213,7 @@ class WindowsMonitor(BehaviorHandler): key = "processes" def __init__(self, *args, **kwargs): + self.task_id = kwargs.pop("task_id") super(WindowsMonitor, self).__init__(*args, **kwargs) self.processes = [] self.behavior = {} @@ -226,8 +227,8 @@ def handles_path(self, path): def parse(self, path): # Invoke parsing of current log file. - parser = BsonParser(open(path, "rb")) - parser.init() + self.fp = open(path, "rb") # TODO: no proper cleanup + parser = BsonParser(self.fp, self.task_id) for event in parser: if event["type"] == "process": @@ -255,8 +256,8 @@ def parse(self, path): # Process the reboot reconstructor. for category, args in reboot.process_apicall(event): - # TODO Improve this where we have to calculate the "real" - # time again even though we already do this in + # TODO Improve this where we have to calculate the + # "real" time again even though we already do this in # MonitorProcessLog. ts = process["first_seen"] + \ datetime.timedelta(0, 0, event["time"] * 1000) @@ -269,7 +270,8 @@ def parse(self, path): } # Indicate that the process has API calls. For more - # information on this matter, see also the __nonzero__ above. + # information on this matter, see also the __nonzero__ + # above. process["calls"].has_apicalls = True yield event diff --git a/cuckoo/processing/procmemory.py b/cuckoo/processing/procmemory.py index 7f13418b84..e4f0e45afc 100644 --- a/cuckoo/processing/procmemory.py +++ b/cuckoo/processing/procmemory.py @@ -52,7 +52,7 @@ def create_idapy(self, process): print>>o, "autoMark(%s, AU_CODE)" % region["addr"] def _fixup_pe_header(self, pe): - """Fixes the PE header from an in-memory representation to an + """Fix the PE header from an in-memory representation to an on-disk representation.""" for section in pe.sections: section.PointerToRawData = section.VirtualAddress diff --git a/cuckoo/processing/procmon.py b/cuckoo/processing/procmon.py index ac7cfe36f0..494f835182 100644 --- a/cuckoo/processing/procmon.py +++ b/cuckoo/processing/procmon.py @@ -8,7 +8,7 @@ from cuckoo.common.abstracts import Processing class ProcmonLog(list): - """Yields each API call event to the parent handler.""" + """Yield each API call event to the parent handler.""" def __init__(self, filepath): list.__init__(self) @@ -32,7 +32,7 @@ def __nonzero__(self): return True class Procmon(Processing): - """Extracts events from procmon.exe output.""" + """Extract events from procmon.exe output.""" key = "procmon" diff --git a/cuckoo/processing/static.py b/cuckoo/processing/static.py index 46b5a2c95f..e5ac96d2bb 100644 --- a/cuckoo/processing/static.py +++ b/cuckoo/processing/static.py @@ -63,14 +63,14 @@ def __init__(self, file_path): self.pe = None def _get_filetype(self, data): - """Gets filetype, uses libmagic if available. + """Get filetype, use libmagic if available. @param data: data to be analyzed. @return: file type or None. """ return sflock.magic.from_buffer(data) def _get_peid_signatures(self): - """Gets PEID signatures. + """Get PEID signatures. @return: matched signatures or None. """ try: @@ -81,7 +81,7 @@ def _get_peid_signatures(self): return None def _get_imported_symbols(self): - """Gets imported symbols. + """Get imported symbols. @return: imported symbols dict or None. """ imports = [] @@ -105,7 +105,7 @@ def _get_imported_symbols(self): return imports def _get_exported_symbols(self): - """Gets exported symbols. + """Get exported symbols. @return: exported symbols dict or None. """ exports = [] @@ -122,7 +122,7 @@ def _get_exported_symbols(self): return exports def _get_sections(self): - """Gets sections. + """Get sections. @return: sections dict or None. """ sections = [] @@ -207,7 +207,7 @@ def _get_versioninfo(self): return infos def _get_imphash(self): - """Gets imphash. + """Get imphash. @return: imphash string or None. """ try: diff --git a/cuckoo/processing/strings.py b/cuckoo/processing/strings.py index baa1454f6c..e255d0414d 100644 --- a/cuckoo/processing/strings.py +++ b/cuckoo/processing/strings.py @@ -29,12 +29,14 @@ def run(self): ) try: - data = open(self.file_path, "r").read(self.MAX_FILESIZE) + data = open(self.file_path, "rb").read(self.MAX_FILESIZE) except (IOError, OSError) as e: raise CuckooProcessingError("Error opening file %s" % e) - strings = re.findall("[\x1f-\x7e]{6,}", data) - for s in re.findall("(?:[\x1f-\x7e][\x00]){6,}", data): + strings = [] + for s in re.findall(b"[\x1f-\x7e]{6,}", data): + strings.append(s.decode("utf-8")) + for s in re.findall(b"(?:[\x1f-\x7e][\x00]){6,}", data): strings.append(s.decode("utf-16le")) # Now limit the amount & length of the strings. @@ -42,4 +44,4 @@ def run(self): for idx, s in enumerate(strings): strings[idx] = s[:self.MAX_STRINGLEN] - return strings + return strings \ No newline at end of file diff --git a/cuckoo/processing/suricata.py b/cuckoo/processing/suricata.py index b0d93871a1..031098fe00 100644 --- a/cuckoo/processing/suricata.py +++ b/cuckoo/processing/suricata.py @@ -25,7 +25,7 @@ class Suricata(Processing): """Suricata processing module.""" # List of Suricata Signatures IDs that should be ignored. - sid_blacklist = [ + sid_blocklist = [ # SURICATA FRAG IPv6 Fragmentation overlap 2200074, @@ -127,7 +127,7 @@ def parse_eve_json(self): if event["event_type"] == "alert": alert = event["alert"] - if alert["signature_id"] in self.sid_blacklist: + if alert["signature_id"] in self.sid_blocklist: log.debug( "Ignoring alert with sid=%d, signature=%s", alert["signature_id"], alert["signature"] diff --git a/cuckoo/processing/virustotal.py b/cuckoo/processing/virustotal.py index 7652a2b0a0..c903a9a866 100644 --- a/cuckoo/processing/virustotal.py +++ b/cuckoo/processing/virustotal.py @@ -18,7 +18,7 @@ log = logging.getLogger(__name__) class VirusTotal(Processing): - """Gets antivirus signatures from VirusTotal.com for various results. + """Get antivirus signatures from VirusTotal.com for various results. Currently obtains VirusTotal results for the target sample or URL and the dropped files. @@ -26,7 +26,7 @@ class VirusTotal(Processing): order = 2 def run(self): - """Runs VirusTotal processing + """Run VirusTotal processing @return: full VirusTotal report. """ self.key = "virustotal" @@ -94,7 +94,7 @@ def scan_url(self, url, summary=False): "\"%s\": %s", url, e.message) def should_scan_file(self, filetype): - """Determines whether a certain filetype should be scanned on + """Determine whether a certain filetype should be scanned on VirusTotal. For example, we're not interested in scanning text files. @param filetype: file type diff --git a/cuckoo/reporting/elasticsearch.py b/cuckoo/reporting/elasticsearch.py index b6701db999..9c2e810b5f 100644 --- a/cuckoo/reporting/elasticsearch.py +++ b/cuckoo/reporting/elasticsearch.py @@ -21,7 +21,7 @@ log = logging.getLogger(__name__) class ElasticSearch(Report): - """Stores report in Elasticsearch.""" + """Store report in Elasticsearch.""" @classmethod def init_once(cls): diff --git a/cuckoo/reporting/feedback.py b/cuckoo/reporting/feedback.py index 6ca53d3f19..44fda322f8 100644 --- a/cuckoo/reporting/feedback.py +++ b/cuckoo/reporting/feedback.py @@ -6,7 +6,7 @@ from cuckoo.core.feedback import CuckooFeedbackObject, CuckooFeedback class Feedback(Report): - """Reports feedback to the Cuckoo Feedback backend if required.""" + """Report feedback to the Cuckoo Feedback backend if required.""" def run(self, results): # Nothing to see here. diff --git a/cuckoo/reporting/jsondump.py b/cuckoo/reporting/jsondump.py index 54f86a8fba..73041e7253 100644 --- a/cuckoo/reporting/jsondump.py +++ b/cuckoo/reporting/jsondump.py @@ -19,10 +19,10 @@ def default(obj): raise TypeError("%r is not JSON serializable" % obj) class JsonDump(Report): - """Saves analysis results in JSON format.""" + """Save analysis results in JSON format.""" def erase_calls(self, results): - """Temporarily removes calls from the report by replacing them with + """Temporarily remove calls from the report by replacing them with empty lists.""" if self.calls: self.calls = None @@ -34,7 +34,7 @@ def erase_calls(self, results): process["calls"] = [] def restore_calls(self, results): - """Restores calls that were temporarily removed in the report by + """Restore calls that were temporarily removed in the report by replacing the calls with the original values.""" if not self.calls: return diff --git a/cuckoo/reporting/misp.py b/cuckoo/reporting/misp.py index 7ac6af21f9..53cfdebd2c 100644 --- a/cuckoo/reporting/misp.py +++ b/cuckoo/reporting/misp.py @@ -2,13 +2,17 @@ # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. +import logging import os.path import shlex import warnings -import logging from cuckoo.common.abstracts import Report from cuckoo.common.exceptions import CuckooProcessingError +from cuckoo.common.safelist import ( + is_safelisted_mispdomain, is_safelisted_mispip, is_safelisted_mispurl, + is_safelisted_misphash +) log = logging.getLogger(__name__) @@ -33,29 +37,34 @@ def all_urls(self, results, event): urls = set() for protocol in ("http_ex", "https_ex"): for entry in results.get("network", {}).get(protocol, []): - urls.add("%s://%s%s" % ( - entry["protocol"], entry["host"], entry["uri"] - )) + if is_safelisted_mispdomain(entry["host"]): + continue + if is_safelisted_mispdomain(entry["host"]): + continue + + url = "%s://%s%s" % ( + entry["protocol"], entry["host"], entry["uri"]) + + if not is_safelisted_mispurl(url): + urls.add(url) self.misp.add_url(event, sorted(list(urls))) def domain_ipaddr(self, results, event): - whitelist = [ - "www.msftncsi.com", "dns.msftncsi.com", "teredo.ipv6.microsoft.com", "time.windows.com", - "www.msftconnecttest.com", "v10.vortex-win.data.microsoft.com","settings-win.data.microsoft.com", - "win10.ipv6.microsoft.com", "sls.update.microsoft.com", "13.74.179.117", "40.81.120.221", - "40.77.226.249", "8.8.8.8", "fs.microsoft.com", "ctldl.windowsupdate.com" - ] - domains, ips = {}, set() for domain in results.get("network", {}).get("domains", []): - if domain["domain"] not in whitelist: - domains[domain["domain"]] = domain["ip"] - ips.add(domain["ip"]) + if is_safelisted_mispip(domain["ip"]): + continue + + if is_safelisted_mispdomain(domain["domain"]): + continue + + domains[domain["domain"]] = domain["ip"] + ips.add(domain["ip"]) ipaddrs = set() for ipaddr in results.get("network", {}).get("hosts", []): - if ipaddr not in ips: + if ipaddr not in ips and not is_safelisted_mispip(ipaddr): ipaddrs.add(ipaddr) self.misp.add_domains_ips(event, domains) @@ -64,39 +73,92 @@ def domain_ipaddr(self, results, event): def family(self, results, event): for config in results.get("metadata", {}).get("cfgextr", []): self.misp.add_detection_name( - event, config["family"], "Sandbox detection" + event, config["family"], "External analysis" ) for cnc in config.get("cnc", []): self.misp.add_url(event, cnc) for url in config.get("url", []): - self.misp.add_url(event, cnc) + self.misp.add_url(event, url) for mutex in config.get("mutex", []): self.misp.add_mutex(event, mutex) for user_agent in config.get("user_agent", []): self.misp.add_useragent(event, user_agent) def signature(self, results, event): - for sig in results["signatures"]: - data = "%s - (%s)" % (sig["description"], ",".join(sig["ttp"])) + for sig in results.get("signatures", []): + + marks = [] + + if sig["ttp"]: + marks.append("%s" % ", ".join(sig["ttp"])) + + for mark in sig.get("marks", []): + if mark["type"] == "generic": + marks.append( + "%s %s" % (mark.get("parent_process", ""), + mark.get("martian_process", "")) + ) + marks.append( + "%s %s" % (mark.get("reg_key", ""), + mark.get("reg_value", "")) + ) + marks.append( + "%s %s" % (mark.get("option", ""), + mark.get("value", "")) + ) + marks.append("%s" % mark.get("domain", "")) + marks.append("%s" % mark.get("description", "")) + marks.append("%s" % mark.get("host", "")) + + elif mark["type"] == "call": + if not mark["call"]["api"] in marks: + marks.append(mark["call"]["api"]) + + elif mark["type"] == "config": + marks.append(mark["config"].get("url", "")) + + else: + marks.append(mark[mark["type"]]) + + markslist = ", ".join([x for x in marks if x and x != " "]) + + data = "%s - (%s)" % (sig["description"], markslist) self.misp.add_internal_comment(event, data) for att, description in sig["ttp"].items(): - if description is None: - log.warning("Description for %s is not found" % (att)) + if not description: + log.warning("Description for %s is not found", att) continue - self.misp.add_internal_comment(event, "TTP: %s, short: %s" % (att, description["short"])) + self.misp.add_internal_comment( + event, "TTP: %s, short: %s" % (att, description["short"]) + ) def run(self, results): - """Submits results to MISP. + """Submit results to MISP. @param results: Cuckoo results dict. """ url = self.options.get("url") apikey = self.options.get("apikey") mode = shlex.split(self.options.get("mode") or "") + score = results.get("info", {}).get("score", 0) + upload_sample = self.options.get("upload_sample") + + if results.get("target", {}).get("category") == "file": + f = results.get("target", {}).get("file", {}) + hash_safelisted = is_safelisted_misphash(f["md5"]) or \ + is_safelisted_misphash(f["sha1"]) or \ + is_safelisted_misphash(f["sha256"]) + + if hash_safelisted: + return + + if score < self.options.get("min_malscore", 0): + return if not url or not apikey: raise CuckooProcessingError( - "Please configure the URL and API key for your MISP instance." + "Please configure the URL and API key for your MISP " + "instance." ) with warnings.catch_warnings(): @@ -105,20 +167,34 @@ def run(self, results): self.misp = pymisp.PyMISP(url, apikey, False, "json") + # Get default settings for a new event + distribution = self.options.get("distribution") or 0 + threat_level = self.options.get("threat_level") or 4 + analysis = self.options.get("analysis") or 0 + tag = self.options.get("tag") or "Cuckoo" + event = self.misp.new_event( - distribution=pymisp.Distribution.all_communities.value, - threat_level_id=pymisp.ThreatLevel.undefined.value, - analysis=pymisp.Analysis.completed.value, - info="Cuckoo Sandbox analysis #%d" % self.task["id"], + distribution=distribution, + threat_level_id=threat_level, + analysis=analysis, + info="Cuckoo Sandbox analysis #%d" % self.task["id"] ) - if results.get("target", {}).get("category") == "file": - self.misp.upload_sample( - filename=os.path.basename(self.task["target"]), - filepath_or_bytes=self.task["target"], - event_id=event["Event"]["id"], - category="External analysis", - ) + # Add a specific tag to flag Cuckoo's event + if tag: + mispresult = self.misp.tag(event["Event"]["uuid"], tag) + if mispresult.has_key("message"): + log.debug("tag event: %s" % mispresult["message"]) + + if upload_sample: + target = results.get("target", {}) + if target.get("category") == "file" and target.get("file"): + self.misp.upload_sample( + filename=os.path.basename(self.task["target"]), + filepath_or_bytes=self.task["target"], + event_id=event["Event"]["id"], + category="External analysis", + ) self.signature(results, event) diff --git a/cuckoo/reporting/mongodb.py b/cuckoo/reporting/mongodb.py index c6df12f385..b170cdc010 100644 --- a/cuckoo/reporting/mongodb.py +++ b/cuckoo/reporting/mongodb.py @@ -12,7 +12,7 @@ from cuckoo.common.objects import File class MongoDB(Report): - """Stores report in MongoDB.""" + """Store report in MongoDB.""" order = 2 # Mongo schema version, used for data migration. @@ -77,7 +77,7 @@ def store_file(self, file_obj, filename=""): return self.db.fs.files.find_one(to_find)["_id"] def run(self, results): - """Writes report. + """Write report. @param results: analysis results dictionary. @raise CuckooReportError: if fails to connect or write to MongoDB. """ diff --git a/cuckoo/reporting/notification.py b/cuckoo/reporting/notification.py index 7a56105d4b..2518d9f882 100644 --- a/cuckoo/reporting/notification.py +++ b/cuckoo/reporting/notification.py @@ -18,7 +18,7 @@ def default(obj): raise TypeError("%r is not JSON serializable" % obj) class Notification(Report): - """Notifies external service about finished analysis via URL.""" + """Notify external service about finished analysis via URL.""" order = 3 def run(self, results): diff --git a/cuckoo/reporting/singlefile.py b/cuckoo/reporting/singlefile.py index 148b7c8272..202b5a7eb9 100644 --- a/cuckoo/reporting/singlefile.py +++ b/cuckoo/reporting/singlefile.py @@ -20,7 +20,7 @@ logging.getLogger("weasyprint").setLevel(logging.ERROR) class SingleFile(Report): - """Stores report in a single-file HTML and/or PDF format.""" + """Store report in a single-file HTML and/or PDF format.""" fonts = [{ "family": "Roboto", @@ -120,14 +120,14 @@ def generate_jinja2_template(self, results): ) def combine_css(self): - """Scans the static/css/ directory and concatenates stylesheets""" + """Scan the static/css/ directory and concatenate stylesheets""" css_includes = [] for filepath in glob.glob("%s/static/css/*.css" % self.path_base): css_includes.append(open(filepath, "rb").read().decode("utf8")) return "\n".join(css_includes) def combine_js(self): - """Scans the static/js/ directory and concatenates js files""" + """Scan the static/js/ directory and concatenate js files""" js_includes = [] # Note: jquery-2.2.4.min.js must be the first file. filepaths = sorted(glob.glob("%s/static/js/*.js" % self.path_base)) diff --git a/cuckoo/web/analysis/templatetags/analysis_tags.py b/cuckoo/web/analysis/templatetags/analysis_tags.py index f5e18ba51e..e34d251a6e 100644 --- a/cuckoo/web/analysis/templatetags/analysis_tags.py +++ b/cuckoo/web/analysis/templatetags/analysis_tags.py @@ -19,7 +19,7 @@ def mongo_id(value): @register.filter def is_dict(value): - """Checks if value is an instance of dict""" + """Check if value is an instance of dict""" return isinstance(value, dict) @register.filter diff --git a/cuckoo/web/analysis/views.py b/cuckoo/web/analysis/views.py index 7b4fcbb1f3..d9d60d9145 100644 --- a/cuckoo/web/analysis/views.py +++ b/cuckoo/web/analysis/views.py @@ -84,7 +84,7 @@ def chunk(request, task_id, pid, pagenum): @require_safe def filtered_chunk(request, task_id, pid, category): - """Filters calls for call category. + """Filter calls for call category. @param task_id: cuckoo task id @param pid: pid you want calls @param category: call category type diff --git a/cuckoo/web/controllers/analysis/analysis.py b/cuckoo/web/controllers/analysis/analysis.py index 0c71128054..5d7a445e76 100644 --- a/cuckoo/web/controllers/analysis/analysis.py +++ b/cuckoo/web/controllers/analysis/analysis.py @@ -34,7 +34,7 @@ def _get_report(task_id): @staticmethod def _get_dnsinfo(report): - """Create DNS information dicts by domain and ip""" + """Create DNS information dicts by domain and ip.""" if "network" in report and "domains" in report["network"]: domainlookups = dict((i["domain"], i["ip"]) for i in report["network"]["domains"]) diff --git a/cuckoo/web/controllers/analysis/api.py b/cuckoo/web/controllers/analysis/api.py index 278f2a805b..829e91c71f 100644 --- a/cuckoo/web/controllers/analysis/api.py +++ b/cuckoo/web/controllers/analysis/api.py @@ -102,7 +102,7 @@ def tasks_info(request, body): @api_get def task_delete(request, task_id): """ - Deletes a task + Delete a task. :param body: required: task_id :return: """ @@ -126,7 +126,7 @@ def task_delete(request, task_id): @api_get def tasks_reschedule(request, task_id, priority=None): """ - Reschedules a task + Reschedule a task. :param body: required: task_id, priority :return: new task_id """ @@ -363,7 +363,7 @@ def tasks_recent(request, body): @api_post def tasks_stats(request, body): """ - Fetches the number of analysis over a + Fetch the number of analysis over a given period for the "failed" and "successful" states. Values are returned in months. diff --git a/cuckoo/web/controllers/analysis/export/export.py b/cuckoo/web/controllers/analysis/export/export.py index 83ebf0f7fc..e12eebd28f 100644 --- a/cuckoo/web/controllers/analysis/export/export.py +++ b/cuckoo/web/controllers/analysis/export/export.py @@ -44,7 +44,7 @@ def estimate_size(task_id, taken_dirs, taken_files): @staticmethod def create(task_id, taken_dirs, taken_files, report=None): """ - Returns a zip file as a file like object. + Return a zip file as a file like object. :param task_id: task id :param taken_dirs: directories to include :param taken_files: files to include diff --git a/cuckoo/web/controllers/cuckoo/api.py b/cuckoo/web/controllers/cuckoo/api.py index 5b36ab17bd..8748cdf1e1 100644 --- a/cuckoo/web/controllers/cuckoo/api.py +++ b/cuckoo/web/controllers/cuckoo/api.py @@ -21,7 +21,7 @@ updates = {} def latest_updates(): - """Updates the latest Cuckoo version & blogposts at maximum once a day.""" + """Update the latest Cuckoo version & blogposts at maximum once a day.""" next_check = datetime.datetime.now() - datetime.timedelta(days=1) if updates and updates["timestamp"] > next_check: return updates @@ -36,7 +36,7 @@ class CuckooApi(object): @api_get def status(request): """ - Returns a variety of information about both + Return a variety of information about both Cuckoo and the operating system. :return: Dictionary """ diff --git a/cuckoo/web/controllers/machines/api.py b/cuckoo/web/controllers/machines/api.py index 50a2b57d17..f1eb67bc2b 100644 --- a/cuckoo/web/controllers/machines/api.py +++ b/cuckoo/web/controllers/machines/api.py @@ -13,7 +13,7 @@ class MachinesApi: @api_get def list(request): """ - Returns a list of all machines currently registered in Cuckoo + Return a list of all machines currently registered in Cuckoo :return: """ data = {} @@ -29,7 +29,7 @@ def list(request): @api_get def view(request, name=None): """ - Returns information about a machine + Return information about a machine :param name: machine name :return: Machine information as a dictionary """ diff --git a/cuckoo/web/controllers/submission/routes.py b/cuckoo/web/controllers/submission/routes.py index f689b00371..69322bea09 100644 --- a/cuckoo/web/controllers/submission/routes.py +++ b/cuckoo/web/controllers/submission/routes.py @@ -10,7 +10,7 @@ from cuckoo.common.exceptions import CuckooOperationalError from cuckoo.core.database import Database from cuckoo.core.submit import SubmitManager -from cuckoo.web.utils import view_error, render_template, dropped_filepath +from cuckoo.web.utils import view_error, render_template, dropped_filepath, binary_filepath log = logging.getLogger(__name__) submit_manager = SubmitManager() @@ -77,7 +77,11 @@ def resubmit(request, task_id): task.target, ], submit_manager.translate_options_to(task.options)) else: - if not os.path.exists(task.target): + file_path = binary_filepath(task_id) + if not file_path or not os.path.exists(file_path): + file_path = task.target + + if not file_path or not os.path.exists(file_path): return view_error( request, "The file you're trying to resubmit " "no longer exists. Please resubmit it altogether!" @@ -87,7 +91,7 @@ def resubmit(request, task_id): # analyses of type "archive". submit_id = submit_manager.pre("files", [{ "name": os.path.basename(task.target), - "data": open(task.target, "rb"), + "data": open(file_path, "rb"), }], submit_manager.translate_options_to(task.options)) return redirect("submission/pre", submit_id=submit_id) diff --git a/cuckoo/web/src/package.json b/cuckoo/web/src/package.json index a582e5a0d5..6c391e7638 100644 --- a/cuckoo/web/src/package.json +++ b/cuckoo/web/src/package.json @@ -37,7 +37,7 @@ "gulp-util": "3.0.7", "gulp-watch": "4.3.10", "gulp-wrap": "0.13.0", - "handlebars": "4.0.5", + "handlebars": "4.7.6", "vinyl-buffer": "1.0.0", "vinyl-source-stream": "1.1.0" } diff --git a/cuckoo/web/src/scripts/analysis_network.js b/cuckoo/web/src/scripts/analysis_network.js index e535a5377a..0bed44bc50 100644 --- a/cuckoo/web/src/scripts/analysis_network.js +++ b/cuckoo/web/src/scripts/analysis_network.js @@ -335,7 +335,7 @@ class RequestDisplay { // this will later be replaced by the ajax call getting the content - CuckooWeb.post("/analysis/api/task/network_http_data/", { + CuckooWeb.api_post("/analysis/api/task/network_http_data/", { "task_id": window.task_id, "protocol": _this.protocol, "request_body": false, diff --git a/cuckoo/web/static/css/main.css b/cuckoo/web/static/css/main.css index c013d94c2c..de6bd26365 100644 --- a/cuckoo/web/static/css/main.css +++ b/cuckoo/web/static/css/main.css @@ -12430,6 +12430,7 @@ pre [id='dashboard.views.index'] .app { padding: 0; + overflow: hidden; } .content diff --git a/cuckoo/web/static/js/cuckoo/analysis_network.js b/cuckoo/web/static/js/cuckoo/analysis_network.js index e6d3d2c4a3..f07668b465 100644 --- a/cuckoo/web/static/js/cuckoo/analysis_network.js +++ b/cuckoo/web/static/js/cuckoo/analysis_network.js @@ -357,7 +357,7 @@ var RequestDisplay = function () { // this will later be replaced by the ajax call getting the content - CuckooWeb.post("/analysis/api/task/network_http_data/", { + CuckooWeb.api_post("/analysis/api/task/network_http_data/", { "task_id": window.task_id, "protocol": _this.protocol, "request_body": false, diff --git a/cuckoo/web/templates/analysis/pages/static/index.html b/cuckoo/web/templates/analysis/pages/static/index.html index 6c68e667dd..e284567a47 100644 --- a/cuckoo/web/templates/analysis/pages/static/index.html +++ b/cuckoo/web/templates/analysis/pages/static/index.html @@ -27,6 +27,12 @@

Static Analysis

{% if "PE32" in report.analysis.target.file.type %} {% include "analysis/pages/static/_pe32.html" %} + {% elif "MS-DOS executable" in report.analysis.target.file.type %} + {% for sig in report.analysis.static.peid_signatures %} + {% if forloop.first and "FSG" in sig %} + {% include "analysis/pages/static/_pe32.html" %} + {% endif %} + {% endfor %} {% elif "ELF" in report.analysis.target.file.type %} {% include "analysis/pages/static/_elf.html" %} {% elif "office" in report.analysis.static %} @@ -66,4 +72,4 @@

Static Analysis

-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/cuckoo/web/utils.py b/cuckoo/web/utils.py index 2656844a7b..46ba0adc51 100644 --- a/cuckoo/web/utils.py +++ b/cuckoo/web/utils.py @@ -120,6 +120,16 @@ def dropped_filepath(task_id, sha1): if dropped["sha1"] == sha1: return dropped["path"] +def binary_filepath(task_id): + record = mongo.db.analysis.find_one({ + "info.id": int(task_id) + }) + + if not record or not record["target"]["file"]: + return + + return record["target"]["file"]["path"] + def normalize_task(task): if task["category"] == "file": task["target"] = os.path.basename(task["target"]) diff --git a/cuckoo/web/web/middle.py b/cuckoo/web/web/middle.py index 68380910cc..ce0c8cffc3 100644 --- a/cuckoo/web/web/middle.py +++ b/cuckoo/web/web/middle.py @@ -4,6 +4,7 @@ # See the file 'docs/LICENSE' for copying permission. from django.shortcuts import redirect +from ipaddress import ip_network, ip_address from cuckoo.common.config import config from cuckoo.misc import version @@ -31,3 +32,26 @@ def process_response(self, request, response): response["Cache-Control"] = "no-cache" response["Expires"] = "0" return response + +class CuckooFileDownloadAuthentication(object): + def process_request(self, request): + if not request.path.startswith(("/file/sample/", "/file/dropped/")): + return + + #if no ALLOWED_FILEDOWNLOAD_SUBNETS in web local_settings, ignore this + try: + from settings import ALLOWED_FILEDOWNLOAD_SUBNETS + except ImportError: + return + + ip = request.META.get("REMOTE_ADDR") + isallowed = False + if ip: + for network in ALLOWED_FILEDOWNLOAD_SUBNETS.split(","): + network = ip_network(unicode(network), strict=False) + if ip_address(unicode(ip)) in network: + isallowed = True + return + + if not isallowed and not request.session.get("auth"): + return redirect("/secret/") diff --git a/cuckoo/web/web/settings.py b/cuckoo/web/web/settings.py index 0a2aeb316e..7de3d650d2 100644 --- a/cuckoo/web/web/settings.py +++ b/cuckoo/web/web/settings.py @@ -110,6 +110,7 @@ "django.middleware.csrf.CsrfViewMiddleware", # Cuckoo Authentication & headers. "web.middle.CuckooAuthentication", + # "web.middle.CuckooFileDownloadAuthentication", "web.middle.CuckooHeaders", # Our custom exception handler. "web.errors.ExceptionMiddleware" diff --git a/cuckoo/web/web/wsgi.py b/cuckoo/web/web/wsgi.py index 5174683407..8d08a7c343 100644 --- a/cuckoo/web/web/wsgi.py +++ b/cuckoo/web/web/wsgi.py @@ -1,5 +1,5 @@ # Copyright (C) 2010-2013 Claudio Guarnieri. -# Copyright (C) 2014-2016 Cuckoo Foundation. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -24,6 +24,7 @@ import cuckoo +from cuckoo.core.startup import ensure_tmpdir, init_console_logging from cuckoo.misc import decide_cwd if os.environ.get("CUCKOO_APP") == "web": @@ -33,6 +34,8 @@ sys.path.insert(0, ".") cuckoo.core.database.Database().connect() + init_console_logging() + ensure_tmpdir() os.environ.setdefault("DJANGO_SETTINGS_MODULE", "web.settings") diff --git a/docs/CHANGELOG b/docs/CHANGELOG index f2d20db8ef..21160ef7e0 100644 --- a/docs/CHANGELOG +++ b/docs/CHANGELOG @@ -78,7 +78,7 @@ Cuckoo Sandbox 2.0-rc1 (2016-01-21) * Added option to skip calls from JSON report * Added option to load the entire process memory dump into IDA Pro * Added some process memory dump analysis improvements -* Added URLs parsing from memory dump and URLs whitelist +* Added URLs parsing from memory dump and URLs safelist * Added tracking and reporting dead IP address/port combinations * Added maliciousness scoring system * Added option to web interface to submit dropped files for analysis diff --git a/docs/book/_files/conf/auxiliary.conf b/docs/book/_files/conf/auxiliary.conf index 39cf076538..d1c8adf62a 100644 --- a/docs/book/_files/conf/auxiliary.conf +++ b/docs/book/_files/conf/auxiliary.conf @@ -42,6 +42,29 @@ script = stuff/mitm.py # be set to bin/cert.p12. certificate = bin/cert.p12 +[replay] +# Enable PCAP replay capabilities. +enabled = yes + +# Specify the path to your local installation of mitmdump. Make sure this +# path is correct. Note that this should be mitmproxy 3.0.5 or higher, +# installed in a separate virtualenv (or similar). +mitmdump = /usr/local/bin/mitmdump + +# Listen port base. Each virtual machine will use its own port to be +# able to make a good distinction between the various running analyses. +# Generally port 51000 should be fine, in this case port 51001, 51002, etc +# will also be used - again, one port per analyses. +port_base = 51000 + +# Path to the certificate to be used by mitmdump. This file will be +# automatically generated for you if you run mitmdump once. It's just that +# you have to copy it from ~/.mitmproxy/mitmproxy-ca-cert.p12 to somewhere +# in the analyzer/windows/ directory. Recommended is to write the certificate +# to analyzer/windows/bin/cert.p12, in that case the following option should +# be set to bin/cert.p12. +certificate = bin/cert.p12 + [services] # Provide extra services accessible through the network of the analysis VM # provided in separate, standalone, Virtual Machines [yes/no]. diff --git a/docs/book/_files/conf/cuckoo.conf b/docs/book/_files/conf/cuckoo.conf index d7da543187..630fc8e2db 100644 --- a/docs/book/_files/conf/cuckoo.conf +++ b/docs/book/_files/conf/cuckoo.conf @@ -4,12 +4,24 @@ # one available. version_check = yes +# Cuckoo will stop at startup if the version check reports vulnerabilities in +# one of Cuckoo's dependencies. This setting ignores the vulnerabilities +# and starts anyway +ignore_vulnerabilities = no + # The authentication token that is required to access the Cuckoo API, using # HTTP Bearer authentication. This will protect the API instance against # unauthorized access and CSRF attacks. It is strongly recommended to set this # to a secure value. api_token = +# The Web secret is used as a very basic, but successful way to provide basic +# authentication to the Cuckoo Web Interface. This is a shared secret amongst +# all users of this Cuckoo instance and will "protect" usage from users outside +# of this instance. Therefore, if you'd like to share this Cuckoo instance with +# the outside world, then don't use the Web secret functionality. +web_secret = + # If turned on, Cuckoo will delete the original file after its analysis # has been completed. delete_original = no @@ -92,14 +104,10 @@ email = # `resultserver_ip` for all your virtual machines in machinery configuration. ip = 192.168.56.1 -# Specify a port number to bind the result server on. +# Specify a port number to bind the result server on. Set to 0 to use a random +# port. port = 2042 -# Force the port chosen above, don't try another one (we can select another -# port dynamically if we can not bind this one, but that is not an option -# in some setups) -force_port = no - # Maximum size of uploaded files from VM (screenshots, dropped files, log). # The value is expressed in bytes, by default 128 MB. upload_max_size = 134217728 diff --git a/docs/book/_files/conf/kvm.conf b/docs/book/_files/conf/kvm.conf index 9ea38f6576..c1dbcaedac 100644 --- a/docs/book/_files/conf/kvm.conf +++ b/docs/book/_files/conf/kvm.conf @@ -1,4 +1,7 @@ [kvm] +# Specify a libvirt URI connection string +dsn = qemu:///system + # Specify a comma-separated list of available machines to be used. For each # specified ID you have to define a dedicated section containing the details # on the respective machine. (E.g. cuckoo1,cuckoo2,cuckoo3) diff --git a/docs/book/_files/conf/processing.conf b/docs/book/_files/conf/processing.conf index 6a3f6b5911..a2bfacf95e 100644 --- a/docs/book/_files/conf/processing.conf +++ b/docs/book/_files/conf/processing.conf @@ -61,10 +61,10 @@ maxioc = 100 [network] enabled = yes -# Allow domain whitelisting -whitelist_dns = no +# Allow domain safelisting +safelist_dns = no -# Allow DNS responses from your configured DNS server for whitelisting to +# Allow DNS responses from your configured DNS server for safelisting to # deactivate when responses come from some other DNS # Can be also multiple like : 8.8.8.8,8.8.4.4 allowed_dns = @@ -170,4 +170,9 @@ scan = no force = no # URL to your IRMA installation # For example : https://your.irma.host -url = +url = +# Probes to use on your IRMA instance +# If not specified, will default to using all available probes +# Expects comma separated list +# For example : ClamAV,F-Secure,Avast,ESET,eScan,Avira,Sophos,McAfee,Kaspersky,GData,Comodo,Bitdefender +probes = diff --git a/docs/book/_files/conf/reporting.conf b/docs/book/_files/conf/reporting.conf index 0faaa0c69c..763731a151 100644 --- a/docs/book/_files/conf/reporting.conf +++ b/docs/book/_files/conf/reporting.conf @@ -31,6 +31,16 @@ apikey = # separated by whitespace. Available modes: maldoc ipaddr hashes url. mode = maldoc ipaddr hashes url +distribution = 0 +analysis = 0 +threat_level = 4 + +# The minimum Cuckoo score for a MISP event to be created +min_malscore = 0 + +tag = Cuckoo +upload_sample = no + [mongodb] enabled = no host = 127.0.0.1 diff --git a/docs/book/_files/conf/routing.conf b/docs/book/_files/conf/routing.conf index 310e4d518b..3cd9e753de 100644 --- a/docs/book/_files/conf/routing.conf +++ b/docs/book/_files/conf/routing.conf @@ -49,6 +49,11 @@ drop = no enabled = no server = 192.168.56.1 +# Redirect TCP ports (should we also support UDP?). If specified, this should +# represent whitespace-separated src:dst pairs. E.g., "80:8080 443:8080" will +# redirect all 80/443 traffic to 8080 on the specified InetSim host. +ports = + [tor] # Route a VM through Tor, requires a local setup of Tor (please refer to our # documentation). diff --git a/docs/book/conf.py b/docs/book/conf.py index 7945cefb7a..8238778c3d 100644 --- a/docs/book/conf.py +++ b/docs/book/conf.py @@ -47,9 +47,9 @@ # built documents. # # The short X.Y version. -version = '2.0.6' +version = '2.0.7' # The full version, including alpha/beta/rc tags. -release = '2.0.6' +release = '2.0.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/book/installation/host/routing.rst b/docs/book/installation/host/routing.rst index 9745dfecb0..faa614ef06 100644 --- a/docs/book/installation/host/routing.rst +++ b/docs/book/installation/host/routing.rst @@ -120,6 +120,11 @@ Rooter and choosing a network routing option for your analysis**. Documentation on starting the ``Cuckoo Rooter`` may be found in the :ref:`cuckoo_rooter_usage` document. +Both global routing and per-analysis routing require ip forwarding to be enabled: + + $ echo 1 | sudo tee -a /proc/sys/net/ipv4/ip_forward + $ sudo sysctl -w net.ipv4.ip_forward=1 + .. _routing_iproute2: Configuring iproute2 diff --git a/docs/book/installation/index.rst b/docs/book/installation/index.rst index 4453636e52..d5051d334f 100644 --- a/docs/book/installation/index.rst +++ b/docs/book/installation/index.rst @@ -10,7 +10,8 @@ Cuckoo has proved to work smoothly on *Mac OS X* and *Microsoft Windows 7* as host as well. The recommended and tested setup for guests are *Windows XP* and *64-bit Windows 7* for Windows analysis, *Mac OS X Yosemite* for Mac OS X analysis, and Debian for Linux Analysis, although Cuckoo should work with -other releases of guest Operating Systems as well. +other releases of guest Operating Systems as well. Don't use +**Windows linux subsystem (WLS)** as system to run Cuckoo Sandbox. .. note:: diff --git a/docs/book/usage/api.rst b/docs/book/usage/api.rst index f0c40d7b0b..2710bf8e17 100644 --- a/docs/book/usage/api.rst +++ b/docs/book/usage/api.rst @@ -175,7 +175,7 @@ each one. For details click on the resource name. +-------------------------------------+------------------------------------------------------------------------------------------------------------------+ | ``GET`` :ref:`files_get` | Returns the content of the binary with the specified SHA256 hash. | +-------------------------------------+------------------------------------------------------------------------------------------------------------------+ -| ``GET`` :ref:`pcap_get` | Returns the content of the PCAP associated with the given task. | +| ``GET`` :ref:`api_pcap_get` | Returns the content of the PCAP associated with the given task. | +-------------------------------------+------------------------------------------------------------------------------------------------------------------+ | ``GET`` :ref:`machines_list` | Returns the list of analysis machines available to Cuckoo. | +-------------------------------------+------------------------------------------------------------------------------------------------------------------+ @@ -912,7 +912,7 @@ Returns details on the file matching either the specified MD5 hash, SHA256 hash * ``200`` - no error * ``404`` - file not found -.. _pcap_get: +.. _api_pcap_get: /pcap/get --------- diff --git a/setup.py b/setup.py index 1b0ca2efbe..a36dc07d44 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2016-2018 Cuckoo Foundation. +# Copyright (C) 2016-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - https://cuckoosandbox.org/. # See the file 'docs/LICENSE' for copying permission. @@ -65,7 +65,7 @@ def githash(): cwd_private = os.path.join("cuckoo", "private") hashes_ignore = ( - "whitelist/domain.txt", + "safelist/domain.txt", ) def update_hashes(): @@ -149,7 +149,7 @@ def do_setup(**kwargs): do_setup( name="Cuckoo", - version="2.0.7a1", + version="2.0.7", author="Stichting Cuckoo Foundation", author_email="cuckoo@cuckoofoundation.org", packages=[ @@ -185,7 +185,7 @@ def do_setup(**kwargs): ], }, install_requires=[ - "alembic==0.8.8", + "alembic==1.0.10", "androguard==3.0.1", "beautifulsoup4==4.5.3", "chardet==2.3.0", @@ -196,8 +196,10 @@ def do_setup(**kwargs): "egghatch>=0.2.3, <0.3", "elasticsearch==5.3.0", "flask==0.12.2", - "flask-sqlalchemy==2.1", + "flask-sqlalchemy==2.4.0", "httpreplay>=0.2.4, <0.3", + "ipaddress>=1.0.22", + "gevent>=1.2, <1.3", "jinja2==2.9.6", "jsbeautifier==1.6.2", "oletools==0.51", @@ -206,13 +208,13 @@ def do_setup(**kwargs): "pillow==3.2", "pyelftools==0.24", "pyguacamole==0.6", - "pymisp==2.4.103", + "pymisp==2.4.106", "pymongo==3.0.3", "python-dateutil==2.4.2", "python-magic==0.4.12", "roach>=0.1.2, <0.2", - "sflock>=0.3.8, <0.4", - "sqlalchemy==1.0.8", + "sflock>=0.3.10, <0.4", + "sqlalchemy==1.3.3", "unicorn==1.0.1", "wakeonlan==0.2.2", "yara-python==3.6.3", @@ -229,7 +231,6 @@ def do_setup(**kwargs): "scapy==2.3.2", ], "distributed": [ - "gevent==1.1.1", "psycopg2==2.6.2", ], "postgresql": [ diff --git a/stuff/vpncheck.py b/stuff/vpncheck.py index 31b464b432..92670186dd 100755 --- a/stuff/vpncheck.py +++ b/stuff/vpncheck.py @@ -16,7 +16,7 @@ SIOCGIFADDR = 0x8915 def get_ip_address(interface): - """Retrieves the local IP address of a network interface.""" + """Retrieve the local IP address of a network interface.""" s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) buf = fcntl.ioctl(s.fileno(), SIOCGIFADDR, struct.pack("256s", interface)) return socket.inet_ntoa(buf[20:24]) diff --git a/tests/files/conf/20c2_plain/cuckoo.conf b/tests/files/conf/20c2_plain/cuckoo.conf index ca14060473..09d98b45f3 100644 --- a/tests/files/conf/20c2_plain/cuckoo.conf +++ b/tests/files/conf/20c2_plain/cuckoo.conf @@ -113,11 +113,6 @@ ip = 192.168.56.1 # Specify a port number to bind the result server on. port = 2042 -# Force the port chosen above, don't try another one (we can select another -# port dynamically if we can not bind this one, but that is not an option -# in some setups) -force_port = no - # Maximum size of uploaded files from VM (screenshots, dropped files, log) # The value is expressed in bytes, by default 10Mb. upload_max_size = 10485760 diff --git a/tests/files/conf/20c2_plain/processing.conf b/tests/files/conf/20c2_plain/processing.conf index d3376b59a7..339f9da419 100644 --- a/tests/files/conf/20c2_plain/processing.conf +++ b/tests/files/conf/20c2_plain/processing.conf @@ -58,10 +58,10 @@ maxioc = 100 [network] enabled = yes -# Allow domain whitelisting -whitelist-dns = no +# Allow domain safelisting +safelist-dns = no -# Allow DNS responses from your configured DNS server for whitelisting to +# Allow DNS responses from your configured DNS server for safelisting to # deactivate when responses come from some other DNS # Can be also multiple like : 8.8.8.8,8.8.4.4 allowed-dns = diff --git a/tests/files/reportsignatures.json b/tests/files/reportsignatures.json new file mode 100644 index 0000000000..001386e48b --- /dev/null +++ b/tests/files/reportsignatures.json @@ -0,0 +1,4131 @@ +[ + { + "families": [], + "description": "Command line console output was observed", + "ttp": {}, + "name": "console_output", + "markcount": 76, + "references": [], + "marks": [ + { + "pid": 2896, + "cid": 30, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The operation completed successfully.\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483088.094, + "tid": 2900, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2932, + "cid": 30, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The operation completed successfully.\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483088.10925, + "tid": 2936, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2992, + "cid": 30, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The operation completed successfully.\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483088.188125, + "tid": 2996, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 182, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", + "console_handle": "0x00000007" + }, + "time": 1558483088.219, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 184, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "echo", + "console_handle": "0x00000007" + }, + "time": 1558483088.219, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 186, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": " WScript.Sleep(50) ", + "console_handle": "0x00000007" + }, + "time": 1558483088.219, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 190, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp/file.vbs ", + "console_handle": "0x00000007" + }, + "time": 1558483088.219, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 220, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", + "console_handle": "0x00000007" + }, + "time": 1558483088.219, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 222, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "cscript", + "console_handle": "0x00000007" + }, + "time": 1558483088.219, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 224, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": " C:\\Users\\ADMINI~1\\AppData\\Local\\Temp/file.vbs ", + "console_handle": "0x00000007" + }, + "time": 1558483088.219, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 273, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", + "console_handle": "0x00000007" + }, + "time": 1558483088.609, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 275, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "del", + "console_handle": "0x00000007" + }, + "time": 1558483088.609, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 277, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": " /F /Q file.js ", + "console_handle": "0x00000007" + }, + "time": 1558483088.609, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 293, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "Could Not Find C:\\Users\\Administrator\\AppData\\Local\\Temp\\file.js\r\n", + "console_handle": "0x0000000b" + }, + "time": 1558483088.625, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 310, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", + "console_handle": "0x00000007" + }, + "time": 1558483088.641, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 312, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "del", + "console_handle": "0x00000007" + }, + "time": 1558483088.641, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 314, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": " /F /Q \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\" ", + "console_handle": "0x00000007" + }, + "time": 1558483088.641, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 330, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483088.703, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 336, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The process cannot access the file because it is being used by another process.\r\n", + "console_handle": "0x0000000b" + }, + "time": 1558483088.703, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 355, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", + "console_handle": "0x00000007" + }, + "time": 1558483088.703, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 357, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "del", + "console_handle": "0x00000007" + }, + "time": 1558483088.719, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 359, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": " /F /Q \"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\YygokMcY.bat\" ", + "console_handle": "0x00000007" + }, + "time": 1558483088.719, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2112, + "cid": 378, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The batch file cannot be found.\r\n", + "console_handle": "0x0000000b" + }, + "time": 1558483088.766, + "tid": 2088, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2216, + "cid": 41, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "Microsoft (R) Windows Script Host Version 5.8\r\nCopyright (C) Microsoft Corporation. All rights reserved.\r\n\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483088.438125, + "tid": 2228, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2556, + "cid": 30, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The operation completed successfully.\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483088.79725, + "tid": 2552, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2588, + "cid": 30, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The operation completed successfully.\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483088.90575, + "tid": 2592, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2624, + "cid": 30, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The operation completed successfully.\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483088.93775, + "tid": 2620, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 181, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", + "console_handle": "0x00000007" + }, + "time": 1558483089.20325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 183, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "echo", + "console_handle": "0x00000007" + }, + "time": 1558483089.20325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 185, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": " WScript.Sleep(50) ", + "console_handle": "0x00000007" + }, + "time": 1558483089.20325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 189, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp/file.vbs ", + "console_handle": "0x00000007" + }, + "time": 1558483089.20325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 221, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", + "console_handle": "0x00000007" + }, + "time": 1558483089.20325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 223, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "cscript", + "console_handle": "0x00000007" + }, + "time": 1558483089.20325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 225, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": " C:\\Users\\ADMINI~1\\AppData\\Local\\Temp/file.vbs ", + "console_handle": "0x00000007" + }, + "time": 1558483089.20325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 279, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", + "console_handle": "0x00000007" + }, + "time": 1558483089.56325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 281, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "del", + "console_handle": "0x00000007" + }, + "time": 1558483089.56325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 283, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": " /F /Q file.js ", + "console_handle": "0x00000007" + }, + "time": 1558483089.56325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 299, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "Could Not Find C:\\Users\\Administrator\\AppData\\Local\\Temp\\file.js\r\n", + "console_handle": "0x0000000b" + }, + "time": 1558483089.56325, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 316, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", + "console_handle": "0x00000007" + }, + "time": 1558483089.57825, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 318, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "del", + "console_handle": "0x00000007" + }, + "time": 1558483089.57825, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 320, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": " /F /Q \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\" ", + "console_handle": "0x00000007" + }, + "time": 1558483089.57825, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 336, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483089.60925, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 342, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "Access is denied.\r\n", + "console_handle": "0x0000000b" + }, + "time": 1558483089.60925, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 360, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "C:\\Users\\Administrator\\AppData\\Local\\Temp>", + "console_handle": "0x00000007" + }, + "time": 1558483089.60925, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 362, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "del", + "console_handle": "0x00000007" + }, + "time": 1558483089.60925, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 364, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": " /F /Q \"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\PeUwoMwM.bat\" ", + "console_handle": "0x00000007" + }, + "time": 1558483089.60925, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2956, + "cid": 383, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The batch file cannot be found.\r\n", + "console_handle": "0x0000000b" + }, + "time": 1558483089.64125, + "tid": 2964, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2148, + "cid": 50, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "Microsoft (R) Windows Script Host Version 5.8\r\nCopyright (C) Microsoft Corporation. All rights reserved.\r\n\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483089.4215, + "tid": 2204, + "flags": {} + }, + "type": "call" + }, + { + "pid": 3056, + "cid": 30, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The operation completed successfully.\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483089.6405, + "tid": 2648, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2484, + "cid": 30, + "call": { + "category": "misc", + "status": 1, + "stacktrace": [], + "api": "WriteConsoleW", + "return_value": 1, + "arguments": { + "buffer": "The operation completed successfully.\r\n", + "console_handle": "0x00000007" + }, + "time": 1558483089.703375, + "tid": 2468, + "flags": {} + }, + "type": "call" + } + ], + "severity": 1 + }, + { + "families": [], + "description": "HTTP traffic contains suspicious features which may be indicative of malware related traffic", + "ttp": {}, + "name": "network_cnc_http", + "markcount": 1, + "references": [], + "marks": [ + { + "suspicious_features": "GET method with no useragent header", + "type": "generic", + "suspicious_request": "GET http://google.com/" + } + ], + "severity": 2 + }, + { + "families": [], + "description": "Performs some HTTP requests", + "ttp": {}, + "name": "network_http", + "markcount": 1, + "references": [], + "marks": [ + { + "category": "request", + "type": "ioc", + "ioc": "GET http://google.com/", + "description": null + } + ], + "severity": 2 + }, + { + "families": [], + "description": "Allocates read-write-execute memory (usually to unpack itself)", + "ttp": {}, + "name": "allocates_rwx", + "markcount": 327, + "references": [], + "marks": [ + { + "pid": 2560, + "cid": 212, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x003f0000", + "allocation_type": 4096, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 213, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 1691648, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01f50000", + "allocation_type": 4096, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 214, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x005b0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 215, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x005c0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 216, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 31457280, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x020f0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 217, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 131072, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x005d0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 218, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x005f0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 223, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x00600000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 224, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x00610000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 225, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x00620000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 226, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01e50000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 227, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01e60000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 228, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01e70000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 229, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01e80000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 230, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01e90000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 241, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01eb0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 242, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01ec0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.96925, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 255, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01ec0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.98425, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 256, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01ed0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.98425, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 259, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01ec0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.98425, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 260, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01ed0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.98425, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 261, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01ee0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.98425, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 262, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01ef0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483086.98425, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 266, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01f00000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 267, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x041c0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 268, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x041d0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 269, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x041e0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 270, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x041f0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 425, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x041f0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 426, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04200000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 432, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x041f0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 433, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04200000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 442, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x041f0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 443, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04200000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 449, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x041f0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 450, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 8192, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04200000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 458, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x041f0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 459, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04200000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 460, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04210000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 461, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04220000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 462, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04230000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.76625, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 481, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04240000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.98425, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 482, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04250000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.98425, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 483, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2560, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x04260000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.98425, + "tid": 2564, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2652, + "cid": 212, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2652, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x003f0000", + "allocation_type": 4096, + "process_handle": "0xffffffff" + }, + "time": 1558483087.516375, + "tid": 2656, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT" + } + }, + "type": "call" + }, + { + "pid": 2652, + "cid": 213, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2652, + "region_size": 1691648, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x01ec0000", + "allocation_type": 4096, + "process_handle": "0xffffffff" + }, + "time": 1558483087.516375, + "tid": 2656, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT" + } + }, + "type": "call" + }, + { + "pid": 2652, + "cid": 214, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2652, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x005b0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.516375, + "tid": 2656, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2652, + "cid": 215, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2652, + "region_size": 4096, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x005c0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.516375, + "tid": 2656, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2652, + "cid": 216, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2652, + "region_size": 31457280, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x020d0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.516375, + "tid": 2656, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + }, + { + "pid": 2652, + "cid": 217, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "NtAllocateVirtualMemory", + "return_value": 0, + "arguments": { + "process_identifier": 2652, + "region_size": 131072, + "stack_dep_bypass": 0, + "stack_pivoted": 0, + "heap_dep_bypass": 0, + "protection": 64, + "base_address": "0x005d0000", + "allocation_type": 12288, + "process_handle": "0xffffffff" + }, + "time": 1558483087.516375, + "tid": 2656, + "flags": { + "protection": "PAGE_EXECUTE_READWRITE", + "allocation_type": "MEM_COMMIT|MEM_RESERVE" + } + }, + "type": "call" + } + ], + "severity": 2 + }, + { + "families": [], + "description": "Creates a service", + "ttp": { + "T1031": { + "short": "Modify Existing Service", + "long": "Windows service configuration information, including the file path to the service's executable or recovery programs/commands, is stored in the Registry. Service configurations can be modified using utilities such as sc.exe and Reg." + } + }, + "name": "creates_service", + "markcount": 1, + "references": [], + "marks": [ + { + "pid": 2560, + "cid": 412, + "call": { + "category": "services", + "status": 1, + "stacktrace": [], + "api": "CreateServiceW", + "return_value": 6645584, + "arguments": { + "service_start_name": "", + "start_type": 2, + "password": "", + "display_name": "UIgAUMzM", + "filepath": "C:\\ProgramData\\JcgAIEgc\\uOYocMso.exe", + "service_name": "UIgAUMzM", + "filepath_r": "C:\\ProgramData\\JcgAIEgc\\uOYocMso.exe", + "desired_access": 983103, + "service_handle": "0x00656750", + "error_control": 0, + "service_type": 16, + "service_manager_handle": "0x006567c8" + }, + "time": 1558483087.45325, + "tid": 2564, + "flags": {} + }, + "type": "call" + } + ], + "severity": 2 + }, + { + "families": [], + "description": "Drops a binary and executes it", + "ttp": { + "T1129": { + "short": "Execution through Module Load", + "long": "The Windows module loader can be instructed to load DLLs from arbitrary local paths and arbitrary Universal Naming Convention (UNC) network paths. This functionality resides in NTDLL.dll and is part of the Windows Native API which is called from functions like CreateProcess(), LoadLibrary(), etc. of the Win32 API." + } + }, + "name": "dropper", + "markcount": 8, + "references": [], + "marks": [ + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\eeokscog\\posooYos.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\ProgramData\\DUUAMwkg\\BeUQQcIE.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\uAgQwAUw.bat", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\YygokMcY.bat", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\CcYUMEMc.bat", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\SAUoIwsA.bat", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\YUMQIYMY.bat", + "description": null + } + ], + "severity": 2 + }, + { + "families": [], + "description": "Drops an executable to the user AppData folder", + "ttp": { + "T1129": { + "short": "Execution through Module Load", + "long": "The Windows module loader can be instructed to load DLLs from arbitrary local paths and arbitrary Universal Naming Convention (UNC) network paths. This functionality resides in NTDLL.dll and is part of the Windows Native API which is called from functions like CreateProcess(), LoadLibrary(), etc. of the Win32 API." + } + }, + "name": "exe_appdata", + "markcount": 30, + "references": [], + "marks": [ + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\tMUy.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\FoUm.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\PUIk.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\tYki.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\gEYu.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\qAYm.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\vkIo.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\mMYq.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\aoQs.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\tEEi.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\Gwgg.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\RkAQ.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ikEa.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\YskU.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\VkMk.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\aEwC.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\gcUw.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\FMck.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\nMkg.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\JAMG.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\wYAC.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\Kowa.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\gksC.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\CwQa.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\xUAW.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\FEQg.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\oIgG.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\jIQo.exe", + "description": null + }, + { + "category": "file", + "type": "ioc", + "ioc": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\lYEc.exe", + "description": null + } + ], + "severity": 2 + }, + { + "families": [], + "description": "A process created a hidden window", + "ttp": { + "T1143": { + "short": "Hidden Window", + "long": "The configurations for how applications run on macOS and OS X are listed in property list (plist) files. One of the tags in these files can be apple.awt.UIElement, which allows for Java applications to prevent the application's icon from appearing in the Dock. A common use for this is when applications run in the system tray, but don't also want to show up in the Dock. However, adversaries can abuse this feature and hide their running window ." + } + }, + "name": "stealth_window", + "markcount": 8, + "references": [], + "marks": [ + { + "pid": 2560, + "cid": 469, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "CreateProcessInternalW", + "return_value": 1, + "arguments": { + "thread_identifier": 2812, + "thread_handle": "0x00000100", + "process_identifier": 2808, + "current_directory": "C:\\Users\\Administrator\\AppData\\Local\\Temp", + "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\uAgQwAUw.bat", + "track": 1, + "command_line": "\"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391\"", + "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\uAgQwAUw.bat", + "stack_pivoted": 0, + "creation_flags": 134217728, + "inherit_handles": 0, + "process_handle": "0x00000108" + }, + "time": 1558483087.81325, + "tid": 2564, + "flags": { + "creation_flags": "CREATE_NO_WINDOW" + } + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 487, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "CreateProcessInternalW", + "return_value": 1, + "arguments": { + "thread_identifier": 2088, + "thread_handle": "0x00000110", + "process_identifier": 2112, + "current_directory": "", + "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\YygokMcY.bat", + "track": 1, + "command_line": "\"\"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\YygokMcY.bat\" \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\"\"", + "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\YygokMcY.bat", + "stack_pivoted": 0, + "creation_flags": 134217728, + "inherit_handles": 0, + "process_handle": "0x0000010c" + }, + "time": 1558483088.03125, + "tid": 2564, + "flags": { + "creation_flags": "CREATE_NO_WINDOW" + } + }, + "type": "call" + }, + { + "pid": 3048, + "cid": 399, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "CreateProcessInternalW", + "return_value": 1, + "arguments": { + "thread_identifier": 2348, + "thread_handle": "0x00000124", + "process_identifier": 2368, + "current_directory": "C:\\Users\\Administrator\\AppData\\Local\\Temp", + "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\CcYUMEMc.bat", + "track": 1, + "command_line": "\"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391\"", + "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\CcYUMEMc.bat", + "stack_pivoted": 0, + "creation_flags": 134217728, + "inherit_handles": 0, + "process_handle": "0x00000128" + }, + "time": 1558483088.48375, + "tid": 3052, + "flags": { + "creation_flags": "CREATE_NO_WINDOW" + } + }, + "type": "call" + }, + { + "pid": 3048, + "cid": 417, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "CreateProcessInternalW", + "return_value": 1, + "arguments": { + "thread_identifier": 2964, + "thread_handle": "0x00000130", + "process_identifier": 2956, + "current_directory": "", + "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\PeUwoMwM.bat", + "track": 1, + "command_line": "\"\"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\PeUwoMwM.bat\" \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\"\"", + "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\PeUwoMwM.bat", + "stack_pivoted": 0, + "creation_flags": 134217728, + "inherit_handles": 0, + "process_handle": "0x0000012c" + }, + "time": 1558483088.93775, + "tid": 3052, + "flags": { + "creation_flags": "CREATE_NO_WINDOW" + } + }, + "type": "call" + }, + { + "pid": 2704, + "cid": 396, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "CreateProcessInternalW", + "return_value": 1, + "arguments": { + "thread_identifier": 2936, + "thread_handle": "0x00000124", + "process_identifier": 1368, + "current_directory": "C:\\Users\\Administrator\\AppData\\Local\\Temp", + "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\SAUoIwsA.bat", + "track": 1, + "command_line": "\"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391\"", + "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\SAUoIwsA.bat", + "stack_pivoted": 0, + "creation_flags": 134217728, + "inherit_handles": 0, + "process_handle": "0x00000128" + }, + "time": 1558483089.28125, + "tid": 2728, + "flags": { + "creation_flags": "CREATE_NO_WINDOW" + } + }, + "type": "call" + }, + { + "pid": 2704, + "cid": 414, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "CreateProcessInternalW", + "return_value": 1, + "arguments": { + "thread_identifier": 2556, + "thread_handle": "0x00000130", + "process_identifier": 2552, + "current_directory": "", + "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\kSIUwkcE.bat", + "track": 1, + "command_line": "\"\"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\kSIUwkcE.bat\" \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\"\"", + "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\kSIUwkcE.bat", + "stack_pivoted": 0, + "creation_flags": 134217728, + "inherit_handles": 0, + "process_handle": "0x0000012c" + }, + "time": 1558483089.60925, + "tid": 2728, + "flags": { + "creation_flags": "CREATE_NO_WINDOW" + } + }, + "type": "call" + }, + { + "pid": 2636, + "cid": 399, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "CreateProcessInternalW", + "return_value": 1, + "arguments": { + "thread_identifier": 2968, + "thread_handle": "0x00000124", + "process_identifier": 2952, + "current_directory": "C:\\Users\\Administrator\\AppData\\Local\\Temp", + "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\YUMQIYMY.bat", + "track": 1, + "command_line": "\"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391\"", + "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\YUMQIYMY.bat", + "stack_pivoted": 0, + "creation_flags": 134217728, + "inherit_handles": 0, + "process_handle": "0x00000128" + }, + "time": 1558483090.10875, + "tid": 2488, + "flags": { + "creation_flags": "CREATE_NO_WINDOW" + } + }, + "type": "call" + }, + { + "pid": 2636, + "cid": 417, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "CreateProcessInternalW", + "return_value": 1, + "arguments": { + "thread_identifier": 2376, + "thread_handle": "0x00000130", + "process_identifier": 3056, + "current_directory": "", + "filepath": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\hyckYQMo.bat", + "track": 1, + "command_line": "\"\"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\hyckYQMo.bat\" \"C:\\Users\\Administrator\\AppData\\Local\\Temp\\malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe\"\"", + "filepath_r": "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\hyckYQMo.bat", + "stack_pivoted": 0, + "creation_flags": 134217728, + "inherit_handles": 0, + "process_handle": "0x0000012c" + }, + "time": 1558483090.35875, + "tid": 2488, + "flags": { + "creation_flags": "CREATE_NO_WINDOW" + } + }, + "type": "call" + } + ], + "severity": 2 + }, + { + "families": [], + "description": "Searches running processes potentially to identify processes for sandbox evasion, code injection or memory dumping", + "ttp": { + "T1057": { + "short": "Process Discovery", + "long": "Adversaries may attempt to get information about running processes on a system. Information obtained could be used to gain an understanding of common software running on systems within the network." + } + }, + "name": "injection_process_search", + "markcount": 40, + "references": [], + "marks": [ + { + "pid": 2560, + "cid": 272, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32FirstW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "[System Process]", + "process_identifier": 0 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 273, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "System", + "process_identifier": 4 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 274, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "smss.exe", + "process_identifier": 268 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 275, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "csrss.exe", + "process_identifier": 344 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 276, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "wininit.exe", + "process_identifier": 376 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 277, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "csrss.exe", + "process_identifier": 396 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 278, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "winlogon.exe", + "process_identifier": 432 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 279, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "services.exe", + "process_identifier": 476 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 280, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "lsass.exe", + "process_identifier": 492 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 281, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "lsm.exe", + "process_identifier": 500 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 282, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "svchost.exe", + "process_identifier": 608 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 283, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "svchost.exe", + "process_identifier": 684 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 284, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "svchost.exe", + "process_identifier": 764 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 285, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "svchost.exe", + "process_identifier": 816 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 286, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "svchost.exe", + "process_identifier": 860 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 287, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "svchost.exe", + "process_identifier": 892 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 288, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "svchost.exe", + "process_identifier": 292 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 289, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "spoolsv.exe", + "process_identifier": 332 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 290, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "svchost.exe", + "process_identifier": 1052 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 291, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "taskhost.exe", + "process_identifier": 1180 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 292, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "userinit.exe", + "process_identifier": 1256 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 293, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "dwm.exe", + "process_identifier": 1300 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 294, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "explorer.exe", + "process_identifier": 1360 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 295, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "svchost.exe", + "process_identifier": 1756 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 296, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "python.exe", + "process_identifier": 1872 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 297, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "conhost.exe", + "process_identifier": 1900 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 298, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "SearchIndexer.exe", + "process_identifier": 1836 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 299, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "SearchProtocolHost.exe", + "process_identifier": 672 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 300, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "SearchFilterHost.exe", + "process_identifier": 776 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 301, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "SearchProtocolHost.exe", + "process_identifier": 1948 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 302, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "mobsync.exe", + "process_identifier": 2116 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 303, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "python.exe", + "process_identifier": 2264 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 304, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "wsqmcons.exe", + "process_identifier": 2308 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 305, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "sdclt.exe", + "process_identifier": 2352 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 306, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "taskhost.exe", + "process_identifier": 2404 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 307, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "malicious-40400b9c1a79189cae13bc79f0f1ee8e41b4bc6c0b07d3974dbfb08fd2adb391.exe", + "process_identifier": 2560 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 308, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "schtasks.exe", + "process_identifier": 2616 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 309, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e0", + "process_name": "conhost.exe", + "process_identifier": 2624 + }, + "time": 1558483087.18825, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 2560, + "cid": 368, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000ec", + "process_name": "posooYos.exe", + "process_identifier": 2652 + }, + "time": 1558483087.29725, + "tid": 2564, + "flags": {} + }, + "type": "call" + }, + { + "pid": 3048, + "cid": 332, + "call": { + "category": "process", + "status": 1, + "stacktrace": [], + "api": "Process32NextW", + "return_value": 1, + "arguments": { + "snapshot_handle": "0x000000e4", + "process_name": "BeUQQcIE.exe", + "process_identifier": 2692 + }, + "time": 1558483088.39075, + "tid": 3052, + "flags": {} + }, + "type": "call" + } + ], + "severity": 2 + }, + { + "families": [], + "description": "The binary likely contains encrypted or compressed data indicative of a packer", + "ttp": { + "T1045": { + "short": "Software Packing", + "long": "Software packing is a method of compressing or encrypting an executable. Packing an executable changes the file signature in an attempt to avoid signature-based detection. Most decompression techniques decompress the executable code in memory." + } + }, + "name": "packer_entropy", + "markcount": 2, + "references": [ + "http://www.forensickb.com/2013/03/file-entropy-explained.html", + "http://virii.es/U/Using%20Entropy%20Analysis%20to%20Find%20Encrypted%20and%20Packed%20Malware.pdf" + ], + "marks": [ + { + "type": "generic", + "section": { + "size_of_data": "0x001a0c00", + "virtual_address": "0x00001000", + "entropy": 7.227448472154554, + "name": ".text", + "virtual_size": "0x001a0b93" + }, + "entropy": 7.227448472154554, + "description": "A section with a high entropy has been found" + }, + { + "type": "generic", + "entropy": 0.9973078073586599, + "description": "Overall entropy of this PE file is high" + } + ], + "severity": 2 + }, + { + "families": [], + "description": "Expresses interest in specific running processes", + "ttp": { + "T1057": { + "short": "Process Discovery", + "long": "Adversaries may attempt to get information about running processes on a system. Information obtained could be used to gain an understanding of common software running on systems within the network." + } + }, + "name": "process_interest", + "markcount": 2, + "references": [], + "marks": [ + { + "category": "process", + "type": "ioc", + "ioc": "beuqqcie.exe", + "description": null + }, + { + "category": "process", + "type": "ioc", + "ioc": "posooyos.exe", + "description": null + } + ], + "severity": 2 + }, + { + "families": [], + "description": "Uses Windows utilities for basic Windows functionality", + "ttp": { + "T1053": { + "short": "Scheduled Task", + "long": "Utilities such as at and schtasks, along with the Windows Task Scheduler, can be used to schedule programs or scripts to be executed at a date and time. A task can also be scheduled on a remote system, provided the proper authentication is met to use RPC and file and printer sharing is turned on. Scheduling a task on a remote system typically required being a member of the Administrators group on the the remote system." + } + }, + "name": "uses_windows_utilities", + "markcount": 3, + "references": [ + "http://blog.jpcert.or.jp/2016/01/windows-commands-abused-by-attackers.html" + ], + "marks": [ + { + "category": "cmdline", + "type": "ioc", + "ioc": "reg add HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced /f /v HideFileExt /t REG_DWORD /d 1", + "description": null + }, + { + "category": "cmdline", + "type": "ioc", + "ioc": "reg add HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced /f /v Hidden /t REG_DWORD /d 2", + "description": null + }, + { + "category": "cmdline", + "type": "ioc", + "ioc": "reg add HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System /v EnableLUA /d 0 /t REG_DWORD /f", + "description": null + } + ], + "severity": 2 + }, + { + "families": [], + "description": "Communicates with host for which no DNS query was performed", + "ttp": {}, + "name": "nolookup_communication", + "markcount": 1, + "references": [], + "marks": [ + { + "host": "200.87.164.69", + "type": "generic" + } + ], + "severity": 3 + }, + { + "families": [], + "description": "Checks for the presence of known windows from debuggers and forensic tools", + "ttp": { + "T1057": { + "short": "Process Discovery", + "long": "Adversaries may attempt to get information about running processes on a system. Information obtained could be used to gain an understanding of common software running on systems within the network." + } + }, + "name": "antidbg_windows", + "markcount": 1, + "references": [], + "marks": [ + { + "pid": 2652, + "cid": 32216, + "call": { + "category": "ui", + "status": 1, + "stacktrace": [], + "api": "FindWindowA", + "return_value": 65832, + "arguments": { + "class_name": "ConsoleWindowClass", + "window_name": "" + }, + "time": 1558483102.703375, + "tid": 2968, + "flags": {} + }, + "type": "call" + } + ], + "severity": 3 + }, + { + "families": [], + "description": "Installs itself for autorun at Windows startup", + "ttp": { + "T1060": { + "short": "Registry Run Keys / Startup Folder", + "long": "Adding an entry to the \"run keys\" in the Registry or startup folder will cause the program referenced to be executed when a user logs in. These programs will be executed under the context of the user and will have the account's associated permissions level." + }, + "T1053": { + "short": "Scheduled Task", + "long": "Utilities such as at and schtasks, along with the Windows Task Scheduler, can be used to schedule programs or scripts to be executed at a date and time. A task can also be scheduled on a remote system, provided the proper authentication is met to use RPC and file and printer sharing is turned on. Scheduling a task on a remote system typically required being a member of the Administrators group on the the remote system." + } + }, + "name": "persistence_autorun", + "markcount": 5, + "references": [], + "marks": [ + { + "type": "generic", + "reg_key": "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Run\\posooYos.exe", + "reg_value": "C:\\Users\\Administrator\\eeokscog\\posooYos.exe" + }, + { + "type": "generic", + "reg_key": "HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Run\\BeUQQcIE.exe", + "reg_value": "C:\\ProgramData\\DUUAMwkg\\BeUQQcIE.exe" + }, + { + "service_name": "UIgAUMzM", + "type": "generic", + "service_path": "C:\\ProgramData\\JcgAIEgc\\uOYocMso.exe" + }, + { + "type": "generic", + "reg_key": "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Run\\posooYos.exe", + "reg_value": "C:\\Users\\Administrator\\eeokscog\\posooYos.exe" + }, + { + "type": "generic", + "reg_key": "HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Run\\BeUQQcIE.exe", + "reg_value": "C:\\ProgramData\\DUUAMwkg\\BeUQQcIE.exe" + } + ], + "severity": 3 + }, + { + "families": [], + "description": "Attempts to modify Explorer settings to prevent file extensions from being displayed", + "ttp": { + "T1158": { + "short": "Hidden Files and Directories", + "long": "To prevent normal users from accidentally changing special files on a system, most operating systems have the concept of a \u2018hidden\u2019 file. These files don\u2019t show up when a user browses the file system with a GUI or when using normal commands on the command line. Users must explicitly ask to show the hidden files either via a series of Graphical User Interface (GUI) prompts or with command line switches (dir /a for Windows and ls \u2013a for Linux and macOS)." + }, + "T1054": { + "short": "Indicator Blocking", + "long": "An adversary may attempt to block indicators or events typically captured by sensors from being gathered and analyzed. This could include modifying sensor settings stored in configuration files and/or Registry keys to disable or maliciously redirect event telemetry." + } + }, + "name": "stealth_hidden_extension", + "markcount": 1, + "references": [], + "marks": [ + { + "category": "registry", + "type": "ioc", + "ioc": "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\HideFileExt", + "description": null + } + ], + "severity": 3 + }, + { + "families": [], + "description": "Attempts to modify Explorer settings to prevent hidden files from being displayed", + "ttp": { + "T1158": { + "short": "Hidden Files and Directories", + "long": "To prevent normal users from accidentally changing special files on a system, most operating systems have the concept of a \u2018hidden\u2019 file. These files don\u2019t show up when a user browses the file system with a GUI or when using normal commands on the command line. Users must explicitly ask to show the hidden files either via a series of Graphical User Interface (GUI) prompts or with command line switches (dir /a for Windows and ls \u2013a for Linux and macOS)." + }, + "T1054": { + "short": "Indicator Blocking", + "long": "An adversary may attempt to block indicators or events typically captured by sensors from being gathered and analyzed. This could include modifying sensor settings stored in configuration files and/or Registry keys to disable or maliciously redirect event telemetry." + } + }, + "name": "stealth_hiddenfile", + "markcount": 1, + "references": [], + "marks": [ + { + "category": "registry", + "type": "ioc", + "ioc": "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\Hidden", + "description": null + } + ], + "severity": 3 + }, + { + "families": [], + "description": "Disables Windows Security features", + "ttp": { + "T1089": { + "short": "Disabling Security Tools", + "long": "Adversaries may disable security tools to avoid possible detection of their tools and activities. This can take the form of killing security software or event logging processes, deleting Registry keys so that tools do not start at run time, or other methods to interfere with security scanning or event reporting." + }, + "T1112": { + "short": "Modify Registry", + "long": "Adversaries may interact with the Windows Registry to hide configuration information within Registry keys, remove information as part of cleaning up, or as part of other techniques to aid in Persistence and Execution." + } + }, + "name": "disables_security", + "markcount": 1, + "references": [], + "marks": [ + { + "type": "generic", + "description": "attempts to disable user access control", + "registry": "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\EnableLUA" + } + ], + "severity": 4 + }, + { + "families": [], + "description": "Connects to an IP address that is no longer responding to requests (legitimate services will remain up-and-running usually)", + "ttp": {}, + "name": "dead_host", + "markcount": 1, + "references": [], + "marks": [ + { + "category": "dead_host", + "type": "ioc", + "ioc": "200.87.164.69:9999", + "description": null + } + ], + "severity": 5 + } +] \ No newline at end of file diff --git a/tests/files/sql/060my.sql b/tests/files/sql/060my.sql index 92fdc33800..01c01962f0 100644 --- a/tests/files/sql/060my.sql +++ b/tests/files/sql/060my.sql @@ -1,9 +1,3 @@ --- MySQL dump 10.13 Distrib 5.5.53, for debian-linux-gnu (x86_64) --- --- Host: localhost Database: cuckoo --- ------------------------------------------------------ --- Server version 5.5.53-0ubuntu0.14.04.1 - /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; @@ -191,3 +185,9 @@ UNLOCK TABLES; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; -- Dump completed on 2016-11-17 2:47:52 + +-- MySQL dump 10.13 Distrib 5.5.53, for debian-linux-gnu (x86_64) +-- +-- Host: localhost Database: cuckoo +-- ------------------------------------------------------ +-- Server version 5.5.53-0ubuntu0.14.04.1 \ No newline at end of file diff --git a/tests/files/sql/11my.sql b/tests/files/sql/11my.sql index b567515bae..8bc76d3fdc 100644 --- a/tests/files/sql/11my.sql +++ b/tests/files/sql/11my.sql @@ -1,9 +1,3 @@ --- MySQL dump 10.14 Distrib 5.5.47-MariaDB, for debian-linux-gnu (x86_64) --- --- Host: localhost Database: cuckoo11 --- ------------------------------------------------------ --- Server version 5.5.47-MariaDB-1ubuntu0.14.04.1 - /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; @@ -290,3 +284,8 @@ UNLOCK TABLES; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; -- Dump completed on 2017-02-07 12:41:40 +-- MySQL dump 10.14 Distrib 5.5.47-MariaDB, for debian-linux-gnu (x86_64) +-- +-- Host: localhost Database: cuckoo11 +-- ------------------------------------------------------ +-- Server version 5.5.47-MariaDB-1ubuntu0.14.04.1 \ No newline at end of file diff --git a/tests/test_api.py b/tests/test_api.py index 797b10783a..7cb8a6f2c5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -157,7 +157,7 @@ def test_create_submit_opts(self): assert t.memory is True assert t.enforce_timeout is True assert t.options == { - "free": "yes", + "free": "yes", "procmemdump": "no" } def test_create_submit_urls(self): diff --git a/tests/test_apps.py b/tests/test_apps.py index 0151c19217..4ed607fec8 100644 --- a/tests/test_apps.py +++ b/tests/test_apps.py @@ -45,10 +45,12 @@ def setup(self): set_cwd(tempfile.mkdtemp()) cuckoo_create() + @mock.patch("cuckoo.machinery.virtualbox.VirtualBox.version") @mock.patch("cuckoo.main.load_signatures") @mock.patch("cuckoo.main.cuckoo_main") - def test_main(self, p, q): + def test_main(self, p, q, vb): p.side_effect = SystemExit(0) + vb.return_value = "9999" # Ensure that the "latest" binary value makes sense so that the # "run community command" exception is not thrown. @@ -56,9 +58,11 @@ def test_main(self, p, q): main.main(("--cwd", cwd(), "-d", "--nolog"), standalone_mode=False) q.assert_called_once() + @mock.patch("cuckoo.machinery.virtualbox.VirtualBox.version") @mock.patch("cuckoo.main.load_signatures") @mock.patch("cuckoo.main.log") - def test_main_exception(self, p, q): + def test_main_exception(self, p, q, vb): + vb.return_value = "9999" q.side_effect = Exception("this is a test") with pytest.raises(SystemExit): main.main( @@ -73,12 +77,14 @@ def test_api(self): p.assert_called_once_with("localhost", 8090, False) if is_linux(): + @mock.patch("cuckoo.main.cleanup_rooter") @mock.patch("cuckoo.main.cuckoo_rooter") - def test_rooter_abort(self, p, capsys): + def test_rooter_abort(self, p, mr, capsys): p.side_effect = KeyboardInterrupt main.main(("--cwd", cwd(), "rooter"), standalone_mode=False) out, _ = capsys.readouterr() assert "Aborting the Cuckoo Rooter" in out + mr.assert_called_once() def test_community(self): with mock.patch("cuckoo.main.fetch_community") as p: @@ -407,13 +413,14 @@ def test_process_many(self, p, q): ("--cwd", cwd(), "process", "instance"), standalone_mode=False ) - p.assert_called_once_with("instance", 0) + p.assert_called_once_with("instance", 0, 0) q.assert_called_once() @mock.patch("cuckoo.apps.apps.Database") @mock.patch("cuckoo.apps.apps.process") @mock.patch("cuckoo.apps.apps.logger") def test_logger(self, p, q, r): + mkdir(cwd(analysis=123)) process_task({ "id": 123, "target": "foo", @@ -499,6 +506,7 @@ def test_process_dodelete(r, s, p): def test_process_log_taskid(p, q): set_cwd(tempfile.mkdtemp()) cuckoo_create() + mkdir(cwd(analysis=12345)) init_console_logging(logging.DEBUG) init_logfile("process-p0.json") @@ -706,7 +714,7 @@ def test_new_directory(self): shutil.rmtree(cwd("yara", "scripts")) shutil.rmtree(cwd("yara", "shellcode")) shutil.rmtree(cwd("stuff")) - shutil.rmtree(cwd("whitelist")) + shutil.rmtree(cwd("safelist")) open(cwd("yara", "index_binaries.yar"), "wb").write("hello") migrate_cwd() # TODO Move this to its own 2.0.2 -> 2.0.3 migration handler. @@ -714,10 +722,10 @@ def test_new_directory(self): assert os.path.exists(cwd("yara", "shellcode", ".gitignore")) # TODO Move this to its own 2.0.3 -> 2.0.4 migration handler. assert os.path.exists(cwd("stuff")) - assert os.path.exists(cwd("whitelist")) - assert open(cwd("whitelist", "domain.txt"), "rb").read().strip() == ( - "# You can add whitelisted domains here." - ) + assert os.path.exists(cwd("safelist")) + + wl = open(cwd("safelist", "domain.txt"), "rb").read().split("\n") + assert wl[0] == "# You can add safelisted domains here." assert os.path.exists(cwd("yara", "dumpmem")) assert not os.path.exists(cwd("yara", "index_binaries.yar")) @@ -754,6 +762,7 @@ class TestCommunitySuggestion(object): def ctx(self): class context(object): log = False + ignore_vuln = True return context @mock.patch("cuckoo.main.green") diff --git a/tests/test_config.py b/tests/test_config.py index cfd0b99be3..a81c7fd03c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -121,7 +121,7 @@ def test_env(): rooter = /tmp/cuckoo-rooter tmppath = [resultserver] -force_port = no +port = 1234 [database] connection = timeout = @@ -169,7 +169,7 @@ def test_boolean_parse(self): """Testing the boolean parsing in the configuration file parsing.""" assert self.cuckoo.get("cuckoo")["version_check"] is True assert self.cuckoo.get("cuckoo")["max_analysis_count"] is not False - assert self.cuckoo.get("resultserver")["force_port"] is False + assert self.cuckoo.get("resultserver")["port"] == 1234 def test_path_parse(self): """Testing the Path parsing in the configuration file parsing.""" @@ -876,13 +876,13 @@ def test_migration_20c1_20c2(): assert cfg["auxiliary"]["reboot"]["enabled"] is True assert cfg["cuckoo"]["routing"]["rt_table"] == "main" assert cfg["cuckoo"]["routing"]["auto_rt"] is True - assert cfg["cuckoo"]["resultserver"]["force_port"] is False + assert cfg["cuckoo"]["resultserver"]["port"] == 2042 assert cfg["cuckoo"]["timeouts"]["critical"] == 60 assert cfg["processing"]["misp"]["enabled"] is False assert cfg["processing"]["misp"]["url"] is None assert cfg["processing"]["misp"]["apikey"] is None assert cfg["processing"]["misp"]["maxioc"] == 100 - assert cfg["processing"]["network"]["whitelist-dns"] is False + assert cfg["processing"]["network"]["safelist-dns"] is False assert cfg["processing"]["network"]["allowed-dns"] is None assert cfg["processing"]["procmemory"]["extract_img"] is True assert cfg["processing"]["procmemory"]["dump_delete"] is False @@ -927,7 +927,7 @@ def test_migration_20c2_200(): """) Files.create(cwd("conf"), "processing.conf", """ [network] -whitelist-dns = yes +safelist-dns = yes allowed-dns = 8.8.8.8 [procmemory] enabled = yes @@ -988,7 +988,7 @@ def test_migration_20c2_200(): """) cfg = Config.from_confdir(cwd("conf"), loose=True) assert "vpn" in cfg - assert "whitelist-dns" in cfg["processing"]["network"] + assert "safelist-dns" in cfg["processing"]["network"] assert "allowed-dns" in cfg["processing"]["network"] cfg = migrate(cfg, "2.0-rc2", "2.0.0") assert cfg["auxiliary"]["mitm"]["script"] == "mitm.py" @@ -1000,9 +1000,9 @@ def test_migration_20c2_200(): assert cfg["cuckoo"]["feedback"]["email"] is None assert cfg["cuckoo"]["processing"]["analysis_size_limit"] == 128*1024*1024 assert cfg["cuckoo"]["resultserver"]["upload_max_size"] == 128*1024*1024 - assert "whitelist-dns" not in cfg["processing"]["network"] + assert "safelist-dns" not in cfg["processing"]["network"] assert "allowed-dns" not in cfg["processing"]["network"] - assert cfg["processing"]["network"]["whitelist_dns"] is True + assert cfg["processing"]["network"]["safelist_dns"] is True assert cfg["processing"]["procmemory"]["extract_dll"] is False assert cfg["processing"]["network"]["allowed_dns"] == "8.8.8.8" assert cfg["processing"]["virustotal"]["enabled"] is False @@ -1160,7 +1160,7 @@ def test_migration_205_206(): assert cfg["virtualbox"]["controlports"] == "5000-5050" assert cfg["routing"]["inetsim"]["ports"] is None -def test_migration_206_210(): +def test_migration_206_207(): set_cwd(tempfile.mkdtemp()) Folders.create(cwd(), "conf") @@ -1177,15 +1177,24 @@ def test_migration_206_210(): [kvm] machines = cuckoo1 interface = virbr0 + """) + Files.create(cwd("conf"), "reporting.conf", """ +[misp] """) cfg = Config.from_confdir(cwd("conf"), loose=True) - cfg = migrate(cfg, "2.0.6", "2.1.0") + cfg = migrate(cfg, "2.0.6", "2.0.7") assert cfg["auxiliary"]["replay"]["certificate"] == "bin/cert.p12" assert cfg["cuckoo"]["cuckoo"]["api_token"] is None assert cfg["cuckoo"]["cuckoo"]["web_secret"] is None assert cfg["processing"]["irma"]["probes"] is None assert cfg["kvm"]["kvm"]["dsn"] == "qemu:///system" + assert cfg["reporting"]["misp"]["distribution"] == 0 + assert cfg["reporting"]["misp"]["analysis"] == 0 + assert cfg["reporting"]["misp"]["threat_level"] == 4 + assert cfg["reporting"]["misp"]["min_malscore"] == 0 + assert cfg["reporting"]["misp"]["tag"] == "Cuckoo" + assert cfg["reporting"]["misp"]["upload_sample"] is False class FullMigration(object): diff --git a/tests/test_database.py b/tests/test_database.py index b91a8be73c..a77a66b718 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -17,7 +17,7 @@ from cuckoo.misc import set_cwd, cwd, mkdir class DatabaseEngine(object): - """Tests database stuff.""" + """Test database stuff.""" URI = None def setup_class(self): @@ -273,9 +273,17 @@ class TestPostgreSQL(DatabaseEngine): class TestMySQL(DatabaseEngine): URI = "mysql://cuckoo:cuckoo@localhost/cuckootest" +def parse_mysql_dump(dump): + for c in dump.split(";"): + c = c.strip() + + if c and not c.startswith("--"): + c = "%s;" % c + yield c + @pytest.mark.skipif("sys.platform != 'linux2'") class DatabaseMigrationEngine(object): - """Tests database migration(s).""" + """Test database migration(s).""" URI = None SRC = None @@ -430,7 +438,8 @@ class TestDatabaseMigration060MySQL(DatabaseMigration060): @staticmethod def execute_script(cls, script): - cls.s.execute(script) + for command in parse_mysql_dump(script): + cls.s.execute(command) @staticmethod def migrate(cls): @@ -510,7 +519,8 @@ class TestDatabaseMigration11MySQL(DatabaseMigration11): @staticmethod def execute_script(cls, script): - cls.s.execute(script) + for command in parse_mysql_dump(script): + cls.s.execute(command) @mock.patch("cuckoo.core.database.create_engine") @mock.patch("cuckoo.core.database.sessionmaker") diff --git a/tests/test_init.py b/tests/test_init.py index a374b59ee1..a3aa9a897d 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -60,7 +60,7 @@ def test_venv_new_unicode(self): write_supervisor_conf(None) def test_cuckoo_init(self): - """Tests that 'cuckoo init' works with a new CWD.""" + """Test that 'cuckoo init' works with a new CWD.""" with pytest.raises(SystemExit): main.main( ("--cwd", cwd(), "--nolog", "init"), @@ -77,7 +77,7 @@ def test_cuckoo_init(self): assert os.path.exists(os.path.join(cwd(), "storage", "baseline")) def test_cuckoo_init_main(self): - """Tests that 'cuckoo' works with a new CWD.""" + """Test that 'cuckoo' works with a new CWD.""" main.main( ("--cwd", cwd(), "--nolog"), standalone_mode=False @@ -94,8 +94,9 @@ def test_cuckoo_init_main_nosigs(self, p): assert os.path.exists(os.path.join(cwd(), "stuff", "mitm.py")) p.assert_not_called() - def test_cuckoo_init_no_resultserver(self): - """Tests that 'cuckoo init' doesn't launch the ResultServer.""" + @mock.patch("cuckoo.main.check_version") + def test_cuckoo_init_no_resultserver(self, cv): + """Test that 'cuckoo init' doesn't launch the ResultServer.""" with pytest.raises(SystemExit): main.main( ("--cwd", cwd(), "--nolog", "init"), @@ -371,6 +372,12 @@ def lookup_config(s): p.Template = Template write_cuckoo_conf(cfg) + print cfg["cuckoo"] + # Force port was removed/now unused for backwards compatibility + cfg["cuckoo"]["resultserver"].pop("force_port", None) + + # Pool size is a hidden option for now + cfg["cuckoo"]["resultserver"].pop("pool_size", None) for key, value in cfg.items(): for key2, value2 in value.items(): for key3, value3 in value2.items(): diff --git a/tests/test_log.py b/tests/test_log.py index 8775194371..4072665382 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -18,7 +18,7 @@ db = Database() def reset_logging(): - """Resets the logging module to its initial state so that we can + """Reset the logging module to its initial state so that we can re-register all kinds of logging logic for unit testing purposes.""" logging.root = logging.RootLogger(logging.WARNING) logging.Logger.root = logging.root @@ -76,7 +76,7 @@ def test_process_json_logging(): init_yara() init_logfile("process-p0.json") - def process_tasks(instance, maxcount): + def process_tasks(instance, maxcount, timeout): logger("foo bar", action="hello.world", status="success") with mock.patch("cuckoo.main.Database"): diff --git a/tests/test_machinery.py b/tests/test_machinery.py index 3f787d839e..066efd4cf3 100644 --- a/tests/test_machinery.py +++ b/tests/test_machinery.py @@ -13,7 +13,7 @@ CuckooMachineError, CuckooCriticalError, CuckooMachineSnapshotError, CuckooDependencyError, CuckooMissingMachineError ) -from cuckoo.common.files import Folders, Files +from cuckoo.common.files import Folders from cuckoo.common.objects import Dictionary from cuckoo.core.database import Database from cuckoo.core.init import write_cuckoo_conf @@ -446,8 +446,10 @@ def test_stop_success(self): self.m._wait_status = mock.MagicMock(return_value=None) with mock.patch("cuckoo.machinery.virtualbox.Popen") as p: - p.poll.return_value = True - p.returncode = 0 + proc = mock.MagicMock() + proc.returncode = 0 + proc.poll.return_value = True + p.return_value = proc self.m.stop("label") p.assert_called_once_with( @@ -459,6 +461,10 @@ def test_stop_success(self): ) with mock.patch("cuckoo.machinery.virtualbox.Popen") as p: + proc = mock.MagicMock() + proc.returncode = 0 + proc.poll.return_value = True + p.return_value = proc self.m._status.return_value = self.m.SAVED self.m.stop("label") p.assert_not_called() diff --git a/tests/test_misc.py b/tests/test_misc.py index 1c3a78b6bf..4d1ef2efa6 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -111,7 +111,7 @@ def test_platforms(): assert sys.platform in ("win32", "linux2", "darwin") def test_popen(): - """Ensures that Popen is working properly.""" + """Ensure that Popen is working properly.""" with mock.patch("subprocess.Popen") as p: p.return_value = None Popen(["foo", "bar"]) diff --git a/tests/test_processing.py b/tests/test_processing.py index 8414686368..09d738087c 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 Cuckoo Foundation. +# Copyright (C) 2016-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -52,6 +52,22 @@ db = Database() class TestProcessing(object): + + def setup(self): + self.tmpdirs = [] + + def teardown(self): + for path in self.tmpdirs: + try: + shutil.rmtree(path) + except: + pass + + def mkdtemp(self): + path = tempfile.mkdtemp() + self.tmpdirs.append(path) + return path + def test_init(self): p = Processing() p.set_options({ @@ -61,7 +77,7 @@ def test_init(self): assert p.options.foo == "bar" def test_debug(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() init_console_logging() @@ -124,7 +140,7 @@ def test_static_none(self): assert s.run() is None def test_archive_pdf(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) s = Static() s.set_task({ @@ -141,7 +157,7 @@ def test_archive_pdf(self): assert "%48%65" in s.run()["pdf"][0]["javascript"][0]["orig_code"] def test_pdf(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) s = Static() s.set_task({ @@ -157,7 +173,7 @@ def test_pdf(self): assert "var x = unescape" in r["javascript"][0]["orig_code"] def test_pdf_stringjs(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) s = Static() s.set_task({ @@ -173,7 +189,7 @@ def test_pdf_stringjs(self): assert "app.alert({" in r["javascript"][0]["orig_code"] def test_pdf_ignorefake(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) s = Static() s.set_task({ @@ -191,7 +207,7 @@ def test_pdf_ignorefake(self): @mock.patch("cuckoo.processing.static.dispatch") def test_pdf_workercrash(self, md): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) md.return_value = None s = Static() @@ -208,7 +224,7 @@ def test_pdf_workercrash(self, md): assert r["pdf"] == [] def test_phishing0_pdf(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) s = Static() s.set_task({ @@ -224,7 +240,7 @@ def test_phishing0_pdf(self): @mock.patch("cuckoo.processing.static.dispatch") def test_pdf_mock(self, p): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) s = Static() s.set_task({ @@ -243,7 +259,7 @@ def test_pdf_mock(self, p): ) def test_pdf_metadata(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) s = Static() s.set_task({ @@ -273,7 +289,7 @@ def test_pdf_metadata(self): } def test_pdf_attach(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) s = Static() s.set_task({ @@ -316,7 +332,7 @@ def test_pdf_parse_string(self): assert p("\xfe\xff\x00h\x00t\x00t\x00p\x00:\x00/\x00/") == "http://" def test_office(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() init_yara() @@ -472,7 +488,7 @@ def test_strings(self): def test_screenshot_tesseract(self, p): s = Screenshots() # Use an empty directory so no actual screenshot analysis is done. - s.shots_path = tempfile.mkdtemp() + s.shots_path = self.mkdtemp() s.set_options({ "tesseract": None, }) @@ -528,7 +544,7 @@ def test_screenshot_truncated(self, p): assert s.run() == [] def test_targetinfo(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() init_yara() @@ -625,7 +641,7 @@ def test_ignore_notesseract(self, p, q): os.unlink(shotpath) def test_virustotal_nokey(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create(cfg={ "processing": { "virustotal": { @@ -638,7 +654,7 @@ def test_virustotal_nokey(self): e.match("API key not configured") def test_virustotal_invalidcategory(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() with pytest.raises(CuckooProcessingError) as e: v = VirusTotal() @@ -650,9 +666,25 @@ def test_virustotal_invalidcategory(self): @pytest.mark.skipif(not HAVE_VOLATILITY, reason="No Volatility installed") class TestVolatility(object): + + def setup(self): + self.tmpdirs = [] + + def teardown(self): + for path in self.tmpdirs: + try: + shutil.rmtree(path) + except: + pass + + def mkdtemp(self): + path = tempfile.mkdtemp() + self.tmpdirs.append(path) + return path + @mock.patch("cuckoo.processing.memory.log") def test_no_mempath(self, p): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) m = Memory() m.memory_path = None assert m.run() is None @@ -661,7 +693,7 @@ def test_no_mempath(self, p): @mock.patch("cuckoo.processing.memory.log") def test_invalid_mempath(self, p): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) m = Memory() m.memory_path = "notafile" assert m.run() is None @@ -670,7 +702,7 @@ def test_invalid_mempath(self, p): @mock.patch("cuckoo.processing.memory.log") def test_empty_mempath(self, p): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) m = Memory() m.memory_path = Files.temp_put("") assert m.run() is None @@ -679,7 +711,7 @@ def test_empty_mempath(self, p): @mock.patch("cuckoo.processing.memory.VolatilityManager") def test_global_osprofile(self, p): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create(cfg={ "memory": { "basic": { @@ -696,7 +728,7 @@ def test_global_osprofile(self, p): @mock.patch("cuckoo.processing.memory.VolatilityManager") def test_vm_osprofile(self, p): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create(cfg={ "memory": { "basic": { @@ -737,7 +769,7 @@ def test_wrong_profile(self, p, q): @mock.patch("volatility.utils.load_as") def test_plugin_enabled(self, p): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create(cfg={ "memory": { "pslist": { @@ -775,8 +807,23 @@ def test_s(self): assert obj_s(vol_obj.NoneObject()) is None class TestProcessingMachineInfo(object): + def setup(self): + self.tmpdirs = [] + + def teardown(self): + for path in self.tmpdirs: + try: + shutil.rmtree(path) + except: + pass + + def mkdtemp(self): + path = tempfile.mkdtemp() + self.tmpdirs.append(path) + return path + def test_machine_info_empty(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) rp = RunProcessing({ "id": 1, }) @@ -784,7 +831,7 @@ def test_machine_info_empty(self): assert rp.machine == {} def test_machine_info_cuckoo1(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() rp = RunProcessing({ @@ -800,7 +847,7 @@ def test_machine_info_cuckoo1(self): assert rp.machine["ip"] == "192.168.56.101" def test_machine_info_cuckoo2(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() rp = RunProcessing({ @@ -816,6 +863,21 @@ def test_machine_info_cuckoo2(self): } class TestBehavior(object): + def setup(self): + self.tmpdirs = [] + + def teardown(self): + for path in self.tmpdirs: + try: + shutil.rmtree(path) + except: + pass + + def mkdtemp(self): + path = tempfile.mkdtemp() + self.tmpdirs.append(path) + return path + def test_process_tree_regular(self): pt = ProcessTree(None) @@ -888,7 +950,7 @@ def test_process_tree_pid_reuse(self): assert not obj[2]["children"] def test_bson_limit(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() ba = BehaviorAnalysis() @@ -914,7 +976,7 @@ def test_bson_limit(self): ] def test_extract_scripts(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() init_yara() @@ -968,7 +1030,7 @@ def test_extract_scripts(self): assert open(out[1]["raw"], "rb").read() == 'echo "Recursive"' def test_stap_log(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() init_yara() @@ -1066,6 +1128,7 @@ def test_stap_log(self): } class TestPcap(object): + @classmethod def setup_class(cls): set_cwd(tempfile.mkdtemp()) @@ -1223,15 +1286,30 @@ def test_network_dns(self): assert expected_types == types class TestPcapAdditional(object): + def setup(self): + self.tmpdirs = [] + + def teardown(self): + for path in self.tmpdirs: + try: + shutil.rmtree(path) + except: + pass + + def mkdtemp(self): + path = tempfile.mkdtemp() + self.tmpdirs.append(path) + return path + @mock.patch("cuckoo.processing.network.resolve") def test_resolve_dns(self, p): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() p.return_value = "1.2.3.4" assert Pcap(None, {})._dns_gethostbyname("google.com") != "" def test_icmp_ignore_resultserver(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create() p = Pcap(None, {}) pkt = dpkt.icmp.ICMP.Echo() @@ -1249,7 +1327,7 @@ def test_icmp_ignore_resultserver(self): assert len(p.icmp_requests) == 1 def test_no_sorted_pcap(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create(cfg={ "cuckoo": { "processing": { @@ -1269,7 +1347,7 @@ def test_no_sorted_pcap(self): assert not os.path.exists(cwd("dump_sorted.pcap", analysis=1)) def test_yes_sorted_pcap(self): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create({ "cuckoo": { "network": { @@ -1304,7 +1382,7 @@ def test_duplicate_dns_requests(self): @mock.patch("cuckoo.processing.network.log") def test_empty_pcap(self, p): - set_cwd(tempfile.mkdtemp()) + set_cwd(self.mkdtemp()) cuckoo_create(cfg={ "cuckoo": { "processing": { @@ -1325,9 +1403,24 @@ def test_empty_pcap(self, p): p.warning.assert_not_called() class TestPcap2(object): + def setup(self): + self.tmpdirs = [] + + def teardown(self): + for path in self.tmpdirs: + try: + shutil.rmtree(path) + except: + pass + + def mkdtemp(self): + path = tempfile.mkdtemp() + self.tmpdirs.append(path) + return path + def test_smtp_ex(self): obj = Pcap2( - "tests/files/pcap/smtp.pcap", None, tempfile.mkdtemp() + "tests/files/pcap/smtp.pcap", None, self.mkdtemp() ).run() assert len(obj["smtp_ex"]) == 1 @@ -1346,7 +1439,7 @@ def test_smtp_ex(self): def test_http_status(self): obj = Pcap2( - "tests/files/pcap/status-code.pcap", None, tempfile.mkdtemp() + "tests/files/pcap/status-code.pcap", None, self.mkdtemp() ).run() assert len(obj["http_ex"]) == 1 assert not obj["https_ex"] @@ -1354,7 +1447,7 @@ def test_http_status(self): def test_http_nostatus(self): obj = Pcap2( - "tests/files/pcap/not-http.pcap", None, tempfile.mkdtemp() + "tests/files/pcap/not-http.pcap", None, self.mkdtemp() ).run() assert len(obj["http_ex"]) == 1 @@ -1439,45 +1532,67 @@ def create(): s.process_pcap_binary = create s.run() -def test_static_extracted(): - set_cwd(tempfile.mkdtemp()) - cuckoo_create(cfg={ - "processing": { - "analysisinfo": { - "enabled": False, +class TestExtracted(object): + def setup(self): + self.tmpdirs = [] + + def teardown(self): + for path in self.tmpdirs: + try: + shutil.rmtree(path) + except: + pass + + def mkdtemp(self): + path = tempfile.mkdtemp() + self.tmpdirs.append(path) + return path + + def test_static_extracted(self): + set_cwd(self.mkdtemp()) + cuckoo_create(cfg={ + "processing": { + "analysisinfo": { + "enabled": False, + }, + "debug": { + "enabled": False, + } }, - "debug": { - "enabled": False, + }) + mkdir(cwd(analysis=1)) + shutil.copy("tests/files/createproc1.docm", cwd("binary", analysis=1)) + + open(cwd("yara", "office", "ole.yar"), "wb").write(""" + rule OleInside { + strings: + $s1 = "Win32_Process" + condition: + filename matches /word\/vbaProject.bin/ and $s1 } - }, - }) - mkdir(cwd(analysis=1)) - shutil.copy("tests/files/createproc1.docm", cwd("binary", analysis=1)) - - open(cwd("yara", "office", "ole.yar"), "wb").write(""" - rule OleInside { - strings: - $s1 = "Win32_Process" - condition: - filename matches /word\/vbaProject.bin/ and $s1 - } - """) - init_yara() - - class OleInsideExtractor(Extractor): - def handle_yara(self, filepath, match): - return ( - match.category == "office" and - match.yara[0].name == "OleInside" - ) - - ExtractManager._instances = {} - ExtractManager.extractors = OleInsideExtractor, - - results = RunProcessing(Dictionary({ - "id": 1, - "category": "file", - "target": "tests/files/createproc1.docm", - })).run() - - assert len(results["extracted"]) == 1 + """) + init_yara() + + class OleInsideExtractor(Extractor): + def handle_yara(self, filepath, match): + return ( + match.category == "office" and + match.yara[0].name == "OleInside" + ) + + class X(object): + @staticmethod + def p(): + return [Extracted, Static] + + with mock.patch("cuckoo.processing.plugins", new_callable=X.p) as p: + ExtractManager._instances = {} + ExtractManager.extractors = OleInsideExtractor, + + results = RunProcessing(Dictionary({ + "id": 1, + "category": "file", + "target": "tests/files/createproc1.docm", + })).run() + + assert len(results["extracted"]) == 1 diff --git a/tests/test_reporting.py b/tests/test_reporting.py index e4bc902c70..3378d56c44 100644 --- a/tests/test_reporting.py +++ b/tests/test_reporting.py @@ -2,6 +2,7 @@ # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. +import json import mock import os.path import pytest @@ -20,6 +21,7 @@ from cuckoo.reporting.mongodb import MongoDB from cuckoo.reporting.singlefile import SingleFile + def test_init(): p = Report() p.set_options({ @@ -114,9 +116,95 @@ def test_empty_mattermost(): task(1, {}, conf, {}) assert len(responses.calls) == 2 +@responses.activate +def test_min_malscore_misp_low(): + """Try to send event with low malscore.""" + set_cwd(tempfile.mkdtemp()) + conf = { + "misp": { + "enabled": True, + "url": "https://misphost", + "apikey": "A"*32, + "mode": "", + "min_malscore": 5 + } + } + + with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: + rsps.add( + responses.GET, "https://misphost/servers/getPyMISPVersion.json", + json={ + "version": "2.4.103" + } + ) + + rsps.add( + responses.GET, "https://misphost/attributes/describeTypes.json", + json={ + "result": { + "categories": None, + "types": None, + "category_type_mappings": None, + "sane_defaults": True, + }, + }, + ) + rsps.add( + responses.POST, "https://misphost/events", + json={ + "response": None, + }, + ) + + task(2, {}, conf, {"info": {"score": 2}}) + assert len(rsps.calls) == 0 + +@responses.activate +def test_min_malscore_misp(): + """Try to send event with low malscore.""" + set_cwd(tempfile.mkdtemp()) + conf = { + "misp": { + "enabled": True, + "url": "https://misphost", + "apikey": "A"*32, + "mode": "", + "min_malscore": 5 + } + } + + with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: + rsps.add( + responses.GET, "https://misphost/servers/getPyMISPVersion.json", + json={ + "version": "2.4.103" + } + ) + + rsps.add( + responses.GET, "https://misphost/attributes/describeTypes.json", + json={ + "result": { + "categories": None, + "types": None, + "category_type_mappings": None, + "sane_defaults": True, + }, + }, + ) + rsps.add( + responses.POST, "https://misphost/events", + json={ + "response": None, + }, + ) + + task(2, {}, conf, {"info": {"score": 6}}) + assert len(rsps.calls) == 3 + @responses.activate def test_empty_misp(): - """Merely connects to MISP and creates the new event.""" + """Merely connect to MISP and create the new event.""" set_cwd(tempfile.mkdtemp()) conf = { "misp": { @@ -129,12 +217,12 @@ def test_empty_misp(): with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps: rsps.add( - responses.GET, "https://misphost/servers/getVersion.json", + responses.GET, "https://misphost/servers/getPyMISPVersion.json", json={ - "version": "2.4.56", - "perm_sync": True, - }, + "version": "2.4.103" + } ) + rsps.add( responses.GET, "https://misphost/attributes/describeTypes.json", json={ @@ -177,33 +265,33 @@ def test_misp_sample_hashes(): comment="File submitted to Cuckoo" ) -def test_misp_maldoc(): +def test_misp_signatures(): r = MISP() r.misp = mock.MagicMock() - r.misp.add_url.return_value = None - - r.maldoc_network({ - "signatures": [ - { - "name": "foobar", - }, - { - "name": "malicious_document_urls", - "marks": [ - { - "category": "file", - }, - { - "category": "url", - "ioc": "url_ioc", - } - ], - }, - ], - }, "event") - r.misp.add_url.assert_called_once_with("event", ["url_ioc"]) + r.misp.add_internal_comment.return_value = None + + with open("tests/files/reportsignatures.json", "rb") as fp: + signatures = json.load(fp) + + r.signature({"signatures": signatures}, "event") + + assert r.misp.add_internal_comment.call_count == 36 + r.misp.add_internal_comment.assert_has_calls([ + mock.call("event", "Creates a service - (T1031, CreateServiceW)"), + mock.call("event", "Searches running processes potentially to identify" + " processes for sandbox evasion, code injection or" + " memory dumping -" + " (T1057, Process32FirstW, Process32NextW)"), + mock.call("event", "TTP: T1054, short: Indicator Blocking"), + mock.call("event", "Disables Windows Security features -" + " (T1089, T1112, attempts to disable user access" + " control)"), + mock.call("event", "Communicates with host for which no DNS query was" + " performed - (200.87.164.69)") + ], any_order=True) def test_misp_all_urls(): + set_cwd(tempfile.mkdtemp()) r = MISP() r.misp = mock.MagicMock() r.misp.add_url.return_value = None @@ -233,6 +321,7 @@ def test_misp_all_urls(): ) def test_misp_domain_ipaddr(): + set_cwd(tempfile.mkdtemp()) r = MISP() r.misp = mock.MagicMock() r.misp.add_domains_ips.return_value = None @@ -246,16 +335,18 @@ def test_misp_domain_ipaddr(): "ip": "1.2.3.4", }, { - # TODO Now that we have global whitelisting, this - # custom-made support for the MISP reporting module should - # probably be removed. "domain": "time.windows.com", "ip": "1.2.3.4", }, + { + "domain": "www.msftncsi.com", + "ip": "95.101.2.42" + } ], "hosts": [ "2.3.4.5", "3.4.5.6", + "8.8.8.8" ], }, }, "event") @@ -268,6 +359,50 @@ def test_misp_domain_ipaddr(): "event", ["2.3.4.5", "3.4.5.6"], ) +def test_misp_family(): + r = MISP() + r.misp = mock.MagicMock() + r.misp.add_detection_name.return_value = None + r.misp.add_url.return_value = None + r.misp.add_mutex.return_value = None + r.misp.add_useragent.return_value = None + + r.family({ + "metadata": { + "cfgextr": [ + { + "family": "3x4mpl3", + "cnc": ["example.com/gate.php"] + }, + { + "family": "3x4mpl3_2", + "url": ["http://example.org"] + }, + { + "family": "3x4mpl3_3", + "mutex": ["@@@@@@"], + "user_agent": ["M3mebr0wz0r V42"] + } + ] + } + }, "event") + + assert r.misp.add_detection_name.call_count == 3 + r.misp.add_detection_name.assert_has_calls([ + mock.call("event", "3x4mpl3", "External analysis"), + mock.call("event", "3x4mpl3_2", "External analysis"), + mock.call("event", "3x4mpl3_3", "External analysis") + ]) + + assert r.misp.add_url.call_count == 2 + r.misp.add_url.assert_has_calls([ + mock.call("event", "example.com/gate.php"), + mock.call("event", "http://example.org") + ]) + + r.misp.add_mutex.assert_called_once_with("event", "@@@@@@") + r.misp.add_useragent.assert_called_once_with("event", "M3mebr0wz0r V42") + @mock.patch("cuckoo.reporting.mongodb.mongo") def test_mongodb_init_once_new(p): p.init.return_value = True diff --git a/tests/test_resultserver.py b/tests/test_resultserver.py index b79ced476f..fa9900f4b5 100644 --- a/tests/test_resultserver.py +++ b/tests/test_resultserver.py @@ -1,85 +1,256 @@ -# Copyright (C) 2017 Cuckoo Foundation. +# Copyright (C) 2017-2018 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. -import logging +from __future__ import print_function + +# Testing TODO: +# - Socket timeout, cleanup +# - Task cleanup +# - Invalid path tests +# - Double LOG command + +import errno +import json import mock +import platform import pytest +import shutil +import socket import tempfile from cuckoo.common.exceptions import CuckooOperationalError -from cuckoo.core.log import task_log_start, task_log_stop -from cuckoo.core.resultserver import ResultHandler, FileUpload -from cuckoo.core.startup import init_logging +from cuckoo.common.files import Folders +from cuckoo.core.resultserver import FileUpload, LogHandler, BsonStore +from cuckoo.core.resultserver import GeventResultServerWorker +from cuckoo.core.resultserver import HandlerContext, netlog_sanitize_fname +from cuckoo.core.resultserver import RESULT_DIRECTORIES, MAX_NETLOG_LINE from cuckoo.main import cuckoo_create -from cuckoo.misc import mkdir, set_cwd, cwd +from cuckoo.misc import set_cwd, cwd -@mock.patch("cuckoo.core.resultserver.select") -def test_open_process_log_unicode(p): - set_cwd(tempfile.mkdtemp()) +@pytest.fixture(scope='module') +def cuckoo_cwd(): + """Create a temporary Cuckoo working directory""" + path = tempfile.mkdtemp() + print('Temporary path:', path) + set_cwd(path) cuckoo_create() - mkdir(cwd(analysis=1)) - mkdir(cwd("logs", analysis=1)) + anal_path = cwd(analysis=1) + Folders.create(anal_path, RESULT_DIRECTORIES) + yield path + shutil.rmtree(path) - request = server = mock.MagicMock() +def mock_handler_context(klass, path, lines, data, version=None): + class FakeContext: + storagepath = path + buf = '' + task_id = 1 - class Handler(ResultHandler): - storagepath = cwd(analysis=1) + def read_newline(self): + if not lines: + raise EOFError + return lines.pop(0) - def handle(self): - pass + def read(self, size=None): + if not data: + raise EOFError + return data.pop(0) - init_logging(logging.DEBUG) + def copy_to_fd(self, fd, max_size=None): + while True: + try: + fd.write(self.read()) + except EOFError: + break - try: - task_log_start(1) - Handler(request, (None, None), server).open_process_log({ - "pid": 1, "ppid": 2, "process_name": u"\u202e", "track": True, - }) - finally: - task_log_stop(1) + ctx = FakeContext() + ctx.sock = mock.Mock() + h = klass(1, ctx, version) + h.init() + h.handle() + h.close() + return h -class TestFileUpload(object): - def fileupload(self, handler): - set_cwd(tempfile.mkdtemp()) - cuckoo_create() - mkdir(cwd(analysis=1)) - mkdir(cwd("logs", analysis=1)) - - handler.storagepath = cwd(analysis=1) - fu = FileUpload(handler, None) - fu.init() - for x in fu: - pass - fu.close() +class TestHandlerContext(object): + def test_pointless_busywork(self): + sock = mock.Mock() + h = HandlerContext(1, 'does-not-exist', sock) + assert repr(h) == '' - def test_success(self): - class Handler(object): - reads = [ - "this", "is", "a", "test", None - ] + h.cancel() + sock.shutdown.assert_called_with(socket.SHUT_RD) - def read_newline(self, strip): - return "logs/1.log" + # Should not raise + sock.shutdown.side_effect = socket.error() + h.cancel() + + err = socket.error() + err.errno = errno.ECONNRESET + sock.recv.side_effect = err + assert h.read() == '' + + err = socket.error() + err.errno = errno.EPIPE + sock.recv.side_effect = err + with pytest.raises(socket.error) as e: + h.read() + + def test_long_line(self): + sock = mock.Mock() + h = HandlerContext(1, 'does-not-exist', sock) + sock.recv.return_value = 'A' * (MAX_NETLOG_LINE + 1) + with pytest.raises(CuckooOperationalError) as e: + h.read_newline() + assert h.buf is sock.recv.return_value - def read_any(self): - return self.reads.pop(0) + def test_line_eof(self): + sock = mock.Mock() + h = HandlerContext(1, 'does-not-exist', sock) + sock.recv.return_value = '' + with pytest.raises(EOFError) as e: + h.read_newline() - self.fileupload(Handler()) + def test_buffer(self): + sock = mock.Mock() + h = HandlerContext(1, 'does-not-exist', sock) + sock.recv.return_value = 'first\nsecond\nthird' + assert h.read_newline() == 'first' + assert h.buf == 'second\nthird' + assert h.drain_buffer() == 'second\nthird' + assert h.buf is None - with open(cwd("logs", "1.log", analysis=1), "rb") as f: + def test_copy_limited(self): + sock = mock.Mock() + fd = mock.Mock() + h = HandlerContext(1, 'does-not-exist', sock) + sock.recv.side_effect = ['A' * 64, ''] + h.copy_to_fd(fd, 32) + fd.write.assert_has_calls([mock.call('A' * 32), + mock.call('... (truncated)')]) + assert fd.flush.called + +@pytest.mark.usefixtures('cuckoo_cwd') +class TestFileUpload(object): + @pytest.mark.order1 + def test_success_noversion(self): + fu = mock_handler_context(FileUpload, + cwd(analysis=1), + ['files/1.exe'], + ['this', 'is', 'a', 'test']) + + with open(cwd("files", "1.exe", analysis=1), "rb") as f: assert f.read() == "thisisatest" - def invalid_path(self, path): - class Handler(object): - def read_newline(self, strip): - return path + with open(fu.filelog) as f: + lines = f.readlines() + blob = json.loads(lines[-1]) + assert blob['filepath'] is None + assert blob['path'] == "files/1.exe" + assert blob["pids"] == [] + @pytest.mark.order2 + def test_overwrite(self): with pytest.raises(CuckooOperationalError) as e: - self.fileupload(Handler()) + mock_handler_context(FileUpload, + cwd(analysis=1), + ['files/1.exe'], + []) + e.match("overwrite an existing file") + + @mock.patch('cuckoo.core.resultserver.open_exclusive') + def test_open_error(self, open_exclusive): + err = OSError() + err.errno = errno.EACCES + open_exclusive.side_effect = err + with pytest.raises(OSError): + mock_handler_context(FileUpload, + cwd(analysis=1), + ['files/any.exe'], + []) + + def test_success_v2(self): + fu = mock_handler_context(FileUpload, + cwd(analysis=1), + ['files/2.exe', 'C:\\RealFilename.exe', + '11 12'], + ['second', 'test'], + 2) + + with open(cwd("files", "2.exe", analysis=1), "rb") as f: + assert f.read() == "secondtest" + + with open(fu.filelog) as f: + lines = f.readlines() + blob = json.loads(lines[-1]) + assert blob['filepath'] == 'C:\\RealFilename.exe' + assert blob['path'] == "files/2.exe" + assert blob["pids"] == [11, 12] + + def invalid_path(self, path): + with pytest.raises(CuckooOperationalError) as e: + mock_handler_context(FileUpload, cwd(analysis=1), [path], []) e.match("banned path") def test_invalid_paths(self): + self.invalid_path("dummy") + self.invalid_path("notallowed/path.exe") + self.invalid_path("shots/notallowed/path.jpg") + self.invalid_path("reports/report.json") self.invalid_path("/tmp/foobar") self.invalid_path("../hello") self.invalid_path("../../foobar") + + def test_banned_names(self): + assert netlog_sanitize_fname("files/file1.exe") == "files/file1.exe" + assert netlog_sanitize_fname("files/p\x00ath.exe") == "files/pXath.exe" + assert netlog_sanitize_fname("files/path.exe:$DATA") == "files/path.exeX$DATA" + assert netlog_sanitize_fname("files/file.\x00.exe:$DATA") == "files/file.X.exeX$DATA" + +@pytest.mark.usefixtures('cuckoo_cwd') +class TestLogHandler(object): + @pytest.mark.order1 + def test_success(self): + mock_handler_context(LogHandler, + cwd(analysis=1), + [], + ['first\n', 'second\n']) + + with open(cwd("analysis.log", analysis=1), "rb") as f: + if platform.system() == "Windows": + assert f.read() == "first\r\nsecond\r\n" + else: + assert f.read() == "first\nsecond\n" + +@pytest.mark.usefixtures('cuckoo_cwd') +class TestBsonStore(object): + def test_success(self): + mock_handler_context(BsonStore, + cwd(analysis=1), + [], + ['\x01\x00\x00\x00', 'a'], + 1) + + with open(cwd("logs/1.bson", analysis=1), "rb") as f: + assert f.read() == "\x01\x00\x00\x00a" + + def test_unversioned(self): + h = mock_handler_context(BsonStore, cwd(analysis=1), [], [], None) + assert h.fd is None + +# Work in progress +@pytest.mark.usefixtures('cuckoo_cwd') +class TestWorkerServer(object): + def test_unregistered(self): + g = GeventResultServerWorker(('127.0.0.1', 1)) + sock = mock.Mock() + sock.recv.side_effect = IOError + g.handle(sock, ('127.0.0.1', 41337)) + # + + def test_negotiate(self): + g = GeventResultServerWorker(('127.0.0.1', 1)) + g.add_task(1, '127.0.0.1') + assert g.tasks == {'127.0.0.1': 1} + sock = mock.Mock() + sock.recv.side_effect = ["FILE\n", "files/example.txt\n", "hello", ""] + g.handle(sock, ('127.0.0.1', 41337)) diff --git a/tests/test_rooter.py b/tests/test_rooter.py index 2950bdf4d2..ea21cc90ab 100644 --- a/tests/test_rooter.py +++ b/tests/test_rooter.py @@ -76,13 +76,17 @@ def test_vpn_disable(): def test_forward_drop(): with mock.patch("cuckoo.apps.rooter.run") as p: r.forward_drop() - p.assert_called_once_with(None, "-P", "FORWARD", "DROP") + p.assert_called_once_with( + None, "-P", "FORWARD", "DROP", "-m", "comment", "--comment", + "cuckoo-rooter" + ) def test_enable_nat(): with mock.patch("cuckoo.apps.rooter.run") as p: r.enable_nat("foo") p.assert_called_once_with( - None, "-t", "nat", "-A", "POSTROUTING", "-o", "foo", "-j", "MASQUERADE" + None, "-t", "nat", "-A", "POSTROUTING", "-o", "foo", "-j", + "MASQUERADE", "-m", "comment", "--comment", "cuckoo-rooter" ) @mock.patch("cuckoo.apps.rooter.run") @@ -94,7 +98,8 @@ def test_disable_nat(p): assert p.call_count == 2 assert p.call_list[0] == p.call_list[1] p.assert_any_call( - None, "-t", "nat", "-D", "POSTROUTING", "-o", "foo", "-j", "MASQUERADE" + None, "-t", "nat", "-D", "POSTROUTING", "-o", "foo", "-j", + "MASQUERADE", "-m", "comment", "--comment", "cuckoo-rooter" ) # TODO init_rttable diff --git a/tests/test_signatures.py b/tests/test_signatures.py index be7f599a83..e574153d03 100644 --- a/tests/test_signatures.py +++ b/tests/test_signatures.py @@ -159,7 +159,7 @@ class sig_windows_platform(object): assert rs.should_enable_signature(sig_windows_platform()) def test_signature_order(): - class sig(object): + class sig(Signature): enabled = True minimum = "2.0.0" maximum = None @@ -181,6 +181,8 @@ class sig3(sig): name = "sig3" order = 2 + set_cwd(tempfile.mkdtemp()) + cuckoo_create() with mock.patch("cuckoo.core.plugins.cuckoo") as p: p.signatures = sig1, sig2, sig3 RunSignatures.init_once() @@ -191,7 +193,7 @@ class sig3(sig): assert isinstance(rs.signatures[2], sig1) class test_call_signature(): - class sig(object): + class sig(Signature): enabled = True name = "sig" minimum = "2.0.0" @@ -206,6 +208,8 @@ def __init__(self, caller): def on_signature(self, sig): pass + set_cwd(tempfile.mkdtemp()) + cuckoo_create() with mock.patch("cuckoo.core.plugins.cuckoo") as p: p.signatures = sig, RunSignatures.init_once() @@ -350,7 +354,7 @@ def test_on_yara(): "vmware1": [(0, 0)], } - class sig1(object): + class sig1(Signature): name = "sig1" @property @@ -377,7 +381,7 @@ def on_extract(self, match): rs = RunSignatures(results) - rs.signatures = sig1(), + rs.signatures = sig1(rs), rs.run() assert sig1.on_yara.call_count == 3 @@ -481,3 +485,80 @@ def test_check_command_line(self): }) r.check_command_line("foo") == "foo" r.check_command_line("ar$", regex=True) == "bar" + +def classes(objects): + return [s.__class__ for s in objects] + +def lazy_compare(unsorted, sorted_lst): + items = sorted(unsorted, key=lambda v: repr(v)) + assert items == sorted_lst + +class TestRunSignatures(object): + _fake_results = { + "behavior": { + "processes": [{ + "process_path": "C:\\Temp\\malware.exe", + "pid": 123, + "calls": [ + {"category": "system", "api": "LdrLoadDll"}, + {"category": "registry", "api": "GetSystemTimeAsFileTime"}, + {"category": "xundefined", "api": "XUndefined"} + ] + }] + } + } + + def _runner(self, signatures): + class TestRunSignatures(RunSignatures): + available_signatures = signatures + return TestRunSignatures(self._fake_results) + + def test_yield_calls(self): + pass + + def test_dispatch_building(self): + class SignatureAll(Signature): + def on_call(self, call, proc): + pass + class SignatureDispatch(Signature): + def on_call_LdrLoadDll(self, call, proc): + pass + class SignatureAPI(Signature): + filter_apinames = ["GetSystemTimeAsFileTime"] + def on_call(self, call, proc): + pass + class SignatureCategory(Signature): + filter_categories = ["system"] + def on_call(self, call, proc): + pass + class SignatureDummy(Signature): + pass + + r = self._runner([SignatureAPI, SignatureAll, SignatureCategory, + SignatureDispatch, SignatureDummy]) + assert len(r.signatures) == 5 + lazy_compare( + r.call_for_api.keys(), + ["GetSystemTimeAsFileTime", "LdrLoadDll"] + ) + assert classes(r.call_for_api["LdrLoadDll"]) == [SignatureDispatch] + assert classes(r.call_for_cat["system"]) == [SignatureCategory] + assert SignatureDummy not in classes(r.call_always) + for v in r.call_for_api.values(): + assert SignatureDummy not in classes(v) + for v in r.call_for_cat.values(): + assert SignatureDummy not in classes(v) + assert not r.api_sigs + r.run() + lazy_compare( + r.api_sigs.keys(), + ["GetSystemTimeAsFileTime", "LdrLoadDll", "XUndefined"] + ) + lazy_compare( + classes(r.api_sigs["GetSystemTimeAsFileTime"]), + [SignatureAPI, SignatureAll] + ) + lazy_compare( + classes(r.api_sigs["LdrLoadDll"]), + [SignatureAll, SignatureCategory, SignatureDispatch] + ) diff --git a/tests/test_startup.py b/tests/test_startup.py index a14fb87471..419792fcb7 100644 --- a/tests/test_startup.py +++ b/tests/test_startup.py @@ -88,7 +88,6 @@ def log(fmt, *args): logs = "\n".join(logs) assert "KVM" in logs assert "Xen" in logs - assert "CreatesExe" in logs assert "SystemMetrics" in logs @mock.patch("cuckoo.core.startup.cuckoo") diff --git a/tests/test_submit.py b/tests/test_submit.py index 10d6517176..dade71be12 100644 --- a/tests/test_submit.py +++ b/tests/test_submit.py @@ -38,7 +38,7 @@ def setup(self): self.submit_manager = SubmitManager() def test_pre_file(self): - """Tests the submission of a plaintext file""" + """Test the submission of a plaintext file""" assert self.submit_manager.pre(submit_type="files", data=[{ "name": "foo.txt", "data": open("tests/files/foo.txt", "rb").read() @@ -54,7 +54,7 @@ def test_pre_file(self): assert filedata == open("tests/files/foo.txt", "rb").read() def test_pre_url(self): - """Tests the submission of URLs (http/https)""" + """Test the submission of URLs (http/https)""" assert self.submit_manager.pre(submit_type="strings", data=[ "http://theguardian.com/", "https://news.ycombinator.com/", @@ -83,7 +83,7 @@ def test_invalid_strings(self): @responses.activate def test_pre_hash(self): - """Tests the submission of a VirusTotal hash.""" + """Test the submission of a VirusTotal hash.""" with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps: rsps.add( responses.GET, VirusTotalAPI.HASH_DOWNLOAD, body="A"*1024*1024 @@ -418,6 +418,14 @@ def test_option_translations_from(): "remotecontrol": "yes", } + assert sm.translate_options_from({}, { + "simulated-human-interaction": False, + "function": "DoStuff", + "json.calls": "0" + }) == { + "human": 0, "function": "DoStuff", "json.calls": "0" + } + def test_option_translations_to(): sm = SubmitManager() @@ -440,3 +448,11 @@ def test_option_translations_to(): }) == { "remote-control": True, } + + assert sm.translate_options_to({ + "human": "0", "function": "DoStuff", "json.calls": "0" + }) == { + "simulated-human-interaction": False, + "function": "DoStuff", + "json.calls": "0" + } diff --git a/tests/test_utils.py b/tests/test_utils.py index 01383b3a83..6f58a27e11 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -15,8 +15,10 @@ import cuckoo from cuckoo.common.exceptions import CuckooOperationalError -from cuckoo.common.files import Folders, Files, Storage, temppath -from cuckoo.common.whitelist import is_whitelisted_domain +from cuckoo.common.files import ( + Folders, Files, Storage, temppath, open_exclusive +) +from cuckoo.common.safelist import is_safelisted_domain from cuckoo.common import utils from cuckoo.main import cuckoo_create from cuckoo.misc import set_cwd, getuser @@ -26,19 +28,19 @@ def setup(self): self.tmp_dir = tempfile.gettempdir() def test_root_folder(self): - """Tests a single folder creation based on the root parameter.""" + """Test single folder creation based on the root parameter.""" Folders.create(os.path.join(self.tmp_dir, "foo")) assert os.path.exists(os.path.join(self.tmp_dir, "foo")) os.rmdir(os.path.join(self.tmp_dir, "foo")) def test_single_folder(self): - """Tests a single folder creation.""" + """Test single folder creation.""" Folders.create(self.tmp_dir, "foo") assert os.path.exists(os.path.join(self.tmp_dir, "foo")) os.rmdir(os.path.join(self.tmp_dir, "foo")) def test_multiple_folders(self): - """Tests multiple folders creation.""" + """Test multiple folder creation.""" Folders.create(self.tmp_dir, ["foo", "bar"]) assert os.path.exists(os.path.join(self.tmp_dir, "foo")) assert os.path.exists(os.path.join(self.tmp_dir, "bar")) @@ -46,7 +48,7 @@ def test_multiple_folders(self): os.rmdir(os.path.join(self.tmp_dir, "bar")) def test_copy_folder(self): - """Tests recursive folder copy""" + """Test recursive folder copy.""" dirpath = tempfile.mkdtemp() set_cwd(dirpath) @@ -54,21 +56,21 @@ def test_copy_folder(self): assert os.path.isfile("%s/reports/report.json" % dirpath) def test_duplicate_folder(self): - """Tests a duplicate folder creation.""" + """Test a duplicate folder creation.""" Folders.create(self.tmp_dir, "foo") assert os.path.exists(os.path.join(self.tmp_dir, "foo")) Folders.create(self.tmp_dir, "foo") os.rmdir(os.path.join(self.tmp_dir, "foo")) def test_delete_folder(self): - """Tests folder deletion #1.""" + """Test folder deletion #1.""" Folders.create(self.tmp_dir, "foo") assert os.path.exists(os.path.join(self.tmp_dir, "foo")) Folders.delete(os.path.join(self.tmp_dir, "foo")) assert not os.path.exists(os.path.join(self.tmp_dir, "foo")) def test_delete_folder2(self): - """Tests folder deletion #2.""" + """Test folder deletion #2.""" Folders.create(self.tmp_dir, "foo") assert os.path.exists(os.path.join(self.tmp_dir, "foo")) Folders.delete(self.tmp_dir, "foo") @@ -433,7 +435,15 @@ def test_list_of(): assert utils.list_of_ints([1, 2]) is True assert utils.list_of_ints([lambda x: x]) is False -def test_is_whitelisted_domain(): - assert is_whitelisted_domain("java.com") is True - assert is_whitelisted_domain("java2.com") is False - assert is_whitelisted_domain("crl.microsoft.com") is True +def test_is_safelisted_domain(): + assert is_safelisted_domain("java.com") is True + assert is_safelisted_domain("java2.com") is False + assert is_safelisted_domain("crl.microsoft.com") is True + +def test_open_exclusive(): + fpath = os.path.join(tempfile.mkdtemp(), "yeet.exclusive") + with open(fpath, "wb") as fp: + fp.write("42421337Test") + + with pytest.raises(OSError): + open_exclusive(fpath, bufsize=1) diff --git a/tests/test_web.py b/tests/test_web.py index d625d771c1..40e56d5d16 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -1274,6 +1274,11 @@ def test_status(self, p, client): "enabled": True, }, }, + "cuckoo": { + "cuckoo": { + "ignore_vulnerabilities": True, + }, + } }) db.connect() r = client.get("/cuckoo/api/status/") @@ -1282,7 +1287,14 @@ def test_status(self, p, client): def test_api_status200(self, client): set_cwd(tempfile.mkdtemp()) - cuckoo_create() + + cuckoo_create(cfg={ + "cuckoo": { + "cuckoo": { + "ignore_vulnerabilities": True, + }, + }, + }) Database().connect() r = client.get("/cuckoo/api/status") assert r.status_code == 200 diff --git a/tests/utils.py b/tests/utils.py index 323861f3cd..7d32016f34 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -21,7 +21,7 @@ def __exit__(self, type_, value, traceback): os.chdir(self.origpath) def init_analysis(task_id, package, *filename): - """Initializes an analysis with an "encrypted" binary from tests/files/.""" + """Initialize an analysis with an "encrypted" binary from tests/files/.""" mkdir(cwd(analysis=task_id)) content = open(os.path.join("tests", "files", *filename), "rb").read() open(cwd("binary", analysis=task_id), "wb").write(content[::-1])